blob: b1a263137a23391a1e19c2589f35fdaf2c4f514f [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
Christoph Hellwigd2852032014-01-17 12:06:53 +01002 * Copyright (C) 1999 Eric Youngdale
3 * Copyright (C) 2014 Christoph Hellwig
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 *
5 * SCSI queueing library.
6 * Initial versions: Eric Youngdale (eric@andante.org).
7 * Based upon conversations with large numbers
8 * of people at Linux Expo.
9 */
10
11#include <linux/bio.h>
James Bottomleyd3f46f32008-01-15 11:11:46 -060012#include <linux/bitops.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070013#include <linux/blkdev.h>
14#include <linux/completion.h>
15#include <linux/kernel.h>
Paul Gortmaker09703662011-05-27 09:37:25 -040016#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/mempool.h>
18#include <linux/slab.h>
19#include <linux/init.h>
20#include <linux/pci.h>
21#include <linux/delay.h>
James Bottomleyfaead262006-02-14 10:42:07 -060022#include <linux/hardirq.h>
Jens Axboec6132da2007-10-16 11:08:49 +020023#include <linux/scatterlist.h>
Christoph Hellwigd2852032014-01-17 12:06:53 +010024#include <linux/blk-mq.h>
Hannes Reineckef1569ff2014-10-24 14:27:07 +020025#include <linux/ratelimit.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070026
27#include <scsi/scsi.h>
Christoph Hellwigbeb40482006-06-10 18:01:03 +020028#include <scsi/scsi_cmnd.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <scsi/scsi_dbg.h>
30#include <scsi/scsi_device.h>
31#include <scsi/scsi_driver.h>
32#include <scsi/scsi_eh.h>
33#include <scsi/scsi_host.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070034
Christoph Hellwig3b5382c2014-05-06 12:25:40 +020035#include <trace/events/scsi.h>
36
Linus Torvalds1da177e2005-04-16 15:20:36 -070037#include "scsi_priv.h"
38#include "scsi_logging.h"
39
40
Tobias Klauser6391a112006-06-08 22:23:48 -070041#define SG_MEMPOOL_NR ARRAY_SIZE(scsi_sg_pools)
Jens Axboe59725112007-04-02 10:06:42 +020042#define SG_MEMPOOL_SIZE 2
Linus Torvalds1da177e2005-04-16 15:20:36 -070043
44struct scsi_host_sg_pool {
45 size_t size;
Jens Axboea8474ce2007-08-07 09:02:51 +020046 char *name;
Christoph Lametere18b8902006-12-06 20:33:20 -080047 struct kmem_cache *slab;
Linus Torvalds1da177e2005-04-16 15:20:36 -070048 mempool_t *pool;
49};
50
Mark Rustad037e6d82014-10-14 06:38:53 -070051#define SP(x) { .size = x, "sgpool-" __stringify(x) }
James Bottomleyd3f46f32008-01-15 11:11:46 -060052#if (SCSI_MAX_SG_SEGMENTS < 32)
53#error SCSI_MAX_SG_SEGMENTS is too small (must be 32 or greater)
54#endif
Adrian Bunk52c1da32005-06-23 22:05:33 -070055static struct scsi_host_sg_pool scsi_sg_pools[] = {
Linus Torvalds1da177e2005-04-16 15:20:36 -070056 SP(8),
57 SP(16),
FUJITA Tomonorifd820f42007-09-18 12:14:37 +020058#if (SCSI_MAX_SG_SEGMENTS > 32)
James Bottomleyd3f46f32008-01-15 11:11:46 -060059 SP(32),
FUJITA Tomonorifd820f42007-09-18 12:14:37 +020060#if (SCSI_MAX_SG_SEGMENTS > 64)
James Bottomleyd3f46f32008-01-15 11:11:46 -060061 SP(64),
62#if (SCSI_MAX_SG_SEGMENTS > 128)
Linus Torvalds1da177e2005-04-16 15:20:36 -070063 SP(128),
James Bottomleyd3f46f32008-01-15 11:11:46 -060064#if (SCSI_MAX_SG_SEGMENTS > 256)
65#error SCSI_MAX_SG_SEGMENTS is too large (256 MAX)
FUJITA Tomonorifd820f42007-09-18 12:14:37 +020066#endif
67#endif
68#endif
James Bottomleyd3f46f32008-01-15 11:11:46 -060069#endif
70 SP(SCSI_MAX_SG_SEGMENTS)
Jens Axboea8474ce2007-08-07 09:02:51 +020071};
Linus Torvalds1da177e2005-04-16 15:20:36 -070072#undef SP
73
Martin K. Petersen7027ad72008-07-17 17:08:48 -040074struct kmem_cache *scsi_sdb_cache;
Boaz Harrosh6f9a35e2007-12-13 13:50:53 +020075
Jens Axboea488e742010-04-16 21:13:15 +020076/*
77 * When to reinvoke queueing after a resource shortage. It's 3 msecs to
78 * not change behaviour from the previous unplug mechanism, experimentation
79 * may prove this needs changing.
80 */
81#define SCSI_QUEUE_DELAY 3
82
Christoph Hellwigde3e8bf2014-01-23 13:55:38 +010083static void
84scsi_set_blocked(struct scsi_cmnd *cmd, int reason)
Linus Torvalds1da177e2005-04-16 15:20:36 -070085{
86 struct Scsi_Host *host = cmd->device->host;
87 struct scsi_device *device = cmd->device;
Mike Christief0c0a372008-08-17 15:24:38 -050088 struct scsi_target *starget = scsi_target(device);
Linus Torvalds1da177e2005-04-16 15:20:36 -070089
90 /*
Tejun Heo d8c37e72005-05-14 00:46:08 +090091 * Set the appropriate busy bit for the device/host.
Linus Torvalds1da177e2005-04-16 15:20:36 -070092 *
93 * If the host/device isn't busy, assume that something actually
94 * completed, and that we should be able to queue a command now.
95 *
96 * Note that the prior mid-layer assumption that any host could
97 * always queue at least one command is now broken. The mid-layer
98 * will implement a user specifiable stall (see
99 * scsi_host.max_host_blocked and scsi_device.max_device_blocked)
100 * if a command is requeued with no other commands outstanding
101 * either for the device or for the host.
102 */
Mike Christief0c0a372008-08-17 15:24:38 -0500103 switch (reason) {
104 case SCSI_MLQUEUE_HOST_BUSY:
Christoph Hellwigcd9070c2014-01-23 12:07:41 +0100105 atomic_set(&host->host_blocked, host->max_host_blocked);
Mike Christief0c0a372008-08-17 15:24:38 -0500106 break;
107 case SCSI_MLQUEUE_DEVICE_BUSY:
James Smart573e5912011-07-06 12:45:17 -0400108 case SCSI_MLQUEUE_EH_RETRY:
Christoph Hellwigcd9070c2014-01-23 12:07:41 +0100109 atomic_set(&device->device_blocked,
110 device->max_device_blocked);
Mike Christief0c0a372008-08-17 15:24:38 -0500111 break;
112 case SCSI_MLQUEUE_TARGET_BUSY:
Christoph Hellwigcd9070c2014-01-23 12:07:41 +0100113 atomic_set(&starget->target_blocked,
114 starget->max_target_blocked);
Mike Christief0c0a372008-08-17 15:24:38 -0500115 break;
116 }
Christoph Hellwigde3e8bf2014-01-23 13:55:38 +0100117}
118
Christoph Hellwigd2852032014-01-17 12:06:53 +0100119static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd)
120{
121 struct scsi_device *sdev = cmd->device;
122 struct request_queue *q = cmd->request->q;
123
124 blk_mq_requeue_request(cmd->request);
125 blk_mq_kick_requeue_list(q);
126 put_device(&sdev->sdev_gendev);
127}
128
Christoph Hellwigde3e8bf2014-01-23 13:55:38 +0100129/**
130 * __scsi_queue_insert - private queue insertion
131 * @cmd: The SCSI command being requeued
132 * @reason: The reason for the requeue
133 * @unbusy: Whether the queue should be unbusied
134 *
135 * This is a private queue insertion. The public interface
136 * scsi_queue_insert() always assumes the queue should be unbusied
137 * because it's always called before the completion. This function is
138 * for a requeue after completion, which should only occur in this
139 * file.
140 */
141static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy)
142{
143 struct scsi_device *device = cmd->device;
144 struct request_queue *q = device->request_queue;
145 unsigned long flags;
146
147 SCSI_LOG_MLQUEUE(1, scmd_printk(KERN_INFO, cmd,
148 "Inserting command %p into mlqueue\n", cmd));
149
150 scsi_set_blocked(cmd, reason);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151
152 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700153 * Decrement the counters, since these commands are no longer
154 * active on the host/device.
155 */
James Bottomley4f5299a2009-01-02 10:42:21 -0600156 if (unbusy)
157 scsi_device_unbusy(device);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158
159 /*
Tejun Heo a1bf9d1d2005-04-24 02:08:52 -0500160 * Requeue this command. It will go before all other commands
Bart Van Asscheb4854622012-06-29 15:36:07 +0000161 * that are already in the queue. Schedule requeue work under
162 * lock such that the kblockd_schedule_work() call happens
163 * before blk_cleanup_queue() finishes.
Jens Axboea488e742010-04-16 21:13:15 +0200164 */
Alan Stern644373a2014-03-28 10:51:15 -0700165 cmd->result = 0;
Christoph Hellwigd2852032014-01-17 12:06:53 +0100166 if (q->mq_ops) {
167 scsi_mq_requeue_cmd(cmd);
168 return;
169 }
Tejun Heo a1bf9d1d2005-04-24 02:08:52 -0500170 spin_lock_irqsave(q->queue_lock, flags);
James Bottomley59897da2005-09-14 12:57:42 -0400171 blk_requeue_request(q, cmd->request);
Jens Axboe59c3d452014-04-08 09:15:35 -0600172 kblockd_schedule_work(&device->requeue_work);
Bart Van Asscheb4854622012-06-29 15:36:07 +0000173 spin_unlock_irqrestore(q->queue_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700174}
175
James Bottomley4f5299a2009-01-02 10:42:21 -0600176/*
177 * Function: scsi_queue_insert()
178 *
179 * Purpose: Insert a command in the midlevel queue.
180 *
181 * Arguments: cmd - command that we are adding to queue.
182 * reason - why we are inserting command to queue.
183 *
184 * Lock status: Assumed that lock is not held upon entry.
185 *
186 * Returns: Nothing.
187 *
188 * Notes: We do this for one of two cases. Either the host is busy
189 * and it cannot accept any more commands for the time being,
190 * or the device returned QUEUE_FULL and can accept no more
191 * commands.
192 * Notes: This could be called either from an interrupt context or a
193 * normal process context.
194 */
Bart Van Assche84feb162012-06-29 15:35:05 +0000195void scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
James Bottomley4f5299a2009-01-02 10:42:21 -0600196{
Bart Van Assche84feb162012-06-29 15:35:05 +0000197 __scsi_queue_insert(cmd, reason, 1);
James Bottomley4f5299a2009-01-02 10:42:21 -0600198}
James Bottomley39216032005-06-15 18:48:29 -0500199/**
James Bottomley33aa6872005-08-28 11:31:14 -0500200 * scsi_execute - insert request and wait for the result
James Bottomley39216032005-06-15 18:48:29 -0500201 * @sdev: scsi device
202 * @cmd: scsi command
203 * @data_direction: data direction
204 * @buffer: data buffer
205 * @bufflen: len of buffer
206 * @sense: optional sense buffer
207 * @timeout: request timeout in seconds
208 * @retries: number of times to retry request
James Bottomley33aa6872005-08-28 11:31:14 -0500209 * @flags: or into request flags;
FUJITA Tomonorif4f4e472008-12-04 14:24:39 +0900210 * @resid: optional residual length
James Bottomley39216032005-06-15 18:48:29 -0500211 *
Michael Opdenacker59c51592007-05-09 08:57:56 +0200212 * returns the req->errors value which is the scsi_cmnd result
James Bottomleyea73a9f2005-08-28 11:33:52 -0500213 * field.
Rob Landleyeb448202007-11-03 13:30:39 -0500214 */
James Bottomley33aa6872005-08-28 11:31:14 -0500215int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
216 int data_direction, void *buffer, unsigned bufflen,
Martin K. Petersen2bfad212014-04-09 22:20:48 -0400217 unsigned char *sense, int timeout, int retries, u64 flags,
FUJITA Tomonorif4f4e472008-12-04 14:24:39 +0900218 int *resid)
James Bottomley39216032005-06-15 18:48:29 -0500219{
220 struct request *req;
221 int write = (data_direction == DMA_TO_DEVICE);
222 int ret = DRIVER_ERROR << 24;
223
224 req = blk_get_request(sdev->request_queue, write, __GFP_WAIT);
Joe Lawrencea492f072014-08-28 08:15:21 -0600225 if (IS_ERR(req))
James Bottomleybfe159a2011-07-07 15:45:40 -0500226 return ret;
Jens Axboef27b0872014-06-06 07:57:37 -0600227 blk_rq_set_block_pc(req);
James Bottomley39216032005-06-15 18:48:29 -0500228
229 if (bufflen && blk_rq_map_kern(sdev->request_queue, req,
230 buffer, bufflen, __GFP_WAIT))
231 goto out;
232
233 req->cmd_len = COMMAND_SIZE(cmd[0]);
234 memcpy(req->cmd, cmd, req->cmd_len);
235 req->sense = sense;
236 req->sense_len = 0;
Mike Christie17e01f22005-11-11 05:31:37 -0600237 req->retries = retries;
James Bottomley39216032005-06-15 18:48:29 -0500238 req->timeout = timeout;
Jens Axboe4aff5e22006-08-10 08:44:47 +0200239 req->cmd_flags |= flags | REQ_QUIET | REQ_PREEMPT;
James Bottomley39216032005-06-15 18:48:29 -0500240
241 /*
242 * head injection *required* here otherwise quiesce won't work
243 */
244 blk_execute_rq(req->q, NULL, req, 1);
245
Alan Sternbdb2b8c2008-06-24 14:03:14 -0400246 /*
247 * Some devices (USB mass-storage in particular) may transfer
248 * garbage data together with a residue indicating that the data
249 * is invalid. Prevent the garbage from being misinterpreted
250 * and prevent security leaks by zeroing out the excess data.
251 */
Tejun Heoc3a4d782009-05-07 22:24:37 +0900252 if (unlikely(req->resid_len > 0 && req->resid_len <= bufflen))
253 memset(buffer + (bufflen - req->resid_len), 0, req->resid_len);
Alan Sternbdb2b8c2008-06-24 14:03:14 -0400254
FUJITA Tomonorif4f4e472008-12-04 14:24:39 +0900255 if (resid)
Tejun Heoc3a4d782009-05-07 22:24:37 +0900256 *resid = req->resid_len;
James Bottomley39216032005-06-15 18:48:29 -0500257 ret = req->errors;
258 out:
259 blk_put_request(req);
260
261 return ret;
262}
James Bottomley33aa6872005-08-28 11:31:14 -0500263EXPORT_SYMBOL(scsi_execute);
James Bottomley39216032005-06-15 18:48:29 -0500264
Lin Ming9b214932013-03-23 11:42:25 +0800265int scsi_execute_req_flags(struct scsi_device *sdev, const unsigned char *cmd,
James Bottomleyea73a9f2005-08-28 11:33:52 -0500266 int data_direction, void *buffer, unsigned bufflen,
FUJITA Tomonorif4f4e472008-12-04 14:24:39 +0900267 struct scsi_sense_hdr *sshdr, int timeout, int retries,
Martin K. Petersen2bfad212014-04-09 22:20:48 -0400268 int *resid, u64 flags)
James Bottomleyea73a9f2005-08-28 11:33:52 -0500269{
270 char *sense = NULL;
akpm@osdl.org1ccb48b2005-06-26 00:12:51 -0700271 int result;
272
James Bottomleyea73a9f2005-08-28 11:33:52 -0500273 if (sshdr) {
Jes Sorensen24669f752006-01-16 10:31:18 -0500274 sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_NOIO);
James Bottomleyea73a9f2005-08-28 11:33:52 -0500275 if (!sense)
276 return DRIVER_ERROR << 24;
James Bottomleyea73a9f2005-08-28 11:33:52 -0500277 }
akpm@osdl.org1ccb48b2005-06-26 00:12:51 -0700278 result = scsi_execute(sdev, cmd, data_direction, buffer, bufflen,
Lin Ming9b214932013-03-23 11:42:25 +0800279 sense, timeout, retries, flags, resid);
James Bottomleyea73a9f2005-08-28 11:33:52 -0500280 if (sshdr)
James Bottomleye5143852005-08-09 11:55:36 -0500281 scsi_normalize_sense(sense, SCSI_SENSE_BUFFERSIZE, sshdr);
James Bottomleyea73a9f2005-08-28 11:33:52 -0500282
283 kfree(sense);
284 return result;
285}
Lin Ming9b214932013-03-23 11:42:25 +0800286EXPORT_SYMBOL(scsi_execute_req_flags);
James Bottomleyea73a9f2005-08-28 11:33:52 -0500287
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288/*
289 * Function: scsi_init_cmd_errh()
290 *
291 * Purpose: Initialize cmd fields related to error handling.
292 *
293 * Arguments: cmd - command that is ready to be queued.
294 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700295 * Notes: This function has the job of initializing a number of
296 * fields related to error handling. Typically this will
297 * be called once for each command, as required.
298 */
Christoph Hellwig631c2282006-07-08 20:42:15 +0200299static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700300{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 cmd->serial_number = 0;
Boaz Harrosh30b0c372007-12-13 13:47:40 +0200302 scsi_set_resid(cmd, 0);
FUJITA Tomonorib80ca4f2008-01-13 15:46:13 +0900303 memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700304 if (cmd->cmd_len == 0)
Boaz Harroshdb4742d2008-04-30 11:27:26 +0300305 cmd->cmd_len = scsi_command_size(cmd->cmnd);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306}
307
308void scsi_device_unbusy(struct scsi_device *sdev)
309{
310 struct Scsi_Host *shost = sdev->host;
Mike Christief0c0a372008-08-17 15:24:38 -0500311 struct scsi_target *starget = scsi_target(sdev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 unsigned long flags;
313
Christoph Hellwig74665012014-01-22 15:29:29 +0100314 atomic_dec(&shost->host_busy);
Christoph Hellwig2ccbb002014-03-26 10:35:00 +0100315 if (starget->can_queue > 0)
316 atomic_dec(&starget->target_busy);
Christoph Hellwig74665012014-01-22 15:29:29 +0100317
James Bottomley939647e2005-09-18 15:05:20 -0500318 if (unlikely(scsi_host_in_recovery(shost) &&
Christoph Hellwig74665012014-01-22 15:29:29 +0100319 (shost->host_failed || shost->host_eh_scheduled))) {
320 spin_lock_irqsave(shost->host_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321 scsi_eh_wakeup(shost);
Christoph Hellwig74665012014-01-22 15:29:29 +0100322 spin_unlock_irqrestore(shost->host_lock, flags);
323 }
324
Christoph Hellwig71e75c92014-04-11 19:07:01 +0200325 atomic_dec(&sdev->device_busy);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700326}
327
Christoph Hellwigd2852032014-01-17 12:06:53 +0100328static void scsi_kick_queue(struct request_queue *q)
329{
330 if (q->mq_ops)
331 blk_mq_start_hw_queues(q);
332 else
333 blk_run_queue(q);
334}
335
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336/*
337 * Called for single_lun devices on IO completion. Clear starget_sdev_user,
338 * and call blk_run_queue for all the scsi_devices on the target -
339 * including current_sdev first.
340 *
341 * Called with *no* scsi locks held.
342 */
343static void scsi_single_lun_run(struct scsi_device *current_sdev)
344{
345 struct Scsi_Host *shost = current_sdev->host;
346 struct scsi_device *sdev, *tmp;
347 struct scsi_target *starget = scsi_target(current_sdev);
348 unsigned long flags;
349
350 spin_lock_irqsave(shost->host_lock, flags);
351 starget->starget_sdev_user = NULL;
352 spin_unlock_irqrestore(shost->host_lock, flags);
353
354 /*
355 * Call blk_run_queue for all LUNs on the target, starting with
356 * current_sdev. We race with others (to set starget_sdev_user),
357 * but in most cases, we will be first. Ideally, each LU on the
358 * target would get some limited time or requests on the target.
359 */
Christoph Hellwigd2852032014-01-17 12:06:53 +0100360 scsi_kick_queue(current_sdev->request_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361
362 spin_lock_irqsave(shost->host_lock, flags);
363 if (starget->starget_sdev_user)
364 goto out;
365 list_for_each_entry_safe(sdev, tmp, &starget->devices,
366 same_target_siblings) {
367 if (sdev == current_sdev)
368 continue;
369 if (scsi_device_get(sdev))
370 continue;
371
372 spin_unlock_irqrestore(shost->host_lock, flags);
Christoph Hellwigd2852032014-01-17 12:06:53 +0100373 scsi_kick_queue(sdev->request_queue);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374 spin_lock_irqsave(shost->host_lock, flags);
375
376 scsi_device_put(sdev);
377 }
378 out:
379 spin_unlock_irqrestore(shost->host_lock, flags);
380}
381
Christoph Hellwigcd9070c2014-01-23 12:07:41 +0100382static inline bool scsi_device_is_busy(struct scsi_device *sdev)
Kiyoshi Ueda9d112512008-10-04 14:11:06 -0400383{
Christoph Hellwigcd9070c2014-01-23 12:07:41 +0100384 if (atomic_read(&sdev->device_busy) >= sdev->queue_depth)
385 return true;
386 if (atomic_read(&sdev->device_blocked) > 0)
387 return true;
388 return false;
Kiyoshi Ueda9d112512008-10-04 14:11:06 -0400389}
390
Christoph Hellwigcd9070c2014-01-23 12:07:41 +0100391static inline bool scsi_target_is_busy(struct scsi_target *starget)
Mike Christief0c0a372008-08-17 15:24:38 -0500392{
Christoph Hellwig2ccbb002014-03-26 10:35:00 +0100393 if (starget->can_queue > 0) {
394 if (atomic_read(&starget->target_busy) >= starget->can_queue)
395 return true;
396 if (atomic_read(&starget->target_blocked) > 0)
397 return true;
398 }
Christoph Hellwigcd9070c2014-01-23 12:07:41 +0100399 return false;
Mike Christief0c0a372008-08-17 15:24:38 -0500400}
401
Christoph Hellwigcd9070c2014-01-23 12:07:41 +0100402static inline bool scsi_host_is_busy(struct Scsi_Host *shost)
Kiyoshi Ueda9d112512008-10-04 14:11:06 -0400403{
Christoph Hellwigcd9070c2014-01-23 12:07:41 +0100404 if (shost->can_queue > 0 &&
405 atomic_read(&shost->host_busy) >= shost->can_queue)
406 return true;
407 if (atomic_read(&shost->host_blocked) > 0)
408 return true;
409 if (shost->host_self_blocked)
410 return true;
411 return false;
Kiyoshi Ueda9d112512008-10-04 14:11:06 -0400412}
413
Christoph Hellwig21a05df2014-02-20 14:20:54 -0800414static void scsi_starved_list_run(struct Scsi_Host *shost)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700415{
Mike Christie2a3a59e2008-11-11 13:42:35 -0600416 LIST_HEAD(starved_list);
Christoph Hellwig21a05df2014-02-20 14:20:54 -0800417 struct scsi_device *sdev;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418 unsigned long flags;
419
Linus Torvalds1da177e2005-04-16 15:20:36 -0700420 spin_lock_irqsave(shost->host_lock, flags);
Mike Christie2a3a59e2008-11-11 13:42:35 -0600421 list_splice_init(&shost->starved_list, &starved_list);
422
423 while (!list_empty(&starved_list)) {
James Bottomleye2eb7242013-07-02 15:05:26 +0200424 struct request_queue *slq;
425
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 /*
427 * As long as shost is accepting commands and we have
428 * starved queues, call blk_run_queue. scsi_request_fn
429 * drops the queue_lock and can add us back to the
430 * starved_list.
431 *
432 * host_lock protects the starved_list and starved_entry.
433 * scsi_request_fn must get the host_lock before checking
434 * or modifying starved_list or starved_entry.
435 */
Mike Christie2a3a59e2008-11-11 13:42:35 -0600436 if (scsi_host_is_busy(shost))
Mike Christief0c0a372008-08-17 15:24:38 -0500437 break;
Mike Christief0c0a372008-08-17 15:24:38 -0500438
Mike Christie2a3a59e2008-11-11 13:42:35 -0600439 sdev = list_entry(starved_list.next,
440 struct scsi_device, starved_entry);
441 list_del_init(&sdev->starved_entry);
Mike Christief0c0a372008-08-17 15:24:38 -0500442 if (scsi_target_is_busy(scsi_target(sdev))) {
443 list_move_tail(&sdev->starved_entry,
444 &shost->starved_list);
445 continue;
446 }
447
James Bottomleye2eb7242013-07-02 15:05:26 +0200448 /*
449 * Once we drop the host lock, a racing scsi_remove_device()
450 * call may remove the sdev from the starved list and destroy
451 * it and the queue. Mitigate by taking a reference to the
452 * queue and never touching the sdev again after we drop the
453 * host lock. Note: if __scsi_remove_device() invokes
454 * blk_cleanup_queue() before the queue is run from this
455 * function then blk_run_queue() will return immediately since
456 * blk_cleanup_queue() marks the queue with QUEUE_FLAG_DYING.
457 */
458 slq = sdev->request_queue;
459 if (!blk_get_queue(slq))
460 continue;
461 spin_unlock_irqrestore(shost->host_lock, flags);
462
Christoph Hellwigd2852032014-01-17 12:06:53 +0100463 scsi_kick_queue(slq);
James Bottomleye2eb7242013-07-02 15:05:26 +0200464 blk_put_queue(slq);
465
466 spin_lock_irqsave(shost->host_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 }
Mike Christie2a3a59e2008-11-11 13:42:35 -0600468 /* put any unprocessed entries back */
469 list_splice(&starved_list, &shost->starved_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470 spin_unlock_irqrestore(shost->host_lock, flags);
Christoph Hellwig21a05df2014-02-20 14:20:54 -0800471}
472
473/*
474 * Function: scsi_run_queue()
475 *
476 * Purpose: Select a proper request queue to serve next
477 *
478 * Arguments: q - last request's queue
479 *
480 * Returns: Nothing
481 *
482 * Notes: The previous command was completely finished, start
483 * a new one if possible.
484 */
485static void scsi_run_queue(struct request_queue *q)
486{
487 struct scsi_device *sdev = q->queuedata;
488
489 if (scsi_target(sdev)->single_lun)
490 scsi_single_lun_run(sdev);
491 if (!list_empty(&sdev->host->starved_list))
492 scsi_starved_list_run(sdev->host);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493
Christoph Hellwigd2852032014-01-17 12:06:53 +0100494 if (q->mq_ops)
495 blk_mq_start_stopped_hw_queues(q, false);
496 else
497 blk_run_queue(q);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498}
499
Jens Axboe9937a5e2011-05-17 11:04:44 +0200500void scsi_requeue_run_queue(struct work_struct *work)
501{
502 struct scsi_device *sdev;
503 struct request_queue *q;
504
505 sdev = container_of(work, struct scsi_device, requeue_work);
506 q = sdev->request_queue;
507 scsi_run_queue(q);
508}
509
Linus Torvalds1da177e2005-04-16 15:20:36 -0700510/*
511 * Function: scsi_requeue_command()
512 *
513 * Purpose: Handle post-processing of completed commands.
514 *
515 * Arguments: q - queue to operate on
516 * cmd - command that may need to be requeued.
517 *
518 * Returns: Nothing
519 *
520 * Notes: After command completion, there may be blocks left
521 * over which weren't finished by the previous command
522 * this can be for a number of reasons - the main one is
523 * I/O errors in the middle of the request, in which case
524 * we need to request the blocks that come after the bad
525 * sector.
James Bottomleye91442b2005-09-09 10:44:16 -0500526 * Notes: Upon return, cmd is a stale pointer.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700527 */
528static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
529{
Bart Van Assche940f5d42012-06-29 15:34:26 +0000530 struct scsi_device *sdev = cmd->device;
James Bottomleye91442b2005-09-09 10:44:16 -0500531 struct request *req = cmd->request;
Tejun Heo 283369c2005-04-24 02:06:36 -0500532 unsigned long flags;
533
Tejun Heo 283369c2005-04-24 02:06:36 -0500534 spin_lock_irqsave(q->queue_lock, flags);
Christoph Hellwig134997a2014-02-20 14:20:58 -0800535 blk_unprep_request(req);
536 req->special = NULL;
537 scsi_put_command(cmd);
James Bottomleye91442b2005-09-09 10:44:16 -0500538 blk_requeue_request(q, req);
Tejun Heo 283369c2005-04-24 02:06:36 -0500539 spin_unlock_irqrestore(q->queue_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540
541 scsi_run_queue(q);
Bart Van Assche940f5d42012-06-29 15:34:26 +0000542
543 put_device(&sdev->sdev_gendev);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544}
545
Linus Torvalds1da177e2005-04-16 15:20:36 -0700546void scsi_run_host_queues(struct Scsi_Host *shost)
547{
548 struct scsi_device *sdev;
549
550 shost_for_each_device(sdev, shost)
551 scsi_run_queue(sdev->request_queue);
552}
553
Jens Axboea8474ce2007-08-07 09:02:51 +0200554static inline unsigned int scsi_sgtable_index(unsigned short nents)
555{
556 unsigned int index;
557
James Bottomleyd3f46f32008-01-15 11:11:46 -0600558 BUG_ON(nents > SCSI_MAX_SG_SEGMENTS);
559
560 if (nents <= 8)
Jens Axboea8474ce2007-08-07 09:02:51 +0200561 index = 0;
James Bottomleyd3f46f32008-01-15 11:11:46 -0600562 else
563 index = get_count_order(nents) - 3;
Jens Axboea8474ce2007-08-07 09:02:51 +0200564
565 return index;
566}
567
Jens Axboe5ed79592007-11-15 09:13:11 +0100568static void scsi_sg_free(struct scatterlist *sgl, unsigned int nents)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569{
570 struct scsi_host_sg_pool *sgp;
Jens Axboe5ed79592007-11-15 09:13:11 +0100571
572 sgp = scsi_sg_pools + scsi_sgtable_index(nents);
573 mempool_free(sgl, sgp->pool);
574}
575
576static struct scatterlist *scsi_sg_alloc(unsigned int nents, gfp_t gfp_mask)
577{
578 struct scsi_host_sg_pool *sgp;
579
580 sgp = scsi_sg_pools + scsi_sgtable_index(nents);
581 return mempool_alloc(sgp->pool, gfp_mask);
582}
583
Christoph Hellwigd2852032014-01-17 12:06:53 +0100584static void scsi_free_sgtable(struct scsi_data_buffer *sdb, bool mq)
Christoph Hellwigc53c6d62014-04-15 14:38:31 +0200585{
Christoph Hellwigd2852032014-01-17 12:06:53 +0100586 if (mq && sdb->table.nents <= SCSI_MAX_SG_SEGMENTS)
587 return;
588 __sg_free_table(&sdb->table, SCSI_MAX_SG_SEGMENTS, mq, scsi_sg_free);
Christoph Hellwigc53c6d62014-04-15 14:38:31 +0200589}
590
Christoph Hellwig3c356bd2014-09-05 18:20:23 -0700591static int scsi_alloc_sgtable(struct scsi_data_buffer *sdb, int nents, bool mq)
Jens Axboe5ed79592007-11-15 09:13:11 +0100592{
Christoph Hellwigd2852032014-01-17 12:06:53 +0100593 struct scatterlist *first_chunk = NULL;
Jens Axboe5ed79592007-11-15 09:13:11 +0100594 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595
Boaz Harrosh30b0c372007-12-13 13:47:40 +0200596 BUG_ON(!nents);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597
Christoph Hellwigd2852032014-01-17 12:06:53 +0100598 if (mq) {
599 if (nents <= SCSI_MAX_SG_SEGMENTS) {
600 sdb->table.nents = nents;
601 sg_init_table(sdb->table.sgl, sdb->table.nents);
602 return 0;
603 }
604 first_chunk = sdb->table.sgl;
605 }
606
Boaz Harrosh30b0c372007-12-13 13:47:40 +0200607 ret = __sg_alloc_table(&sdb->table, nents, SCSI_MAX_SG_SEGMENTS,
Christoph Hellwig70a0f2c2015-01-05 20:29:38 +0100608 first_chunk, GFP_ATOMIC, scsi_sg_alloc);
Jens Axboe5ed79592007-11-15 09:13:11 +0100609 if (unlikely(ret))
Christoph Hellwigd2852032014-01-17 12:06:53 +0100610 scsi_free_sgtable(sdb, mq);
Jens Axboea8474ce2007-08-07 09:02:51 +0200611 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700612}
613
Christoph Hellwigd2852032014-01-17 12:06:53 +0100614static void scsi_uninit_cmd(struct scsi_cmnd *cmd)
615{
616 if (cmd->request->cmd_type == REQ_TYPE_FS) {
617 struct scsi_driver *drv = scsi_cmd_to_driver(cmd);
618
619 if (drv->uninit_command)
620 drv->uninit_command(cmd);
621 }
622}
623
624static void scsi_mq_free_sgtables(struct scsi_cmnd *cmd)
625{
626 if (cmd->sdb.table.nents)
627 scsi_free_sgtable(&cmd->sdb, true);
628 if (cmd->request->next_rq && cmd->request->next_rq->special)
629 scsi_free_sgtable(cmd->request->next_rq->special, true);
630 if (scsi_prot_sg_count(cmd))
631 scsi_free_sgtable(cmd->prot_sdb, true);
632}
633
634static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd)
635{
636 struct scsi_device *sdev = cmd->device;
Kashyap.Desai@avagotech.com64bdcbc2014-08-20 19:24:33 +0530637 struct Scsi_Host *shost = sdev->host;
Christoph Hellwigd2852032014-01-17 12:06:53 +0100638 unsigned long flags;
639
Christoph Hellwigd2852032014-01-17 12:06:53 +0100640 scsi_mq_free_sgtables(cmd);
641 scsi_uninit_cmd(cmd);
642
Kashyap.Desai@avagotech.com64bdcbc2014-08-20 19:24:33 +0530643 if (shost->use_cmd_list) {
644 BUG_ON(list_empty(&cmd->list));
645 spin_lock_irqsave(&sdev->list_lock, flags);
646 list_del_init(&cmd->list);
647 spin_unlock_irqrestore(&sdev->list_lock, flags);
648 }
Christoph Hellwigd2852032014-01-17 12:06:53 +0100649}
650
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651/*
652 * Function: scsi_release_buffers()
653 *
Christoph Hellwigc682adf2014-05-01 16:51:02 +0200654 * Purpose: Free resources allocate for a scsi_command.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700655 *
656 * Arguments: cmd - command that we are bailing.
657 *
658 * Lock status: Assumed that no lock is held upon entry.
659 *
660 * Returns: Nothing
661 *
662 * Notes: In the event that an upper level driver rejects a
663 * command, we must release resources allocated during
664 * the __init_io() function. Primarily this would involve
Christoph Hellwigc682adf2014-05-01 16:51:02 +0200665 * the scatter-gather table.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666 */
Christoph Hellwigf1bea552014-04-15 12:26:54 +0200667static void scsi_release_buffers(struct scsi_cmnd *cmd)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668{
Christoph Hellwigc682adf2014-05-01 16:51:02 +0200669 if (cmd->sdb.table.nents)
Christoph Hellwigd2852032014-01-17 12:06:53 +0100670 scsi_free_sgtable(&cmd->sdb, false);
Christoph Hellwigc682adf2014-05-01 16:51:02 +0200671
672 memset(&cmd->sdb, 0, sizeof(cmd->sdb));
673
674 if (scsi_prot_sg_count(cmd))
Christoph Hellwigd2852032014-01-17 12:06:53 +0100675 scsi_free_sgtable(cmd->prot_sdb, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700676}
677
Christoph Hellwigc682adf2014-05-01 16:51:02 +0200678static void scsi_release_bidi_buffers(struct scsi_cmnd *cmd)
679{
680 struct scsi_data_buffer *bidi_sdb = cmd->request->next_rq->special;
681
Christoph Hellwigd2852032014-01-17 12:06:53 +0100682 scsi_free_sgtable(bidi_sdb, false);
Christoph Hellwigc682adf2014-05-01 16:51:02 +0200683 kmem_cache_free(scsi_sdb_cache, bidi_sdb);
684 cmd->request->next_rq->special = NULL;
685}
686
Christoph Hellwigf6d47e72014-02-16 06:16:13 -0800687static bool scsi_end_request(struct request *req, int error,
688 unsigned int bytes, unsigned int bidi_bytes)
689{
690 struct scsi_cmnd *cmd = req->special;
691 struct scsi_device *sdev = cmd->device;
692 struct request_queue *q = sdev->request_queue;
Christoph Hellwigf6d47e72014-02-16 06:16:13 -0800693
694 if (blk_update_request(req, error, bytes))
695 return true;
696
697 /* Bidi request must be completed as a whole */
698 if (unlikely(bidi_bytes) &&
699 blk_update_request(req->next_rq, error, bidi_bytes))
700 return true;
701
702 if (blk_queue_add_random(q))
703 add_disk_randomness(req->rq_disk);
704
Christoph Hellwigd2852032014-01-17 12:06:53 +0100705 if (req->mq_ctx) {
706 /*
Christoph Hellwigc8a446a2014-09-13 16:40:10 -0700707 * In the MQ case the command gets freed by __blk_mq_end_request,
Christoph Hellwigd2852032014-01-17 12:06:53 +0100708 * so we have to do all cleanup that depends on it earlier.
709 *
710 * We also can't kick the queues from irq context, so we
711 * will have to defer it to a workqueue.
712 */
713 scsi_mq_uninit_cmd(cmd);
Christoph Hellwigf6d47e72014-02-16 06:16:13 -0800714
Christoph Hellwigc8a446a2014-09-13 16:40:10 -0700715 __blk_mq_end_request(req, error);
Christoph Hellwigd2852032014-01-17 12:06:53 +0100716
717 if (scsi_target(sdev)->single_lun ||
718 !list_empty(&sdev->host->starved_list))
719 kblockd_schedule_work(&sdev->requeue_work);
720 else
721 blk_mq_start_stopped_hw_queues(q, true);
Christoph Hellwigd2852032014-01-17 12:06:53 +0100722 } else {
723 unsigned long flags;
724
Daniel Gryniewiczf81426a2014-09-16 10:41:13 -0400725 if (bidi_bytes)
726 scsi_release_bidi_buffers(cmd);
727
Christoph Hellwigd2852032014-01-17 12:06:53 +0100728 spin_lock_irqsave(q->queue_lock, flags);
729 blk_finish_request(req, error);
730 spin_unlock_irqrestore(q->queue_lock, flags);
731
Christoph Hellwigd2852032014-01-17 12:06:53 +0100732 scsi_release_buffers(cmd);
Christoph Hellwigbb3ec622014-09-05 18:02:09 -0700733
734 scsi_put_command(cmd);
735 scsi_run_queue(q);
Christoph Hellwigd2852032014-01-17 12:06:53 +0100736 }
737
Christoph Hellwigbb3ec622014-09-05 18:02:09 -0700738 put_device(&sdev->sdev_gendev);
Christoph Hellwigf6d47e72014-02-16 06:16:13 -0800739 return false;
740}
741
Hannes Reinecke0f7f6232013-07-01 15:16:23 +0200742/**
743 * __scsi_error_from_host_byte - translate SCSI error code into errno
744 * @cmd: SCSI command (unused)
745 * @result: scsi error code
746 *
747 * Translate SCSI error code into standard UNIX errno.
748 * Return values:
749 * -ENOLINK temporary transport failure
750 * -EREMOTEIO permanent target failure, do not retry
751 * -EBADE permanent nexus failure, retry on other path
Hannes Reineckea9d6ceb82013-07-01 15:16:25 +0200752 * -ENOSPC No write space available
Hannes Reinecke7e782af2013-07-01 15:16:26 +0200753 * -ENODATA Medium error
Hannes Reinecke0f7f6232013-07-01 15:16:23 +0200754 * -EIO unspecified I/O error
755 */
Hannes Reinecke63583cc2011-01-18 10:13:11 +0100756static int __scsi_error_from_host_byte(struct scsi_cmnd *cmd, int result)
757{
758 int error = 0;
759
760 switch(host_byte(result)) {
761 case DID_TRANSPORT_FAILFAST:
762 error = -ENOLINK;
763 break;
764 case DID_TARGET_FAILURE:
Moger, Babu2082ebc2012-01-24 20:38:46 +0000765 set_host_byte(cmd, DID_OK);
Hannes Reinecke63583cc2011-01-18 10:13:11 +0100766 error = -EREMOTEIO;
767 break;
768 case DID_NEXUS_FAILURE:
Moger, Babu2082ebc2012-01-24 20:38:46 +0000769 set_host_byte(cmd, DID_OK);
Hannes Reinecke63583cc2011-01-18 10:13:11 +0100770 error = -EBADE;
771 break;
Hannes Reineckea9d6ceb82013-07-01 15:16:25 +0200772 case DID_ALLOC_FAILURE:
773 set_host_byte(cmd, DID_OK);
774 error = -ENOSPC;
775 break;
Hannes Reinecke7e782af2013-07-01 15:16:26 +0200776 case DID_MEDIUM_ERROR:
777 set_host_byte(cmd, DID_OK);
778 error = -ENODATA;
779 break;
Hannes Reinecke63583cc2011-01-18 10:13:11 +0100780 default:
781 error = -EIO;
782 break;
783 }
784
785 return error;
786}
787
Linus Torvalds1da177e2005-04-16 15:20:36 -0700788/*
789 * Function: scsi_io_completion()
790 *
791 * Purpose: Completion processing for block device I/O requests.
792 *
793 * Arguments: cmd - command that is finished.
794 *
795 * Lock status: Assumed that no lock is held upon entry.
796 *
797 * Returns: Nothing
798 *
Christoph Hellwigbc85dc52014-05-01 16:51:03 +0200799 * Notes: We will finish off the specified number of sectors. If we
800 * are done, the command block will be released and the queue
801 * function will be goosed. If we are not done then we have to
Alan Sternb60af5b2008-11-03 15:56:47 -0500802 * figure out what to do next:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803 *
Alan Sternb60af5b2008-11-03 15:56:47 -0500804 * a) We can call scsi_requeue_command(). The request
805 * will be unprepared and put back on the queue. Then
806 * a new command will be created for it. This should
807 * be used if we made forward progress, or if we want
808 * to switch from READ(10) to READ(6) for example.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809 *
Christoph Hellwigbc85dc52014-05-01 16:51:03 +0200810 * b) We can call __scsi_queue_insert(). The request will
Alan Sternb60af5b2008-11-03 15:56:47 -0500811 * be put back on the queue and retried using the same
812 * command as before, possibly after a delay.
813 *
Christoph Hellwigf6d47e72014-02-16 06:16:13 -0800814 * c) We can call scsi_end_request() with -EIO to fail
Alan Sternb60af5b2008-11-03 15:56:47 -0500815 * the remainder of the request.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700816 */
Luben Tuikov03aba2f2006-06-23 09:39:09 -0700817void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818{
819 int result = cmd->result;
Jens Axboe165125e2007-07-24 09:28:11 +0200820 struct request_queue *q = cmd->device->request_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821 struct request *req = cmd->request;
James Bottomleyfa8e36c2008-04-02 18:11:52 -0500822 int error = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823 struct scsi_sense_hdr sshdr;
Hannes Reinecke4753cbc2014-10-24 14:26:52 +0200824 bool sense_valid = false;
Hannes Reineckec11c0042014-10-24 14:27:01 +0200825 int sense_deferred = 0, level = 0;
Alan Sternb60af5b2008-11-03 15:56:47 -0500826 enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY,
827 ACTION_DELAYED_RETRY} action;
Eiichi Tsukataee60b2c2014-02-11 14:29:52 +0900828 unsigned long wait_for = (cmd->allowed + 1) * req->timeout;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700829
Linus Torvalds1da177e2005-04-16 15:20:36 -0700830 if (result) {
831 sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
832 if (sense_valid)
833 sense_deferred = scsi_sense_is_deferred(&sshdr);
834 }
Christoph Hellwig631c2282006-07-08 20:42:15 +0200835
Christoph Hellwig33659eb2010-08-07 18:17:56 +0200836 if (req->cmd_type == REQ_TYPE_BLOCK_PC) { /* SG_IO ioctl from block level */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837 if (result) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838 if (sense_valid && req->sense) {
839 /*
840 * SG_IO wants current and deferred errors
841 */
842 int len = 8 + cmd->sense_buffer[7];
843
844 if (len > SCSI_SENSE_BUFFERSIZE)
845 len = SCSI_SENSE_BUFFERSIZE;
846 memcpy(req->sense, cmd->sense_buffer, len);
847 req->sense_len = len;
848 }
James Bottomleyfa8e36c2008-04-02 18:11:52 -0500849 if (!sense_deferred)
Hannes Reinecke63583cc2011-01-18 10:13:11 +0100850 error = __scsi_error_from_host_byte(cmd, result);
Pete Wyckoffb22f6872007-03-13 16:53:28 -0400851 }
Mike Snitzer27c41972012-05-31 15:05:33 -0400852 /*
853 * __scsi_error_from_host_byte may have reset the host_byte
854 */
855 req->errors = cmd->result;
FUJITA Tomonorie6bb7a962009-05-11 17:56:08 +0900856
857 req->resid_len = scsi_get_resid(cmd);
858
Boaz Harrosh6f9a35e2007-12-13 13:50:53 +0200859 if (scsi_bidi_cmnd(cmd)) {
FUJITA Tomonorie6bb7a962009-05-11 17:56:08 +0900860 /*
861 * Bidi commands Must be complete as a whole,
862 * both sides at once.
863 */
864 req->next_rq->resid_len = scsi_in(cmd)->resid;
Christoph Hellwigf6d47e72014-02-16 06:16:13 -0800865 if (scsi_end_request(req, 0, blk_rq_bytes(req),
866 blk_rq_bytes(req->next_rq)))
867 BUG();
Boaz Harrosh6f9a35e2007-12-13 13:50:53 +0200868 return;
869 }
James Bottomley89fb4cd2014-07-03 19:17:34 +0200870 } else if (blk_rq_bytes(req) == 0 && result && !sense_deferred) {
871 /*
872 * Certain non BLOCK_PC requests are commands that don't
873 * actually transfer anything (FLUSH), so cannot use
874 * good_bytes != blk_rq_bytes(req) as the signal for an error.
875 * This sets the error explicitly for the problem case.
876 */
877 error = __scsi_error_from_host_byte(cmd, result);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878 }
879
Christoph Hellwig33659eb2010-08-07 18:17:56 +0200880 /* no bidi support for !REQ_TYPE_BLOCK_PC yet */
881 BUG_ON(blk_bidi_rq(req));
Boaz Harrosh30b0c372007-12-13 13:47:40 +0200882
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 * Next deal with any sectors which we were able to correctly
885 * handle.
886 */
Hannes Reinecke91921e02014-06-25 16:39:59 +0200887 SCSI_LOG_HLCOMPLETE(1, scmd_printk(KERN_INFO, cmd,
888 "%u sectors total, %d bytes done.\n",
889 blk_rq_sectors(req), good_bytes));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700890
James Bottomleya9bddd72009-03-30 16:55:51 +0000891 /*
892 * Recovered errors need reporting, but they're always treated
893 * as success, so fiddle the result code here. For BLOCK_PC
894 * we already took a copy of the original into rq->errors which
895 * is what gets returned to the user
896 */
Douglas Gilberte7efe592010-01-03 13:51:15 -0500897 if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) {
898 /* if ATA PASS-THROUGH INFORMATION AVAILABLE skip
899 * print since caller wants ATA registers. Only occurs on
900 * SCSI ATA PASS_THROUGH commands when CK_COND=1
901 */
902 if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d))
903 ;
904 else if (!(req->cmd_flags & REQ_QUIET))
Hannes Reinecked811b842014-10-24 14:26:45 +0200905 scsi_print_sense(cmd);
James Bottomleya9bddd72009-03-30 16:55:51 +0000906 result = 0;
907 /* BLOCK_PC may have set error */
908 error = 0;
909 }
910
911 /*
Christoph Hellwigbc85dc52014-05-01 16:51:03 +0200912 * If we finished all bytes in the request we are done now.
James Bottomleyd6b0c532006-07-02 10:06:28 -0500913 */
Christoph Hellwigf6d47e72014-02-16 06:16:13 -0800914 if (!scsi_end_request(req, error, good_bytes, 0))
915 return;
Christoph Hellwigbc85dc52014-05-01 16:51:03 +0200916
917 /*
918 * Kill remainder if no retrys.
919 */
920 if (error && scsi_noretry_cmd(cmd)) {
Christoph Hellwigf6d47e72014-02-16 06:16:13 -0800921 if (scsi_end_request(req, error, blk_rq_bytes(req), 0))
922 BUG();
923 return;
Christoph Hellwigbc85dc52014-05-01 16:51:03 +0200924 }
925
926 /*
927 * If there had been no error, but we have leftover bytes in the
928 * requeues just queue the command up again.
929 */
930 if (result == 0)
931 goto requeue;
Luben Tuikov03aba2f2006-06-23 09:39:09 -0700932
Hannes Reinecke63583cc2011-01-18 10:13:11 +0100933 error = __scsi_error_from_host_byte(cmd, result);
Martin K. Petersen3e695f82009-01-04 03:04:31 -0500934
Alan Sternb60af5b2008-11-03 15:56:47 -0500935 if (host_byte(result) == DID_RESET) {
936 /* Third party bus reset or reset for error recovery
937 * reasons. Just retry the command and see what
938 * happens.
939 */
940 action = ACTION_RETRY;
941 } else if (sense_valid && !sense_deferred) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700942 switch (sshdr.sense_key) {
943 case UNIT_ATTENTION:
944 if (cmd->device->removable) {
Luben Tuikov03aba2f2006-06-23 09:39:09 -0700945 /* Detected disc change. Set a bit
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946 * and quietly refuse further access.
947 */
948 cmd->device->changed = 1;
Alan Sternb60af5b2008-11-03 15:56:47 -0500949 action = ACTION_FAIL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950 } else {
Luben Tuikov03aba2f2006-06-23 09:39:09 -0700951 /* Must have been a power glitch, or a
952 * bus reset. Could not have been a
953 * media change, so we just retry the
Alan Sternb60af5b2008-11-03 15:56:47 -0500954 * command and see what happens.
Luben Tuikov03aba2f2006-06-23 09:39:09 -0700955 */
Alan Sternb60af5b2008-11-03 15:56:47 -0500956 action = ACTION_RETRY;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700957 }
958 break;
959 case ILLEGAL_REQUEST:
Luben Tuikov03aba2f2006-06-23 09:39:09 -0700960 /* If we had an ILLEGAL REQUEST returned, then
961 * we may have performed an unsupported
962 * command. The only thing this should be
963 * would be a ten byte read where only a six
964 * byte read was supported. Also, on a system
965 * where READ CAPACITY failed, we may have
966 * read past the end of the disk.
967 */
Jens Axboe26a68012005-11-29 21:03:34 +0100968 if ((cmd->device->use_10_for_rw &&
969 sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
Linus Torvalds1da177e2005-04-16 15:20:36 -0700970 (cmd->cmnd[0] == READ_10 ||
971 cmd->cmnd[0] == WRITE_10)) {
Alan Sternb60af5b2008-11-03 15:56:47 -0500972 /* This will issue a new 6-byte command. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973 cmd->device->use_10_for_rw = 0;
Alan Sternb60af5b2008-11-03 15:56:47 -0500974 action = ACTION_REPREP;
Martin K. Petersen3e695f82009-01-04 03:04:31 -0500975 } else if (sshdr.asc == 0x10) /* DIX */ {
Martin K. Petersen3e695f82009-01-04 03:04:31 -0500976 action = ACTION_FAIL;
977 error = -EILSEQ;
Martin K. Petersenc98a0eb2011-03-08 02:07:15 -0500978 /* INVALID COMMAND OPCODE or INVALID FIELD IN CDB */
Martin K. Petersen5db44862012-09-18 12:19:32 -0400979 } else if (sshdr.asc == 0x20 || sshdr.asc == 0x24) {
Martin K. Petersenc98a0eb2011-03-08 02:07:15 -0500980 action = ACTION_FAIL;
Martin K. Petersen66a651a2012-02-13 15:38:22 -0500981 error = -EREMOTEIO;
Alan Sternb60af5b2008-11-03 15:56:47 -0500982 } else
983 action = ACTION_FAIL;
984 break;
Martin K. Petersen511e44f2008-07-17 04:28:33 -0400985 case ABORTED_COMMAND:
James Bottomley126c09822009-02-19 21:48:54 +0000986 action = ACTION_FAIL;
Maurizio Lombardie6c11db2014-07-10 09:41:53 +0200987 if (sshdr.asc == 0x10) /* DIF */
Martin K. Petersen3e695f82009-01-04 03:04:31 -0500988 error = -EILSEQ;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989 break;
990 case NOT_READY:
Luben Tuikov03aba2f2006-06-23 09:39:09 -0700991 /* If the device is in the process of becoming
James Bottomleyf3e93f72006-04-25 16:48:30 -0500992 * ready, or has a temporary blockage, retry.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 */
James Bottomleyf3e93f72006-04-25 16:48:30 -0500994 if (sshdr.asc == 0x04) {
995 switch (sshdr.ascq) {
996 case 0x01: /* becoming ready */
997 case 0x04: /* format in progress */
998 case 0x05: /* rebuild in progress */
999 case 0x06: /* recalculation in progress */
1000 case 0x07: /* operation in progress */
1001 case 0x08: /* Long write in progress */
1002 case 0x09: /* self test in progress */
Martin K. Petersend8705f12009-11-26 12:00:41 -05001003 case 0x14: /* space allocation in progress */
Alan Sternb60af5b2008-11-03 15:56:47 -05001004 action = ACTION_DELAYED_RETRY;
James Bottomleyf3e93f72006-04-25 16:48:30 -05001005 break;
Alan Stern3dbf6a52008-12-15 10:31:28 -05001006 default:
Alan Stern3dbf6a52008-12-15 10:31:28 -05001007 action = ACTION_FAIL;
1008 break;
James Bottomleyf3e93f72006-04-25 16:48:30 -05001009 }
Maurizio Lombardie6c11db2014-07-10 09:41:53 +02001010 } else
Alan Sternb60af5b2008-11-03 15:56:47 -05001011 action = ACTION_FAIL;
Alan Sternb60af5b2008-11-03 15:56:47 -05001012 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001013 case VOLUME_OVERFLOW:
Luben Tuikov03aba2f2006-06-23 09:39:09 -07001014 /* See SSC3rXX or current. */
Alan Sternb60af5b2008-11-03 15:56:47 -05001015 action = ACTION_FAIL;
1016 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001017 default:
Alan Sternb60af5b2008-11-03 15:56:47 -05001018 action = ACTION_FAIL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001019 break;
1020 }
Maurizio Lombardie6c11db2014-07-10 09:41:53 +02001021 } else
Alan Sternb60af5b2008-11-03 15:56:47 -05001022 action = ACTION_FAIL;
Alan Sternb60af5b2008-11-03 15:56:47 -05001023
Eiichi Tsukataee60b2c2014-02-11 14:29:52 +09001024 if (action != ACTION_FAIL &&
Maurizio Lombardie6c11db2014-07-10 09:41:53 +02001025 time_before(cmd->jiffies_at_alloc + wait_for, jiffies))
Eiichi Tsukataee60b2c2014-02-11 14:29:52 +09001026 action = ACTION_FAIL;
Eiichi Tsukataee60b2c2014-02-11 14:29:52 +09001027
Alan Sternb60af5b2008-11-03 15:56:47 -05001028 switch (action) {
1029 case ACTION_FAIL:
1030 /* Give up and fail the remainder of the request */
Hannes Reineckef1569ff2014-10-24 14:27:07 +02001031 if (!(req->cmd_flags & REQ_QUIET)) {
1032 static DEFINE_RATELIMIT_STATE(_rs,
1033 DEFAULT_RATELIMIT_INTERVAL,
1034 DEFAULT_RATELIMIT_BURST);
1035
1036 if (unlikely(scsi_logging_level))
1037 level = SCSI_LOG_LEVEL(SCSI_LOG_MLCOMPLETE_SHIFT,
1038 SCSI_LOG_MLCOMPLETE_BITS);
1039
1040 /*
1041 * if logging is enabled the failure will be printed
1042 * in scsi_log_completion(), so avoid duplicate messages
1043 */
1044 if (!level && __ratelimit(&_rs)) {
1045 scsi_print_result(cmd, NULL, FAILED);
1046 if (driver_byte(result) & DRIVER_SENSE)
1047 scsi_print_sense(cmd);
1048 scsi_print_command(cmd);
1049 }
James Bottomley3173d8c2005-09-04 11:32:05 -05001050 }
Christoph Hellwigf6d47e72014-02-16 06:16:13 -08001051 if (!scsi_end_request(req, error, blk_rq_err_bytes(req), 0))
1052 return;
Christoph Hellwigbc85dc52014-05-01 16:51:03 +02001053 /*FALLTHRU*/
Alan Sternb60af5b2008-11-03 15:56:47 -05001054 case ACTION_REPREP:
Christoph Hellwigbc85dc52014-05-01 16:51:03 +02001055 requeue:
Alan Sternb60af5b2008-11-03 15:56:47 -05001056 /* Unprep the request and put it back at the head of the queue.
1057 * A new command will be prepared and issued.
1058 */
Christoph Hellwigd2852032014-01-17 12:06:53 +01001059 if (q->mq_ops) {
1060 cmd->request->cmd_flags &= ~REQ_DONTPREP;
1061 scsi_mq_uninit_cmd(cmd);
1062 scsi_mq_requeue_cmd(cmd);
1063 } else {
1064 scsi_release_buffers(cmd);
1065 scsi_requeue_command(q, cmd);
1066 }
Alan Sternb60af5b2008-11-03 15:56:47 -05001067 break;
1068 case ACTION_RETRY:
1069 /* Retry the same command immediately */
James Bottomley4f5299a2009-01-02 10:42:21 -06001070 __scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, 0);
Alan Sternb60af5b2008-11-03 15:56:47 -05001071 break;
1072 case ACTION_DELAYED_RETRY:
1073 /* Retry the same command after a delay */
James Bottomley4f5299a2009-01-02 10:42:21 -06001074 __scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, 0);
Alan Sternb60af5b2008-11-03 15:56:47 -05001075 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001076 }
1077}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001078
Christoph Hellwig3c356bd2014-09-05 18:20:23 -07001079static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001080{
Boaz Harrosh6f9a35e2007-12-13 13:50:53 +02001081 int count;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001082
1083 /*
Christoph Hellwig3b003152006-11-04 20:10:55 +01001084 * If sg table allocation fails, requeue request later.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085 */
Boaz Harrosh30b0c372007-12-13 13:47:40 +02001086 if (unlikely(scsi_alloc_sgtable(sdb, req->nr_phys_segments,
Christoph Hellwig3c356bd2014-09-05 18:20:23 -07001087 req->mq_ctx != NULL)))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001088 return BLKPREP_DEFER;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001089
Linus Torvalds1da177e2005-04-16 15:20:36 -07001090 /*
1091 * Next, walk the list, and fill in the addresses and sizes of
1092 * each segment.
1093 */
Boaz Harrosh30b0c372007-12-13 13:47:40 +02001094 count = blk_rq_map_sg(req->q, req, sdb->table.sgl);
1095 BUG_ON(count > sdb->table.nents);
1096 sdb->table.nents = count;
Tejun Heo1011c1b2009-05-07 22:24:45 +09001097 sdb->length = blk_rq_bytes(req);
Rusty Russell4a03d902007-11-19 11:28:48 +11001098 return BLKPREP_OK;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099}
Boaz Harrosh6f9a35e2007-12-13 13:50:53 +02001100
1101/*
1102 * Function: scsi_init_io()
1103 *
1104 * Purpose: SCSI I/O initialize function.
1105 *
1106 * Arguments: cmd - Command descriptor we wish to initialize
1107 *
1108 * Returns: 0 on success
1109 * BLKPREP_DEFER if the failure is retryable
1110 * BLKPREP_KILL if the failure is fatal
1111 */
Christoph Hellwig3c356bd2014-09-05 18:20:23 -07001112int scsi_init_io(struct scsi_cmnd *cmd)
Boaz Harrosh6f9a35e2007-12-13 13:50:53 +02001113{
Christoph Hellwig5e012aa2014-04-15 12:24:55 +02001114 struct scsi_device *sdev = cmd->device;
Martin K. Petersen13f05c82010-09-10 20:50:10 +02001115 struct request *rq = cmd->request;
Christoph Hellwigd2852032014-01-17 12:06:53 +01001116 bool is_mq = (rq->mq_ctx != NULL);
Christoph Hellwig635d98b2014-06-28 11:51:01 +02001117 int error;
Martin K. Petersen13f05c82010-09-10 20:50:10 +02001118
Christoph Hellwig635d98b2014-06-28 11:51:01 +02001119 BUG_ON(!rq->nr_phys_segments);
1120
Christoph Hellwig3c356bd2014-09-05 18:20:23 -07001121 error = scsi_init_sgtable(rq, &cmd->sdb);
Boaz Harrosh6f9a35e2007-12-13 13:50:53 +02001122 if (error)
1123 goto err_exit;
1124
Martin K. Petersen13f05c82010-09-10 20:50:10 +02001125 if (blk_bidi_rq(rq)) {
Christoph Hellwigd2852032014-01-17 12:06:53 +01001126 if (!rq->q->mq_ops) {
1127 struct scsi_data_buffer *bidi_sdb =
1128 kmem_cache_zalloc(scsi_sdb_cache, GFP_ATOMIC);
1129 if (!bidi_sdb) {
1130 error = BLKPREP_DEFER;
1131 goto err_exit;
1132 }
1133
1134 rq->next_rq->special = bidi_sdb;
Boaz Harrosh6f9a35e2007-12-13 13:50:53 +02001135 }
1136
Christoph Hellwig3c356bd2014-09-05 18:20:23 -07001137 error = scsi_init_sgtable(rq->next_rq, rq->next_rq->special);
Boaz Harrosh6f9a35e2007-12-13 13:50:53 +02001138 if (error)
1139 goto err_exit;
1140 }
1141
Martin K. Petersen13f05c82010-09-10 20:50:10 +02001142 if (blk_integrity_rq(rq)) {
Martin K. Petersen7027ad72008-07-17 17:08:48 -04001143 struct scsi_data_buffer *prot_sdb = cmd->prot_sdb;
1144 int ivecs, count;
1145
Ewan D. Milne91724c22015-01-15 10:02:12 -05001146 if (prot_sdb == NULL) {
1147 /*
1148 * This can happen if someone (e.g. multipath)
1149 * queues a command to a device on an adapter
1150 * that does not support DIX.
1151 */
1152 WARN_ON_ONCE(1);
1153 error = BLKPREP_KILL;
1154 goto err_exit;
1155 }
1156
Martin K. Petersen13f05c82010-09-10 20:50:10 +02001157 ivecs = blk_rq_count_integrity_sg(rq->q, rq->bio);
Martin K. Petersen7027ad72008-07-17 17:08:48 -04001158
Christoph Hellwig3c356bd2014-09-05 18:20:23 -07001159 if (scsi_alloc_sgtable(prot_sdb, ivecs, is_mq)) {
Martin K. Petersen7027ad72008-07-17 17:08:48 -04001160 error = BLKPREP_DEFER;
1161 goto err_exit;
1162 }
1163
Martin K. Petersen13f05c82010-09-10 20:50:10 +02001164 count = blk_rq_map_integrity_sg(rq->q, rq->bio,
Martin K. Petersen7027ad72008-07-17 17:08:48 -04001165 prot_sdb->table.sgl);
1166 BUG_ON(unlikely(count > ivecs));
Martin K. Petersen13f05c82010-09-10 20:50:10 +02001167 BUG_ON(unlikely(count > queue_max_integrity_segments(rq->q)));
Martin K. Petersen7027ad72008-07-17 17:08:48 -04001168
1169 cmd->prot_sdb = prot_sdb;
1170 cmd->prot_sdb->table.nents = count;
1171 }
1172
Christoph Hellwigd2852032014-01-17 12:06:53 +01001173 return BLKPREP_OK;
Boaz Harrosh6f9a35e2007-12-13 13:50:53 +02001174err_exit:
Christoph Hellwigd2852032014-01-17 12:06:53 +01001175 if (is_mq) {
1176 scsi_mq_free_sgtables(cmd);
1177 } else {
1178 scsi_release_buffers(cmd);
1179 cmd->request->special = NULL;
1180 scsi_put_command(cmd);
1181 put_device(&sdev->sdev_gendev);
1182 }
Boaz Harrosh6f9a35e2007-12-13 13:50:53 +02001183 return error;
1184}
Boaz Harroshbb52d822007-12-13 16:14:27 -08001185EXPORT_SYMBOL(scsi_init_io);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001186
Christoph Hellwig3b003152006-11-04 20:10:55 +01001187static struct scsi_cmnd *scsi_get_cmd_from_req(struct scsi_device *sdev,
1188 struct request *req)
1189{
1190 struct scsi_cmnd *cmd;
1191
1192 if (!req->special) {
Christoph Hellwig04796332014-02-20 14:20:55 -08001193 /* Bail if we can't get a reference to the device */
1194 if (!get_device(&sdev->sdev_gendev))
Christoph Hellwig3b003152006-11-04 20:10:55 +01001195 return NULL;
Christoph Hellwig04796332014-02-20 14:20:55 -08001196
1197 cmd = scsi_get_command(sdev, GFP_ATOMIC);
1198 if (unlikely(!cmd)) {
1199 put_device(&sdev->sdev_gendev);
1200 return NULL;
1201 }
Christoph Hellwig3b003152006-11-04 20:10:55 +01001202 req->special = cmd;
1203 } else {
1204 cmd = req->special;
1205 }
1206
1207 /* pull a tag out of the request if we have one */
1208 cmd->tag = req->tag;
1209 cmd->request = req;
1210
Boaz Harrosh64a87b22008-04-30 11:19:47 +03001211 cmd->cmnd = req->cmd;
Martin K. Petersen72f7d322011-03-08 02:03:59 -05001212 cmd->prot_op = SCSI_PROT_NORMAL;
Boaz Harrosh64a87b22008-04-30 11:19:47 +03001213
Christoph Hellwig3b003152006-11-04 20:10:55 +01001214 return cmd;
1215}
1216
Christoph Hellwig4f1e5762014-06-28 12:36:28 +02001217static int scsi_setup_blk_pc_cmnd(struct scsi_device *sdev, struct request *req)
James Bottomley7b163182005-12-15 20:17:02 -06001218{
Christoph Hellwiga1b73fc2014-05-01 16:51:04 +02001219 struct scsi_cmnd *cmd = req->special;
Christoph Hellwig3b003152006-11-04 20:10:55 +01001220
1221 /*
1222 * BLOCK_PC requests may transfer data, in which case they must
1223 * a bio attached to them. Or they might contain a SCSI command
1224 * that does not transfer data, in which case they may optionally
1225 * submit a request without an attached bio.
1226 */
1227 if (req->bio) {
Christoph Hellwig3c356bd2014-09-05 18:20:23 -07001228 int ret = scsi_init_io(cmd);
Christoph Hellwig3b003152006-11-04 20:10:55 +01001229 if (unlikely(ret))
1230 return ret;
1231 } else {
Tejun Heob0790412009-05-07 22:24:42 +09001232 BUG_ON(blk_rq_bytes(req));
Christoph Hellwig3b003152006-11-04 20:10:55 +01001233
Boaz Harrosh30b0c372007-12-13 13:47:40 +02001234 memset(&cmd->sdb, 0, sizeof(cmd->sdb));
Christoph Hellwig3b003152006-11-04 20:10:55 +01001235 }
James Bottomley7b163182005-12-15 20:17:02 -06001236
James Bottomley7b163182005-12-15 20:17:02 -06001237 cmd->cmd_len = req->cmd_len;
Tejun Heob0790412009-05-07 22:24:42 +09001238 cmd->transfersize = blk_rq_bytes(req);
James Bottomley7b163182005-12-15 20:17:02 -06001239 cmd->allowed = req->retries;
Christoph Hellwig3b003152006-11-04 20:10:55 +01001240 return BLKPREP_OK;
1241}
1242
1243/*
Christoph Hellwig3868cf82014-06-28 11:58:42 +02001244 * Setup a REQ_TYPE_FS command. These are simple request from filesystems
1245 * that still need to be translated to SCSI CDBs from the ULD.
Christoph Hellwig3b003152006-11-04 20:10:55 +01001246 */
Christoph Hellwig3868cf82014-06-28 11:58:42 +02001247static int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req)
Christoph Hellwig3b003152006-11-04 20:10:55 +01001248{
Christoph Hellwiga1b73fc2014-05-01 16:51:04 +02001249 struct scsi_cmnd *cmd = req->special;
Chandra Seetharamana6a8d9f2008-05-01 14:49:46 -07001250
1251 if (unlikely(sdev->scsi_dh_data && sdev->scsi_dh_data->scsi_dh
1252 && sdev->scsi_dh_data->scsi_dh->prep_fn)) {
Christoph Hellwiga1b73fc2014-05-01 16:51:04 +02001253 int ret = sdev->scsi_dh_data->scsi_dh->prep_fn(sdev, req);
Chandra Seetharamana6a8d9f2008-05-01 14:49:46 -07001254 if (ret != BLKPREP_OK)
1255 return ret;
1256 }
1257
Boaz Harrosh64a87b22008-04-30 11:19:47 +03001258 memset(cmd->cmnd, 0, BLK_MAX_CDB);
Christoph Hellwig3868cf82014-06-28 11:58:42 +02001259 return scsi_cmd_to_driver(cmd)->init_command(cmd);
James Bottomley7b163182005-12-15 20:17:02 -06001260}
James Bottomley7b163182005-12-15 20:17:02 -06001261
Christoph Hellwig6af7a4f2014-07-08 13:16:17 +02001262static int scsi_setup_cmnd(struct scsi_device *sdev, struct request *req)
1263{
1264 struct scsi_cmnd *cmd = req->special;
1265
1266 if (!blk_rq_bytes(req))
1267 cmd->sc_data_direction = DMA_NONE;
1268 else if (rq_data_dir(req) == WRITE)
1269 cmd->sc_data_direction = DMA_TO_DEVICE;
1270 else
1271 cmd->sc_data_direction = DMA_FROM_DEVICE;
1272
1273 switch (req->cmd_type) {
1274 case REQ_TYPE_FS:
1275 return scsi_setup_fs_cmnd(sdev, req);
1276 case REQ_TYPE_BLOCK_PC:
1277 return scsi_setup_blk_pc_cmnd(sdev, req);
1278 default:
1279 return BLKPREP_KILL;
1280 }
1281}
1282
Christoph Hellwiga1b73fc2014-05-01 16:51:04 +02001283static int
1284scsi_prep_state_check(struct scsi_device *sdev, struct request *req)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001285{
Christoph Hellwig3b003152006-11-04 20:10:55 +01001286 int ret = BLKPREP_OK;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001287
1288 /*
Christoph Hellwig3b003152006-11-04 20:10:55 +01001289 * If the device is not in running state we will reject some
1290 * or all commands.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001291 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001292 if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
Christoph Hellwig3b003152006-11-04 20:10:55 +01001293 switch (sdev->sdev_state) {
1294 case SDEV_OFFLINE:
Mike Christie1b8d2622012-05-17 23:56:56 -05001295 case SDEV_TRANSPORT_OFFLINE:
Christoph Hellwig3b003152006-11-04 20:10:55 +01001296 /*
1297 * If the device is offline we refuse to process any
1298 * commands. The device must be brought online
1299 * before trying any recovery commands.
1300 */
1301 sdev_printk(KERN_ERR, sdev,
1302 "rejecting I/O to offline device\n");
1303 ret = BLKPREP_KILL;
1304 break;
1305 case SDEV_DEL:
1306 /*
1307 * If the device is fully deleted, we refuse to
1308 * process any commands as well.
1309 */
James Bottomley9ccfc752005-10-02 11:45:08 -05001310 sdev_printk(KERN_ERR, sdev,
1311 "rejecting I/O to dead device\n");
Christoph Hellwig3b003152006-11-04 20:10:55 +01001312 ret = BLKPREP_KILL;
1313 break;
Christoph Hellwig3b003152006-11-04 20:10:55 +01001314 case SDEV_BLOCK:
James Bottomley6f4267e2008-08-22 16:53:31 -05001315 case SDEV_CREATED_BLOCK:
Bart Van Asschebba0bdd2015-03-04 10:31:47 +01001316 ret = BLKPREP_DEFER;
1317 break;
1318 case SDEV_QUIESCE:
Christoph Hellwig3b003152006-11-04 20:10:55 +01001319 /*
1320 * If the devices is blocked we defer normal commands.
1321 */
1322 if (!(req->cmd_flags & REQ_PREEMPT))
1323 ret = BLKPREP_DEFER;
1324 break;
1325 default:
1326 /*
1327 * For any other not fully online state we only allow
1328 * special commands. In particular any user initiated
1329 * command is not allowed.
1330 */
1331 if (!(req->cmd_flags & REQ_PREEMPT))
1332 ret = BLKPREP_KILL;
1333 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001334 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335 }
James Bottomley7f9a6bc2007-08-04 10:06:25 -05001336 return ret;
1337}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001338
Christoph Hellwiga1b73fc2014-05-01 16:51:04 +02001339static int
1340scsi_prep_return(struct request_queue *q, struct request *req, int ret)
James Bottomley7f9a6bc2007-08-04 10:06:25 -05001341{
1342 struct scsi_device *sdev = q->queuedata;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001343
Christoph Hellwig3b003152006-11-04 20:10:55 +01001344 switch (ret) {
1345 case BLKPREP_KILL:
1346 req->errors = DID_NO_CONNECT << 16;
James Bottomley7f9a6bc2007-08-04 10:06:25 -05001347 /* release the command and kill it */
1348 if (req->special) {
1349 struct scsi_cmnd *cmd = req->special;
1350 scsi_release_buffers(cmd);
1351 scsi_put_command(cmd);
Christoph Hellwig68c03d92014-04-15 12:24:56 +02001352 put_device(&sdev->sdev_gendev);
James Bottomley7f9a6bc2007-08-04 10:06:25 -05001353 req->special = NULL;
1354 }
Christoph Hellwig3b003152006-11-04 20:10:55 +01001355 break;
1356 case BLKPREP_DEFER:
1357 /*
Tejun Heo9934c8c2009-05-08 11:54:16 +09001358 * If we defer, the blk_peek_request() returns NULL, but the
Jens Axboea488e742010-04-16 21:13:15 +02001359 * queue must be restarted, so we schedule a callback to happen
1360 * shortly.
Christoph Hellwig3b003152006-11-04 20:10:55 +01001361 */
Christoph Hellwig71e75c92014-04-11 19:07:01 +02001362 if (atomic_read(&sdev->device_busy) == 0)
Jens Axboea488e742010-04-16 21:13:15 +02001363 blk_delay_queue(q, SCSI_QUEUE_DELAY);
Christoph Hellwig3b003152006-11-04 20:10:55 +01001364 break;
1365 default:
1366 req->cmd_flags |= REQ_DONTPREP;
1367 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001368
Christoph Hellwig3b003152006-11-04 20:10:55 +01001369 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001370}
James Bottomley7f9a6bc2007-08-04 10:06:25 -05001371
Christoph Hellwiga1b73fc2014-05-01 16:51:04 +02001372static int scsi_prep_fn(struct request_queue *q, struct request *req)
James Bottomley7f9a6bc2007-08-04 10:06:25 -05001373{
1374 struct scsi_device *sdev = q->queuedata;
Christoph Hellwiga1b73fc2014-05-01 16:51:04 +02001375 struct scsi_cmnd *cmd;
1376 int ret;
James Bottomley7f9a6bc2007-08-04 10:06:25 -05001377
Christoph Hellwiga1b73fc2014-05-01 16:51:04 +02001378 ret = scsi_prep_state_check(sdev, req);
1379 if (ret != BLKPREP_OK)
1380 goto out;
1381
1382 cmd = scsi_get_cmd_from_req(sdev, req);
1383 if (unlikely(!cmd)) {
1384 ret = BLKPREP_DEFER;
1385 goto out;
1386 }
1387
Christoph Hellwig6af7a4f2014-07-08 13:16:17 +02001388 ret = scsi_setup_cmnd(sdev, req);
Christoph Hellwiga1b73fc2014-05-01 16:51:04 +02001389out:
James Bottomley7f9a6bc2007-08-04 10:06:25 -05001390 return scsi_prep_return(q, req, ret);
1391}
Christoph Hellwiga1b73fc2014-05-01 16:51:04 +02001392
1393static void scsi_unprep_fn(struct request_queue *q, struct request *req)
1394{
Christoph Hellwigd2852032014-01-17 12:06:53 +01001395 scsi_uninit_cmd(req->special);
Christoph Hellwiga1b73fc2014-05-01 16:51:04 +02001396}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001397
1398/*
1399 * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
1400 * return 0.
1401 *
1402 * Called with the queue_lock held.
1403 */
1404static inline int scsi_dev_queue_ready(struct request_queue *q,
1405 struct scsi_device *sdev)
1406{
Christoph Hellwig71e75c92014-04-11 19:07:01 +02001407 unsigned int busy;
1408
1409 busy = atomic_inc_return(&sdev->device_busy) - 1;
Christoph Hellwigcd9070c2014-01-23 12:07:41 +01001410 if (atomic_read(&sdev->device_blocked)) {
Christoph Hellwig71e75c92014-04-11 19:07:01 +02001411 if (busy)
1412 goto out_dec;
1413
Linus Torvalds1da177e2005-04-16 15:20:36 -07001414 /*
1415 * unblock after device_blocked iterates to zero
1416 */
Christoph Hellwigcd9070c2014-01-23 12:07:41 +01001417 if (atomic_dec_return(&sdev->device_blocked) > 0) {
Christoph Hellwigd2852032014-01-17 12:06:53 +01001418 /*
1419 * For the MQ case we take care of this in the caller.
1420 */
1421 if (!q->mq_ops)
1422 blk_delay_queue(q, SCSI_QUEUE_DELAY);
Christoph Hellwig71e75c92014-04-11 19:07:01 +02001423 goto out_dec;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001424 }
Christoph Hellwig71e75c92014-04-11 19:07:01 +02001425 SCSI_LOG_MLQUEUE(3, sdev_printk(KERN_INFO, sdev,
1426 "unblocking device at zero depth\n"));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001427 }
Christoph Hellwig71e75c92014-04-11 19:07:01 +02001428
1429 if (busy >= sdev->queue_depth)
1430 goto out_dec;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001431
1432 return 1;
Christoph Hellwig71e75c92014-04-11 19:07:01 +02001433out_dec:
1434 atomic_dec(&sdev->device_busy);
1435 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436}
1437
Mike Christief0c0a372008-08-17 15:24:38 -05001438/*
1439 * scsi_target_queue_ready: checks if there we can send commands to target
1440 * @sdev: scsi device on starget to check.
Mike Christief0c0a372008-08-17 15:24:38 -05001441 */
1442static inline int scsi_target_queue_ready(struct Scsi_Host *shost,
1443 struct scsi_device *sdev)
1444{
1445 struct scsi_target *starget = scsi_target(sdev);
Christoph Hellwig7ae65c02014-01-22 14:49:41 +01001446 unsigned int busy;
Mike Christief0c0a372008-08-17 15:24:38 -05001447
1448 if (starget->single_lun) {
Christoph Hellwig7ae65c02014-01-22 14:49:41 +01001449 spin_lock_irq(shost->host_lock);
Mike Christief0c0a372008-08-17 15:24:38 -05001450 if (starget->starget_sdev_user &&
Christoph Hellwig7ae65c02014-01-22 14:49:41 +01001451 starget->starget_sdev_user != sdev) {
1452 spin_unlock_irq(shost->host_lock);
1453 return 0;
1454 }
Mike Christief0c0a372008-08-17 15:24:38 -05001455 starget->starget_sdev_user = sdev;
Christoph Hellwig7ae65c02014-01-22 14:49:41 +01001456 spin_unlock_irq(shost->host_lock);
Mike Christief0c0a372008-08-17 15:24:38 -05001457 }
1458
Christoph Hellwig2ccbb002014-03-26 10:35:00 +01001459 if (starget->can_queue <= 0)
1460 return 1;
1461
Christoph Hellwig7ae65c02014-01-22 14:49:41 +01001462 busy = atomic_inc_return(&starget->target_busy) - 1;
Christoph Hellwigcd9070c2014-01-23 12:07:41 +01001463 if (atomic_read(&starget->target_blocked) > 0) {
Christoph Hellwig7ae65c02014-01-22 14:49:41 +01001464 if (busy)
1465 goto starved;
1466
Mike Christief0c0a372008-08-17 15:24:38 -05001467 /*
1468 * unblock after target_blocked iterates to zero
1469 */
Christoph Hellwigcd9070c2014-01-23 12:07:41 +01001470 if (atomic_dec_return(&starget->target_blocked) > 0)
Christoph Hellwig7ae65c02014-01-22 14:49:41 +01001471 goto out_dec;
Christoph Hellwigcf68d332014-01-22 14:36:32 +01001472
1473 SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget,
1474 "unblocking target at zero depth\n"));
Mike Christief0c0a372008-08-17 15:24:38 -05001475 }
1476
Christoph Hellwig2ccbb002014-03-26 10:35:00 +01001477 if (busy >= starget->can_queue)
Christoph Hellwig7ae65c02014-01-22 14:49:41 +01001478 goto starved;
Mike Christief0c0a372008-08-17 15:24:38 -05001479
Christoph Hellwig7ae65c02014-01-22 14:49:41 +01001480 return 1;
1481
1482starved:
1483 spin_lock_irq(shost->host_lock);
1484 list_move_tail(&sdev->starved_entry, &shost->starved_list);
Christoph Hellwigcf68d332014-01-22 14:36:32 +01001485 spin_unlock_irq(shost->host_lock);
Christoph Hellwig7ae65c02014-01-22 14:49:41 +01001486out_dec:
Christoph Hellwig2ccbb002014-03-26 10:35:00 +01001487 if (starget->can_queue > 0)
1488 atomic_dec(&starget->target_busy);
Christoph Hellwig7ae65c02014-01-22 14:49:41 +01001489 return 0;
Mike Christief0c0a372008-08-17 15:24:38 -05001490}
1491
Linus Torvalds1da177e2005-04-16 15:20:36 -07001492/*
1493 * scsi_host_queue_ready: if we can send requests to shost, return 1 else
1494 * return 0. We must end up running the queue again whenever 0 is
1495 * returned, else IO can hang.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001496 */
1497static inline int scsi_host_queue_ready(struct request_queue *q,
1498 struct Scsi_Host *shost,
1499 struct scsi_device *sdev)
1500{
Christoph Hellwig74665012014-01-22 15:29:29 +01001501 unsigned int busy;
Christoph Hellwigcf68d332014-01-22 14:36:32 +01001502
James Bottomley939647e2005-09-18 15:05:20 -05001503 if (scsi_host_in_recovery(shost))
Christoph Hellwig74665012014-01-22 15:29:29 +01001504 return 0;
1505
1506 busy = atomic_inc_return(&shost->host_busy) - 1;
Christoph Hellwigcd9070c2014-01-23 12:07:41 +01001507 if (atomic_read(&shost->host_blocked) > 0) {
Christoph Hellwig74665012014-01-22 15:29:29 +01001508 if (busy)
1509 goto starved;
1510
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511 /*
1512 * unblock after host_blocked iterates to zero
1513 */
Christoph Hellwigcd9070c2014-01-23 12:07:41 +01001514 if (atomic_dec_return(&shost->host_blocked) > 0)
Christoph Hellwig74665012014-01-22 15:29:29 +01001515 goto out_dec;
Christoph Hellwigcf68d332014-01-22 14:36:32 +01001516
1517 SCSI_LOG_MLQUEUE(3,
1518 shost_printk(KERN_INFO, shost,
1519 "unblocking host at zero depth\n"));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001520 }
Christoph Hellwig74665012014-01-22 15:29:29 +01001521
1522 if (shost->can_queue > 0 && busy >= shost->can_queue)
1523 goto starved;
1524 if (shost->host_self_blocked)
1525 goto starved;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526
1527 /* We're OK to process the command, so we can't be starved */
Christoph Hellwig74665012014-01-22 15:29:29 +01001528 if (!list_empty(&sdev->starved_entry)) {
1529 spin_lock_irq(shost->host_lock);
1530 if (!list_empty(&sdev->starved_entry))
1531 list_del_init(&sdev->starved_entry);
1532 spin_unlock_irq(shost->host_lock);
1533 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001534
Christoph Hellwig74665012014-01-22 15:29:29 +01001535 return 1;
1536
1537starved:
1538 spin_lock_irq(shost->host_lock);
1539 if (list_empty(&sdev->starved_entry))
1540 list_add_tail(&sdev->starved_entry, &shost->starved_list);
Christoph Hellwigcf68d332014-01-22 14:36:32 +01001541 spin_unlock_irq(shost->host_lock);
Christoph Hellwig74665012014-01-22 15:29:29 +01001542out_dec:
1543 atomic_dec(&shost->host_busy);
1544 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001545}
1546
1547/*
Kiyoshi Ueda6c5121b2008-10-04 14:11:35 -04001548 * Busy state exporting function for request stacking drivers.
1549 *
1550 * For efficiency, no lock is taken to check the busy state of
1551 * shost/starget/sdev, since the returned value is not guaranteed and
1552 * may be changed after request stacking drivers call the function,
1553 * regardless of taking lock or not.
1554 *
Bart Van Assche67bd9412012-06-29 15:33:22 +00001555 * When scsi can't dispatch I/Os anymore and needs to kill I/Os scsi
1556 * needs to return 'not busy'. Otherwise, request stacking drivers
1557 * may hold requests forever.
Kiyoshi Ueda6c5121b2008-10-04 14:11:35 -04001558 */
1559static int scsi_lld_busy(struct request_queue *q)
1560{
1561 struct scsi_device *sdev = q->queuedata;
1562 struct Scsi_Host *shost;
Kiyoshi Ueda6c5121b2008-10-04 14:11:35 -04001563
Bart Van Assche3f3299d2012-11-28 13:42:38 +01001564 if (blk_queue_dying(q))
Kiyoshi Ueda6c5121b2008-10-04 14:11:35 -04001565 return 0;
1566
1567 shost = sdev->host;
Kiyoshi Ueda6c5121b2008-10-04 14:11:35 -04001568
Jun'ichi Nomurab7e94a12012-05-22 18:57:17 +09001569 /*
1570 * Ignore host/starget busy state.
1571 * Since block layer does not have a concept of fairness across
1572 * multiple queues, congestion of host/starget needs to be handled
1573 * in SCSI layer.
1574 */
1575 if (scsi_host_in_recovery(shost) || scsi_device_is_busy(sdev))
Kiyoshi Ueda6c5121b2008-10-04 14:11:35 -04001576 return 1;
1577
1578 return 0;
1579}
1580
1581/*
James Bottomleye91442b2005-09-09 10:44:16 -05001582 * Kill a request for a dead device
Linus Torvalds1da177e2005-04-16 15:20:36 -07001583 */
Jens Axboe165125e2007-07-24 09:28:11 +02001584static void scsi_kill_request(struct request *req, struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001585{
James Bottomleye91442b2005-09-09 10:44:16 -05001586 struct scsi_cmnd *cmd = req->special;
Jiri Slaby03b14702009-09-23 16:15:35 +02001587 struct scsi_device *sdev;
1588 struct scsi_target *starget;
1589 struct Scsi_Host *shost;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001590
Tejun Heo9934c8c2009-05-08 11:54:16 +09001591 blk_start_request(req);
James Bottomley788ce432005-09-09 13:40:23 -05001592
Hannes Reinecke74571812011-11-09 08:39:24 +01001593 scmd_printk(KERN_INFO, cmd, "killing request\n");
1594
Jiri Slaby03b14702009-09-23 16:15:35 +02001595 sdev = cmd->device;
1596 starget = scsi_target(sdev);
1597 shost = sdev->host;
James Bottomleye91442b2005-09-09 10:44:16 -05001598 scsi_init_cmd_errh(cmd);
1599 cmd->result = DID_NO_CONNECT << 16;
1600 atomic_inc(&cmd->device->iorequest_cnt);
Tejun Heoe36e0c82006-04-11 17:27:53 +09001601
1602 /*
1603 * SCSI request completion path will do scsi_device_unbusy(),
1604 * bump busy counts. To bump the counters, we need to dance
1605 * with the locks as normal issue path does.
1606 */
Christoph Hellwig71e75c92014-04-11 19:07:01 +02001607 atomic_inc(&sdev->device_busy);
Christoph Hellwig74665012014-01-22 15:29:29 +01001608 atomic_inc(&shost->host_busy);
Christoph Hellwig2ccbb002014-03-26 10:35:00 +01001609 if (starget->can_queue > 0)
1610 atomic_inc(&starget->target_busy);
Tejun Heoe36e0c82006-04-11 17:27:53 +09001611
Jens Axboe242f9dc2008-09-14 05:55:09 -07001612 blk_complete_request(req);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001613}
1614
Jens Axboe1aea6432006-01-09 16:03:03 +01001615static void scsi_softirq_done(struct request *rq)
1616{
Jens Axboe242f9dc2008-09-14 05:55:09 -07001617 struct scsi_cmnd *cmd = rq->special;
1618 unsigned long wait_for = (cmd->allowed + 1) * rq->timeout;
Jens Axboe1aea6432006-01-09 16:03:03 +01001619 int disposition;
1620
1621 INIT_LIST_HEAD(&cmd->eh_entry);
1622
Jens Axboe242f9dc2008-09-14 05:55:09 -07001623 atomic_inc(&cmd->device->iodone_cnt);
1624 if (cmd->result)
1625 atomic_inc(&cmd->device->ioerr_cnt);
1626
Jens Axboe1aea6432006-01-09 16:03:03 +01001627 disposition = scsi_decide_disposition(cmd);
1628 if (disposition != SUCCESS &&
1629 time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
1630 sdev_printk(KERN_ERR, cmd->device,
1631 "timing out command, waited %lus\n",
1632 wait_for/HZ);
1633 disposition = SUCCESS;
1634 }
Hannes Reinecke91921e02014-06-25 16:39:59 +02001635
Jens Axboe1aea6432006-01-09 16:03:03 +01001636 scsi_log_completion(cmd, disposition);
1637
1638 switch (disposition) {
1639 case SUCCESS:
1640 scsi_finish_command(cmd);
1641 break;
1642 case NEEDS_RETRY:
Christoph Hellwig596f4822007-01-02 12:56:00 +01001643 scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);
Jens Axboe1aea6432006-01-09 16:03:03 +01001644 break;
1645 case ADD_TO_MLQUEUE:
1646 scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
1647 break;
1648 default:
1649 if (!scsi_eh_scmd_add(cmd, 0))
1650 scsi_finish_command(cmd);
1651 }
1652}
1653
Christoph Hellwig3b5382c2014-05-06 12:25:40 +02001654/**
Christoph Hellwig82042a22014-09-05 18:23:07 -07001655 * scsi_dispatch_command - Dispatch a command to the low-level driver.
1656 * @cmd: command block we are dispatching.
1657 *
1658 * Return: nonzero return request was rejected and device's queue needs to be
1659 * plugged.
1660 */
1661static int scsi_dispatch_cmd(struct scsi_cmnd *cmd)
1662{
1663 struct Scsi_Host *host = cmd->device->host;
1664 int rtn = 0;
1665
1666 atomic_inc(&cmd->device->iorequest_cnt);
1667
1668 /* check if the device is still usable */
1669 if (unlikely(cmd->device->sdev_state == SDEV_DEL)) {
1670 /* in SDEV_DEL we error all commands. DID_NO_CONNECT
1671 * returns an immediate error upwards, and signals
1672 * that the device is no longer present */
1673 cmd->result = DID_NO_CONNECT << 16;
1674 goto done;
1675 }
1676
1677 /* Check to see if the scsi lld made this device blocked. */
1678 if (unlikely(scsi_device_blocked(cmd->device))) {
1679 /*
1680 * in blocked state, the command is just put back on
1681 * the device queue. The suspend state has already
1682 * blocked the queue so future requests should not
1683 * occur until the device transitions out of the
1684 * suspend state.
1685 */
1686 SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,
1687 "queuecommand : device blocked\n"));
1688 return SCSI_MLQUEUE_DEVICE_BUSY;
1689 }
1690
1691 /* Store the LUN value in cmnd, if needed. */
1692 if (cmd->device->lun_in_cdb)
1693 cmd->cmnd[1] = (cmd->cmnd[1] & 0x1f) |
1694 (cmd->device->lun << 5 & 0xe0);
1695
1696 scsi_log_send(cmd);
1697
1698 /*
1699 * Before we queue this command, check if the command
1700 * length exceeds what the host adapter can handle.
1701 */
1702 if (cmd->cmd_len > cmd->device->host->max_cmd_len) {
1703 SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,
1704 "queuecommand : command too long. "
1705 "cdb_size=%d host->max_cmd_len=%d\n",
1706 cmd->cmd_len, cmd->device->host->max_cmd_len));
1707 cmd->result = (DID_ABORT << 16);
1708 goto done;
1709 }
1710
1711 if (unlikely(host->shost_state == SHOST_DEL)) {
1712 cmd->result = (DID_NO_CONNECT << 16);
1713 goto done;
1714
1715 }
1716
1717 trace_scsi_dispatch_cmd_start(cmd);
1718 rtn = host->hostt->queuecommand(host, cmd);
1719 if (rtn) {
1720 trace_scsi_dispatch_cmd_error(cmd, rtn);
1721 if (rtn != SCSI_MLQUEUE_DEVICE_BUSY &&
1722 rtn != SCSI_MLQUEUE_TARGET_BUSY)
1723 rtn = SCSI_MLQUEUE_HOST_BUSY;
1724
1725 SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,
1726 "queuecommand : request rejected\n"));
1727 }
1728
1729 return rtn;
1730 done:
1731 cmd->scsi_done(cmd);
1732 return 0;
1733}
1734
1735/**
Christoph Hellwig3b5382c2014-05-06 12:25:40 +02001736 * scsi_done - Invoke completion on finished SCSI command.
1737 * @cmd: The SCSI Command for which a low-level device driver (LLDD) gives
1738 * ownership back to SCSI Core -- i.e. the LLDD has finished with it.
1739 *
1740 * Description: This function is the mid-level's (SCSI Core) interrupt routine,
1741 * which regains ownership of the SCSI command (de facto) from a LLDD, and
1742 * calls blk_complete_request() for further processing.
1743 *
1744 * This function is interrupt context safe.
1745 */
1746static void scsi_done(struct scsi_cmnd *cmd)
1747{
1748 trace_scsi_dispatch_cmd_done(cmd);
1749 blk_complete_request(cmd->request);
1750}
1751
Linus Torvalds1da177e2005-04-16 15:20:36 -07001752/*
1753 * Function: scsi_request_fn()
1754 *
1755 * Purpose: Main strategy routine for SCSI.
1756 *
1757 * Arguments: q - Pointer to actual queue.
1758 *
1759 * Returns: Nothing
1760 *
1761 * Lock status: IO request lock assumed to be held when called.
1762 */
1763static void scsi_request_fn(struct request_queue *q)
Bart Van Assche613be1f2014-02-20 14:20:56 -08001764 __releases(q->queue_lock)
1765 __acquires(q->queue_lock)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001766{
1767 struct scsi_device *sdev = q->queuedata;
1768 struct Scsi_Host *shost;
1769 struct scsi_cmnd *cmd;
1770 struct request *req;
1771
Linus Torvalds1da177e2005-04-16 15:20:36 -07001772 /*
1773 * To start with, we keep looping until the queue is empty, or until
1774 * the host is no longer able to accept any more requests.
1775 */
1776 shost = sdev->host;
Jens Axboea488e742010-04-16 21:13:15 +02001777 for (;;) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001778 int rtn;
1779 /*
1780 * get next queueable request. We do this early to make sure
Hannes Reinecke91921e02014-06-25 16:39:59 +02001781 * that the request is fully prepared even if we cannot
Linus Torvalds1da177e2005-04-16 15:20:36 -07001782 * accept it.
1783 */
Tejun Heo9934c8c2009-05-08 11:54:16 +09001784 req = blk_peek_request(q);
Christoph Hellwig71e75c92014-04-11 19:07:01 +02001785 if (!req)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001786 break;
1787
1788 if (unlikely(!scsi_device_online(sdev))) {
James Bottomley9ccfc752005-10-02 11:45:08 -05001789 sdev_printk(KERN_ERR, sdev,
1790 "rejecting I/O to offline device\n");
James Bottomleye91442b2005-09-09 10:44:16 -05001791 scsi_kill_request(req, q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001792 continue;
1793 }
1794
Christoph Hellwig71e75c92014-04-11 19:07:01 +02001795 if (!scsi_dev_queue_ready(q, sdev))
1796 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001797
1798 /*
1799 * Remove the request from the request list.
1800 */
1801 if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
Tejun Heo9934c8c2009-05-08 11:54:16 +09001802 blk_start_request(req);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001803
Christoph Hellwigcf68d332014-01-22 14:36:32 +01001804 spin_unlock_irq(q->queue_lock);
James Bottomleye91442b2005-09-09 10:44:16 -05001805 cmd = req->special;
1806 if (unlikely(cmd == NULL)) {
1807 printk(KERN_CRIT "impossible request in %s.\n"
1808 "please mail a stack trace to "
Jens Axboe4aff5e22006-08-10 08:44:47 +02001809 "linux-scsi@vger.kernel.org\n",
Harvey Harrisoncadbd4a2008-07-03 23:47:27 -07001810 __func__);
Jens Axboe4aff5e22006-08-10 08:44:47 +02001811 blk_dump_rq_flags(req, "foo");
James Bottomleye91442b2005-09-09 10:44:16 -05001812 BUG();
1813 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001814
Mike Christieecefe8a2008-07-11 19:50:35 -05001815 /*
1816 * We hit this when the driver is using a host wide
1817 * tag map. For device level tag maps the queue_depth check
1818 * in the device ready fn would prevent us from trying
1819 * to allocate a tag. Since the map is a shared host resource
1820 * we add the dev to the starved list so it eventually gets
1821 * a run when a tag is freed.
1822 */
Christoph Hellwig125c99b2014-11-03 12:47:47 +01001823 if (blk_queue_tagged(q) && !(req->cmd_flags & REQ_QUEUED)) {
Christoph Hellwigcf68d332014-01-22 14:36:32 +01001824 spin_lock_irq(shost->host_lock);
Mike Christieecefe8a2008-07-11 19:50:35 -05001825 if (list_empty(&sdev->starved_entry))
1826 list_add_tail(&sdev->starved_entry,
1827 &shost->starved_list);
Christoph Hellwigcf68d332014-01-22 14:36:32 +01001828 spin_unlock_irq(shost->host_lock);
Mike Christieecefe8a2008-07-11 19:50:35 -05001829 goto not_ready;
1830 }
1831
Mike Christief0c0a372008-08-17 15:24:38 -05001832 if (!scsi_target_queue_ready(shost, sdev))
1833 goto not_ready;
1834
Linus Torvalds1da177e2005-04-16 15:20:36 -07001835 if (!scsi_host_queue_ready(q, shost, sdev))
Christoph Hellwigcf68d332014-01-22 14:36:32 +01001836 goto host_not_ready;
Christoph Hellwig125c99b2014-11-03 12:47:47 +01001837
1838 if (sdev->simple_tags)
1839 cmd->flags |= SCMD_TAGGED;
1840 else
1841 cmd->flags &= ~SCMD_TAGGED;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001842
Linus Torvalds1da177e2005-04-16 15:20:36 -07001843 /*
1844 * Finally, initialize any error handling parameters, and set up
1845 * the timers for timeouts.
1846 */
1847 scsi_init_cmd_errh(cmd);
1848
1849 /*
1850 * Dispatch the command to the low-level driver.
1851 */
Christoph Hellwig3b5382c2014-05-06 12:25:40 +02001852 cmd->scsi_done = scsi_done;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001853 rtn = scsi_dispatch_cmd(cmd);
Christoph Hellwigd0d3bbf2014-01-22 18:39:04 +01001854 if (rtn) {
1855 scsi_queue_insert(cmd, rtn);
1856 spin_lock_irq(q->queue_lock);
Jens Axboea488e742010-04-16 21:13:15 +02001857 goto out_delay;
Christoph Hellwigd0d3bbf2014-01-22 18:39:04 +01001858 }
1859 spin_lock_irq(q->queue_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001860 }
1861
Bart Van Assche613be1f2014-02-20 14:20:56 -08001862 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001863
Christoph Hellwigcf68d332014-01-22 14:36:32 +01001864 host_not_ready:
Christoph Hellwig2ccbb002014-03-26 10:35:00 +01001865 if (scsi_target(sdev)->can_queue > 0)
1866 atomic_dec(&scsi_target(sdev)->target_busy);
Christoph Hellwigcf68d332014-01-22 14:36:32 +01001867 not_ready:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001868 /*
1869 * lock q, handle tag, requeue req, and decrement device_busy. We
1870 * must return with queue_lock held.
1871 *
1872 * Decrementing device_busy without checking it is OK, as all such
1873 * cases (host limits or settings) should run the queue at some
1874 * later time.
1875 */
1876 spin_lock_irq(q->queue_lock);
1877 blk_requeue_request(q, req);
Christoph Hellwig71e75c92014-04-11 19:07:01 +02001878 atomic_dec(&sdev->device_busy);
Jens Axboea488e742010-04-16 21:13:15 +02001879out_delay:
Guenter Roeck480cadc2014-08-10 05:54:25 -07001880 if (!atomic_read(&sdev->device_busy) && !scsi_device_blocked(sdev))
Jens Axboea488e742010-04-16 21:13:15 +02001881 blk_delay_queue(q, SCSI_QUEUE_DELAY);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001882}
1883
Christoph Hellwigd2852032014-01-17 12:06:53 +01001884static inline int prep_to_mq(int ret)
1885{
1886 switch (ret) {
1887 case BLKPREP_OK:
1888 return 0;
1889 case BLKPREP_DEFER:
1890 return BLK_MQ_RQ_QUEUE_BUSY;
1891 default:
1892 return BLK_MQ_RQ_QUEUE_ERROR;
1893 }
1894}
1895
1896static int scsi_mq_prep_fn(struct request *req)
1897{
1898 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
1899 struct scsi_device *sdev = req->q->queuedata;
1900 struct Scsi_Host *shost = sdev->host;
1901 unsigned char *sense_buf = cmd->sense_buffer;
1902 struct scatterlist *sg;
1903
1904 memset(cmd, 0, sizeof(struct scsi_cmnd));
1905
1906 req->special = cmd;
1907
1908 cmd->request = req;
1909 cmd->device = sdev;
1910 cmd->sense_buffer = sense_buf;
1911
1912 cmd->tag = req->tag;
1913
Christoph Hellwigd2852032014-01-17 12:06:53 +01001914 cmd->cmnd = req->cmd;
1915 cmd->prot_op = SCSI_PROT_NORMAL;
1916
1917 INIT_LIST_HEAD(&cmd->list);
1918 INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler);
1919 cmd->jiffies_at_alloc = jiffies;
1920
Kashyap.Desai@avagotech.com64bdcbc2014-08-20 19:24:33 +05301921 if (shost->use_cmd_list) {
1922 spin_lock_irq(&sdev->list_lock);
1923 list_add_tail(&cmd->list, &sdev->cmd_list);
1924 spin_unlock_irq(&sdev->list_lock);
1925 }
Christoph Hellwigd2852032014-01-17 12:06:53 +01001926
1927 sg = (void *)cmd + sizeof(struct scsi_cmnd) + shost->hostt->cmd_size;
1928 cmd->sdb.table.sgl = sg;
1929
1930 if (scsi_host_get_prot(shost)) {
1931 cmd->prot_sdb = (void *)sg +
Tony Battersby120bb3e2014-12-08 17:20:52 -05001932 min_t(unsigned int,
1933 shost->sg_tablesize, SCSI_MAX_SG_SEGMENTS) *
1934 sizeof(struct scatterlist);
Christoph Hellwigd2852032014-01-17 12:06:53 +01001935 memset(cmd->prot_sdb, 0, sizeof(struct scsi_data_buffer));
1936
1937 cmd->prot_sdb->table.sgl =
1938 (struct scatterlist *)(cmd->prot_sdb + 1);
1939 }
1940
1941 if (blk_bidi_rq(req)) {
1942 struct request *next_rq = req->next_rq;
1943 struct scsi_data_buffer *bidi_sdb = blk_mq_rq_to_pdu(next_rq);
1944
1945 memset(bidi_sdb, 0, sizeof(struct scsi_data_buffer));
1946 bidi_sdb->table.sgl =
1947 (struct scatterlist *)(bidi_sdb + 1);
1948
1949 next_rq->special = bidi_sdb;
1950 }
1951
Christoph Hellwigfe052522014-09-22 15:59:31 +02001952 blk_mq_start_request(req);
1953
Christoph Hellwigd2852032014-01-17 12:06:53 +01001954 return scsi_setup_cmnd(sdev, req);
1955}
1956
1957static void scsi_mq_done(struct scsi_cmnd *cmd)
1958{
1959 trace_scsi_dispatch_cmd_done(cmd);
1960 blk_mq_complete_request(cmd->request);
1961}
1962
Jens Axboe74c45052014-10-29 11:14:52 -06001963static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
1964 const struct blk_mq_queue_data *bd)
Christoph Hellwigd2852032014-01-17 12:06:53 +01001965{
Jens Axboe74c45052014-10-29 11:14:52 -06001966 struct request *req = bd->rq;
Christoph Hellwigd2852032014-01-17 12:06:53 +01001967 struct request_queue *q = req->q;
1968 struct scsi_device *sdev = q->queuedata;
1969 struct Scsi_Host *shost = sdev->host;
1970 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
1971 int ret;
1972 int reason;
1973
1974 ret = prep_to_mq(scsi_prep_state_check(sdev, req));
1975 if (ret)
1976 goto out;
1977
1978 ret = BLK_MQ_RQ_QUEUE_BUSY;
1979 if (!get_device(&sdev->sdev_gendev))
1980 goto out;
1981
1982 if (!scsi_dev_queue_ready(q, sdev))
1983 goto out_put_device;
1984 if (!scsi_target_queue_ready(shost, sdev))
1985 goto out_dec_device_busy;
1986 if (!scsi_host_queue_ready(q, shost, sdev))
1987 goto out_dec_target_busy;
1988
Christoph Hellwigfe052522014-09-22 15:59:31 +02001989
Christoph Hellwigd2852032014-01-17 12:06:53 +01001990 if (!(req->cmd_flags & REQ_DONTPREP)) {
1991 ret = prep_to_mq(scsi_mq_prep_fn(req));
1992 if (ret)
1993 goto out_dec_host_busy;
1994 req->cmd_flags |= REQ_DONTPREP;
Christoph Hellwigfe052522014-09-22 15:59:31 +02001995 } else {
1996 blk_mq_start_request(req);
Christoph Hellwigd2852032014-01-17 12:06:53 +01001997 }
1998
Christoph Hellwig125c99b2014-11-03 12:47:47 +01001999 if (sdev->simple_tags)
2000 cmd->flags |= SCMD_TAGGED;
Christoph Hellwigb1dd2aa2014-10-19 17:13:58 +02002001 else
Christoph Hellwig125c99b2014-11-03 12:47:47 +01002002 cmd->flags &= ~SCMD_TAGGED;
Christoph Hellwigb1dd2aa2014-10-19 17:13:58 +02002003
Christoph Hellwigd2852032014-01-17 12:06:53 +01002004 scsi_init_cmd_errh(cmd);
2005 cmd->scsi_done = scsi_mq_done;
2006
2007 reason = scsi_dispatch_cmd(cmd);
2008 if (reason) {
2009 scsi_set_blocked(cmd, reason);
2010 ret = BLK_MQ_RQ_QUEUE_BUSY;
2011 goto out_dec_host_busy;
2012 }
2013
2014 return BLK_MQ_RQ_QUEUE_OK;
2015
2016out_dec_host_busy:
2017 atomic_dec(&shost->host_busy);
2018out_dec_target_busy:
2019 if (scsi_target(sdev)->can_queue > 0)
2020 atomic_dec(&scsi_target(sdev)->target_busy);
2021out_dec_device_busy:
2022 atomic_dec(&sdev->device_busy);
2023out_put_device:
2024 put_device(&sdev->sdev_gendev);
2025out:
2026 switch (ret) {
2027 case BLK_MQ_RQ_QUEUE_BUSY:
2028 blk_mq_stop_hw_queue(hctx);
2029 if (atomic_read(&sdev->device_busy) == 0 &&
2030 !scsi_device_blocked(sdev))
2031 blk_mq_delay_queue(hctx, SCSI_QUEUE_DELAY);
2032 break;
2033 case BLK_MQ_RQ_QUEUE_ERROR:
2034 /*
2035 * Make sure to release all allocated ressources when
2036 * we hit an error, as we will never see this command
2037 * again.
2038 */
2039 if (req->cmd_flags & REQ_DONTPREP)
2040 scsi_mq_uninit_cmd(cmd);
2041 break;
2042 default:
2043 break;
2044 }
2045 return ret;
2046}
2047
Christoph Hellwig0152fb62014-09-13 16:40:13 -07002048static enum blk_eh_timer_return scsi_timeout(struct request *req,
2049 bool reserved)
2050{
2051 if (reserved)
2052 return BLK_EH_RESET_TIMER;
2053 return scsi_times_out(req);
2054}
2055
Christoph Hellwigd2852032014-01-17 12:06:53 +01002056static int scsi_init_request(void *data, struct request *rq,
2057 unsigned int hctx_idx, unsigned int request_idx,
2058 unsigned int numa_node)
2059{
2060 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
2061
2062 cmd->sense_buffer = kzalloc_node(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL,
2063 numa_node);
2064 if (!cmd->sense_buffer)
2065 return -ENOMEM;
2066 return 0;
2067}
2068
2069static void scsi_exit_request(void *data, struct request *rq,
2070 unsigned int hctx_idx, unsigned int request_idx)
2071{
2072 struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
2073
2074 kfree(cmd->sense_buffer);
2075}
2076
Christoph Hellwigf1bea552014-04-15 12:26:54 +02002077static u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002078{
2079 struct device *host_dev;
2080 u64 bounce_limit = 0xffffffff;
2081
2082 if (shost->unchecked_isa_dma)
2083 return BLK_BOUNCE_ISA;
2084 /*
2085 * Platforms with virtual-DMA translation
2086 * hardware have no practical limit.
2087 */
2088 if (!PCI_DMA_BUS_IS_PHYS)
2089 return BLK_BOUNCE_ANY;
2090
2091 host_dev = scsi_get_device(shost);
2092 if (host_dev && host_dev->dma_mask)
Russell Kinge83b3662014-02-11 17:11:04 +00002093 bounce_limit = (u64)dma_max_pfn(host_dev) << PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002094
2095 return bounce_limit;
2096}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002097
Christoph Hellwigd2852032014-01-17 12:06:53 +01002098static void __scsi_init_queue(struct Scsi_Host *shost, struct request_queue *q)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002099{
Lin Ming6f381fa2012-04-12 13:50:38 +08002100 struct device *dev = shost->dma_dev;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002101
Jens Axboea8474ce2007-08-07 09:02:51 +02002102 /*
2103 * this limit is imposed by hardware restrictions
2104 */
Martin K. Petersen8a783622010-02-26 00:20:39 -05002105 blk_queue_max_segments(q, min_t(unsigned short, shost->sg_tablesize,
2106 SCSI_MAX_SG_CHAIN_SEGMENTS));
Jens Axboea8474ce2007-08-07 09:02:51 +02002107
Martin K. Petersen13f05c82010-09-10 20:50:10 +02002108 if (scsi_host_prot_dma(shost)) {
2109 shost->sg_prot_tablesize =
2110 min_not_zero(shost->sg_prot_tablesize,
2111 (unsigned short)SCSI_MAX_PROT_SG_SEGMENTS);
2112 BUG_ON(shost->sg_prot_tablesize < shost->sg_tablesize);
2113 blk_queue_max_integrity_segments(q, shost->sg_prot_tablesize);
2114 }
2115
Martin K. Petersen086fa5f2010-02-26 00:20:38 -05002116 blk_queue_max_hw_sectors(q, shost->max_sectors);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002117 blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
2118 blk_queue_segment_boundary(q, shost->dma_boundary);
FUJITA Tomonori99c84db2008-02-04 22:28:17 -08002119 dma_set_seg_boundary(dev, shost->dma_boundary);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002120
FUJITA Tomonori860ac562008-02-04 22:28:05 -08002121 blk_queue_max_segment_size(q, dma_get_max_seg_size(dev));
2122
Linus Torvalds1da177e2005-04-16 15:20:36 -07002123 if (!shost->use_clustering)
Martin K. Petersene692cb62010-12-01 19:41:49 +01002124 q->limits.cluster = 0;
James Bottomley465ff312008-01-01 10:00:10 -06002125
2126 /*
2127 * set a reasonable default alignment on word boundaries: the
2128 * host and device may alter it using
2129 * blk_queue_update_dma_alignment() later.
2130 */
2131 blk_queue_dma_alignment(q, 0x03);
Christoph Hellwigd2852032014-01-17 12:06:53 +01002132}
James Bottomley465ff312008-01-01 10:00:10 -06002133
Christoph Hellwigd2852032014-01-17 12:06:53 +01002134struct request_queue *__scsi_alloc_queue(struct Scsi_Host *shost,
2135 request_fn_proc *request_fn)
2136{
2137 struct request_queue *q;
2138
2139 q = blk_init_queue(request_fn, NULL);
2140 if (!q)
2141 return NULL;
2142 __scsi_init_queue(shost, q);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002143 return q;
2144}
FUJITA Tomonorib58d9152006-11-16 19:24:10 +09002145EXPORT_SYMBOL(__scsi_alloc_queue);
2146
2147struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
2148{
2149 struct request_queue *q;
2150
2151 q = __scsi_alloc_queue(sdev->host, scsi_request_fn);
2152 if (!q)
2153 return NULL;
2154
2155 blk_queue_prep_rq(q, scsi_prep_fn);
Christoph Hellwiga1b73fc2014-05-01 16:51:04 +02002156 blk_queue_unprep_rq(q, scsi_unprep_fn);
FUJITA Tomonorib58d9152006-11-16 19:24:10 +09002157 blk_queue_softirq_done(q, scsi_softirq_done);
Jens Axboe242f9dc2008-09-14 05:55:09 -07002158 blk_queue_rq_timed_out(q, scsi_times_out);
Kiyoshi Ueda6c5121b2008-10-04 14:11:35 -04002159 blk_queue_lld_busy(q, scsi_lld_busy);
FUJITA Tomonorib58d9152006-11-16 19:24:10 +09002160 return q;
2161}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002162
Christoph Hellwigd2852032014-01-17 12:06:53 +01002163static struct blk_mq_ops scsi_mq_ops = {
2164 .map_queue = blk_mq_map_queue,
2165 .queue_rq = scsi_queue_rq,
2166 .complete = scsi_softirq_done,
Christoph Hellwig0152fb62014-09-13 16:40:13 -07002167 .timeout = scsi_timeout,
Christoph Hellwigd2852032014-01-17 12:06:53 +01002168 .init_request = scsi_init_request,
2169 .exit_request = scsi_exit_request,
2170};
2171
2172struct request_queue *scsi_mq_alloc_queue(struct scsi_device *sdev)
2173{
2174 sdev->request_queue = blk_mq_init_queue(&sdev->host->tag_set);
2175 if (IS_ERR(sdev->request_queue))
2176 return NULL;
2177
2178 sdev->request_queue->queuedata = sdev;
2179 __scsi_init_queue(sdev->host, sdev->request_queue);
2180 return sdev->request_queue;
2181}
2182
2183int scsi_mq_setup_tags(struct Scsi_Host *shost)
2184{
2185 unsigned int cmd_size, sgl_size, tbl_size;
2186
2187 tbl_size = shost->sg_tablesize;
2188 if (tbl_size > SCSI_MAX_SG_SEGMENTS)
2189 tbl_size = SCSI_MAX_SG_SEGMENTS;
2190 sgl_size = tbl_size * sizeof(struct scatterlist);
2191 cmd_size = sizeof(struct scsi_cmnd) + shost->hostt->cmd_size + sgl_size;
2192 if (scsi_host_get_prot(shost))
2193 cmd_size += sizeof(struct scsi_data_buffer) + sgl_size;
2194
2195 memset(&shost->tag_set, 0, sizeof(shost->tag_set));
2196 shost->tag_set.ops = &scsi_mq_ops;
Bart Van Asscheefec4b92014-10-30 14:45:36 +01002197 shost->tag_set.nr_hw_queues = shost->nr_hw_queues ? : 1;
Christoph Hellwigd2852032014-01-17 12:06:53 +01002198 shost->tag_set.queue_depth = shost->can_queue;
2199 shost->tag_set.cmd_size = cmd_size;
2200 shost->tag_set.numa_node = NUMA_NO_NODE;
2201 shost->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE;
Shaohua Li24391c02015-01-23 14:18:00 -07002202 shost->tag_set.flags |=
2203 BLK_ALLOC_POLICY_TO_MQ_FLAG(shost->hostt->tag_alloc_policy);
Christoph Hellwigd2852032014-01-17 12:06:53 +01002204 shost->tag_set.driver_data = shost;
2205
2206 return blk_mq_alloc_tag_set(&shost->tag_set);
2207}
2208
2209void scsi_mq_destroy_tags(struct Scsi_Host *shost)
2210{
2211 blk_mq_free_tag_set(&shost->tag_set);
2212}
2213
Linus Torvalds1da177e2005-04-16 15:20:36 -07002214/*
2215 * Function: scsi_block_requests()
2216 *
2217 * Purpose: Utility function used by low-level drivers to prevent further
2218 * commands from being queued to the device.
2219 *
2220 * Arguments: shost - Host in question
2221 *
2222 * Returns: Nothing
2223 *
2224 * Lock status: No locks are assumed held.
2225 *
2226 * Notes: There is no timer nor any other means by which the requests
2227 * get unblocked other than the low-level driver calling
2228 * scsi_unblock_requests().
2229 */
2230void scsi_block_requests(struct Scsi_Host *shost)
2231{
2232 shost->host_self_blocked = 1;
2233}
2234EXPORT_SYMBOL(scsi_block_requests);
2235
2236/*
2237 * Function: scsi_unblock_requests()
2238 *
2239 * Purpose: Utility function used by low-level drivers to allow further
2240 * commands from being queued to the device.
2241 *
2242 * Arguments: shost - Host in question
2243 *
2244 * Returns: Nothing
2245 *
2246 * Lock status: No locks are assumed held.
2247 *
2248 * Notes: There is no timer nor any other means by which the requests
2249 * get unblocked other than the low-level driver calling
2250 * scsi_unblock_requests().
2251 *
2252 * This is done as an API function so that changes to the
2253 * internals of the scsi mid-layer won't require wholesale
2254 * changes to drivers that use this feature.
2255 */
2256void scsi_unblock_requests(struct Scsi_Host *shost)
2257{
2258 shost->host_self_blocked = 0;
2259 scsi_run_host_queues(shost);
2260}
2261EXPORT_SYMBOL(scsi_unblock_requests);
2262
2263int __init scsi_init_queue(void)
2264{
2265 int i;
2266
Martin K. Petersen6362abd2008-06-05 23:30:03 -04002267 scsi_sdb_cache = kmem_cache_create("scsi_data_buffer",
2268 sizeof(struct scsi_data_buffer),
2269 0, 0, NULL);
2270 if (!scsi_sdb_cache) {
2271 printk(KERN_ERR "SCSI: can't init scsi sdb cache\n");
FUJITA Tomonorif0787272008-12-14 01:23:45 +09002272 return -ENOMEM;
Boaz Harrosh6f9a35e2007-12-13 13:50:53 +02002273 }
2274
Linus Torvalds1da177e2005-04-16 15:20:36 -07002275 for (i = 0; i < SG_MEMPOOL_NR; i++) {
2276 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
2277 int size = sgp->size * sizeof(struct scatterlist);
2278
2279 sgp->slab = kmem_cache_create(sgp->name, size, 0,
Paul Mundt20c2df82007-07-20 10:11:58 +09002280 SLAB_HWCACHE_ALIGN, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002281 if (!sgp->slab) {
2282 printk(KERN_ERR "SCSI: can't init sg slab %s\n",
2283 sgp->name);
Martin K. Petersen6362abd2008-06-05 23:30:03 -04002284 goto cleanup_sdb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002285 }
2286
Matthew Dobson93d23412006-03-26 01:37:50 -08002287 sgp->pool = mempool_create_slab_pool(SG_MEMPOOL_SIZE,
2288 sgp->slab);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002289 if (!sgp->pool) {
2290 printk(KERN_ERR "SCSI: can't init sg mempool %s\n",
2291 sgp->name);
Martin K. Petersen6362abd2008-06-05 23:30:03 -04002292 goto cleanup_sdb;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002293 }
2294 }
2295
2296 return 0;
FUJITA Tomonori3d9dd6e2008-01-25 23:25:14 +09002297
Martin K. Petersen6362abd2008-06-05 23:30:03 -04002298cleanup_sdb:
FUJITA Tomonori3d9dd6e2008-01-25 23:25:14 +09002299 for (i = 0; i < SG_MEMPOOL_NR; i++) {
2300 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
2301 if (sgp->pool)
2302 mempool_destroy(sgp->pool);
2303 if (sgp->slab)
2304 kmem_cache_destroy(sgp->slab);
2305 }
Martin K. Petersen6362abd2008-06-05 23:30:03 -04002306 kmem_cache_destroy(scsi_sdb_cache);
FUJITA Tomonori3d9dd6e2008-01-25 23:25:14 +09002307
2308 return -ENOMEM;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002309}
2310
2311void scsi_exit_queue(void)
2312{
2313 int i;
2314
Martin K. Petersen6362abd2008-06-05 23:30:03 -04002315 kmem_cache_destroy(scsi_sdb_cache);
Mike Christieaa7b5cd2005-11-11 05:31:41 -06002316
Linus Torvalds1da177e2005-04-16 15:20:36 -07002317 for (i = 0; i < SG_MEMPOOL_NR; i++) {
2318 struct scsi_host_sg_pool *sgp = scsi_sg_pools + i;
2319 mempool_destroy(sgp->pool);
2320 kmem_cache_destroy(sgp->slab);
2321 }
2322}
James Bottomley5baba832006-03-18 14:10:35 -06002323
2324/**
2325 * scsi_mode_select - issue a mode select
2326 * @sdev: SCSI device to be queried
2327 * @pf: Page format bit (1 == standard, 0 == vendor specific)
2328 * @sp: Save page bit (0 == don't save, 1 == save)
2329 * @modepage: mode page being requested
2330 * @buffer: request buffer (may not be smaller than eight bytes)
2331 * @len: length of request buffer.
2332 * @timeout: command timeout
2333 * @retries: number of retries before failing
2334 * @data: returns a structure abstracting the mode header data
Rob Landleyeb448202007-11-03 13:30:39 -05002335 * @sshdr: place to put sense data (or NULL if no sense to be collected).
James Bottomley5baba832006-03-18 14:10:35 -06002336 * must be SCSI_SENSE_BUFFERSIZE big.
2337 *
2338 * Returns zero if successful; negative error number or scsi
2339 * status on error
2340 *
2341 */
2342int
2343scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage,
2344 unsigned char *buffer, int len, int timeout, int retries,
2345 struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
2346{
2347 unsigned char cmd[10];
2348 unsigned char *real_buffer;
2349 int ret;
2350
2351 memset(cmd, 0, sizeof(cmd));
2352 cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0);
2353
2354 if (sdev->use_10_for_ms) {
2355 if (len > 65535)
2356 return -EINVAL;
2357 real_buffer = kmalloc(8 + len, GFP_KERNEL);
2358 if (!real_buffer)
2359 return -ENOMEM;
2360 memcpy(real_buffer + 8, buffer, len);
2361 len += 8;
2362 real_buffer[0] = 0;
2363 real_buffer[1] = 0;
2364 real_buffer[2] = data->medium_type;
2365 real_buffer[3] = data->device_specific;
2366 real_buffer[4] = data->longlba ? 0x01 : 0;
2367 real_buffer[5] = 0;
2368 real_buffer[6] = data->block_descriptor_length >> 8;
2369 real_buffer[7] = data->block_descriptor_length;
2370
2371 cmd[0] = MODE_SELECT_10;
2372 cmd[7] = len >> 8;
2373 cmd[8] = len;
2374 } else {
2375 if (len > 255 || data->block_descriptor_length > 255 ||
2376 data->longlba)
2377 return -EINVAL;
2378
2379 real_buffer = kmalloc(4 + len, GFP_KERNEL);
2380 if (!real_buffer)
2381 return -ENOMEM;
2382 memcpy(real_buffer + 4, buffer, len);
2383 len += 4;
2384 real_buffer[0] = 0;
2385 real_buffer[1] = data->medium_type;
2386 real_buffer[2] = data->device_specific;
2387 real_buffer[3] = data->block_descriptor_length;
2388
2389
2390 cmd[0] = MODE_SELECT;
2391 cmd[4] = len;
2392 }
2393
2394 ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len,
FUJITA Tomonorif4f4e472008-12-04 14:24:39 +09002395 sshdr, timeout, retries, NULL);
James Bottomley5baba832006-03-18 14:10:35 -06002396 kfree(real_buffer);
2397 return ret;
2398}
2399EXPORT_SYMBOL_GPL(scsi_mode_select);
2400
Linus Torvalds1da177e2005-04-16 15:20:36 -07002401/**
Rob Landleyeb448202007-11-03 13:30:39 -05002402 * scsi_mode_sense - issue a mode sense, falling back from 10 to six bytes if necessary.
James Bottomley1cf72692005-08-28 11:27:01 -05002403 * @sdev: SCSI device to be queried
Linus Torvalds1da177e2005-04-16 15:20:36 -07002404 * @dbd: set if mode sense will allow block descriptors to be returned
2405 * @modepage: mode page being requested
2406 * @buffer: request buffer (may not be smaller than eight bytes)
2407 * @len: length of request buffer.
2408 * @timeout: command timeout
2409 * @retries: number of retries before failing
2410 * @data: returns a structure abstracting the mode header data
Rob Landleyeb448202007-11-03 13:30:39 -05002411 * @sshdr: place to put sense data (or NULL if no sense to be collected).
James Bottomley1cf72692005-08-28 11:27:01 -05002412 * must be SCSI_SENSE_BUFFERSIZE big.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002413 *
2414 * Returns zero if unsuccessful, or the header offset (either 4
2415 * or 8 depending on whether a six or ten byte command was
2416 * issued) if successful.
Rob Landleyeb448202007-11-03 13:30:39 -05002417 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002418int
James Bottomley1cf72692005-08-28 11:27:01 -05002419scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002420 unsigned char *buffer, int len, int timeout, int retries,
James Bottomley5baba832006-03-18 14:10:35 -06002421 struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
2422{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002423 unsigned char cmd[12];
2424 int use_10_for_ms;
2425 int header_length;
James Bottomley1cf72692005-08-28 11:27:01 -05002426 int result;
James Bottomleyea73a9f2005-08-28 11:33:52 -05002427 struct scsi_sense_hdr my_sshdr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002428
2429 memset(data, 0, sizeof(*data));
2430 memset(&cmd[0], 0, 12);
2431 cmd[1] = dbd & 0x18; /* allows DBD and LLBA bits */
2432 cmd[2] = modepage;
2433
James Bottomleyea73a9f2005-08-28 11:33:52 -05002434 /* caller might not be interested in sense, but we need it */
2435 if (!sshdr)
2436 sshdr = &my_sshdr;
2437
Linus Torvalds1da177e2005-04-16 15:20:36 -07002438 retry:
James Bottomley1cf72692005-08-28 11:27:01 -05002439 use_10_for_ms = sdev->use_10_for_ms;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002440
2441 if (use_10_for_ms) {
2442 if (len < 8)
2443 len = 8;
2444
2445 cmd[0] = MODE_SENSE_10;
2446 cmd[8] = len;
2447 header_length = 8;
2448 } else {
2449 if (len < 4)
2450 len = 4;
2451
2452 cmd[0] = MODE_SENSE;
2453 cmd[4] = len;
2454 header_length = 4;
2455 }
2456
Linus Torvalds1da177e2005-04-16 15:20:36 -07002457 memset(buffer, 0, len);
2458
James Bottomley1cf72692005-08-28 11:27:01 -05002459 result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len,
FUJITA Tomonorif4f4e472008-12-04 14:24:39 +09002460 sshdr, timeout, retries, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002461
2462 /* This code looks awful: what it's doing is making sure an
2463 * ILLEGAL REQUEST sense return identifies the actual command
2464 * byte as the problem. MODE_SENSE commands can return
2465 * ILLEGAL REQUEST if the code page isn't supported */
2466
James Bottomley1cf72692005-08-28 11:27:01 -05002467 if (use_10_for_ms && !scsi_status_is_good(result) &&
2468 (driver_byte(result) & DRIVER_SENSE)) {
James Bottomleyea73a9f2005-08-28 11:33:52 -05002469 if (scsi_sense_valid(sshdr)) {
2470 if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
2471 (sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002472 /*
2473 * Invalid command operation code
2474 */
James Bottomley1cf72692005-08-28 11:27:01 -05002475 sdev->use_10_for_ms = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002476 goto retry;
2477 }
2478 }
2479 }
2480
James Bottomley1cf72692005-08-28 11:27:01 -05002481 if(scsi_status_is_good(result)) {
Al Viro6d73c852006-02-23 02:03:16 +01002482 if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b &&
2483 (modepage == 6 || modepage == 8))) {
2484 /* Initio breakage? */
2485 header_length = 0;
2486 data->length = 13;
2487 data->medium_type = 0;
2488 data->device_specific = 0;
2489 data->longlba = 0;
2490 data->block_descriptor_length = 0;
2491 } else if(use_10_for_ms) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002492 data->length = buffer[0]*256 + buffer[1] + 2;
2493 data->medium_type = buffer[2];
2494 data->device_specific = buffer[3];
2495 data->longlba = buffer[4] & 0x01;
2496 data->block_descriptor_length = buffer[6]*256
2497 + buffer[7];
2498 } else {
2499 data->length = buffer[0] + 1;
2500 data->medium_type = buffer[1];
2501 data->device_specific = buffer[2];
2502 data->block_descriptor_length = buffer[3];
2503 }
Al Viro6d73c852006-02-23 02:03:16 +01002504 data->header_length = header_length;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002505 }
2506
James Bottomley1cf72692005-08-28 11:27:01 -05002507 return result;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002508}
2509EXPORT_SYMBOL(scsi_mode_sense);
2510
James Bottomley001aac22007-12-02 19:10:40 +02002511/**
2512 * scsi_test_unit_ready - test if unit is ready
2513 * @sdev: scsi device to change the state of.
2514 * @timeout: command timeout
2515 * @retries: number of retries before failing
2516 * @sshdr_external: Optional pointer to struct scsi_sense_hdr for
2517 * returning sense. Make sure that this is cleared before passing
2518 * in.
2519 *
2520 * Returns zero if unsuccessful or an error if TUR failed. For
Tejun Heo9f8a2c22010-12-08 20:57:40 +01002521 * removable media, UNIT_ATTENTION sets ->changed flag.
James Bottomley001aac22007-12-02 19:10:40 +02002522 **/
Linus Torvalds1da177e2005-04-16 15:20:36 -07002523int
James Bottomley001aac22007-12-02 19:10:40 +02002524scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries,
2525 struct scsi_sense_hdr *sshdr_external)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002526{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002527 char cmd[] = {
2528 TEST_UNIT_READY, 0, 0, 0, 0, 0,
2529 };
James Bottomley001aac22007-12-02 19:10:40 +02002530 struct scsi_sense_hdr *sshdr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002531 int result;
James Bottomley001aac22007-12-02 19:10:40 +02002532
2533 if (!sshdr_external)
2534 sshdr = kzalloc(sizeof(*sshdr), GFP_KERNEL);
2535 else
2536 sshdr = sshdr_external;
2537
2538 /* try to eat the UNIT_ATTENTION if there are enough retries */
2539 do {
2540 result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, sshdr,
FUJITA Tomonorif4f4e472008-12-04 14:24:39 +09002541 timeout, retries, NULL);
James Bottomley32c356d2008-08-20 00:58:13 +02002542 if (sdev->removable && scsi_sense_valid(sshdr) &&
2543 sshdr->sense_key == UNIT_ATTENTION)
2544 sdev->changed = 1;
2545 } while (scsi_sense_valid(sshdr) &&
2546 sshdr->sense_key == UNIT_ATTENTION && --retries);
James Bottomley001aac22007-12-02 19:10:40 +02002547
James Bottomley001aac22007-12-02 19:10:40 +02002548 if (!sshdr_external)
2549 kfree(sshdr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002550 return result;
2551}
2552EXPORT_SYMBOL(scsi_test_unit_ready);
2553
2554/**
Rob Landleyeb448202007-11-03 13:30:39 -05002555 * scsi_device_set_state - Take the given device through the device state model.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002556 * @sdev: scsi device to change the state of.
2557 * @state: state to change to.
2558 *
2559 * Returns zero if unsuccessful or an error if the requested
2560 * transition is illegal.
Rob Landleyeb448202007-11-03 13:30:39 -05002561 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002562int
2563scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
2564{
2565 enum scsi_device_state oldstate = sdev->sdev_state;
2566
2567 if (state == oldstate)
2568 return 0;
2569
2570 switch (state) {
2571 case SDEV_CREATED:
James Bottomley6f4267e2008-08-22 16:53:31 -05002572 switch (oldstate) {
2573 case SDEV_CREATED_BLOCK:
2574 break;
2575 default:
2576 goto illegal;
2577 }
2578 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002579
2580 case SDEV_RUNNING:
2581 switch (oldstate) {
2582 case SDEV_CREATED:
2583 case SDEV_OFFLINE:
Mike Christie1b8d2622012-05-17 23:56:56 -05002584 case SDEV_TRANSPORT_OFFLINE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002585 case SDEV_QUIESCE:
2586 case SDEV_BLOCK:
2587 break;
2588 default:
2589 goto illegal;
2590 }
2591 break;
2592
2593 case SDEV_QUIESCE:
2594 switch (oldstate) {
2595 case SDEV_RUNNING:
2596 case SDEV_OFFLINE:
Mike Christie1b8d2622012-05-17 23:56:56 -05002597 case SDEV_TRANSPORT_OFFLINE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002598 break;
2599 default:
2600 goto illegal;
2601 }
2602 break;
2603
2604 case SDEV_OFFLINE:
Mike Christie1b8d2622012-05-17 23:56:56 -05002605 case SDEV_TRANSPORT_OFFLINE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002606 switch (oldstate) {
2607 case SDEV_CREATED:
2608 case SDEV_RUNNING:
2609 case SDEV_QUIESCE:
2610 case SDEV_BLOCK:
2611 break;
2612 default:
2613 goto illegal;
2614 }
2615 break;
2616
2617 case SDEV_BLOCK:
2618 switch (oldstate) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002619 case SDEV_RUNNING:
James Bottomley6f4267e2008-08-22 16:53:31 -05002620 case SDEV_CREATED_BLOCK:
2621 break;
2622 default:
2623 goto illegal;
2624 }
2625 break;
2626
2627 case SDEV_CREATED_BLOCK:
2628 switch (oldstate) {
2629 case SDEV_CREATED:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002630 break;
2631 default:
2632 goto illegal;
2633 }
2634 break;
2635
2636 case SDEV_CANCEL:
2637 switch (oldstate) {
2638 case SDEV_CREATED:
2639 case SDEV_RUNNING:
Alan Stern9ea72902006-06-23 14:25:34 -04002640 case SDEV_QUIESCE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002641 case SDEV_OFFLINE:
Mike Christie1b8d2622012-05-17 23:56:56 -05002642 case SDEV_TRANSPORT_OFFLINE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002643 case SDEV_BLOCK:
2644 break;
2645 default:
2646 goto illegal;
2647 }
2648 break;
2649
2650 case SDEV_DEL:
2651 switch (oldstate) {
Brian King309bd272006-06-27 11:10:43 -05002652 case SDEV_CREATED:
2653 case SDEV_RUNNING:
2654 case SDEV_OFFLINE:
Mike Christie1b8d2622012-05-17 23:56:56 -05002655 case SDEV_TRANSPORT_OFFLINE:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002656 case SDEV_CANCEL:
Bart Van Assche0516c082013-07-02 15:06:33 +02002657 case SDEV_CREATED_BLOCK:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002658 break;
2659 default:
2660 goto illegal;
2661 }
2662 break;
2663
2664 }
2665 sdev->sdev_state = state;
2666 return 0;
2667
2668 illegal:
Hannes Reinecke91921e02014-06-25 16:39:59 +02002669 SCSI_LOG_ERROR_RECOVERY(1,
James Bottomley9ccfc752005-10-02 11:45:08 -05002670 sdev_printk(KERN_ERR, sdev,
Hannes Reinecke91921e02014-06-25 16:39:59 +02002671 "Illegal state transition %s->%s",
James Bottomley9ccfc752005-10-02 11:45:08 -05002672 scsi_device_state_name(oldstate),
2673 scsi_device_state_name(state))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002674 );
2675 return -EINVAL;
2676}
2677EXPORT_SYMBOL(scsi_device_set_state);
2678
2679/**
Jeff Garzika341cd02007-10-29 17:15:22 -04002680 * sdev_evt_emit - emit a single SCSI device uevent
2681 * @sdev: associated SCSI device
2682 * @evt: event to emit
2683 *
2684 * Send a single uevent (scsi_event) to the associated scsi_device.
2685 */
2686static void scsi_evt_emit(struct scsi_device *sdev, struct scsi_event *evt)
2687{
2688 int idx = 0;
2689 char *envp[3];
2690
2691 switch (evt->evt_type) {
2692 case SDEV_EVT_MEDIA_CHANGE:
2693 envp[idx++] = "SDEV_MEDIA_CHANGE=1";
2694 break;
Ewan D. Milne279afdf2013-08-08 15:07:48 -04002695 case SDEV_EVT_INQUIRY_CHANGE_REPORTED:
2696 envp[idx++] = "SDEV_UA=INQUIRY_DATA_HAS_CHANGED";
2697 break;
2698 case SDEV_EVT_CAPACITY_CHANGE_REPORTED:
2699 envp[idx++] = "SDEV_UA=CAPACITY_DATA_HAS_CHANGED";
2700 break;
2701 case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED:
2702 envp[idx++] = "SDEV_UA=THIN_PROVISIONING_SOFT_THRESHOLD_REACHED";
2703 break;
2704 case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED:
2705 envp[idx++] = "SDEV_UA=MODE_PARAMETERS_CHANGED";
2706 break;
2707 case SDEV_EVT_LUN_CHANGE_REPORTED:
2708 envp[idx++] = "SDEV_UA=REPORTED_LUNS_DATA_HAS_CHANGED";
2709 break;
Jeff Garzika341cd02007-10-29 17:15:22 -04002710 default:
2711 /* do nothing */
2712 break;
2713 }
2714
2715 envp[idx++] = NULL;
2716
2717 kobject_uevent_env(&sdev->sdev_gendev.kobj, KOBJ_CHANGE, envp);
2718}
2719
2720/**
2721 * sdev_evt_thread - send a uevent for each scsi event
2722 * @work: work struct for scsi_device
2723 *
2724 * Dispatch queued events to their associated scsi_device kobjects
2725 * as uevents.
2726 */
2727void scsi_evt_thread(struct work_struct *work)
2728{
2729 struct scsi_device *sdev;
Ewan D. Milne279afdf2013-08-08 15:07:48 -04002730 enum scsi_device_event evt_type;
Jeff Garzika341cd02007-10-29 17:15:22 -04002731 LIST_HEAD(event_list);
2732
2733 sdev = container_of(work, struct scsi_device, event_work);
2734
Ewan D. Milne279afdf2013-08-08 15:07:48 -04002735 for (evt_type = SDEV_EVT_FIRST; evt_type <= SDEV_EVT_LAST; evt_type++)
2736 if (test_and_clear_bit(evt_type, sdev->pending_events))
2737 sdev_evt_send_simple(sdev, evt_type, GFP_KERNEL);
2738
Jeff Garzika341cd02007-10-29 17:15:22 -04002739 while (1) {
2740 struct scsi_event *evt;
2741 struct list_head *this, *tmp;
2742 unsigned long flags;
2743
2744 spin_lock_irqsave(&sdev->list_lock, flags);
2745 list_splice_init(&sdev->event_list, &event_list);
2746 spin_unlock_irqrestore(&sdev->list_lock, flags);
2747
2748 if (list_empty(&event_list))
2749 break;
2750
2751 list_for_each_safe(this, tmp, &event_list) {
2752 evt = list_entry(this, struct scsi_event, node);
2753 list_del(&evt->node);
2754 scsi_evt_emit(sdev, evt);
2755 kfree(evt);
2756 }
2757 }
2758}
2759
2760/**
2761 * sdev_evt_send - send asserted event to uevent thread
2762 * @sdev: scsi_device event occurred on
2763 * @evt: event to send
2764 *
2765 * Assert scsi device event asynchronously.
2766 */
2767void sdev_evt_send(struct scsi_device *sdev, struct scsi_event *evt)
2768{
2769 unsigned long flags;
2770
Kay Sievers4d1566e2008-03-19 13:04:47 +01002771#if 0
2772 /* FIXME: currently this check eliminates all media change events
2773 * for polled devices. Need to update to discriminate between AN
2774 * and polled events */
Jeff Garzika341cd02007-10-29 17:15:22 -04002775 if (!test_bit(evt->evt_type, sdev->supported_events)) {
2776 kfree(evt);
2777 return;
2778 }
Kay Sievers4d1566e2008-03-19 13:04:47 +01002779#endif
Jeff Garzika341cd02007-10-29 17:15:22 -04002780
2781 spin_lock_irqsave(&sdev->list_lock, flags);
2782 list_add_tail(&evt->node, &sdev->event_list);
2783 schedule_work(&sdev->event_work);
2784 spin_unlock_irqrestore(&sdev->list_lock, flags);
2785}
2786EXPORT_SYMBOL_GPL(sdev_evt_send);
2787
2788/**
2789 * sdev_evt_alloc - allocate a new scsi event
2790 * @evt_type: type of event to allocate
2791 * @gfpflags: GFP flags for allocation
2792 *
2793 * Allocates and returns a new scsi_event.
2794 */
2795struct scsi_event *sdev_evt_alloc(enum scsi_device_event evt_type,
2796 gfp_t gfpflags)
2797{
2798 struct scsi_event *evt = kzalloc(sizeof(struct scsi_event), gfpflags);
2799 if (!evt)
2800 return NULL;
2801
2802 evt->evt_type = evt_type;
2803 INIT_LIST_HEAD(&evt->node);
2804
2805 /* evt_type-specific initialization, if any */
2806 switch (evt_type) {
2807 case SDEV_EVT_MEDIA_CHANGE:
Ewan D. Milne279afdf2013-08-08 15:07:48 -04002808 case SDEV_EVT_INQUIRY_CHANGE_REPORTED:
2809 case SDEV_EVT_CAPACITY_CHANGE_REPORTED:
2810 case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED:
2811 case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED:
2812 case SDEV_EVT_LUN_CHANGE_REPORTED:
Jeff Garzika341cd02007-10-29 17:15:22 -04002813 default:
2814 /* do nothing */
2815 break;
2816 }
2817
2818 return evt;
2819}
2820EXPORT_SYMBOL_GPL(sdev_evt_alloc);
2821
2822/**
2823 * sdev_evt_send_simple - send asserted event to uevent thread
2824 * @sdev: scsi_device event occurred on
2825 * @evt_type: type of event to send
2826 * @gfpflags: GFP flags for allocation
2827 *
2828 * Assert scsi device event asynchronously, given an event type.
2829 */
2830void sdev_evt_send_simple(struct scsi_device *sdev,
2831 enum scsi_device_event evt_type, gfp_t gfpflags)
2832{
2833 struct scsi_event *evt = sdev_evt_alloc(evt_type, gfpflags);
2834 if (!evt) {
2835 sdev_printk(KERN_ERR, sdev, "event %d eaten due to OOM\n",
2836 evt_type);
2837 return;
2838 }
2839
2840 sdev_evt_send(sdev, evt);
2841}
2842EXPORT_SYMBOL_GPL(sdev_evt_send_simple);
2843
2844/**
Linus Torvalds1da177e2005-04-16 15:20:36 -07002845 * scsi_device_quiesce - Block user issued commands.
2846 * @sdev: scsi device to quiesce.
2847 *
2848 * This works by trying to transition to the SDEV_QUIESCE state
2849 * (which must be a legal transition). When the device is in this
2850 * state, only special requests will be accepted, all others will
2851 * be deferred. Since special requests may also be requeued requests,
2852 * a successful return doesn't guarantee the device will be
2853 * totally quiescent.
2854 *
2855 * Must be called with user context, may sleep.
2856 *
2857 * Returns zero if unsuccessful or an error if not.
Rob Landleyeb448202007-11-03 13:30:39 -05002858 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002859int
2860scsi_device_quiesce(struct scsi_device *sdev)
2861{
2862 int err = scsi_device_set_state(sdev, SDEV_QUIESCE);
2863 if (err)
2864 return err;
2865
2866 scsi_run_queue(sdev->request_queue);
Christoph Hellwig71e75c92014-04-11 19:07:01 +02002867 while (atomic_read(&sdev->device_busy)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002868 msleep_interruptible(200);
2869 scsi_run_queue(sdev->request_queue);
2870 }
2871 return 0;
2872}
2873EXPORT_SYMBOL(scsi_device_quiesce);
2874
2875/**
2876 * scsi_device_resume - Restart user issued commands to a quiesced device.
2877 * @sdev: scsi device to resume.
2878 *
2879 * Moves the device from quiesced back to running and restarts the
2880 * queues.
2881 *
2882 * Must be called with user context, may sleep.
Rob Landleyeb448202007-11-03 13:30:39 -05002883 */
Dan Williamsa7a20d12012-03-22 17:05:11 -07002884void scsi_device_resume(struct scsi_device *sdev)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002885{
Dan Williamsa7a20d12012-03-22 17:05:11 -07002886 /* check if the device state was mutated prior to resume, and if
2887 * so assume the state is being managed elsewhere (for example
2888 * device deleted during suspend)
2889 */
2890 if (sdev->sdev_state != SDEV_QUIESCE ||
2891 scsi_device_set_state(sdev, SDEV_RUNNING))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002892 return;
2893 scsi_run_queue(sdev->request_queue);
2894}
2895EXPORT_SYMBOL(scsi_device_resume);
2896
2897static void
2898device_quiesce_fn(struct scsi_device *sdev, void *data)
2899{
2900 scsi_device_quiesce(sdev);
2901}
2902
2903void
2904scsi_target_quiesce(struct scsi_target *starget)
2905{
2906 starget_for_each_device(starget, NULL, device_quiesce_fn);
2907}
2908EXPORT_SYMBOL(scsi_target_quiesce);
2909
2910static void
2911device_resume_fn(struct scsi_device *sdev, void *data)
2912{
2913 scsi_device_resume(sdev);
2914}
2915
2916void
2917scsi_target_resume(struct scsi_target *starget)
2918{
2919 starget_for_each_device(starget, NULL, device_resume_fn);
2920}
2921EXPORT_SYMBOL(scsi_target_resume);
2922
2923/**
Rob Landleyeb448202007-11-03 13:30:39 -05002924 * scsi_internal_device_block - internal function to put a device temporarily into the SDEV_BLOCK state
Linus Torvalds1da177e2005-04-16 15:20:36 -07002925 * @sdev: device to block
2926 *
2927 * Block request made by scsi lld's to temporarily stop all
2928 * scsi commands on the specified device. Called from interrupt
2929 * or normal process context.
2930 *
2931 * Returns zero if successful or error if not
2932 *
2933 * Notes:
2934 * This routine transitions the device to the SDEV_BLOCK state
2935 * (which must be a legal transition). When the device is in this
2936 * state, all commands are deferred until the scsi lld reenables
2937 * the device with scsi_device_unblock or device_block_tmo fires.
Rob Landleyeb448202007-11-03 13:30:39 -05002938 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002939int
2940scsi_internal_device_block(struct scsi_device *sdev)
2941{
Jens Axboe165125e2007-07-24 09:28:11 +02002942 struct request_queue *q = sdev->request_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002943 unsigned long flags;
2944 int err = 0;
2945
2946 err = scsi_device_set_state(sdev, SDEV_BLOCK);
James Bottomley6f4267e2008-08-22 16:53:31 -05002947 if (err) {
2948 err = scsi_device_set_state(sdev, SDEV_CREATED_BLOCK);
2949
2950 if (err)
2951 return err;
2952 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002953
2954 /*
2955 * The device has transitioned to SDEV_BLOCK. Stop the
2956 * block layer from calling the midlayer with this device's
2957 * request queue.
2958 */
Christoph Hellwigd2852032014-01-17 12:06:53 +01002959 if (q->mq_ops) {
2960 blk_mq_stop_hw_queues(q);
2961 } else {
2962 spin_lock_irqsave(q->queue_lock, flags);
2963 blk_stop_queue(q);
2964 spin_unlock_irqrestore(q->queue_lock, flags);
2965 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002966
2967 return 0;
2968}
2969EXPORT_SYMBOL_GPL(scsi_internal_device_block);
2970
2971/**
2972 * scsi_internal_device_unblock - resume a device after a block request
2973 * @sdev: device to resume
Mike Christie5d9fb5c2012-05-17 23:56:57 -05002974 * @new_state: state to set devices to after unblocking
Linus Torvalds1da177e2005-04-16 15:20:36 -07002975 *
2976 * Called by scsi lld's or the midlayer to restart the device queue
2977 * for the previously suspended scsi device. Called from interrupt or
2978 * normal process context.
2979 *
2980 * Returns zero if successful or error if not.
2981 *
2982 * Notes:
2983 * This routine transitions the device to the SDEV_RUNNING state
Mike Christie5d9fb5c2012-05-17 23:56:57 -05002984 * or to one of the offline states (which must be a legal transition)
Mike Christied0754982012-05-17 23:56:58 -05002985 * allowing the midlayer to goose the queue for this device.
Rob Landleyeb448202007-11-03 13:30:39 -05002986 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002987int
Mike Christie5d9fb5c2012-05-17 23:56:57 -05002988scsi_internal_device_unblock(struct scsi_device *sdev,
2989 enum scsi_device_state new_state)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002990{
Jens Axboe165125e2007-07-24 09:28:11 +02002991 struct request_queue *q = sdev->request_queue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002992 unsigned long flags;
Mike Christie5d9fb5c2012-05-17 23:56:57 -05002993
2994 /*
2995 * Try to transition the scsi device to SDEV_RUNNING or one of the
2996 * offlined states and goose the device queue if successful.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002997 */
Vikas Chaudhary0e580762012-08-09 04:51:30 -04002998 if ((sdev->sdev_state == SDEV_BLOCK) ||
2999 (sdev->sdev_state == SDEV_TRANSPORT_OFFLINE))
Mike Christie5d9fb5c2012-05-17 23:56:57 -05003000 sdev->sdev_state = new_state;
3001 else if (sdev->sdev_state == SDEV_CREATED_BLOCK) {
3002 if (new_state == SDEV_TRANSPORT_OFFLINE ||
3003 new_state == SDEV_OFFLINE)
3004 sdev->sdev_state = new_state;
3005 else
3006 sdev->sdev_state = SDEV_CREATED;
3007 } else if (sdev->sdev_state != SDEV_CANCEL &&
Mike Christie986fe6c2010-10-06 03:10:59 -05003008 sdev->sdev_state != SDEV_OFFLINE)
Takahiro Yasui5c10e632009-04-29 12:13:02 -04003009 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003010
Christoph Hellwigd2852032014-01-17 12:06:53 +01003011 if (q->mq_ops) {
3012 blk_mq_start_stopped_hw_queues(q, false);
3013 } else {
3014 spin_lock_irqsave(q->queue_lock, flags);
3015 blk_start_queue(q);
3016 spin_unlock_irqrestore(q->queue_lock, flags);
3017 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003018
3019 return 0;
3020}
3021EXPORT_SYMBOL_GPL(scsi_internal_device_unblock);
3022
3023static void
3024device_block(struct scsi_device *sdev, void *data)
3025{
3026 scsi_internal_device_block(sdev);
3027}
3028
3029static int
3030target_block(struct device *dev, void *data)
3031{
3032 if (scsi_is_target_device(dev))
3033 starget_for_each_device(to_scsi_target(dev), NULL,
3034 device_block);
3035 return 0;
3036}
3037
3038void
3039scsi_target_block(struct device *dev)
3040{
3041 if (scsi_is_target_device(dev))
3042 starget_for_each_device(to_scsi_target(dev), NULL,
3043 device_block);
3044 else
3045 device_for_each_child(dev, NULL, target_block);
3046}
3047EXPORT_SYMBOL_GPL(scsi_target_block);
3048
3049static void
3050device_unblock(struct scsi_device *sdev, void *data)
3051{
Mike Christie5d9fb5c2012-05-17 23:56:57 -05003052 scsi_internal_device_unblock(sdev, *(enum scsi_device_state *)data);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003053}
3054
3055static int
3056target_unblock(struct device *dev, void *data)
3057{
3058 if (scsi_is_target_device(dev))
Mike Christie5d9fb5c2012-05-17 23:56:57 -05003059 starget_for_each_device(to_scsi_target(dev), data,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003060 device_unblock);
3061 return 0;
3062}
3063
3064void
Mike Christie5d9fb5c2012-05-17 23:56:57 -05003065scsi_target_unblock(struct device *dev, enum scsi_device_state new_state)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003066{
3067 if (scsi_is_target_device(dev))
Mike Christie5d9fb5c2012-05-17 23:56:57 -05003068 starget_for_each_device(to_scsi_target(dev), &new_state,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003069 device_unblock);
3070 else
Mike Christie5d9fb5c2012-05-17 23:56:57 -05003071 device_for_each_child(dev, &new_state, target_unblock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003072}
3073EXPORT_SYMBOL_GPL(scsi_target_unblock);
Guennadi Liakhovetskicdb8c2a2006-04-02 21:57:43 +02003074
3075/**
3076 * scsi_kmap_atomic_sg - find and atomically map an sg-elemnt
Rob Landleyeb448202007-11-03 13:30:39 -05003077 * @sgl: scatter-gather list
Guennadi Liakhovetskicdb8c2a2006-04-02 21:57:43 +02003078 * @sg_count: number of segments in sg
3079 * @offset: offset in bytes into sg, on return offset into the mapped area
3080 * @len: bytes to map, on return number of bytes mapped
3081 *
3082 * Returns virtual address of the start of the mapped page
3083 */
Jens Axboec6132da2007-10-16 11:08:49 +02003084void *scsi_kmap_atomic_sg(struct scatterlist *sgl, int sg_count,
Guennadi Liakhovetskicdb8c2a2006-04-02 21:57:43 +02003085 size_t *offset, size_t *len)
3086{
3087 int i;
3088 size_t sg_len = 0, len_complete = 0;
Jens Axboec6132da2007-10-16 11:08:49 +02003089 struct scatterlist *sg;
Guennadi Liakhovetskicdb8c2a2006-04-02 21:57:43 +02003090 struct page *page;
3091
Andrew Morton22cfefb2007-02-05 16:39:03 -08003092 WARN_ON(!irqs_disabled());
3093
Jens Axboec6132da2007-10-16 11:08:49 +02003094 for_each_sg(sgl, sg, sg_count, i) {
Guennadi Liakhovetskicdb8c2a2006-04-02 21:57:43 +02003095 len_complete = sg_len; /* Complete sg-entries */
Jens Axboec6132da2007-10-16 11:08:49 +02003096 sg_len += sg->length;
Guennadi Liakhovetskicdb8c2a2006-04-02 21:57:43 +02003097 if (sg_len > *offset)
3098 break;
3099 }
3100
3101 if (unlikely(i == sg_count)) {
Andrew Morton169e1a22006-04-18 21:09:08 -07003102 printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, "
3103 "elements %d\n",
Harvey Harrisoncadbd4a2008-07-03 23:47:27 -07003104 __func__, sg_len, *offset, sg_count);
Guennadi Liakhovetskicdb8c2a2006-04-02 21:57:43 +02003105 WARN_ON(1);
3106 return NULL;
3107 }
3108
3109 /* Offset starting from the beginning of first page in this sg-entry */
Jens Axboec6132da2007-10-16 11:08:49 +02003110 *offset = *offset - len_complete + sg->offset;
Guennadi Liakhovetskicdb8c2a2006-04-02 21:57:43 +02003111
3112 /* Assumption: contiguous pages can be accessed as "page + i" */
Jens Axboe45711f12007-10-22 21:19:53 +02003113 page = nth_page(sg_page(sg), (*offset >> PAGE_SHIFT));
Guennadi Liakhovetskicdb8c2a2006-04-02 21:57:43 +02003114 *offset &= ~PAGE_MASK;
3115
3116 /* Bytes in this sg-entry from *offset to the end of the page */
3117 sg_len = PAGE_SIZE - *offset;
3118 if (*len > sg_len)
3119 *len = sg_len;
3120
Cong Wang77dfce02011-11-25 23:14:23 +08003121 return kmap_atomic(page);
Guennadi Liakhovetskicdb8c2a2006-04-02 21:57:43 +02003122}
3123EXPORT_SYMBOL(scsi_kmap_atomic_sg);
3124
3125/**
Rob Landleyeb448202007-11-03 13:30:39 -05003126 * scsi_kunmap_atomic_sg - atomically unmap a virtual address, previously mapped with scsi_kmap_atomic_sg
Guennadi Liakhovetskicdb8c2a2006-04-02 21:57:43 +02003127 * @virt: virtual address to be unmapped
3128 */
3129void scsi_kunmap_atomic_sg(void *virt)
3130{
Cong Wang77dfce02011-11-25 23:14:23 +08003131 kunmap_atomic(virt);
Guennadi Liakhovetskicdb8c2a2006-04-02 21:57:43 +02003132}
3133EXPORT_SYMBOL(scsi_kunmap_atomic_sg);
Aaron Lu6f4c8272013-01-23 15:09:32 +08003134
3135void sdev_disable_disk_events(struct scsi_device *sdev)
3136{
3137 atomic_inc(&sdev->disk_events_disable_depth);
3138}
3139EXPORT_SYMBOL(sdev_disable_disk_events);
3140
3141void sdev_enable_disk_events(struct scsi_device *sdev)
3142{
3143 if (WARN_ON_ONCE(atomic_read(&sdev->disk_events_disable_depth) <= 0))
3144 return;
3145 atomic_dec(&sdev->disk_events_disable_depth);
3146}
3147EXPORT_SYMBOL(sdev_enable_disk_events);