blob: 5adfc4739cfddd1ddb3ecc2294e61f536b82b5ba [file] [log] [blame]
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001/*
2 * Copyright (C) 2016 CNEX Labs
3 * Initial release: Javier Gonzalez <javier@cnexlabs.com>
4 * Matias Bjorling <matias@cnexlabs.com>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License version
8 * 2 as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
14 *
15 * pblk-read.c - pblk's read path
16 */
17
18#include "pblk.h"
19
20/*
21 * There is no guarantee that the value read from cache has not been updated and
22 * resides at another location in the cache. We guarantee though that if the
23 * value is read from the cache, it belongs to the mapped lba. In order to
24 * guarantee and order between writes and reads are ordered, a flush must be
25 * issued.
26 */
27static int pblk_read_from_cache(struct pblk *pblk, struct bio *bio,
28 sector_t lba, struct ppa_addr ppa,
Javier González75cb8e92017-07-28 15:13:16 +020029 int bio_iter, bool advanced_bio)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020030{
31#ifdef CONFIG_NVM_DEBUG
32 /* Callers must ensure that the ppa points to a cache address */
33 BUG_ON(pblk_ppa_empty(ppa));
34 BUG_ON(!pblk_addr_in_cache(ppa));
35#endif
36
Javier González75cb8e92017-07-28 15:13:16 +020037 return pblk_rb_copy_to_bio(&pblk->rwb, bio, lba, ppa,
38 bio_iter, advanced_bio);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020039}
40
41static void pblk_read_ppalist_rq(struct pblk *pblk, struct nvm_rq *rqd,
Javier González84454e62017-10-13 14:46:13 +020042 sector_t blba, unsigned long *read_bitmap)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020043{
Javier Gonzáleza4809fe2017-10-13 14:46:22 +020044 struct pblk_sec_meta *meta_list = rqd->meta_list;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020045 struct bio *bio = rqd->bio;
46 struct ppa_addr ppas[PBLK_MAX_REQ_ADDRS];
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020047 int nr_secs = rqd->nr_ppas;
Javier González75cb8e92017-07-28 15:13:16 +020048 bool advanced_bio = false;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020049 int i, j = 0;
50
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020051 pblk_lookup_l2p_seq(pblk, ppas, blba, nr_secs);
52
53 for (i = 0; i < nr_secs; i++) {
54 struct ppa_addr p = ppas[i];
55 sector_t lba = blba + i;
56
57retry:
58 if (pblk_ppa_empty(p)) {
59 WARN_ON(test_and_set_bit(i, read_bitmap));
Javier Gonzáleza4809fe2017-10-13 14:46:22 +020060 meta_list[i].lba = cpu_to_le64(ADDR_EMPTY);
Javier González75cb8e92017-07-28 15:13:16 +020061
62 if (unlikely(!advanced_bio)) {
63 bio_advance(bio, (i) * PBLK_EXPOSED_PAGE_SIZE);
64 advanced_bio = true;
65 }
66
67 goto next;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020068 }
69
70 /* Try to read from write buffer. The address is later checked
71 * on the write buffer to prevent retrieving overwritten data.
72 */
73 if (pblk_addr_in_cache(p)) {
Javier González75cb8e92017-07-28 15:13:16 +020074 if (!pblk_read_from_cache(pblk, bio, lba, p, i,
75 advanced_bio)) {
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020076 pblk_lookup_l2p_seq(pblk, &p, lba, 1);
77 goto retry;
78 }
79 WARN_ON(test_and_set_bit(i, read_bitmap));
Javier Gonzáleza4809fe2017-10-13 14:46:22 +020080 meta_list[i].lba = cpu_to_le64(lba);
Javier González75cb8e92017-07-28 15:13:16 +020081 advanced_bio = true;
Javier Gonzálezdb7ada32017-06-26 11:57:13 +020082#ifdef CONFIG_NVM_DEBUG
83 atomic_long_inc(&pblk->cache_reads);
84#endif
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020085 } else {
86 /* Read from media non-cached sectors */
87 rqd->ppa_list[j++] = p;
88 }
89
Javier González75cb8e92017-07-28 15:13:16 +020090next:
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020091 if (advanced_bio)
92 bio_advance(bio, PBLK_EXPOSED_PAGE_SIZE);
93 }
94
Javier Gonzálezf9c10152017-06-26 11:57:20 +020095 if (pblk_io_aligned(pblk, nr_secs))
96 rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_SEQUENTIAL);
97 else
98 rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM);
99
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200100#ifdef CONFIG_NVM_DEBUG
101 atomic_long_add(nr_secs, &pblk->inflight_reads);
102#endif
103}
104
105static int pblk_submit_read_io(struct pblk *pblk, struct nvm_rq *rqd)
106{
107 int err;
108
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200109 err = pblk_submit_io(pblk, rqd);
110 if (err)
111 return NVM_IO_ERR;
112
113 return NVM_IO_OK;
114}
115
Javier González03a34b22018-06-01 15:04:18 +0200116static void pblk_read_check_seq(struct pblk *pblk, struct nvm_rq *rqd,
117 sector_t blba)
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200118{
Javier González03a34b22018-06-01 15:04:18 +0200119 struct pblk_sec_meta *meta_lba_list = rqd->meta_list;
120 int nr_lbas = rqd->nr_ppas;
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200121 int i;
122
123 for (i = 0; i < nr_lbas; i++) {
Javier González310df582018-06-01 15:04:17 +0200124 u64 lba = le64_to_cpu(meta_lba_list[i].lba);
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200125
126 if (lba == ADDR_EMPTY)
127 continue;
128
Javier González03a34b22018-06-01 15:04:18 +0200129 if (lba != blba + i) {
130#ifdef CONFIG_NVM_DEBUG
131 struct ppa_addr *p;
132
133 p = (nr_lbas == 1) ? &rqd->ppa_list[i] : &rqd->ppa_addr;
134 print_ppa(&pblk->dev->geo, p, "seq", i);
135#endif
136 pr_err("pblk: corrupted read LBA (%llu/%llu)\n",
137 lba, (u64)blba + i);
138 WARN_ON(1);
139 }
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200140 }
141}
142
Javier González310df582018-06-01 15:04:17 +0200143/*
144 * There can be holes in the lba list.
145 */
Javier González03a34b22018-06-01 15:04:18 +0200146static void pblk_read_check_rand(struct pblk *pblk, struct nvm_rq *rqd,
147 u64 *lba_list, int nr_lbas)
Javier González310df582018-06-01 15:04:17 +0200148{
Javier González03a34b22018-06-01 15:04:18 +0200149 struct pblk_sec_meta *meta_lba_list = rqd->meta_list;
Javier González310df582018-06-01 15:04:17 +0200150 int i, j;
151
152 for (i = 0, j = 0; i < nr_lbas; i++) {
153 u64 lba = lba_list[i];
154 u64 meta_lba;
155
156 if (lba == ADDR_EMPTY)
157 continue;
158
Javier González03a34b22018-06-01 15:04:18 +0200159 meta_lba = le64_to_cpu(meta_lba_list[j].lba);
Javier González310df582018-06-01 15:04:17 +0200160
161 if (lba != meta_lba) {
Javier González03a34b22018-06-01 15:04:18 +0200162#ifdef CONFIG_NVM_DEBUG
163 struct ppa_addr *p;
164 int nr_ppas = rqd->nr_ppas;
165
166 p = (nr_ppas == 1) ? &rqd->ppa_list[j] : &rqd->ppa_addr;
167 print_ppa(&pblk->dev->geo, p, "seq", j);
168#endif
Javier González310df582018-06-01 15:04:17 +0200169 pr_err("pblk: corrupted read LBA (%llu/%llu)\n",
170 lba, meta_lba);
171 WARN_ON(1);
172 }
Javier González03a34b22018-06-01 15:04:18 +0200173
174 j++;
Javier González310df582018-06-01 15:04:17 +0200175 }
Javier González03a34b22018-06-01 15:04:18 +0200176
177 WARN_ONCE(j != rqd->nr_ppas, "pblk: corrupted random request\n");
Javier González310df582018-06-01 15:04:17 +0200178}
179
Javier González7bd4d372017-10-13 14:46:23 +0200180static void pblk_read_put_rqd_kref(struct pblk *pblk, struct nvm_rq *rqd)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200181{
Javier González7bd4d372017-10-13 14:46:23 +0200182 struct ppa_addr *ppa_list;
183 int i;
184
185 ppa_list = (rqd->nr_ppas > 1) ? rqd->ppa_list : &rqd->ppa_addr;
186
187 for (i = 0; i < rqd->nr_ppas; i++) {
188 struct ppa_addr ppa = ppa_list[i];
189 struct pblk_line *line;
190
Javier Gonzálezb1bcfda2018-01-05 14:16:06 +0100191 line = &pblk->lines[pblk_ppa_to_line(ppa)];
Javier González7bd4d372017-10-13 14:46:23 +0200192 kref_put(&line->ref, pblk_line_put_wq);
193 }
194}
195
196static void pblk_end_user_read(struct bio *bio)
197{
198#ifdef CONFIG_NVM_DEBUG
199 WARN_ONCE(bio->bi_status, "pblk: corrupted read bio\n");
200#endif
201 bio_endio(bio);
202 bio_put(bio);
203}
204
205static void __pblk_end_io_read(struct pblk *pblk, struct nvm_rq *rqd,
206 bool put_line)
207{
Javier González998ba622018-01-05 14:16:20 +0100208 struct nvm_tgt_dev *dev = pblk->dev;
Javier González084ec9b2017-06-26 16:27:13 -0600209 struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200210 struct bio *bio = rqd->bio;
Javier González998ba622018-01-05 14:16:20 +0100211 unsigned long start_time = r_ctx->start_time;
212
213 generic_end_io_acct(dev->q, READ, &pblk->disk->part0, start_time);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200214
215 if (rqd->error)
216 pblk_log_read_err(pblk, rqd);
217#ifdef CONFIG_NVM_DEBUG
218 else
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200219 WARN_ONCE(bio->bi_status, "pblk: corrupted read error\n");
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200220#endif
221
Javier González03a34b22018-06-01 15:04:18 +0200222 pblk_read_check_seq(pblk, rqd, r_ctx->lba);
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200223
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200224 bio_put(bio);
Javier González7bd4d372017-10-13 14:46:23 +0200225 if (r_ctx->private)
226 pblk_end_user_read((struct bio *)r_ctx->private);
Javier González084ec9b2017-06-26 16:27:13 -0600227
Javier González7bd4d372017-10-13 14:46:23 +0200228 if (put_line)
229 pblk_read_put_rqd_kref(pblk, rqd);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200230
231#ifdef CONFIG_NVM_DEBUG
232 atomic_long_add(rqd->nr_ppas, &pblk->sync_reads);
233 atomic_long_sub(rqd->nr_ppas, &pblk->inflight_reads);
234#endif
235
Javier Gonzáleze2cddf22017-10-13 14:46:19 +0200236 pblk_free_rqd(pblk, rqd, PBLK_READ);
Javier González588726d32017-06-26 11:57:29 +0200237 atomic_dec(&pblk->inflight_io);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200238}
239
Javier González7bd4d372017-10-13 14:46:23 +0200240static void pblk_end_io_read(struct nvm_rq *rqd)
241{
242 struct pblk *pblk = rqd->private;
243
244 __pblk_end_io_read(pblk, rqd, true);
245}
246
Javier González998ba622018-01-05 14:16:20 +0100247static int pblk_partial_read_bio(struct pblk *pblk, struct nvm_rq *rqd,
248 unsigned int bio_init_idx,
249 unsigned long *read_bitmap)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200250{
251 struct bio *new_bio, *bio = rqd->bio;
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200252 struct pblk_sec_meta *meta_list = rqd->meta_list;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200253 struct bio_vec src_bv, dst_bv;
254 void *ppa_ptr = NULL;
255 void *src_p, *dst_p;
256 dma_addr_t dma_ppa_list = 0;
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200257 __le64 *lba_list_mem, *lba_list_media;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200258 int nr_secs = rqd->nr_ppas;
259 int nr_holes = nr_secs - bitmap_weight(read_bitmap, nr_secs);
260 int i, ret, hole;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200261
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200262 /* Re-use allocated memory for intermediate lbas */
263 lba_list_mem = (((void *)rqd->ppa_list) + pblk_dma_ppa_size);
264 lba_list_media = (((void *)rqd->ppa_list) + 2 * pblk_dma_ppa_size);
265
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200266 new_bio = bio_alloc(GFP_KERNEL, nr_holes);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200267
268 if (pblk_bio_add_pages(pblk, new_bio, GFP_KERNEL, nr_holes))
269 goto err;
270
271 if (nr_holes != new_bio->bi_vcnt) {
272 pr_err("pblk: malformed bio\n");
273 goto err;
274 }
275
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200276 for (i = 0; i < nr_secs; i++)
277 lba_list_mem[i] = meta_list[i].lba;
278
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200279 new_bio->bi_iter.bi_sector = 0; /* internal bio */
280 bio_set_op_attrs(new_bio, REQ_OP_READ, 0);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200281
282 rqd->bio = new_bio;
283 rqd->nr_ppas = nr_holes;
Javier Gonzálezf9c10152017-06-26 11:57:20 +0200284 rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200285
Javier González0f9248c2017-10-13 14:46:24 +0200286 if (unlikely(nr_holes == 1)) {
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200287 ppa_ptr = rqd->ppa_list;
288 dma_ppa_list = rqd->dma_ppa_list;
289 rqd->ppa_addr = rqd->ppa_list[0];
290 }
291
Javier González1a94b2d2017-10-13 14:46:47 +0200292 ret = pblk_submit_io_sync(pblk, rqd);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200293 if (ret) {
294 bio_put(rqd->bio);
Javier González1a94b2d2017-10-13 14:46:47 +0200295 pr_err("pblk: sync read IO submission failed\n");
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200296 goto err;
297 }
298
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200299 if (rqd->error) {
300 atomic_long_inc(&pblk->read_failed);
301#ifdef CONFIG_NVM_DEBUG
302 pblk_print_failed_rqd(pblk, rqd, rqd->error);
303#endif
304 }
305
Javier González0f9248c2017-10-13 14:46:24 +0200306 if (unlikely(nr_holes == 1)) {
Javier González7bd4d372017-10-13 14:46:23 +0200307 struct ppa_addr ppa;
308
309 ppa = rqd->ppa_addr;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200310 rqd->ppa_list = ppa_ptr;
311 rqd->dma_ppa_list = dma_ppa_list;
Javier González7bd4d372017-10-13 14:46:23 +0200312 rqd->ppa_list[0] = ppa;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200313 }
314
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200315 for (i = 0; i < nr_secs; i++) {
316 lba_list_media[i] = meta_list[i].lba;
317 meta_list[i].lba = lba_list_mem[i];
318 }
319
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200320 /* Fill the holes in the original bio */
321 i = 0;
322 hole = find_first_zero_bit(read_bitmap, nr_secs);
323 do {
Javier Gonzálezb1bcfda2018-01-05 14:16:06 +0100324 int line_id = pblk_ppa_to_line(rqd->ppa_list[i]);
Javier González7bd4d372017-10-13 14:46:23 +0200325 struct pblk_line *line = &pblk->lines[line_id];
326
327 kref_put(&line->ref, pblk_line_put);
328
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200329 meta_list[hole].lba = lba_list_media[i];
330
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200331 src_bv = new_bio->bi_io_vec[i++];
332 dst_bv = bio->bi_io_vec[bio_init_idx + hole];
333
334 src_p = kmap_atomic(src_bv.bv_page);
335 dst_p = kmap_atomic(dst_bv.bv_page);
336
337 memcpy(dst_p + dst_bv.bv_offset,
338 src_p + src_bv.bv_offset,
339 PBLK_EXPOSED_PAGE_SIZE);
340
341 kunmap_atomic(src_p);
342 kunmap_atomic(dst_p);
343
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400344 mempool_free(src_bv.bv_page, &pblk->page_bio_pool);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200345
346 hole = find_next_zero_bit(read_bitmap, nr_secs, hole + 1);
347 } while (hole < nr_secs);
348
349 bio_put(new_bio);
350
351 /* Complete the original bio and associated request */
Javier González7bd4d372017-10-13 14:46:23 +0200352 bio_endio(bio);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200353 rqd->bio = bio;
354 rqd->nr_ppas = nr_secs;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200355
Javier González7bd4d372017-10-13 14:46:23 +0200356 __pblk_end_io_read(pblk, rqd, false);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200357 return NVM_IO_OK;
358
359err:
Javier González998ba622018-01-05 14:16:20 +0100360 pr_err("pblk: failed to perform partial read\n");
361
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200362 /* Free allocated pages in new bio */
363 pblk_bio_free_pages(pblk, bio, 0, new_bio->bi_vcnt);
Javier González7bd4d372017-10-13 14:46:23 +0200364 __pblk_end_io_read(pblk, rqd, false);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200365 return NVM_IO_ERR;
366}
367
368static void pblk_read_rq(struct pblk *pblk, struct nvm_rq *rqd,
Javier González84454e62017-10-13 14:46:13 +0200369 sector_t lba, unsigned long *read_bitmap)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200370{
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200371 struct pblk_sec_meta *meta_list = rqd->meta_list;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200372 struct bio *bio = rqd->bio;
373 struct ppa_addr ppa;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200374
375 pblk_lookup_l2p_seq(pblk, &ppa, lba, 1);
376
377#ifdef CONFIG_NVM_DEBUG
378 atomic_long_inc(&pblk->inflight_reads);
379#endif
380
381retry:
382 if (pblk_ppa_empty(ppa)) {
383 WARN_ON(test_and_set_bit(0, read_bitmap));
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200384 meta_list[0].lba = cpu_to_le64(ADDR_EMPTY);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200385 return;
386 }
387
388 /* Try to read from write buffer. The address is later checked on the
389 * write buffer to prevent retrieving overwritten data.
390 */
391 if (pblk_addr_in_cache(ppa)) {
Javier González75cb8e92017-07-28 15:13:16 +0200392 if (!pblk_read_from_cache(pblk, bio, lba, ppa, 0, 1)) {
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200393 pblk_lookup_l2p_seq(pblk, &ppa, lba, 1);
394 goto retry;
395 }
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200396
Javier González7bd4d372017-10-13 14:46:23 +0200397 WARN_ON(test_and_set_bit(0, read_bitmap));
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200398 meta_list[0].lba = cpu_to_le64(lba);
399
Javier Gonzálezdb7ada32017-06-26 11:57:13 +0200400#ifdef CONFIG_NVM_DEBUG
Javier González7bd4d372017-10-13 14:46:23 +0200401 atomic_long_inc(&pblk->cache_reads);
Javier Gonzálezdb7ada32017-06-26 11:57:13 +0200402#endif
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200403 } else {
404 rqd->ppa_addr = ppa;
405 }
Javier Gonzálezf9c10152017-06-26 11:57:20 +0200406
407 rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200408}
409
410int pblk_submit_read(struct pblk *pblk, struct bio *bio)
411{
412 struct nvm_tgt_dev *dev = pblk->dev;
Javier González998ba622018-01-05 14:16:20 +0100413 struct request_queue *q = dev->q;
Javier González84454e62017-10-13 14:46:13 +0200414 sector_t blba = pblk_get_lba(bio);
Dan Carpenter5bf1e1ee2017-04-21 16:48:40 -0600415 unsigned int nr_secs = pblk_get_secs(bio);
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200416 struct pblk_g_ctx *r_ctx;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200417 struct nvm_rq *rqd;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200418 unsigned int bio_init_idx;
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200419 unsigned long read_bitmap; /* Max 64 ppas per request */
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200420 int ret = NVM_IO_ERR;
421
Javier González84454e62017-10-13 14:46:13 +0200422 /* logic error: lba out-of-bounds. Ignore read request */
423 if (blba >= pblk->rl.nr_secs || nr_secs > PBLK_MAX_REQ_ADDRS) {
424 WARN(1, "pblk: read lba out of bounds (lba:%llu, nr:%d)\n",
425 (unsigned long long)blba, nr_secs);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200426 return NVM_IO_ERR;
Javier González84454e62017-10-13 14:46:13 +0200427 }
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200428
Javier González998ba622018-01-05 14:16:20 +0100429 generic_start_io_acct(q, READ, bio_sectors(bio), &pblk->disk->part0);
430
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200431 bitmap_zero(&read_bitmap, nr_secs);
432
Javier Gonzáleze2cddf22017-10-13 14:46:19 +0200433 rqd = pblk_alloc_rqd(pblk, PBLK_READ);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200434
435 rqd->opcode = NVM_OP_PREAD;
436 rqd->bio = bio;
437 rqd->nr_ppas = nr_secs;
438 rqd->private = pblk;
439 rqd->end_io = pblk_end_io_read;
440
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200441 r_ctx = nvm_rq_to_pdu(rqd);
Javier González998ba622018-01-05 14:16:20 +0100442 r_ctx->start_time = jiffies;
Javier Gonzáleza4809fe2017-10-13 14:46:22 +0200443 r_ctx->lba = blba;
444
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200445 /* Save the index for this bio's start. This is needed in case
446 * we need to fill a partial read.
447 */
448 bio_init_idx = pblk_get_bi_idx(bio);
449
Javier González63e38092017-06-26 11:57:24 +0200450 rqd->meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL,
451 &rqd->dma_meta_list);
452 if (!rqd->meta_list) {
453 pr_err("pblk: not able to allocate ppa list\n");
454 goto fail_rqd_free;
455 }
456
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200457 if (nr_secs > 1) {
Javier González63e38092017-06-26 11:57:24 +0200458 rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size;
459 rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200460
Javier González84454e62017-10-13 14:46:13 +0200461 pblk_read_ppalist_rq(pblk, rqd, blba, &read_bitmap);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200462 } else {
Javier González84454e62017-10-13 14:46:13 +0200463 pblk_read_rq(pblk, rqd, blba, &read_bitmap);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200464 }
465
466 bio_get(bio);
467 if (bitmap_full(&read_bitmap, nr_secs)) {
468 bio_endio(bio);
Javier González588726d32017-06-26 11:57:29 +0200469 atomic_inc(&pblk->inflight_io);
Javier González7bd4d372017-10-13 14:46:23 +0200470 __pblk_end_io_read(pblk, rqd, false);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200471 return NVM_IO_OK;
472 }
473
474 /* All sectors are to be read from the device */
475 if (bitmap_empty(&read_bitmap, rqd->nr_ppas)) {
476 struct bio *int_bio = NULL;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200477
478 /* Clone read bio to deal with read errors internally */
Kent Overstreetb906bbb2018-05-20 18:25:50 -0400479 int_bio = bio_clone_fast(bio, GFP_KERNEL, &pblk_bio_set);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200480 if (!int_bio) {
481 pr_err("pblk: could not clone read bio\n");
Javier González998ba622018-01-05 14:16:20 +0100482 goto fail_end_io;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200483 }
484
485 rqd->bio = int_bio;
Javier González084ec9b2017-06-26 16:27:13 -0600486 r_ctx->private = bio;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200487
488 ret = pblk_submit_read_io(pblk, rqd);
489 if (ret) {
490 pr_err("pblk: read IO submission failed\n");
491 if (int_bio)
492 bio_put(int_bio);
Javier González998ba622018-01-05 14:16:20 +0100493 goto fail_end_io;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200494 }
495
496 return NVM_IO_OK;
497 }
498
499 /* The read bio request could be partially filled by the write buffer,
500 * but there are some holes that need to be read from the drive.
501 */
Javier González998ba622018-01-05 14:16:20 +0100502 return pblk_partial_read_bio(pblk, rqd, bio_init_idx, &read_bitmap);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200503
504fail_rqd_free:
Javier Gonzáleze2cddf22017-10-13 14:46:19 +0200505 pblk_free_rqd(pblk, rqd, PBLK_READ);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200506 return ret;
Javier González998ba622018-01-05 14:16:20 +0100507fail_end_io:
508 __pblk_end_io_read(pblk, rqd, false);
509 return ret;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200510}
511
512static int read_ppalist_rq_gc(struct pblk *pblk, struct nvm_rq *rqd,
513 struct pblk_line *line, u64 *lba_list,
Javier Gonzálezd3401212017-10-13 14:46:14 +0200514 u64 *paddr_list_gc, unsigned int nr_secs)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200515{
Javier Gonzálezd3401212017-10-13 14:46:14 +0200516 struct ppa_addr ppa_list_l2p[PBLK_MAX_REQ_ADDRS];
517 struct ppa_addr ppa_gc;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200518 int valid_secs = 0;
519 int i;
520
Javier Gonzálezd3401212017-10-13 14:46:14 +0200521 pblk_lookup_l2p_rand(pblk, ppa_list_l2p, lba_list, nr_secs);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200522
523 for (i = 0; i < nr_secs; i++) {
Javier Gonzálezd3401212017-10-13 14:46:14 +0200524 if (lba_list[i] == ADDR_EMPTY)
525 continue;
526
527 ppa_gc = addr_to_gen_ppa(pblk, paddr_list_gc[i], line->id);
528 if (!pblk_ppa_comp(ppa_list_l2p[i], ppa_gc)) {
529 paddr_list_gc[i] = lba_list[i] = ADDR_EMPTY;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200530 continue;
531 }
532
Javier Gonzálezd3401212017-10-13 14:46:14 +0200533 rqd->ppa_list[valid_secs++] = ppa_list_l2p[i];
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200534 }
535
536#ifdef CONFIG_NVM_DEBUG
537 atomic_long_add(valid_secs, &pblk->inflight_reads);
538#endif
Javier Gonzálezd3401212017-10-13 14:46:14 +0200539
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200540 return valid_secs;
541}
542
543static int read_rq_gc(struct pblk *pblk, struct nvm_rq *rqd,
Javier Gonzálezd3401212017-10-13 14:46:14 +0200544 struct pblk_line *line, sector_t lba,
545 u64 paddr_gc)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200546{
Javier Gonzálezd3401212017-10-13 14:46:14 +0200547 struct ppa_addr ppa_l2p, ppa_gc;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200548 int valid_secs = 0;
549
Dan Carpenter659226e2017-04-21 16:49:56 -0600550 if (lba == ADDR_EMPTY)
551 goto out;
552
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200553 /* logic error: lba out-of-bounds */
Dan Carpenter2a79efd2017-04-15 20:55:52 +0200554 if (lba >= pblk->rl.nr_secs) {
555 WARN(1, "pblk: read lba out of bounds\n");
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200556 goto out;
557 }
558
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200559 spin_lock(&pblk->trans_lock);
Javier Gonzálezd3401212017-10-13 14:46:14 +0200560 ppa_l2p = pblk_trans_map_get(pblk, lba);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200561 spin_unlock(&pblk->trans_lock);
562
Javier Gonzálezd3401212017-10-13 14:46:14 +0200563 ppa_gc = addr_to_gen_ppa(pblk, paddr_gc, line->id);
564 if (!pblk_ppa_comp(ppa_l2p, ppa_gc))
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200565 goto out;
566
Javier Gonzálezd3401212017-10-13 14:46:14 +0200567 rqd->ppa_addr = ppa_l2p;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200568 valid_secs = 1;
569
570#ifdef CONFIG_NVM_DEBUG
571 atomic_long_inc(&pblk->inflight_reads);
572#endif
573
574out:
575 return valid_secs;
576}
577
Javier Gonzálezd3401212017-10-13 14:46:14 +0200578int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200579{
580 struct nvm_tgt_dev *dev = pblk->dev;
581 struct nvm_geo *geo = &dev->geo;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200582 struct bio *bio;
583 struct nvm_rq rqd;
Javier Gonzálezd3401212017-10-13 14:46:14 +0200584 int data_len;
585 int ret = NVM_IO_OK;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200586
587 memset(&rqd, 0, sizeof(struct nvm_rq));
588
Javier González63e38092017-06-26 11:57:24 +0200589 rqd.meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL,
590 &rqd.dma_meta_list);
591 if (!rqd.meta_list)
Javier Gonzálezd3401212017-10-13 14:46:14 +0200592 return -ENOMEM;
Javier González63e38092017-06-26 11:57:24 +0200593
Javier Gonzálezd3401212017-10-13 14:46:14 +0200594 if (gc_rq->nr_secs > 1) {
Javier González63e38092017-06-26 11:57:24 +0200595 rqd.ppa_list = rqd.meta_list + pblk_dma_meta_size;
596 rqd.dma_ppa_list = rqd.dma_meta_list + pblk_dma_meta_size;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200597
Javier Gonzálezd3401212017-10-13 14:46:14 +0200598 gc_rq->secs_to_gc = read_ppalist_rq_gc(pblk, &rqd, gc_rq->line,
599 gc_rq->lba_list,
600 gc_rq->paddr_list,
601 gc_rq->nr_secs);
602 if (gc_rq->secs_to_gc == 1)
Javier González63e38092017-06-26 11:57:24 +0200603 rqd.ppa_addr = rqd.ppa_list[0];
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200604 } else {
Javier Gonzálezd3401212017-10-13 14:46:14 +0200605 gc_rq->secs_to_gc = read_rq_gc(pblk, &rqd, gc_rq->line,
606 gc_rq->lba_list[0],
607 gc_rq->paddr_list[0]);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200608 }
609
Javier Gonzálezd3401212017-10-13 14:46:14 +0200610 if (!(gc_rq->secs_to_gc))
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200611 goto out;
612
Javier Gonzáleze46f4e42018-03-30 00:05:10 +0200613 data_len = (gc_rq->secs_to_gc) * geo->csecs;
Javier Gonzálezd3401212017-10-13 14:46:14 +0200614 bio = pblk_bio_map_addr(pblk, gc_rq->data, gc_rq->secs_to_gc, data_len,
Javier González7d327a92017-10-13 14:46:02 +0200615 PBLK_VMALLOC_META, GFP_KERNEL);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200616 if (IS_ERR(bio)) {
617 pr_err("pblk: could not allocate GC bio (%lu)\n", PTR_ERR(bio));
618 goto err_free_dma;
619 }
620
621 bio->bi_iter.bi_sector = 0; /* internal bio */
622 bio_set_op_attrs(bio, REQ_OP_READ, 0);
623
624 rqd.opcode = NVM_OP_PREAD;
Javier Gonzálezd3401212017-10-13 14:46:14 +0200625 rqd.nr_ppas = gc_rq->secs_to_gc;
Javier Gonzálezf9c10152017-06-26 11:57:20 +0200626 rqd.flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200627 rqd.bio = bio;
628
Javier González1a94b2d2017-10-13 14:46:47 +0200629 if (pblk_submit_io_sync(pblk, &rqd)) {
Javier Gonzálezd3401212017-10-13 14:46:14 +0200630 ret = -EIO;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200631 pr_err("pblk: GC read request failed\n");
Javier González7d327a92017-10-13 14:46:02 +0200632 goto err_free_bio;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200633 }
634
Javier González03a34b22018-06-01 15:04:18 +0200635 pblk_read_check_rand(pblk, &rqd, gc_rq->lba_list, gc_rq->nr_secs);
Javier González310df582018-06-01 15:04:17 +0200636
Javier González588726d32017-06-26 11:57:29 +0200637 atomic_dec(&pblk->inflight_io);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200638
639 if (rqd.error) {
640 atomic_long_inc(&pblk->read_failed_gc);
641#ifdef CONFIG_NVM_DEBUG
642 pblk_print_failed_rqd(pblk, &rqd, rqd.error);
643#endif
644 }
645
646#ifdef CONFIG_NVM_DEBUG
Javier Gonzálezd3401212017-10-13 14:46:14 +0200647 atomic_long_add(gc_rq->secs_to_gc, &pblk->sync_reads);
648 atomic_long_add(gc_rq->secs_to_gc, &pblk->recov_gc_reads);
649 atomic_long_sub(gc_rq->secs_to_gc, &pblk->inflight_reads);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200650#endif
651
652out:
Javier González63e38092017-06-26 11:57:24 +0200653 nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list);
Javier Gonzálezd3401212017-10-13 14:46:14 +0200654 return ret;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200655
Javier González7d327a92017-10-13 14:46:02 +0200656err_free_bio:
657 bio_put(bio);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200658err_free_dma:
Javier González63e38092017-06-26 11:57:24 +0200659 nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list);
Javier Gonzálezd3401212017-10-13 14:46:14 +0200660 return ret;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200661}