Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2016 CNEX Labs |
| 3 | * Initial release: Javier Gonzalez <javier@cnexlabs.com> |
| 4 | * Matias Bjorling <matias@cnexlabs.com> |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License version |
| 8 | * 2 as published by the Free Software Foundation. |
| 9 | * |
| 10 | * This program is distributed in the hope that it will be useful, but |
| 11 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 13 | * General Public License for more details. |
| 14 | * |
| 15 | * pblk-read.c - pblk's read path |
| 16 | */ |
| 17 | |
| 18 | #include "pblk.h" |
| 19 | |
| 20 | /* |
| 21 | * There is no guarantee that the value read from cache has not been updated and |
| 22 | * resides at another location in the cache. We guarantee though that if the |
| 23 | * value is read from the cache, it belongs to the mapped lba. In order to |
| 24 | * guarantee and order between writes and reads are ordered, a flush must be |
| 25 | * issued. |
| 26 | */ |
| 27 | static int pblk_read_from_cache(struct pblk *pblk, struct bio *bio, |
| 28 | sector_t lba, struct ppa_addr ppa, |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 29 | int bio_iter, bool advanced_bio) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 30 | { |
| 31 | #ifdef CONFIG_NVM_DEBUG |
| 32 | /* Callers must ensure that the ppa points to a cache address */ |
| 33 | BUG_ON(pblk_ppa_empty(ppa)); |
| 34 | BUG_ON(!pblk_addr_in_cache(ppa)); |
| 35 | #endif |
| 36 | |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 37 | return pblk_rb_copy_to_bio(&pblk->rwb, bio, lba, ppa, |
| 38 | bio_iter, advanced_bio); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 39 | } |
| 40 | |
| 41 | static void pblk_read_ppalist_rq(struct pblk *pblk, struct nvm_rq *rqd, |
Javier González | 84454e6 | 2017-10-13 14:46:13 +0200 | [diff] [blame] | 42 | sector_t blba, unsigned long *read_bitmap) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 43 | { |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 44 | struct pblk_sec_meta *meta_list = rqd->meta_list; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 45 | struct bio *bio = rqd->bio; |
| 46 | struct ppa_addr ppas[PBLK_MAX_REQ_ADDRS]; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 47 | int nr_secs = rqd->nr_ppas; |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 48 | bool advanced_bio = false; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 49 | int i, j = 0; |
| 50 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 51 | pblk_lookup_l2p_seq(pblk, ppas, blba, nr_secs); |
| 52 | |
| 53 | for (i = 0; i < nr_secs; i++) { |
| 54 | struct ppa_addr p = ppas[i]; |
| 55 | sector_t lba = blba + i; |
| 56 | |
| 57 | retry: |
| 58 | if (pblk_ppa_empty(p)) { |
| 59 | WARN_ON(test_and_set_bit(i, read_bitmap)); |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 60 | meta_list[i].lba = cpu_to_le64(ADDR_EMPTY); |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 61 | |
| 62 | if (unlikely(!advanced_bio)) { |
| 63 | bio_advance(bio, (i) * PBLK_EXPOSED_PAGE_SIZE); |
| 64 | advanced_bio = true; |
| 65 | } |
| 66 | |
| 67 | goto next; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 68 | } |
| 69 | |
| 70 | /* Try to read from write buffer. The address is later checked |
| 71 | * on the write buffer to prevent retrieving overwritten data. |
| 72 | */ |
| 73 | if (pblk_addr_in_cache(p)) { |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 74 | if (!pblk_read_from_cache(pblk, bio, lba, p, i, |
| 75 | advanced_bio)) { |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 76 | pblk_lookup_l2p_seq(pblk, &p, lba, 1); |
| 77 | goto retry; |
| 78 | } |
| 79 | WARN_ON(test_and_set_bit(i, read_bitmap)); |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 80 | meta_list[i].lba = cpu_to_le64(lba); |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 81 | advanced_bio = true; |
Javier González | db7ada3 | 2017-06-26 11:57:13 +0200 | [diff] [blame] | 82 | #ifdef CONFIG_NVM_DEBUG |
| 83 | atomic_long_inc(&pblk->cache_reads); |
| 84 | #endif |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 85 | } else { |
| 86 | /* Read from media non-cached sectors */ |
| 87 | rqd->ppa_list[j++] = p; |
| 88 | } |
| 89 | |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 90 | next: |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 91 | if (advanced_bio) |
| 92 | bio_advance(bio, PBLK_EXPOSED_PAGE_SIZE); |
| 93 | } |
| 94 | |
Javier González | f9c1015 | 2017-06-26 11:57:20 +0200 | [diff] [blame] | 95 | if (pblk_io_aligned(pblk, nr_secs)) |
| 96 | rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_SEQUENTIAL); |
| 97 | else |
| 98 | rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM); |
| 99 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 100 | #ifdef CONFIG_NVM_DEBUG |
| 101 | atomic_long_add(nr_secs, &pblk->inflight_reads); |
| 102 | #endif |
| 103 | } |
| 104 | |
| 105 | static int pblk_submit_read_io(struct pblk *pblk, struct nvm_rq *rqd) |
| 106 | { |
| 107 | int err; |
| 108 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 109 | err = pblk_submit_io(pblk, rqd); |
| 110 | if (err) |
| 111 | return NVM_IO_ERR; |
| 112 | |
| 113 | return NVM_IO_OK; |
| 114 | } |
| 115 | |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 116 | static void pblk_read_check_seq(struct pblk *pblk, struct nvm_rq *rqd, |
| 117 | sector_t blba) |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 118 | { |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 119 | struct pblk_sec_meta *meta_lba_list = rqd->meta_list; |
| 120 | int nr_lbas = rqd->nr_ppas; |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 121 | int i; |
| 122 | |
| 123 | for (i = 0; i < nr_lbas; i++) { |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 124 | u64 lba = le64_to_cpu(meta_lba_list[i].lba); |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 125 | |
| 126 | if (lba == ADDR_EMPTY) |
| 127 | continue; |
| 128 | |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 129 | if (lba != blba + i) { |
| 130 | #ifdef CONFIG_NVM_DEBUG |
| 131 | struct ppa_addr *p; |
| 132 | |
| 133 | p = (nr_lbas == 1) ? &rqd->ppa_list[i] : &rqd->ppa_addr; |
| 134 | print_ppa(&pblk->dev->geo, p, "seq", i); |
| 135 | #endif |
| 136 | pr_err("pblk: corrupted read LBA (%llu/%llu)\n", |
| 137 | lba, (u64)blba + i); |
| 138 | WARN_ON(1); |
| 139 | } |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 140 | } |
| 141 | } |
| 142 | |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 143 | /* |
| 144 | * There can be holes in the lba list. |
| 145 | */ |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 146 | static void pblk_read_check_rand(struct pblk *pblk, struct nvm_rq *rqd, |
| 147 | u64 *lba_list, int nr_lbas) |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 148 | { |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 149 | struct pblk_sec_meta *meta_lba_list = rqd->meta_list; |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 150 | int i, j; |
| 151 | |
| 152 | for (i = 0, j = 0; i < nr_lbas; i++) { |
| 153 | u64 lba = lba_list[i]; |
| 154 | u64 meta_lba; |
| 155 | |
| 156 | if (lba == ADDR_EMPTY) |
| 157 | continue; |
| 158 | |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 159 | meta_lba = le64_to_cpu(meta_lba_list[j].lba); |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 160 | |
| 161 | if (lba != meta_lba) { |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 162 | #ifdef CONFIG_NVM_DEBUG |
| 163 | struct ppa_addr *p; |
| 164 | int nr_ppas = rqd->nr_ppas; |
| 165 | |
| 166 | p = (nr_ppas == 1) ? &rqd->ppa_list[j] : &rqd->ppa_addr; |
| 167 | print_ppa(&pblk->dev->geo, p, "seq", j); |
| 168 | #endif |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 169 | pr_err("pblk: corrupted read LBA (%llu/%llu)\n", |
| 170 | lba, meta_lba); |
| 171 | WARN_ON(1); |
| 172 | } |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 173 | |
| 174 | j++; |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 175 | } |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 176 | |
| 177 | WARN_ONCE(j != rqd->nr_ppas, "pblk: corrupted random request\n"); |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 178 | } |
| 179 | |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 180 | static void pblk_read_put_rqd_kref(struct pblk *pblk, struct nvm_rq *rqd) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 181 | { |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 182 | struct ppa_addr *ppa_list; |
| 183 | int i; |
| 184 | |
| 185 | ppa_list = (rqd->nr_ppas > 1) ? rqd->ppa_list : &rqd->ppa_addr; |
| 186 | |
| 187 | for (i = 0; i < rqd->nr_ppas; i++) { |
| 188 | struct ppa_addr ppa = ppa_list[i]; |
| 189 | struct pblk_line *line; |
| 190 | |
Javier González | b1bcfda | 2018-01-05 14:16:06 +0100 | [diff] [blame] | 191 | line = &pblk->lines[pblk_ppa_to_line(ppa)]; |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 192 | kref_put(&line->ref, pblk_line_put_wq); |
| 193 | } |
| 194 | } |
| 195 | |
| 196 | static void pblk_end_user_read(struct bio *bio) |
| 197 | { |
| 198 | #ifdef CONFIG_NVM_DEBUG |
| 199 | WARN_ONCE(bio->bi_status, "pblk: corrupted read bio\n"); |
| 200 | #endif |
| 201 | bio_endio(bio); |
| 202 | bio_put(bio); |
| 203 | } |
| 204 | |
| 205 | static void __pblk_end_io_read(struct pblk *pblk, struct nvm_rq *rqd, |
| 206 | bool put_line) |
| 207 | { |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 208 | struct nvm_tgt_dev *dev = pblk->dev; |
Javier González | 084ec9b | 2017-06-26 16:27:13 -0600 | [diff] [blame] | 209 | struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 210 | struct bio *bio = rqd->bio; |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 211 | unsigned long start_time = r_ctx->start_time; |
| 212 | |
| 213 | generic_end_io_acct(dev->q, READ, &pblk->disk->part0, start_time); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 214 | |
| 215 | if (rqd->error) |
| 216 | pblk_log_read_err(pblk, rqd); |
| 217 | #ifdef CONFIG_NVM_DEBUG |
| 218 | else |
Christoph Hellwig | 4e4cbee | 2017-06-03 09:38:06 +0200 | [diff] [blame] | 219 | WARN_ONCE(bio->bi_status, "pblk: corrupted read error\n"); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 220 | #endif |
| 221 | |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 222 | pblk_read_check_seq(pblk, rqd, r_ctx->lba); |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 223 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 224 | bio_put(bio); |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 225 | if (r_ctx->private) |
| 226 | pblk_end_user_read((struct bio *)r_ctx->private); |
Javier González | 084ec9b | 2017-06-26 16:27:13 -0600 | [diff] [blame] | 227 | |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 228 | if (put_line) |
| 229 | pblk_read_put_rqd_kref(pblk, rqd); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 230 | |
| 231 | #ifdef CONFIG_NVM_DEBUG |
| 232 | atomic_long_add(rqd->nr_ppas, &pblk->sync_reads); |
| 233 | atomic_long_sub(rqd->nr_ppas, &pblk->inflight_reads); |
| 234 | #endif |
| 235 | |
Javier González | e2cddf2 | 2017-10-13 14:46:19 +0200 | [diff] [blame] | 236 | pblk_free_rqd(pblk, rqd, PBLK_READ); |
Javier González | 588726d3 | 2017-06-26 11:57:29 +0200 | [diff] [blame] | 237 | atomic_dec(&pblk->inflight_io); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 238 | } |
| 239 | |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 240 | static void pblk_end_io_read(struct nvm_rq *rqd) |
| 241 | { |
| 242 | struct pblk *pblk = rqd->private; |
| 243 | |
| 244 | __pblk_end_io_read(pblk, rqd, true); |
| 245 | } |
| 246 | |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 247 | static int pblk_partial_read_bio(struct pblk *pblk, struct nvm_rq *rqd, |
| 248 | unsigned int bio_init_idx, |
| 249 | unsigned long *read_bitmap) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 250 | { |
| 251 | struct bio *new_bio, *bio = rqd->bio; |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 252 | struct pblk_sec_meta *meta_list = rqd->meta_list; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 253 | struct bio_vec src_bv, dst_bv; |
| 254 | void *ppa_ptr = NULL; |
| 255 | void *src_p, *dst_p; |
| 256 | dma_addr_t dma_ppa_list = 0; |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 257 | __le64 *lba_list_mem, *lba_list_media; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 258 | int nr_secs = rqd->nr_ppas; |
| 259 | int nr_holes = nr_secs - bitmap_weight(read_bitmap, nr_secs); |
| 260 | int i, ret, hole; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 261 | |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 262 | /* Re-use allocated memory for intermediate lbas */ |
| 263 | lba_list_mem = (((void *)rqd->ppa_list) + pblk_dma_ppa_size); |
| 264 | lba_list_media = (((void *)rqd->ppa_list) + 2 * pblk_dma_ppa_size); |
| 265 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 266 | new_bio = bio_alloc(GFP_KERNEL, nr_holes); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 267 | |
| 268 | if (pblk_bio_add_pages(pblk, new_bio, GFP_KERNEL, nr_holes)) |
| 269 | goto err; |
| 270 | |
| 271 | if (nr_holes != new_bio->bi_vcnt) { |
| 272 | pr_err("pblk: malformed bio\n"); |
| 273 | goto err; |
| 274 | } |
| 275 | |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 276 | for (i = 0; i < nr_secs; i++) |
| 277 | lba_list_mem[i] = meta_list[i].lba; |
| 278 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 279 | new_bio->bi_iter.bi_sector = 0; /* internal bio */ |
| 280 | bio_set_op_attrs(new_bio, REQ_OP_READ, 0); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 281 | |
| 282 | rqd->bio = new_bio; |
| 283 | rqd->nr_ppas = nr_holes; |
Javier González | f9c1015 | 2017-06-26 11:57:20 +0200 | [diff] [blame] | 284 | rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 285 | |
Javier González | 0f9248c | 2017-10-13 14:46:24 +0200 | [diff] [blame] | 286 | if (unlikely(nr_holes == 1)) { |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 287 | ppa_ptr = rqd->ppa_list; |
| 288 | dma_ppa_list = rqd->dma_ppa_list; |
| 289 | rqd->ppa_addr = rqd->ppa_list[0]; |
| 290 | } |
| 291 | |
Javier González | 1a94b2d | 2017-10-13 14:46:47 +0200 | [diff] [blame] | 292 | ret = pblk_submit_io_sync(pblk, rqd); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 293 | if (ret) { |
| 294 | bio_put(rqd->bio); |
Javier González | 1a94b2d | 2017-10-13 14:46:47 +0200 | [diff] [blame] | 295 | pr_err("pblk: sync read IO submission failed\n"); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 296 | goto err; |
| 297 | } |
| 298 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 299 | if (rqd->error) { |
| 300 | atomic_long_inc(&pblk->read_failed); |
| 301 | #ifdef CONFIG_NVM_DEBUG |
| 302 | pblk_print_failed_rqd(pblk, rqd, rqd->error); |
| 303 | #endif |
| 304 | } |
| 305 | |
Javier González | 0f9248c | 2017-10-13 14:46:24 +0200 | [diff] [blame] | 306 | if (unlikely(nr_holes == 1)) { |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 307 | struct ppa_addr ppa; |
| 308 | |
| 309 | ppa = rqd->ppa_addr; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 310 | rqd->ppa_list = ppa_ptr; |
| 311 | rqd->dma_ppa_list = dma_ppa_list; |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 312 | rqd->ppa_list[0] = ppa; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 313 | } |
| 314 | |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 315 | for (i = 0; i < nr_secs; i++) { |
| 316 | lba_list_media[i] = meta_list[i].lba; |
| 317 | meta_list[i].lba = lba_list_mem[i]; |
| 318 | } |
| 319 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 320 | /* Fill the holes in the original bio */ |
| 321 | i = 0; |
| 322 | hole = find_first_zero_bit(read_bitmap, nr_secs); |
| 323 | do { |
Javier González | b1bcfda | 2018-01-05 14:16:06 +0100 | [diff] [blame] | 324 | int line_id = pblk_ppa_to_line(rqd->ppa_list[i]); |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 325 | struct pblk_line *line = &pblk->lines[line_id]; |
| 326 | |
| 327 | kref_put(&line->ref, pblk_line_put); |
| 328 | |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 329 | meta_list[hole].lba = lba_list_media[i]; |
| 330 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 331 | src_bv = new_bio->bi_io_vec[i++]; |
| 332 | dst_bv = bio->bi_io_vec[bio_init_idx + hole]; |
| 333 | |
| 334 | src_p = kmap_atomic(src_bv.bv_page); |
| 335 | dst_p = kmap_atomic(dst_bv.bv_page); |
| 336 | |
| 337 | memcpy(dst_p + dst_bv.bv_offset, |
| 338 | src_p + src_bv.bv_offset, |
| 339 | PBLK_EXPOSED_PAGE_SIZE); |
| 340 | |
| 341 | kunmap_atomic(src_p); |
| 342 | kunmap_atomic(dst_p); |
| 343 | |
Kent Overstreet | b906bbb | 2018-05-20 18:25:50 -0400 | [diff] [blame] | 344 | mempool_free(src_bv.bv_page, &pblk->page_bio_pool); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 345 | |
| 346 | hole = find_next_zero_bit(read_bitmap, nr_secs, hole + 1); |
| 347 | } while (hole < nr_secs); |
| 348 | |
| 349 | bio_put(new_bio); |
| 350 | |
| 351 | /* Complete the original bio and associated request */ |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 352 | bio_endio(bio); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 353 | rqd->bio = bio; |
| 354 | rqd->nr_ppas = nr_secs; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 355 | |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 356 | __pblk_end_io_read(pblk, rqd, false); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 357 | return NVM_IO_OK; |
| 358 | |
| 359 | err: |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 360 | pr_err("pblk: failed to perform partial read\n"); |
| 361 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 362 | /* Free allocated pages in new bio */ |
| 363 | pblk_bio_free_pages(pblk, bio, 0, new_bio->bi_vcnt); |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 364 | __pblk_end_io_read(pblk, rqd, false); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 365 | return NVM_IO_ERR; |
| 366 | } |
| 367 | |
| 368 | static void pblk_read_rq(struct pblk *pblk, struct nvm_rq *rqd, |
Javier González | 84454e6 | 2017-10-13 14:46:13 +0200 | [diff] [blame] | 369 | sector_t lba, unsigned long *read_bitmap) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 370 | { |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 371 | struct pblk_sec_meta *meta_list = rqd->meta_list; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 372 | struct bio *bio = rqd->bio; |
| 373 | struct ppa_addr ppa; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 374 | |
| 375 | pblk_lookup_l2p_seq(pblk, &ppa, lba, 1); |
| 376 | |
| 377 | #ifdef CONFIG_NVM_DEBUG |
| 378 | atomic_long_inc(&pblk->inflight_reads); |
| 379 | #endif |
| 380 | |
| 381 | retry: |
| 382 | if (pblk_ppa_empty(ppa)) { |
| 383 | WARN_ON(test_and_set_bit(0, read_bitmap)); |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 384 | meta_list[0].lba = cpu_to_le64(ADDR_EMPTY); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 385 | return; |
| 386 | } |
| 387 | |
| 388 | /* Try to read from write buffer. The address is later checked on the |
| 389 | * write buffer to prevent retrieving overwritten data. |
| 390 | */ |
| 391 | if (pblk_addr_in_cache(ppa)) { |
Javier González | 75cb8e9 | 2017-07-28 15:13:16 +0200 | [diff] [blame] | 392 | if (!pblk_read_from_cache(pblk, bio, lba, ppa, 0, 1)) { |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 393 | pblk_lookup_l2p_seq(pblk, &ppa, lba, 1); |
| 394 | goto retry; |
| 395 | } |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 396 | |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 397 | WARN_ON(test_and_set_bit(0, read_bitmap)); |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 398 | meta_list[0].lba = cpu_to_le64(lba); |
| 399 | |
Javier González | db7ada3 | 2017-06-26 11:57:13 +0200 | [diff] [blame] | 400 | #ifdef CONFIG_NVM_DEBUG |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 401 | atomic_long_inc(&pblk->cache_reads); |
Javier González | db7ada3 | 2017-06-26 11:57:13 +0200 | [diff] [blame] | 402 | #endif |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 403 | } else { |
| 404 | rqd->ppa_addr = ppa; |
| 405 | } |
Javier González | f9c1015 | 2017-06-26 11:57:20 +0200 | [diff] [blame] | 406 | |
| 407 | rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 408 | } |
| 409 | |
| 410 | int pblk_submit_read(struct pblk *pblk, struct bio *bio) |
| 411 | { |
| 412 | struct nvm_tgt_dev *dev = pblk->dev; |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 413 | struct request_queue *q = dev->q; |
Javier González | 84454e6 | 2017-10-13 14:46:13 +0200 | [diff] [blame] | 414 | sector_t blba = pblk_get_lba(bio); |
Dan Carpenter | 5bf1e1ee | 2017-04-21 16:48:40 -0600 | [diff] [blame] | 415 | unsigned int nr_secs = pblk_get_secs(bio); |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 416 | struct pblk_g_ctx *r_ctx; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 417 | struct nvm_rq *rqd; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 418 | unsigned int bio_init_idx; |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 419 | unsigned long read_bitmap; /* Max 64 ppas per request */ |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 420 | int ret = NVM_IO_ERR; |
| 421 | |
Javier González | 84454e6 | 2017-10-13 14:46:13 +0200 | [diff] [blame] | 422 | /* logic error: lba out-of-bounds. Ignore read request */ |
| 423 | if (blba >= pblk->rl.nr_secs || nr_secs > PBLK_MAX_REQ_ADDRS) { |
| 424 | WARN(1, "pblk: read lba out of bounds (lba:%llu, nr:%d)\n", |
| 425 | (unsigned long long)blba, nr_secs); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 426 | return NVM_IO_ERR; |
Javier González | 84454e6 | 2017-10-13 14:46:13 +0200 | [diff] [blame] | 427 | } |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 428 | |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 429 | generic_start_io_acct(q, READ, bio_sectors(bio), &pblk->disk->part0); |
| 430 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 431 | bitmap_zero(&read_bitmap, nr_secs); |
| 432 | |
Javier González | e2cddf2 | 2017-10-13 14:46:19 +0200 | [diff] [blame] | 433 | rqd = pblk_alloc_rqd(pblk, PBLK_READ); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 434 | |
| 435 | rqd->opcode = NVM_OP_PREAD; |
| 436 | rqd->bio = bio; |
| 437 | rqd->nr_ppas = nr_secs; |
| 438 | rqd->private = pblk; |
| 439 | rqd->end_io = pblk_end_io_read; |
| 440 | |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 441 | r_ctx = nvm_rq_to_pdu(rqd); |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 442 | r_ctx->start_time = jiffies; |
Javier González | a4809fe | 2017-10-13 14:46:22 +0200 | [diff] [blame] | 443 | r_ctx->lba = blba; |
| 444 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 445 | /* Save the index for this bio's start. This is needed in case |
| 446 | * we need to fill a partial read. |
| 447 | */ |
| 448 | bio_init_idx = pblk_get_bi_idx(bio); |
| 449 | |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 450 | rqd->meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, |
| 451 | &rqd->dma_meta_list); |
| 452 | if (!rqd->meta_list) { |
| 453 | pr_err("pblk: not able to allocate ppa list\n"); |
| 454 | goto fail_rqd_free; |
| 455 | } |
| 456 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 457 | if (nr_secs > 1) { |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 458 | rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size; |
| 459 | rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 460 | |
Javier González | 84454e6 | 2017-10-13 14:46:13 +0200 | [diff] [blame] | 461 | pblk_read_ppalist_rq(pblk, rqd, blba, &read_bitmap); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 462 | } else { |
Javier González | 84454e6 | 2017-10-13 14:46:13 +0200 | [diff] [blame] | 463 | pblk_read_rq(pblk, rqd, blba, &read_bitmap); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 464 | } |
| 465 | |
| 466 | bio_get(bio); |
| 467 | if (bitmap_full(&read_bitmap, nr_secs)) { |
| 468 | bio_endio(bio); |
Javier González | 588726d3 | 2017-06-26 11:57:29 +0200 | [diff] [blame] | 469 | atomic_inc(&pblk->inflight_io); |
Javier González | 7bd4d37 | 2017-10-13 14:46:23 +0200 | [diff] [blame] | 470 | __pblk_end_io_read(pblk, rqd, false); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 471 | return NVM_IO_OK; |
| 472 | } |
| 473 | |
| 474 | /* All sectors are to be read from the device */ |
| 475 | if (bitmap_empty(&read_bitmap, rqd->nr_ppas)) { |
| 476 | struct bio *int_bio = NULL; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 477 | |
| 478 | /* Clone read bio to deal with read errors internally */ |
Kent Overstreet | b906bbb | 2018-05-20 18:25:50 -0400 | [diff] [blame] | 479 | int_bio = bio_clone_fast(bio, GFP_KERNEL, &pblk_bio_set); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 480 | if (!int_bio) { |
| 481 | pr_err("pblk: could not clone read bio\n"); |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 482 | goto fail_end_io; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 483 | } |
| 484 | |
| 485 | rqd->bio = int_bio; |
Javier González | 084ec9b | 2017-06-26 16:27:13 -0600 | [diff] [blame] | 486 | r_ctx->private = bio; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 487 | |
| 488 | ret = pblk_submit_read_io(pblk, rqd); |
| 489 | if (ret) { |
| 490 | pr_err("pblk: read IO submission failed\n"); |
| 491 | if (int_bio) |
| 492 | bio_put(int_bio); |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 493 | goto fail_end_io; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 494 | } |
| 495 | |
| 496 | return NVM_IO_OK; |
| 497 | } |
| 498 | |
| 499 | /* The read bio request could be partially filled by the write buffer, |
| 500 | * but there are some holes that need to be read from the drive. |
| 501 | */ |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 502 | return pblk_partial_read_bio(pblk, rqd, bio_init_idx, &read_bitmap); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 503 | |
| 504 | fail_rqd_free: |
Javier González | e2cddf2 | 2017-10-13 14:46:19 +0200 | [diff] [blame] | 505 | pblk_free_rqd(pblk, rqd, PBLK_READ); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 506 | return ret; |
Javier González | 998ba62 | 2018-01-05 14:16:20 +0100 | [diff] [blame] | 507 | fail_end_io: |
| 508 | __pblk_end_io_read(pblk, rqd, false); |
| 509 | return ret; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 510 | } |
| 511 | |
| 512 | static int read_ppalist_rq_gc(struct pblk *pblk, struct nvm_rq *rqd, |
| 513 | struct pblk_line *line, u64 *lba_list, |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 514 | u64 *paddr_list_gc, unsigned int nr_secs) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 515 | { |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 516 | struct ppa_addr ppa_list_l2p[PBLK_MAX_REQ_ADDRS]; |
| 517 | struct ppa_addr ppa_gc; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 518 | int valid_secs = 0; |
| 519 | int i; |
| 520 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 521 | pblk_lookup_l2p_rand(pblk, ppa_list_l2p, lba_list, nr_secs); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 522 | |
| 523 | for (i = 0; i < nr_secs; i++) { |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 524 | if (lba_list[i] == ADDR_EMPTY) |
| 525 | continue; |
| 526 | |
| 527 | ppa_gc = addr_to_gen_ppa(pblk, paddr_list_gc[i], line->id); |
| 528 | if (!pblk_ppa_comp(ppa_list_l2p[i], ppa_gc)) { |
| 529 | paddr_list_gc[i] = lba_list[i] = ADDR_EMPTY; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 530 | continue; |
| 531 | } |
| 532 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 533 | rqd->ppa_list[valid_secs++] = ppa_list_l2p[i]; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 534 | } |
| 535 | |
| 536 | #ifdef CONFIG_NVM_DEBUG |
| 537 | atomic_long_add(valid_secs, &pblk->inflight_reads); |
| 538 | #endif |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 539 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 540 | return valid_secs; |
| 541 | } |
| 542 | |
| 543 | static int read_rq_gc(struct pblk *pblk, struct nvm_rq *rqd, |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 544 | struct pblk_line *line, sector_t lba, |
| 545 | u64 paddr_gc) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 546 | { |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 547 | struct ppa_addr ppa_l2p, ppa_gc; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 548 | int valid_secs = 0; |
| 549 | |
Dan Carpenter | 659226e | 2017-04-21 16:49:56 -0600 | [diff] [blame] | 550 | if (lba == ADDR_EMPTY) |
| 551 | goto out; |
| 552 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 553 | /* logic error: lba out-of-bounds */ |
Dan Carpenter | 2a79efd | 2017-04-15 20:55:52 +0200 | [diff] [blame] | 554 | if (lba >= pblk->rl.nr_secs) { |
| 555 | WARN(1, "pblk: read lba out of bounds\n"); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 556 | goto out; |
| 557 | } |
| 558 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 559 | spin_lock(&pblk->trans_lock); |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 560 | ppa_l2p = pblk_trans_map_get(pblk, lba); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 561 | spin_unlock(&pblk->trans_lock); |
| 562 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 563 | ppa_gc = addr_to_gen_ppa(pblk, paddr_gc, line->id); |
| 564 | if (!pblk_ppa_comp(ppa_l2p, ppa_gc)) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 565 | goto out; |
| 566 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 567 | rqd->ppa_addr = ppa_l2p; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 568 | valid_secs = 1; |
| 569 | |
| 570 | #ifdef CONFIG_NVM_DEBUG |
| 571 | atomic_long_inc(&pblk->inflight_reads); |
| 572 | #endif |
| 573 | |
| 574 | out: |
| 575 | return valid_secs; |
| 576 | } |
| 577 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 578 | int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 579 | { |
| 580 | struct nvm_tgt_dev *dev = pblk->dev; |
| 581 | struct nvm_geo *geo = &dev->geo; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 582 | struct bio *bio; |
| 583 | struct nvm_rq rqd; |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 584 | int data_len; |
| 585 | int ret = NVM_IO_OK; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 586 | |
| 587 | memset(&rqd, 0, sizeof(struct nvm_rq)); |
| 588 | |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 589 | rqd.meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, |
| 590 | &rqd.dma_meta_list); |
| 591 | if (!rqd.meta_list) |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 592 | return -ENOMEM; |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 593 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 594 | if (gc_rq->nr_secs > 1) { |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 595 | rqd.ppa_list = rqd.meta_list + pblk_dma_meta_size; |
| 596 | rqd.dma_ppa_list = rqd.dma_meta_list + pblk_dma_meta_size; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 597 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 598 | gc_rq->secs_to_gc = read_ppalist_rq_gc(pblk, &rqd, gc_rq->line, |
| 599 | gc_rq->lba_list, |
| 600 | gc_rq->paddr_list, |
| 601 | gc_rq->nr_secs); |
| 602 | if (gc_rq->secs_to_gc == 1) |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 603 | rqd.ppa_addr = rqd.ppa_list[0]; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 604 | } else { |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 605 | gc_rq->secs_to_gc = read_rq_gc(pblk, &rqd, gc_rq->line, |
| 606 | gc_rq->lba_list[0], |
| 607 | gc_rq->paddr_list[0]); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 608 | } |
| 609 | |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 610 | if (!(gc_rq->secs_to_gc)) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 611 | goto out; |
| 612 | |
Javier González | e46f4e4 | 2018-03-30 00:05:10 +0200 | [diff] [blame] | 613 | data_len = (gc_rq->secs_to_gc) * geo->csecs; |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 614 | bio = pblk_bio_map_addr(pblk, gc_rq->data, gc_rq->secs_to_gc, data_len, |
Javier González | 7d327a9 | 2017-10-13 14:46:02 +0200 | [diff] [blame] | 615 | PBLK_VMALLOC_META, GFP_KERNEL); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 616 | if (IS_ERR(bio)) { |
| 617 | pr_err("pblk: could not allocate GC bio (%lu)\n", PTR_ERR(bio)); |
| 618 | goto err_free_dma; |
| 619 | } |
| 620 | |
| 621 | bio->bi_iter.bi_sector = 0; /* internal bio */ |
| 622 | bio_set_op_attrs(bio, REQ_OP_READ, 0); |
| 623 | |
| 624 | rqd.opcode = NVM_OP_PREAD; |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 625 | rqd.nr_ppas = gc_rq->secs_to_gc; |
Javier González | f9c1015 | 2017-06-26 11:57:20 +0200 | [diff] [blame] | 626 | rqd.flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 627 | rqd.bio = bio; |
| 628 | |
Javier González | 1a94b2d | 2017-10-13 14:46:47 +0200 | [diff] [blame] | 629 | if (pblk_submit_io_sync(pblk, &rqd)) { |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 630 | ret = -EIO; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 631 | pr_err("pblk: GC read request failed\n"); |
Javier González | 7d327a9 | 2017-10-13 14:46:02 +0200 | [diff] [blame] | 632 | goto err_free_bio; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 633 | } |
| 634 | |
Javier González | 03a34b2 | 2018-06-01 15:04:18 +0200 | [diff] [blame] | 635 | pblk_read_check_rand(pblk, &rqd, gc_rq->lba_list, gc_rq->nr_secs); |
Javier González | 310df58 | 2018-06-01 15:04:17 +0200 | [diff] [blame] | 636 | |
Javier González | 588726d3 | 2017-06-26 11:57:29 +0200 | [diff] [blame] | 637 | atomic_dec(&pblk->inflight_io); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 638 | |
| 639 | if (rqd.error) { |
| 640 | atomic_long_inc(&pblk->read_failed_gc); |
| 641 | #ifdef CONFIG_NVM_DEBUG |
| 642 | pblk_print_failed_rqd(pblk, &rqd, rqd.error); |
| 643 | #endif |
| 644 | } |
| 645 | |
| 646 | #ifdef CONFIG_NVM_DEBUG |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 647 | atomic_long_add(gc_rq->secs_to_gc, &pblk->sync_reads); |
| 648 | atomic_long_add(gc_rq->secs_to_gc, &pblk->recov_gc_reads); |
| 649 | atomic_long_sub(gc_rq->secs_to_gc, &pblk->inflight_reads); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 650 | #endif |
| 651 | |
| 652 | out: |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 653 | nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list); |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 654 | return ret; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 655 | |
Javier González | 7d327a9 | 2017-10-13 14:46:02 +0200 | [diff] [blame] | 656 | err_free_bio: |
| 657 | bio_put(bio); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 658 | err_free_dma: |
Javier González | 63e3809 | 2017-06-26 11:57:24 +0200 | [diff] [blame] | 659 | nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list); |
Javier González | d340121 | 2017-10-13 14:46:14 +0200 | [diff] [blame] | 660 | return ret; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 661 | } |