Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2016 CNEX Labs |
| 3 | * Initial release: Javier Gonzalez <javier@cnexlabs.com> |
| 4 | * Matias Bjorling <matias@cnexlabs.com> |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License version |
| 8 | * 2 as published by the Free Software Foundation. |
| 9 | * |
| 10 | * This program is distributed in the hope that it will be useful, but |
| 11 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 13 | * General Public License for more details. |
| 14 | * |
| 15 | * pblk-write.c - pblk's write path from write buffer to media |
| 16 | */ |
| 17 | |
| 18 | #include "pblk.h" |
| 19 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 20 | static unsigned long pblk_end_w_bio(struct pblk *pblk, struct nvm_rq *rqd, |
| 21 | struct pblk_c_ctx *c_ctx) |
| 22 | { |
| 23 | struct nvm_tgt_dev *dev = pblk->dev; |
| 24 | struct bio *original_bio; |
| 25 | unsigned long ret; |
| 26 | int i; |
| 27 | |
| 28 | for (i = 0; i < c_ctx->nr_valid; i++) { |
| 29 | struct pblk_w_ctx *w_ctx; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 30 | |
| 31 | w_ctx = pblk_rb_w_ctx(&pblk->rwb, c_ctx->sentry + i); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 32 | while ((original_bio = bio_list_pop(&w_ctx->bios))) |
| 33 | bio_endio(original_bio); |
| 34 | } |
| 35 | |
Javier González | cd8ddbf | 2017-10-13 14:46:03 +0200 | [diff] [blame] | 36 | if (c_ctx->nr_padded) |
| 37 | pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid, |
| 38 | c_ctx->nr_padded); |
| 39 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 40 | #ifdef CONFIG_NVM_DEBUG |
Javier González | e0e12a7 | 2017-10-13 14:46:04 +0200 | [diff] [blame] | 41 | atomic_long_add(rqd->nr_ppas, &pblk->sync_writes); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 42 | #endif |
| 43 | |
| 44 | ret = pblk_rb_sync_advance(&pblk->rwb, c_ctx->nr_valid); |
| 45 | |
Javier González | 56c7641 | 2017-07-07 21:08:53 +0200 | [diff] [blame] | 46 | nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 47 | |
| 48 | bio_put(rqd->bio); |
| 49 | pblk_free_rqd(pblk, rqd, WRITE); |
| 50 | |
| 51 | return ret; |
| 52 | } |
| 53 | |
| 54 | static unsigned long pblk_end_queued_w_bio(struct pblk *pblk, |
| 55 | struct nvm_rq *rqd, |
| 56 | struct pblk_c_ctx *c_ctx) |
| 57 | { |
| 58 | list_del(&c_ctx->list); |
| 59 | return pblk_end_w_bio(pblk, rqd, c_ctx); |
| 60 | } |
| 61 | |
| 62 | static void pblk_complete_write(struct pblk *pblk, struct nvm_rq *rqd, |
| 63 | struct pblk_c_ctx *c_ctx) |
| 64 | { |
| 65 | struct pblk_c_ctx *c, *r; |
| 66 | unsigned long flags; |
| 67 | unsigned long pos; |
| 68 | |
| 69 | #ifdef CONFIG_NVM_DEBUG |
| 70 | atomic_long_sub(c_ctx->nr_valid, &pblk->inflight_writes); |
| 71 | #endif |
| 72 | |
| 73 | pblk_up_rq(pblk, rqd->ppa_list, rqd->nr_ppas, c_ctx->lun_bitmap); |
| 74 | |
| 75 | pos = pblk_rb_sync_init(&pblk->rwb, &flags); |
| 76 | if (pos == c_ctx->sentry) { |
| 77 | pos = pblk_end_w_bio(pblk, rqd, c_ctx); |
| 78 | |
| 79 | retry: |
| 80 | list_for_each_entry_safe(c, r, &pblk->compl_list, list) { |
| 81 | rqd = nvm_rq_from_c_ctx(c); |
| 82 | if (c->sentry == pos) { |
| 83 | pos = pblk_end_queued_w_bio(pblk, rqd, c); |
| 84 | goto retry; |
| 85 | } |
| 86 | } |
| 87 | } else { |
| 88 | WARN_ON(nvm_rq_from_c_ctx(c_ctx) != rqd); |
| 89 | list_add_tail(&c_ctx->list, &pblk->compl_list); |
| 90 | } |
| 91 | pblk_rb_sync_end(&pblk->rwb, &flags); |
| 92 | } |
| 93 | |
| 94 | /* When a write fails, we are not sure whether the block has grown bad or a page |
| 95 | * range is more susceptible to write errors. If a high number of pages fail, we |
| 96 | * assume that the block is bad and we mark it accordingly. In all cases, we |
| 97 | * remap and resubmit the failed entries as fast as possible; if a flush is |
| 98 | * waiting on a completion, the whole stack would stall otherwise. |
| 99 | */ |
| 100 | static void pblk_end_w_fail(struct pblk *pblk, struct nvm_rq *rqd) |
| 101 | { |
| 102 | void *comp_bits = &rqd->ppa_status; |
| 103 | struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd); |
| 104 | struct pblk_rec_ctx *recovery; |
| 105 | struct ppa_addr *ppa_list = rqd->ppa_list; |
| 106 | int nr_ppas = rqd->nr_ppas; |
| 107 | unsigned int c_entries; |
| 108 | int bit, ret; |
| 109 | |
| 110 | if (unlikely(nr_ppas == 1)) |
| 111 | ppa_list = &rqd->ppa_addr; |
| 112 | |
| 113 | recovery = mempool_alloc(pblk->rec_pool, GFP_ATOMIC); |
Javier González | 2942f50 | 2017-10-13 14:46:10 +0200 | [diff] [blame] | 114 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 115 | INIT_LIST_HEAD(&recovery->failed); |
| 116 | |
| 117 | bit = -1; |
| 118 | while ((bit = find_next_bit(comp_bits, nr_ppas, bit + 1)) < nr_ppas) { |
| 119 | struct pblk_rb_entry *entry; |
| 120 | struct ppa_addr ppa; |
| 121 | |
| 122 | /* Logic error */ |
| 123 | if (bit > c_ctx->nr_valid) { |
Dan Carpenter | 2a79efd | 2017-04-15 20:55:52 +0200 | [diff] [blame] | 124 | WARN_ONCE(1, "pblk: corrupted write request\n"); |
Javier González | 33db9fd | 2017-04-22 01:32:47 +0200 | [diff] [blame] | 125 | mempool_free(recovery, pblk->rec_pool); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 126 | goto out; |
| 127 | } |
| 128 | |
| 129 | ppa = ppa_list[bit]; |
| 130 | entry = pblk_rb_sync_scan_entry(&pblk->rwb, &ppa); |
| 131 | if (!entry) { |
| 132 | pr_err("pblk: could not scan entry on write failure\n"); |
Javier González | 33db9fd | 2017-04-22 01:32:47 +0200 | [diff] [blame] | 133 | mempool_free(recovery, pblk->rec_pool); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 134 | goto out; |
| 135 | } |
| 136 | |
| 137 | /* The list is filled first and emptied afterwards. No need for |
| 138 | * protecting it with a lock |
| 139 | */ |
| 140 | list_add_tail(&entry->index, &recovery->failed); |
| 141 | } |
| 142 | |
| 143 | c_entries = find_first_bit(comp_bits, nr_ppas); |
| 144 | ret = pblk_recov_setup_rq(pblk, c_ctx, recovery, comp_bits, c_entries); |
| 145 | if (ret) { |
| 146 | pr_err("pblk: could not recover from write failure\n"); |
Javier González | 33db9fd | 2017-04-22 01:32:47 +0200 | [diff] [blame] | 147 | mempool_free(recovery, pblk->rec_pool); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 148 | goto out; |
| 149 | } |
| 150 | |
| 151 | INIT_WORK(&recovery->ws_rec, pblk_submit_rec); |
Javier González | ef57649 | 2017-06-26 11:57:28 +0200 | [diff] [blame] | 152 | queue_work(pblk->close_wq, &recovery->ws_rec); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 153 | |
| 154 | out: |
| 155 | pblk_complete_write(pblk, rqd, c_ctx); |
| 156 | } |
| 157 | |
| 158 | static void pblk_end_io_write(struct nvm_rq *rqd) |
| 159 | { |
| 160 | struct pblk *pblk = rqd->private; |
| 161 | struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd); |
| 162 | |
| 163 | if (rqd->error) { |
| 164 | pblk_log_write_err(pblk, rqd); |
| 165 | return pblk_end_w_fail(pblk, rqd); |
| 166 | } |
| 167 | #ifdef CONFIG_NVM_DEBUG |
| 168 | else |
Christoph Hellwig | 4e4cbee | 2017-06-03 09:38:06 +0200 | [diff] [blame] | 169 | WARN_ONCE(rqd->bio->bi_status, "pblk: corrupted write error\n"); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 170 | #endif |
| 171 | |
| 172 | pblk_complete_write(pblk, rqd, c_ctx); |
Javier González | 588726d3 | 2017-06-26 11:57:29 +0200 | [diff] [blame] | 173 | atomic_dec(&pblk->inflight_io); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 174 | } |
| 175 | |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 176 | static void pblk_end_io_write_meta(struct nvm_rq *rqd) |
| 177 | { |
| 178 | struct pblk *pblk = rqd->private; |
| 179 | struct nvm_tgt_dev *dev = pblk->dev; |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 180 | struct pblk_g_ctx *m_ctx = nvm_rq_to_pdu(rqd); |
| 181 | struct pblk_line *line = m_ctx->private; |
| 182 | struct pblk_emeta *emeta = line->emeta; |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 183 | int sync; |
| 184 | |
Javier González | 3eaa11e | 2017-07-07 21:08:52 +0200 | [diff] [blame] | 185 | pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas); |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 186 | |
| 187 | if (rqd->error) { |
| 188 | pblk_log_write_err(pblk, rqd); |
Javier González | ee8d5c1 | 2017-06-30 17:56:40 +0200 | [diff] [blame] | 189 | pr_err("pblk: metadata I/O failed. Line %d\n", line->id); |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 190 | } |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 191 | |
| 192 | sync = atomic_add_return(rqd->nr_ppas, &emeta->sync); |
| 193 | if (sync == emeta->nr_entries) |
Javier González | b84ae4a8 | 2017-10-13 14:46:07 +0200 | [diff] [blame] | 194 | pblk_gen_run_ws(pblk, line, NULL, pblk_line_close_ws, |
| 195 | GFP_ATOMIC, pblk->close_wq); |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 196 | |
Javier González | 3eaa11e | 2017-07-07 21:08:52 +0200 | [diff] [blame] | 197 | nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list); |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 198 | pblk_free_rqd(pblk, rqd, READ); |
Javier González | 588726d3 | 2017-06-26 11:57:29 +0200 | [diff] [blame] | 199 | |
| 200 | atomic_dec(&pblk->inflight_io); |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 201 | } |
| 202 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 203 | static int pblk_alloc_w_rq(struct pblk *pblk, struct nvm_rq *rqd, |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 204 | unsigned int nr_secs, |
| 205 | nvm_end_io_fn(*end_io)) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 206 | { |
| 207 | struct nvm_tgt_dev *dev = pblk->dev; |
| 208 | |
| 209 | /* Setup write request */ |
| 210 | rqd->opcode = NVM_OP_PWRITE; |
| 211 | rqd->nr_ppas = nr_secs; |
| 212 | rqd->flags = pblk_set_progr_mode(pblk, WRITE); |
| 213 | rqd->private = pblk; |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 214 | rqd->end_io = end_io; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 215 | |
| 216 | rqd->meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, |
| 217 | &rqd->dma_meta_list); |
| 218 | if (!rqd->meta_list) |
| 219 | return -ENOMEM; |
| 220 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 221 | rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size; |
| 222 | rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size; |
| 223 | |
| 224 | return 0; |
| 225 | } |
| 226 | |
| 227 | static int pblk_setup_w_rq(struct pblk *pblk, struct nvm_rq *rqd, |
Javier González | d624f37 | 2017-06-26 11:57:15 +0200 | [diff] [blame] | 228 | struct pblk_c_ctx *c_ctx, struct ppa_addr *erase_ppa) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 229 | { |
| 230 | struct pblk_line_meta *lm = &pblk->lm; |
Javier González | d624f37 | 2017-06-26 11:57:15 +0200 | [diff] [blame] | 231 | struct pblk_line *e_line = pblk_line_get_erase(pblk); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 232 | unsigned int valid = c_ctx->nr_valid; |
| 233 | unsigned int padded = c_ctx->nr_padded; |
| 234 | unsigned int nr_secs = valid + padded; |
| 235 | unsigned long *lun_bitmap; |
| 236 | int ret = 0; |
| 237 | |
| 238 | lun_bitmap = kzalloc(lm->lun_bitmap_len, GFP_KERNEL); |
Javier González | d624f37 | 2017-06-26 11:57:15 +0200 | [diff] [blame] | 239 | if (!lun_bitmap) |
| 240 | return -ENOMEM; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 241 | c_ctx->lun_bitmap = lun_bitmap; |
| 242 | |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 243 | ret = pblk_alloc_w_rq(pblk, rqd, nr_secs, pblk_end_io_write); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 244 | if (ret) { |
| 245 | kfree(lun_bitmap); |
Javier González | d624f37 | 2017-06-26 11:57:15 +0200 | [diff] [blame] | 246 | return ret; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 247 | } |
| 248 | |
Javier González | 588726d3 | 2017-06-26 11:57:29 +0200 | [diff] [blame] | 249 | if (likely(!e_line || !atomic_read(&e_line->left_eblks))) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 250 | pblk_map_rq(pblk, rqd, c_ctx->sentry, lun_bitmap, valid, 0); |
| 251 | else |
| 252 | pblk_map_erase_rq(pblk, rqd, c_ctx->sentry, lun_bitmap, |
Javier González | d624f37 | 2017-06-26 11:57:15 +0200 | [diff] [blame] | 253 | valid, erase_ppa); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 254 | |
Javier González | d624f37 | 2017-06-26 11:57:15 +0200 | [diff] [blame] | 255 | return 0; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 256 | } |
| 257 | |
| 258 | int pblk_setup_w_rec_rq(struct pblk *pblk, struct nvm_rq *rqd, |
| 259 | struct pblk_c_ctx *c_ctx) |
| 260 | { |
| 261 | struct pblk_line_meta *lm = &pblk->lm; |
| 262 | unsigned long *lun_bitmap; |
| 263 | int ret; |
| 264 | |
| 265 | lun_bitmap = kzalloc(lm->lun_bitmap_len, GFP_KERNEL); |
| 266 | if (!lun_bitmap) |
| 267 | return -ENOMEM; |
| 268 | |
| 269 | c_ctx->lun_bitmap = lun_bitmap; |
| 270 | |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 271 | ret = pblk_alloc_w_rq(pblk, rqd, rqd->nr_ppas, pblk_end_io_write); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 272 | if (ret) |
| 273 | return ret; |
| 274 | |
| 275 | pblk_map_rq(pblk, rqd, c_ctx->sentry, lun_bitmap, c_ctx->nr_valid, 0); |
| 276 | |
| 277 | rqd->ppa_status = (u64)0; |
| 278 | rqd->flags = pblk_set_progr_mode(pblk, WRITE); |
| 279 | |
| 280 | return ret; |
| 281 | } |
| 282 | |
| 283 | static int pblk_calc_secs_to_sync(struct pblk *pblk, unsigned int secs_avail, |
| 284 | unsigned int secs_to_flush) |
| 285 | { |
| 286 | int secs_to_sync; |
| 287 | |
| 288 | secs_to_sync = pblk_calc_secs(pblk, secs_avail, secs_to_flush); |
| 289 | |
| 290 | #ifdef CONFIG_NVM_DEBUG |
| 291 | if ((!secs_to_sync && secs_to_flush) |
| 292 | || (secs_to_sync < 0) |
| 293 | || (secs_to_sync > secs_avail && !secs_to_flush)) { |
| 294 | pr_err("pblk: bad sector calculation (a:%d,s:%d,f:%d)\n", |
| 295 | secs_avail, secs_to_sync, secs_to_flush); |
| 296 | } |
| 297 | #endif |
| 298 | |
| 299 | return secs_to_sync; |
| 300 | } |
| 301 | |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 302 | static inline int pblk_valid_meta_ppa(struct pblk *pblk, |
| 303 | struct pblk_line *meta_line, |
| 304 | struct ppa_addr *ppa_list, int nr_ppas) |
| 305 | { |
| 306 | struct nvm_tgt_dev *dev = pblk->dev; |
| 307 | struct nvm_geo *geo = &dev->geo; |
| 308 | struct pblk_line *data_line; |
| 309 | struct ppa_addr ppa, ppa_opt; |
| 310 | u64 paddr; |
| 311 | int i; |
| 312 | |
| 313 | data_line = &pblk->lines[pblk_dev_ppa_to_line(ppa_list[0])]; |
| 314 | paddr = pblk_lookup_page(pblk, meta_line); |
| 315 | ppa = addr_to_gen_ppa(pblk, paddr, 0); |
| 316 | |
| 317 | if (test_bit(pblk_ppa_to_pos(geo, ppa), data_line->blk_bitmap)) |
| 318 | return 1; |
| 319 | |
| 320 | /* Schedule a metadata I/O that is half the distance from the data I/O |
| 321 | * with regards to the number of LUNs forming the pblk instance. This |
| 322 | * balances LUN conflicts across every I/O. |
| 323 | * |
| 324 | * When the LUN configuration changes (e.g., due to GC), this distance |
| 325 | * can align, which would result on a LUN deadlock. In this case, modify |
| 326 | * the distance to not be optimal, but allow metadata I/Os to succeed. |
| 327 | */ |
| 328 | ppa_opt = addr_to_gen_ppa(pblk, paddr + data_line->meta_distance, 0); |
| 329 | if (unlikely(ppa_opt.ppa == ppa.ppa)) { |
| 330 | data_line->meta_distance--; |
| 331 | return 0; |
| 332 | } |
| 333 | |
| 334 | for (i = 0; i < nr_ppas; i += pblk->min_write_pgs) |
| 335 | if (ppa_list[i].g.ch == ppa_opt.g.ch && |
| 336 | ppa_list[i].g.lun == ppa_opt.g.lun) |
| 337 | return 1; |
| 338 | |
| 339 | if (test_bit(pblk_ppa_to_pos(geo, ppa_opt), data_line->blk_bitmap)) { |
| 340 | for (i = 0; i < nr_ppas; i += pblk->min_write_pgs) |
| 341 | if (ppa_list[i].g.ch == ppa.g.ch && |
| 342 | ppa_list[i].g.lun == ppa.g.lun) |
| 343 | return 0; |
| 344 | |
| 345 | return 1; |
| 346 | } |
| 347 | |
| 348 | return 0; |
| 349 | } |
| 350 | |
| 351 | int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line) |
| 352 | { |
| 353 | struct nvm_tgt_dev *dev = pblk->dev; |
| 354 | struct nvm_geo *geo = &dev->geo; |
| 355 | struct pblk_line_mgmt *l_mg = &pblk->l_mg; |
| 356 | struct pblk_line_meta *lm = &pblk->lm; |
| 357 | struct pblk_emeta *emeta = meta_line->emeta; |
| 358 | struct pblk_g_ctx *m_ctx; |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 359 | struct bio *bio; |
| 360 | struct nvm_rq *rqd; |
| 361 | void *data; |
| 362 | u64 paddr; |
| 363 | int rq_ppas = pblk->min_write_pgs; |
| 364 | int id = meta_line->id; |
| 365 | int rq_len; |
| 366 | int i, j; |
| 367 | int ret; |
| 368 | |
| 369 | rqd = pblk_alloc_rqd(pblk, READ); |
Javier González | 2942f50 | 2017-10-13 14:46:10 +0200 | [diff] [blame] | 370 | |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 371 | m_ctx = nvm_rq_to_pdu(rqd); |
| 372 | m_ctx->private = meta_line; |
| 373 | |
| 374 | rq_len = rq_ppas * geo->sec_size; |
| 375 | data = ((void *)emeta->buf) + emeta->mem; |
| 376 | |
Javier González | de54e70 | 2017-06-30 17:56:39 +0200 | [diff] [blame] | 377 | bio = pblk_bio_map_addr(pblk, data, rq_ppas, rq_len, |
| 378 | l_mg->emeta_alloc_type, GFP_KERNEL); |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 379 | if (IS_ERR(bio)) { |
| 380 | ret = PTR_ERR(bio); |
| 381 | goto fail_free_rqd; |
| 382 | } |
| 383 | bio->bi_iter.bi_sector = 0; /* internal bio */ |
| 384 | bio_set_op_attrs(bio, REQ_OP_WRITE, 0); |
| 385 | rqd->bio = bio; |
| 386 | |
| 387 | ret = pblk_alloc_w_rq(pblk, rqd, rq_ppas, pblk_end_io_write_meta); |
| 388 | if (ret) |
| 389 | goto fail_free_bio; |
| 390 | |
| 391 | for (i = 0; i < rqd->nr_ppas; ) { |
| 392 | spin_lock(&meta_line->lock); |
| 393 | paddr = __pblk_alloc_page(pblk, meta_line, rq_ppas); |
| 394 | spin_unlock(&meta_line->lock); |
| 395 | for (j = 0; j < rq_ppas; j++, i++, paddr++) |
| 396 | rqd->ppa_list[i] = addr_to_gen_ppa(pblk, paddr, id); |
| 397 | } |
| 398 | |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 399 | emeta->mem += rq_len; |
| 400 | if (emeta->mem >= lm->emeta_len[0]) { |
| 401 | spin_lock(&l_mg->close_lock); |
| 402 | list_del(&meta_line->list); |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 403 | spin_unlock(&l_mg->close_lock); |
| 404 | } |
| 405 | |
Javier González | 3eaa11e | 2017-07-07 21:08:52 +0200 | [diff] [blame] | 406 | pblk_down_page(pblk, rqd->ppa_list, rqd->nr_ppas); |
| 407 | |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 408 | ret = pblk_submit_io(pblk, rqd); |
| 409 | if (ret) { |
| 410 | pr_err("pblk: emeta I/O submission failed: %d\n", ret); |
| 411 | goto fail_rollback; |
| 412 | } |
| 413 | |
| 414 | return NVM_IO_OK; |
| 415 | |
| 416 | fail_rollback: |
Javier González | 3eaa11e | 2017-07-07 21:08:52 +0200 | [diff] [blame] | 417 | pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas); |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 418 | spin_lock(&l_mg->close_lock); |
| 419 | pblk_dealloc_page(pblk, meta_line, rq_ppas); |
| 420 | list_add(&meta_line->list, &meta_line->list); |
| 421 | spin_unlock(&l_mg->close_lock); |
Javier González | 3eaa11e | 2017-07-07 21:08:52 +0200 | [diff] [blame] | 422 | |
| 423 | nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list); |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 424 | fail_free_bio: |
Javier González | 55e836d | 2017-10-13 14:46:16 +0200 | [diff] [blame] | 425 | bio_put(bio); |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 426 | fail_free_rqd: |
| 427 | pblk_free_rqd(pblk, rqd, READ); |
| 428 | return ret; |
| 429 | } |
| 430 | |
| 431 | static int pblk_sched_meta_io(struct pblk *pblk, struct ppa_addr *prev_list, |
| 432 | int prev_n) |
| 433 | { |
| 434 | struct pblk_line_meta *lm = &pblk->lm; |
| 435 | struct pblk_line_mgmt *l_mg = &pblk->l_mg; |
| 436 | struct pblk_line *meta_line; |
| 437 | |
| 438 | spin_lock(&l_mg->close_lock); |
| 439 | retry: |
| 440 | if (list_empty(&l_mg->emeta_list)) { |
| 441 | spin_unlock(&l_mg->close_lock); |
| 442 | return 0; |
| 443 | } |
| 444 | meta_line = list_first_entry(&l_mg->emeta_list, struct pblk_line, list); |
Javier González | e72ec1d | 2017-10-13 14:46:09 +0200 | [diff] [blame] | 445 | if (meta_line->emeta->mem >= lm->emeta_len[0]) |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 446 | goto retry; |
| 447 | spin_unlock(&l_mg->close_lock); |
| 448 | |
| 449 | if (!pblk_valid_meta_ppa(pblk, meta_line, prev_list, prev_n)) |
| 450 | return 0; |
| 451 | |
| 452 | return pblk_submit_meta_io(pblk, meta_line); |
| 453 | } |
| 454 | |
Javier González | d624f37 | 2017-06-26 11:57:15 +0200 | [diff] [blame] | 455 | static int pblk_submit_io_set(struct pblk *pblk, struct nvm_rq *rqd) |
| 456 | { |
| 457 | struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd); |
| 458 | struct ppa_addr erase_ppa; |
| 459 | int err; |
| 460 | |
| 461 | ppa_set_empty(&erase_ppa); |
| 462 | |
| 463 | /* Assign lbas to ppas and populate request structure */ |
| 464 | err = pblk_setup_w_rq(pblk, rqd, c_ctx, &erase_ppa); |
| 465 | if (err) { |
| 466 | pr_err("pblk: could not setup write request: %d\n", err); |
| 467 | return NVM_IO_ERR; |
| 468 | } |
| 469 | |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 470 | if (likely(ppa_empty(erase_ppa))) { |
| 471 | /* Submit metadata write for previous data line */ |
| 472 | err = pblk_sched_meta_io(pblk, rqd->ppa_list, rqd->nr_ppas); |
| 473 | if (err) { |
| 474 | pr_err("pblk: metadata I/O submission failed: %d", err); |
| 475 | return NVM_IO_ERR; |
| 476 | } |
Javier González | d624f37 | 2017-06-26 11:57:15 +0200 | [diff] [blame] | 477 | |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 478 | /* Submit data write for current data line */ |
| 479 | err = pblk_submit_io(pblk, rqd); |
| 480 | if (err) { |
| 481 | pr_err("pblk: data I/O submission failed: %d\n", err); |
| 482 | return NVM_IO_ERR; |
| 483 | } |
| 484 | } else { |
| 485 | /* Submit data write for current data line */ |
| 486 | err = pblk_submit_io(pblk, rqd); |
| 487 | if (err) { |
| 488 | pr_err("pblk: data I/O submission failed: %d\n", err); |
| 489 | return NVM_IO_ERR; |
| 490 | } |
Javier González | d624f37 | 2017-06-26 11:57:15 +0200 | [diff] [blame] | 491 | |
Javier González | dd2a434 | 2017-06-26 11:57:17 +0200 | [diff] [blame] | 492 | /* Submit available erase for next data line */ |
| 493 | if (pblk_blk_erase_async(pblk, erase_ppa)) { |
| 494 | struct pblk_line *e_line = pblk_line_get_erase(pblk); |
| 495 | struct nvm_tgt_dev *dev = pblk->dev; |
| 496 | struct nvm_geo *geo = &dev->geo; |
| 497 | int bit; |
| 498 | |
| 499 | atomic_inc(&e_line->left_eblks); |
| 500 | bit = pblk_ppa_to_pos(geo, erase_ppa); |
| 501 | WARN_ON(!test_and_clear_bit(bit, e_line->erase_bitmap)); |
| 502 | } |
Javier González | d624f37 | 2017-06-26 11:57:15 +0200 | [diff] [blame] | 503 | } |
| 504 | |
| 505 | return NVM_IO_OK; |
| 506 | } |
| 507 | |
| 508 | static void pblk_free_write_rqd(struct pblk *pblk, struct nvm_rq *rqd) |
| 509 | { |
| 510 | struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd); |
| 511 | struct bio *bio = rqd->bio; |
| 512 | |
| 513 | if (c_ctx->nr_padded) |
Javier González | cd8ddbf | 2017-10-13 14:46:03 +0200 | [diff] [blame] | 514 | pblk_bio_free_pages(pblk, bio, c_ctx->nr_valid, |
| 515 | c_ctx->nr_padded); |
Javier González | d624f37 | 2017-06-26 11:57:15 +0200 | [diff] [blame] | 516 | } |
| 517 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 518 | static int pblk_submit_write(struct pblk *pblk) |
| 519 | { |
| 520 | struct bio *bio; |
| 521 | struct nvm_rq *rqd; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 522 | unsigned int secs_avail, secs_to_sync, secs_to_com; |
| 523 | unsigned int secs_to_flush; |
| 524 | unsigned long pos; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 525 | |
| 526 | /* If there are no sectors in the cache, flushes (bios without data) |
| 527 | * will be cleared on the cache threads |
| 528 | */ |
| 529 | secs_avail = pblk_rb_read_count(&pblk->rwb); |
| 530 | if (!secs_avail) |
| 531 | return 1; |
| 532 | |
| 533 | secs_to_flush = pblk_rb_sync_point_count(&pblk->rwb); |
| 534 | if (!secs_to_flush && secs_avail < pblk->min_write_pgs) |
| 535 | return 1; |
| 536 | |
Javier González | 875d94f | 2017-10-13 14:46:18 +0200 | [diff] [blame^] | 537 | secs_to_sync = pblk_calc_secs_to_sync(pblk, secs_avail, secs_to_flush); |
| 538 | if (secs_to_sync > pblk->max_write_pgs) { |
| 539 | pr_err("pblk: bad buffer sync calculation\n"); |
| 540 | return 1; |
| 541 | } |
| 542 | |
| 543 | secs_to_com = (secs_to_sync > secs_avail) ? secs_avail : secs_to_sync; |
| 544 | pos = pblk_rb_read_commit(&pblk->rwb, secs_to_com); |
| 545 | |
| 546 | bio = bio_alloc(GFP_KERNEL, secs_to_sync); |
Javier González | 2942f50 | 2017-10-13 14:46:10 +0200 | [diff] [blame] | 547 | |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 548 | bio->bi_iter.bi_sector = 0; /* internal bio */ |
| 549 | bio_set_op_attrs(bio, REQ_OP_WRITE, 0); |
Javier González | 2942f50 | 2017-10-13 14:46:10 +0200 | [diff] [blame] | 550 | |
| 551 | rqd = pblk_alloc_rqd(pblk, WRITE); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 552 | rqd->bio = bio; |
| 553 | |
Javier González | 875d94f | 2017-10-13 14:46:18 +0200 | [diff] [blame^] | 554 | if (pblk_rb_read_to_bio(&pblk->rwb, rqd, pos, secs_to_sync, |
Javier González | d624f37 | 2017-06-26 11:57:15 +0200 | [diff] [blame] | 555 | secs_avail)) { |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 556 | pr_err("pblk: corrupted write bio\n"); |
| 557 | goto fail_put_bio; |
| 558 | } |
| 559 | |
Javier González | d624f37 | 2017-06-26 11:57:15 +0200 | [diff] [blame] | 560 | if (pblk_submit_io_set(pblk, rqd)) |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 561 | goto fail_free_bio; |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 562 | |
| 563 | #ifdef CONFIG_NVM_DEBUG |
| 564 | atomic_long_add(secs_to_sync, &pblk->sub_writes); |
| 565 | #endif |
| 566 | |
| 567 | return 0; |
| 568 | |
| 569 | fail_free_bio: |
Javier González | d624f37 | 2017-06-26 11:57:15 +0200 | [diff] [blame] | 570 | pblk_free_write_rqd(pblk, rqd); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 571 | fail_put_bio: |
| 572 | bio_put(bio); |
Javier González | a4bd217 | 2017-04-15 20:55:50 +0200 | [diff] [blame] | 573 | pblk_free_rqd(pblk, rqd, WRITE); |
| 574 | |
| 575 | return 1; |
| 576 | } |
| 577 | |
| 578 | int pblk_write_ts(void *data) |
| 579 | { |
| 580 | struct pblk *pblk = data; |
| 581 | |
| 582 | while (!kthread_should_stop()) { |
| 583 | if (!pblk_submit_write(pblk)) |
| 584 | continue; |
| 585 | set_current_state(TASK_INTERRUPTIBLE); |
| 586 | io_schedule(); |
| 587 | } |
| 588 | |
| 589 | return 0; |
| 590 | } |