blob: 140a26edd1d3d292d91c524844626b913c1b966b [file] [log] [blame]
Javier Gonzáleza4bd2172017-04-15 20:55:50 +02001/*
2 * Copyright (C) 2016 CNEX Labs
3 * Initial release: Javier Gonzalez <javier@cnexlabs.com>
4 * Matias Bjorling <matias@cnexlabs.com>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License version
8 * 2 as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
14 *
15 * pblk-write.c - pblk's write path from write buffer to media
16 */
17
18#include "pblk.h"
19
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020020static unsigned long pblk_end_w_bio(struct pblk *pblk, struct nvm_rq *rqd,
21 struct pblk_c_ctx *c_ctx)
22{
23 struct nvm_tgt_dev *dev = pblk->dev;
24 struct bio *original_bio;
25 unsigned long ret;
26 int i;
27
28 for (i = 0; i < c_ctx->nr_valid; i++) {
29 struct pblk_w_ctx *w_ctx;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020030
31 w_ctx = pblk_rb_w_ctx(&pblk->rwb, c_ctx->sentry + i);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020032 while ((original_bio = bio_list_pop(&w_ctx->bios)))
33 bio_endio(original_bio);
34 }
35
Javier Gonzálezcd8ddbf2017-10-13 14:46:03 +020036 if (c_ctx->nr_padded)
37 pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid,
38 c_ctx->nr_padded);
39
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020040#ifdef CONFIG_NVM_DEBUG
Javier Gonzáleze0e12a72017-10-13 14:46:04 +020041 atomic_long_add(rqd->nr_ppas, &pblk->sync_writes);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020042#endif
43
44 ret = pblk_rb_sync_advance(&pblk->rwb, c_ctx->nr_valid);
45
Javier González56c76412017-07-07 21:08:53 +020046 nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +020047
48 bio_put(rqd->bio);
49 pblk_free_rqd(pblk, rqd, WRITE);
50
51 return ret;
52}
53
54static unsigned long pblk_end_queued_w_bio(struct pblk *pblk,
55 struct nvm_rq *rqd,
56 struct pblk_c_ctx *c_ctx)
57{
58 list_del(&c_ctx->list);
59 return pblk_end_w_bio(pblk, rqd, c_ctx);
60}
61
62static void pblk_complete_write(struct pblk *pblk, struct nvm_rq *rqd,
63 struct pblk_c_ctx *c_ctx)
64{
65 struct pblk_c_ctx *c, *r;
66 unsigned long flags;
67 unsigned long pos;
68
69#ifdef CONFIG_NVM_DEBUG
70 atomic_long_sub(c_ctx->nr_valid, &pblk->inflight_writes);
71#endif
72
73 pblk_up_rq(pblk, rqd->ppa_list, rqd->nr_ppas, c_ctx->lun_bitmap);
74
75 pos = pblk_rb_sync_init(&pblk->rwb, &flags);
76 if (pos == c_ctx->sentry) {
77 pos = pblk_end_w_bio(pblk, rqd, c_ctx);
78
79retry:
80 list_for_each_entry_safe(c, r, &pblk->compl_list, list) {
81 rqd = nvm_rq_from_c_ctx(c);
82 if (c->sentry == pos) {
83 pos = pblk_end_queued_w_bio(pblk, rqd, c);
84 goto retry;
85 }
86 }
87 } else {
88 WARN_ON(nvm_rq_from_c_ctx(c_ctx) != rqd);
89 list_add_tail(&c_ctx->list, &pblk->compl_list);
90 }
91 pblk_rb_sync_end(&pblk->rwb, &flags);
92}
93
94/* When a write fails, we are not sure whether the block has grown bad or a page
95 * range is more susceptible to write errors. If a high number of pages fail, we
96 * assume that the block is bad and we mark it accordingly. In all cases, we
97 * remap and resubmit the failed entries as fast as possible; if a flush is
98 * waiting on a completion, the whole stack would stall otherwise.
99 */
100static void pblk_end_w_fail(struct pblk *pblk, struct nvm_rq *rqd)
101{
102 void *comp_bits = &rqd->ppa_status;
103 struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);
104 struct pblk_rec_ctx *recovery;
105 struct ppa_addr *ppa_list = rqd->ppa_list;
106 int nr_ppas = rqd->nr_ppas;
107 unsigned int c_entries;
108 int bit, ret;
109
110 if (unlikely(nr_ppas == 1))
111 ppa_list = &rqd->ppa_addr;
112
113 recovery = mempool_alloc(pblk->rec_pool, GFP_ATOMIC);
Javier González2942f502017-10-13 14:46:10 +0200114
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200115 INIT_LIST_HEAD(&recovery->failed);
116
117 bit = -1;
118 while ((bit = find_next_bit(comp_bits, nr_ppas, bit + 1)) < nr_ppas) {
119 struct pblk_rb_entry *entry;
120 struct ppa_addr ppa;
121
122 /* Logic error */
123 if (bit > c_ctx->nr_valid) {
Dan Carpenter2a79efd2017-04-15 20:55:52 +0200124 WARN_ONCE(1, "pblk: corrupted write request\n");
Javier González33db9fd2017-04-22 01:32:47 +0200125 mempool_free(recovery, pblk->rec_pool);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200126 goto out;
127 }
128
129 ppa = ppa_list[bit];
130 entry = pblk_rb_sync_scan_entry(&pblk->rwb, &ppa);
131 if (!entry) {
132 pr_err("pblk: could not scan entry on write failure\n");
Javier González33db9fd2017-04-22 01:32:47 +0200133 mempool_free(recovery, pblk->rec_pool);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200134 goto out;
135 }
136
137 /* The list is filled first and emptied afterwards. No need for
138 * protecting it with a lock
139 */
140 list_add_tail(&entry->index, &recovery->failed);
141 }
142
143 c_entries = find_first_bit(comp_bits, nr_ppas);
144 ret = pblk_recov_setup_rq(pblk, c_ctx, recovery, comp_bits, c_entries);
145 if (ret) {
146 pr_err("pblk: could not recover from write failure\n");
Javier González33db9fd2017-04-22 01:32:47 +0200147 mempool_free(recovery, pblk->rec_pool);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200148 goto out;
149 }
150
151 INIT_WORK(&recovery->ws_rec, pblk_submit_rec);
Javier Gonzálezef576492017-06-26 11:57:28 +0200152 queue_work(pblk->close_wq, &recovery->ws_rec);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200153
154out:
155 pblk_complete_write(pblk, rqd, c_ctx);
156}
157
158static void pblk_end_io_write(struct nvm_rq *rqd)
159{
160 struct pblk *pblk = rqd->private;
161 struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);
162
163 if (rqd->error) {
164 pblk_log_write_err(pblk, rqd);
165 return pblk_end_w_fail(pblk, rqd);
166 }
167#ifdef CONFIG_NVM_DEBUG
168 else
Christoph Hellwig4e4cbee2017-06-03 09:38:06 +0200169 WARN_ONCE(rqd->bio->bi_status, "pblk: corrupted write error\n");
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200170#endif
171
172 pblk_complete_write(pblk, rqd, c_ctx);
Javier González588726d32017-06-26 11:57:29 +0200173 atomic_dec(&pblk->inflight_io);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200174}
175
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200176static void pblk_end_io_write_meta(struct nvm_rq *rqd)
177{
178 struct pblk *pblk = rqd->private;
179 struct nvm_tgt_dev *dev = pblk->dev;
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200180 struct pblk_g_ctx *m_ctx = nvm_rq_to_pdu(rqd);
181 struct pblk_line *line = m_ctx->private;
182 struct pblk_emeta *emeta = line->emeta;
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200183 int sync;
184
Javier González3eaa11e2017-07-07 21:08:52 +0200185 pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas);
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200186
187 if (rqd->error) {
188 pblk_log_write_err(pblk, rqd);
Javier Gonzálezee8d5c12017-06-30 17:56:40 +0200189 pr_err("pblk: metadata I/O failed. Line %d\n", line->id);
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200190 }
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200191
192 sync = atomic_add_return(rqd->nr_ppas, &emeta->sync);
193 if (sync == emeta->nr_entries)
Javier Gonzálezb84ae4a82017-10-13 14:46:07 +0200194 pblk_gen_run_ws(pblk, line, NULL, pblk_line_close_ws,
195 GFP_ATOMIC, pblk->close_wq);
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200196
Javier González3eaa11e2017-07-07 21:08:52 +0200197 nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list);
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200198 pblk_free_rqd(pblk, rqd, READ);
Javier González588726d32017-06-26 11:57:29 +0200199
200 atomic_dec(&pblk->inflight_io);
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200201}
202
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200203static int pblk_alloc_w_rq(struct pblk *pblk, struct nvm_rq *rqd,
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200204 unsigned int nr_secs,
205 nvm_end_io_fn(*end_io))
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200206{
207 struct nvm_tgt_dev *dev = pblk->dev;
208
209 /* Setup write request */
210 rqd->opcode = NVM_OP_PWRITE;
211 rqd->nr_ppas = nr_secs;
212 rqd->flags = pblk_set_progr_mode(pblk, WRITE);
213 rqd->private = pblk;
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200214 rqd->end_io = end_io;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200215
216 rqd->meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL,
217 &rqd->dma_meta_list);
218 if (!rqd->meta_list)
219 return -ENOMEM;
220
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200221 rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size;
222 rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size;
223
224 return 0;
225}
226
227static int pblk_setup_w_rq(struct pblk *pblk, struct nvm_rq *rqd,
Javier Gonzálezd624f372017-06-26 11:57:15 +0200228 struct pblk_c_ctx *c_ctx, struct ppa_addr *erase_ppa)
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200229{
230 struct pblk_line_meta *lm = &pblk->lm;
Javier Gonzálezd624f372017-06-26 11:57:15 +0200231 struct pblk_line *e_line = pblk_line_get_erase(pblk);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200232 unsigned int valid = c_ctx->nr_valid;
233 unsigned int padded = c_ctx->nr_padded;
234 unsigned int nr_secs = valid + padded;
235 unsigned long *lun_bitmap;
236 int ret = 0;
237
238 lun_bitmap = kzalloc(lm->lun_bitmap_len, GFP_KERNEL);
Javier Gonzálezd624f372017-06-26 11:57:15 +0200239 if (!lun_bitmap)
240 return -ENOMEM;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200241 c_ctx->lun_bitmap = lun_bitmap;
242
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200243 ret = pblk_alloc_w_rq(pblk, rqd, nr_secs, pblk_end_io_write);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200244 if (ret) {
245 kfree(lun_bitmap);
Javier Gonzálezd624f372017-06-26 11:57:15 +0200246 return ret;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200247 }
248
Javier González588726d32017-06-26 11:57:29 +0200249 if (likely(!e_line || !atomic_read(&e_line->left_eblks)))
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200250 pblk_map_rq(pblk, rqd, c_ctx->sentry, lun_bitmap, valid, 0);
251 else
252 pblk_map_erase_rq(pblk, rqd, c_ctx->sentry, lun_bitmap,
Javier Gonzálezd624f372017-06-26 11:57:15 +0200253 valid, erase_ppa);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200254
Javier Gonzálezd624f372017-06-26 11:57:15 +0200255 return 0;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200256}
257
258int pblk_setup_w_rec_rq(struct pblk *pblk, struct nvm_rq *rqd,
259 struct pblk_c_ctx *c_ctx)
260{
261 struct pblk_line_meta *lm = &pblk->lm;
262 unsigned long *lun_bitmap;
263 int ret;
264
265 lun_bitmap = kzalloc(lm->lun_bitmap_len, GFP_KERNEL);
266 if (!lun_bitmap)
267 return -ENOMEM;
268
269 c_ctx->lun_bitmap = lun_bitmap;
270
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200271 ret = pblk_alloc_w_rq(pblk, rqd, rqd->nr_ppas, pblk_end_io_write);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200272 if (ret)
273 return ret;
274
275 pblk_map_rq(pblk, rqd, c_ctx->sentry, lun_bitmap, c_ctx->nr_valid, 0);
276
277 rqd->ppa_status = (u64)0;
278 rqd->flags = pblk_set_progr_mode(pblk, WRITE);
279
280 return ret;
281}
282
283static int pblk_calc_secs_to_sync(struct pblk *pblk, unsigned int secs_avail,
284 unsigned int secs_to_flush)
285{
286 int secs_to_sync;
287
288 secs_to_sync = pblk_calc_secs(pblk, secs_avail, secs_to_flush);
289
290#ifdef CONFIG_NVM_DEBUG
291 if ((!secs_to_sync && secs_to_flush)
292 || (secs_to_sync < 0)
293 || (secs_to_sync > secs_avail && !secs_to_flush)) {
294 pr_err("pblk: bad sector calculation (a:%d,s:%d,f:%d)\n",
295 secs_avail, secs_to_sync, secs_to_flush);
296 }
297#endif
298
299 return secs_to_sync;
300}
301
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200302static inline int pblk_valid_meta_ppa(struct pblk *pblk,
303 struct pblk_line *meta_line,
304 struct ppa_addr *ppa_list, int nr_ppas)
305{
306 struct nvm_tgt_dev *dev = pblk->dev;
307 struct nvm_geo *geo = &dev->geo;
308 struct pblk_line *data_line;
309 struct ppa_addr ppa, ppa_opt;
310 u64 paddr;
311 int i;
312
313 data_line = &pblk->lines[pblk_dev_ppa_to_line(ppa_list[0])];
314 paddr = pblk_lookup_page(pblk, meta_line);
315 ppa = addr_to_gen_ppa(pblk, paddr, 0);
316
317 if (test_bit(pblk_ppa_to_pos(geo, ppa), data_line->blk_bitmap))
318 return 1;
319
320 /* Schedule a metadata I/O that is half the distance from the data I/O
321 * with regards to the number of LUNs forming the pblk instance. This
322 * balances LUN conflicts across every I/O.
323 *
324 * When the LUN configuration changes (e.g., due to GC), this distance
325 * can align, which would result on a LUN deadlock. In this case, modify
326 * the distance to not be optimal, but allow metadata I/Os to succeed.
327 */
328 ppa_opt = addr_to_gen_ppa(pblk, paddr + data_line->meta_distance, 0);
329 if (unlikely(ppa_opt.ppa == ppa.ppa)) {
330 data_line->meta_distance--;
331 return 0;
332 }
333
334 for (i = 0; i < nr_ppas; i += pblk->min_write_pgs)
335 if (ppa_list[i].g.ch == ppa_opt.g.ch &&
336 ppa_list[i].g.lun == ppa_opt.g.lun)
337 return 1;
338
339 if (test_bit(pblk_ppa_to_pos(geo, ppa_opt), data_line->blk_bitmap)) {
340 for (i = 0; i < nr_ppas; i += pblk->min_write_pgs)
341 if (ppa_list[i].g.ch == ppa.g.ch &&
342 ppa_list[i].g.lun == ppa.g.lun)
343 return 0;
344
345 return 1;
346 }
347
348 return 0;
349}
350
351int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line)
352{
353 struct nvm_tgt_dev *dev = pblk->dev;
354 struct nvm_geo *geo = &dev->geo;
355 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
356 struct pblk_line_meta *lm = &pblk->lm;
357 struct pblk_emeta *emeta = meta_line->emeta;
358 struct pblk_g_ctx *m_ctx;
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200359 struct bio *bio;
360 struct nvm_rq *rqd;
361 void *data;
362 u64 paddr;
363 int rq_ppas = pblk->min_write_pgs;
364 int id = meta_line->id;
365 int rq_len;
366 int i, j;
367 int ret;
368
369 rqd = pblk_alloc_rqd(pblk, READ);
Javier González2942f502017-10-13 14:46:10 +0200370
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200371 m_ctx = nvm_rq_to_pdu(rqd);
372 m_ctx->private = meta_line;
373
374 rq_len = rq_ppas * geo->sec_size;
375 data = ((void *)emeta->buf) + emeta->mem;
376
Javier Gonzálezde54e702017-06-30 17:56:39 +0200377 bio = pblk_bio_map_addr(pblk, data, rq_ppas, rq_len,
378 l_mg->emeta_alloc_type, GFP_KERNEL);
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200379 if (IS_ERR(bio)) {
380 ret = PTR_ERR(bio);
381 goto fail_free_rqd;
382 }
383 bio->bi_iter.bi_sector = 0; /* internal bio */
384 bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
385 rqd->bio = bio;
386
387 ret = pblk_alloc_w_rq(pblk, rqd, rq_ppas, pblk_end_io_write_meta);
388 if (ret)
389 goto fail_free_bio;
390
391 for (i = 0; i < rqd->nr_ppas; ) {
392 spin_lock(&meta_line->lock);
393 paddr = __pblk_alloc_page(pblk, meta_line, rq_ppas);
394 spin_unlock(&meta_line->lock);
395 for (j = 0; j < rq_ppas; j++, i++, paddr++)
396 rqd->ppa_list[i] = addr_to_gen_ppa(pblk, paddr, id);
397 }
398
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200399 emeta->mem += rq_len;
400 if (emeta->mem >= lm->emeta_len[0]) {
401 spin_lock(&l_mg->close_lock);
402 list_del(&meta_line->list);
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200403 spin_unlock(&l_mg->close_lock);
404 }
405
Javier González3eaa11e2017-07-07 21:08:52 +0200406 pblk_down_page(pblk, rqd->ppa_list, rqd->nr_ppas);
407
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200408 ret = pblk_submit_io(pblk, rqd);
409 if (ret) {
410 pr_err("pblk: emeta I/O submission failed: %d\n", ret);
411 goto fail_rollback;
412 }
413
414 return NVM_IO_OK;
415
416fail_rollback:
Javier González3eaa11e2017-07-07 21:08:52 +0200417 pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas);
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200418 spin_lock(&l_mg->close_lock);
419 pblk_dealloc_page(pblk, meta_line, rq_ppas);
420 list_add(&meta_line->list, &meta_line->list);
421 spin_unlock(&l_mg->close_lock);
Javier González3eaa11e2017-07-07 21:08:52 +0200422
423 nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list);
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200424fail_free_bio:
Javier González55e836d2017-10-13 14:46:16 +0200425 bio_put(bio);
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200426fail_free_rqd:
427 pblk_free_rqd(pblk, rqd, READ);
428 return ret;
429}
430
431static int pblk_sched_meta_io(struct pblk *pblk, struct ppa_addr *prev_list,
432 int prev_n)
433{
434 struct pblk_line_meta *lm = &pblk->lm;
435 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
436 struct pblk_line *meta_line;
437
438 spin_lock(&l_mg->close_lock);
439retry:
440 if (list_empty(&l_mg->emeta_list)) {
441 spin_unlock(&l_mg->close_lock);
442 return 0;
443 }
444 meta_line = list_first_entry(&l_mg->emeta_list, struct pblk_line, list);
Javier Gonzáleze72ec1d2017-10-13 14:46:09 +0200445 if (meta_line->emeta->mem >= lm->emeta_len[0])
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200446 goto retry;
447 spin_unlock(&l_mg->close_lock);
448
449 if (!pblk_valid_meta_ppa(pblk, meta_line, prev_list, prev_n))
450 return 0;
451
452 return pblk_submit_meta_io(pblk, meta_line);
453}
454
Javier Gonzálezd624f372017-06-26 11:57:15 +0200455static int pblk_submit_io_set(struct pblk *pblk, struct nvm_rq *rqd)
456{
457 struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);
458 struct ppa_addr erase_ppa;
459 int err;
460
461 ppa_set_empty(&erase_ppa);
462
463 /* Assign lbas to ppas and populate request structure */
464 err = pblk_setup_w_rq(pblk, rqd, c_ctx, &erase_ppa);
465 if (err) {
466 pr_err("pblk: could not setup write request: %d\n", err);
467 return NVM_IO_ERR;
468 }
469
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200470 if (likely(ppa_empty(erase_ppa))) {
471 /* Submit metadata write for previous data line */
472 err = pblk_sched_meta_io(pblk, rqd->ppa_list, rqd->nr_ppas);
473 if (err) {
474 pr_err("pblk: metadata I/O submission failed: %d", err);
475 return NVM_IO_ERR;
476 }
Javier Gonzálezd624f372017-06-26 11:57:15 +0200477
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200478 /* Submit data write for current data line */
479 err = pblk_submit_io(pblk, rqd);
480 if (err) {
481 pr_err("pblk: data I/O submission failed: %d\n", err);
482 return NVM_IO_ERR;
483 }
484 } else {
485 /* Submit data write for current data line */
486 err = pblk_submit_io(pblk, rqd);
487 if (err) {
488 pr_err("pblk: data I/O submission failed: %d\n", err);
489 return NVM_IO_ERR;
490 }
Javier Gonzálezd624f372017-06-26 11:57:15 +0200491
Javier Gonzálezdd2a4342017-06-26 11:57:17 +0200492 /* Submit available erase for next data line */
493 if (pblk_blk_erase_async(pblk, erase_ppa)) {
494 struct pblk_line *e_line = pblk_line_get_erase(pblk);
495 struct nvm_tgt_dev *dev = pblk->dev;
496 struct nvm_geo *geo = &dev->geo;
497 int bit;
498
499 atomic_inc(&e_line->left_eblks);
500 bit = pblk_ppa_to_pos(geo, erase_ppa);
501 WARN_ON(!test_and_clear_bit(bit, e_line->erase_bitmap));
502 }
Javier Gonzálezd624f372017-06-26 11:57:15 +0200503 }
504
505 return NVM_IO_OK;
506}
507
508static void pblk_free_write_rqd(struct pblk *pblk, struct nvm_rq *rqd)
509{
510 struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);
511 struct bio *bio = rqd->bio;
512
513 if (c_ctx->nr_padded)
Javier Gonzálezcd8ddbf2017-10-13 14:46:03 +0200514 pblk_bio_free_pages(pblk, bio, c_ctx->nr_valid,
515 c_ctx->nr_padded);
Javier Gonzálezd624f372017-06-26 11:57:15 +0200516}
517
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200518static int pblk_submit_write(struct pblk *pblk)
519{
520 struct bio *bio;
521 struct nvm_rq *rqd;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200522 unsigned int secs_avail, secs_to_sync, secs_to_com;
523 unsigned int secs_to_flush;
524 unsigned long pos;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200525
526 /* If there are no sectors in the cache, flushes (bios without data)
527 * will be cleared on the cache threads
528 */
529 secs_avail = pblk_rb_read_count(&pblk->rwb);
530 if (!secs_avail)
531 return 1;
532
533 secs_to_flush = pblk_rb_sync_point_count(&pblk->rwb);
534 if (!secs_to_flush && secs_avail < pblk->min_write_pgs)
535 return 1;
536
Javier González875d94f2017-10-13 14:46:18 +0200537 secs_to_sync = pblk_calc_secs_to_sync(pblk, secs_avail, secs_to_flush);
538 if (secs_to_sync > pblk->max_write_pgs) {
539 pr_err("pblk: bad buffer sync calculation\n");
540 return 1;
541 }
542
543 secs_to_com = (secs_to_sync > secs_avail) ? secs_avail : secs_to_sync;
544 pos = pblk_rb_read_commit(&pblk->rwb, secs_to_com);
545
546 bio = bio_alloc(GFP_KERNEL, secs_to_sync);
Javier González2942f502017-10-13 14:46:10 +0200547
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200548 bio->bi_iter.bi_sector = 0; /* internal bio */
549 bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
Javier González2942f502017-10-13 14:46:10 +0200550
551 rqd = pblk_alloc_rqd(pblk, WRITE);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200552 rqd->bio = bio;
553
Javier González875d94f2017-10-13 14:46:18 +0200554 if (pblk_rb_read_to_bio(&pblk->rwb, rqd, pos, secs_to_sync,
Javier Gonzálezd624f372017-06-26 11:57:15 +0200555 secs_avail)) {
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200556 pr_err("pblk: corrupted write bio\n");
557 goto fail_put_bio;
558 }
559
Javier Gonzálezd624f372017-06-26 11:57:15 +0200560 if (pblk_submit_io_set(pblk, rqd))
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200561 goto fail_free_bio;
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200562
563#ifdef CONFIG_NVM_DEBUG
564 atomic_long_add(secs_to_sync, &pblk->sub_writes);
565#endif
566
567 return 0;
568
569fail_free_bio:
Javier Gonzálezd624f372017-06-26 11:57:15 +0200570 pblk_free_write_rqd(pblk, rqd);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200571fail_put_bio:
572 bio_put(bio);
Javier Gonzáleza4bd2172017-04-15 20:55:50 +0200573 pblk_free_rqd(pblk, rqd, WRITE);
574
575 return 1;
576}
577
578int pblk_write_ts(void *data)
579{
580 struct pblk *pblk = data;
581
582 while (!kthread_should_stop()) {
583 if (!pblk_submit_write(pblk))
584 continue;
585 set_current_state(TASK_INTERRUPTIBLE);
586 io_schedule();
587 }
588
589 return 0;
590}