20 static unsigned long pblk_end_w_bio(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_end_w_bio()  argument
24 	struct pblk_rb *rwb = &pblk->rwb;  in pblk_end_w_bio()
51 		pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid,  in pblk_end_w_bio()
55 	atomic_long_add(rqd->nr_ppas, &pblk->sync_writes);  in pblk_end_w_bio()
58 	ret = pblk_rb_sync_advance(&pblk->rwb, c_ctx->nr_valid);  in pblk_end_w_bio()
61 	pblk_free_rqd(pblk, rqd, PBLK_WRITE);  in pblk_end_w_bio()
66 static unsigned long pblk_end_queued_w_bio(struct pblk *pblk,  in pblk_end_queued_w_bio()  argument
71 	return pblk_end_w_bio(pblk, rqd, c_ctx);  in pblk_end_queued_w_bio()
74 static void pblk_complete_write(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_complete_write()  argument
82 	atomic_long_sub(c_ctx->nr_valid, &pblk->inflight_writes);  in pblk_complete_write()
85 	pblk_up_rq(pblk, rqd->ppa_list, rqd->nr_ppas, c_ctx->lun_bitmap);  in pblk_complete_write()
87 	pos = pblk_rb_sync_init(&pblk->rwb, &flags);  in pblk_complete_write()
89 		pos = pblk_end_w_bio(pblk, rqd, c_ctx);  in pblk_complete_write()
92 		list_for_each_entry_safe(c, r, &pblk->compl_list, list) {  in pblk_complete_write()
95 				pos = pblk_end_queued_w_bio(pblk, rqd, c);  in pblk_complete_write()
101 		list_add_tail(&c_ctx->list, &pblk->compl_list);  in pblk_complete_write()
103 	pblk_rb_sync_end(&pblk->rwb, &flags);  in pblk_complete_write()
107 static void pblk_map_remaining(struct pblk *pblk, struct ppa_addr *ppa)  in pblk_map_remaining()  argument
109 	struct nvm_tgt_dev *dev = pblk->dev;  in pblk_map_remaining()
116 	line = &pblk->lines[pblk_ppa_to_line(*ppa)];  in pblk_map_remaining()
120 		paddr = pblk_dev_ppa_to_line_addr(pblk, map_ppa);  in pblk_map_remaining()
143 static void pblk_prepare_resubmit(struct pblk *pblk, unsigned int sentry,  in pblk_prepare_resubmit()  argument
146 	struct pblk_rb *rb = &pblk->rwb;  in pblk_prepare_resubmit()
154 	spin_lock(&pblk->trans_lock);  in pblk_prepare_resubmit()
161 		ppa_l2p = pblk_trans_map_get(pblk, w_ctx->lba);  in pblk_prepare_resubmit()
174 		line = &pblk->lines[pblk_ppa_to_line(w_ctx->ppa)];  in pblk_prepare_resubmit()
179 	spin_unlock(&pblk->trans_lock);  in pblk_prepare_resubmit()
182 static void pblk_queue_resubmit(struct pblk *pblk, struct pblk_c_ctx *c_ctx)  in pblk_queue_resubmit()  argument
195 	spin_lock(&pblk->resubmit_lock);  in pblk_queue_resubmit()
196 	list_add_tail(&r_ctx->list, &pblk->resubmit_list);  in pblk_queue_resubmit()
197 	spin_unlock(&pblk->resubmit_lock);  in pblk_queue_resubmit()
200 	atomic_long_add(c_ctx->nr_valid, &pblk->recov_writes);  in pblk_queue_resubmit()
208 	struct pblk *pblk = recovery->pblk;  in pblk_submit_rec()  local
213 	pblk_log_write_err(pblk, rqd);  in pblk_submit_rec()
220 	pblk_map_remaining(pblk, ppa_list);  in pblk_submit_rec()
221 	pblk_queue_resubmit(pblk, c_ctx);  in pblk_submit_rec()
223 	pblk_up_rq(pblk, rqd->ppa_list, rqd->nr_ppas, c_ctx->lun_bitmap);  in pblk_submit_rec()
225 		pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid,  in pblk_submit_rec()
228 	pblk_free_rqd(pblk, rqd, PBLK_WRITE);  in pblk_submit_rec()
229 	mempool_free(recovery, &pblk->rec_pool);  in pblk_submit_rec()
231 	atomic_dec(&pblk->inflight_io);  in pblk_submit_rec()
235 static void pblk_end_w_fail(struct pblk *pblk, struct nvm_rq *rqd)  in pblk_end_w_fail()  argument
239 	recovery = mempool_alloc(&pblk->rec_pool, GFP_ATOMIC);  in pblk_end_w_fail()
241 		pblk_err(pblk, "could not allocate recovery work\n");  in pblk_end_w_fail()
245 	recovery->pblk = pblk;  in pblk_end_w_fail()
249 	queue_work(pblk->close_wq, &recovery->ws_rec);  in pblk_end_w_fail()
254 	struct pblk *pblk = rqd->private;  in pblk_end_io_write()  local
258 		pblk_end_w_fail(pblk, rqd);  in pblk_end_io_write()
266 	pblk_complete_write(pblk, rqd, c_ctx);  in pblk_end_io_write()
267 	atomic_dec(&pblk->inflight_io);  in pblk_end_io_write()
272 	struct pblk *pblk = rqd->private;  in pblk_end_io_write_meta()  local
278 	pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas);  in pblk_end_io_write_meta()
281 		pblk_log_write_err(pblk, rqd);  in pblk_end_io_write_meta()
282 		pblk_err(pblk, "metadata I/O failed. Line %d\n", line->id);  in pblk_end_io_write_meta()
288 		pblk_gen_run_ws(pblk, line, NULL, pblk_line_close_ws,  in pblk_end_io_write_meta()
289 						GFP_ATOMIC, pblk->close_wq);  in pblk_end_io_write_meta()
291 	pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT);  in pblk_end_io_write_meta()
293 	atomic_dec(&pblk->inflight_io);  in pblk_end_io_write_meta()
296 static int pblk_alloc_w_rq(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_alloc_w_rq()  argument
300 	struct nvm_tgt_dev *dev = pblk->dev;  in pblk_alloc_w_rq()
305 	rqd->flags = pblk_set_progr_mode(pblk, PBLK_WRITE);  in pblk_alloc_w_rq()
306 	rqd->private = pblk;  in pblk_alloc_w_rq()
320 static int pblk_setup_w_rq(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_setup_w_rq()  argument
323 	struct pblk_line_meta *lm = &pblk->lm;  in pblk_setup_w_rq()
324 	struct pblk_line *e_line = pblk_line_get_erase(pblk);  in pblk_setup_w_rq()
337 	ret = pblk_alloc_w_rq(pblk, rqd, nr_secs, pblk_end_io_write);  in pblk_setup_w_rq()
344 		pblk_map_rq(pblk, rqd, c_ctx->sentry, lun_bitmap, valid, 0);  in pblk_setup_w_rq()
346 		pblk_map_erase_rq(pblk, rqd, c_ctx->sentry, lun_bitmap,  in pblk_setup_w_rq()
352 static int pblk_calc_secs_to_sync(struct pblk *pblk, unsigned int secs_avail,  in pblk_calc_secs_to_sync()  argument
357 	secs_to_sync = pblk_calc_secs(pblk, secs_avail, secs_to_flush);  in pblk_calc_secs_to_sync()
363 		pblk_err(pblk, "bad sector calculation (a:%d,s:%d,f:%d)\n",  in pblk_calc_secs_to_sync()
371 int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line)  in pblk_submit_meta_io()  argument
373 	struct nvm_tgt_dev *dev = pblk->dev;  in pblk_submit_meta_io()
375 	struct pblk_line_mgmt *l_mg = &pblk->l_mg;  in pblk_submit_meta_io()
376 	struct pblk_line_meta *lm = &pblk->lm;  in pblk_submit_meta_io()
383 	int rq_ppas = pblk->min_write_pgs;  in pblk_submit_meta_io()
389 	rqd = pblk_alloc_rqd(pblk, PBLK_WRITE_INT);  in pblk_submit_meta_io()
397 	bio = pblk_bio_map_addr(pblk, data, rq_ppas, rq_len,  in pblk_submit_meta_io()
400 		pblk_err(pblk, "failed to map emeta io");  in pblk_submit_meta_io()
408 	ret = pblk_alloc_w_rq(pblk, rqd, rq_ppas, pblk_end_io_write_meta);  in pblk_submit_meta_io()
414 		paddr = __pblk_alloc_page(pblk, meta_line, rq_ppas);  in pblk_submit_meta_io()
417 			rqd->ppa_list[i] = addr_to_gen_ppa(pblk, paddr, id);  in pblk_submit_meta_io()
427 	pblk_down_page(pblk, rqd->ppa_list, rqd->nr_ppas);  in pblk_submit_meta_io()
429 	ret = pblk_submit_io(pblk, rqd);  in pblk_submit_meta_io()
431 		pblk_err(pblk, "emeta I/O submission failed: %d\n", ret);  in pblk_submit_meta_io()
438 	pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas);  in pblk_submit_meta_io()
440 	pblk_dealloc_page(pblk, meta_line, rq_ppas);  in pblk_submit_meta_io()
446 	pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT);  in pblk_submit_meta_io()
450 static inline bool pblk_valid_meta_ppa(struct pblk *pblk,  in pblk_valid_meta_ppa()  argument
454 	struct nvm_tgt_dev *dev = pblk->dev;  in pblk_valid_meta_ppa()
457 	struct pblk_line *data_line = pblk_line_get_data(pblk);  in pblk_valid_meta_ppa()
471 	paddr = pblk_lookup_page(pblk, meta_line);  in pblk_valid_meta_ppa()
472 	ppa = addr_to_gen_ppa(pblk, paddr, 0);  in pblk_valid_meta_ppa()
473 	ppa_opt = addr_to_gen_ppa(pblk, paddr + data_line->meta_distance, 0);  in pblk_valid_meta_ppa()
486 static struct pblk_line *pblk_should_submit_meta_io(struct pblk *pblk,  in pblk_should_submit_meta_io()  argument
489 	struct pblk_line_meta *lm = &pblk->lm;  in pblk_should_submit_meta_io()
490 	struct pblk_line_mgmt *l_mg = &pblk->l_mg;  in pblk_should_submit_meta_io()
504 	if (!pblk_valid_meta_ppa(pblk, meta_line, data_rqd))  in pblk_should_submit_meta_io()
510 static int pblk_submit_io_set(struct pblk *pblk, struct nvm_rq *rqd)  in pblk_submit_io_set()  argument
519 	err = pblk_setup_w_rq(pblk, rqd, &erase_ppa);  in pblk_submit_io_set()
521 		pblk_err(pblk, "could not setup write request: %d\n", err);  in pblk_submit_io_set()
525 	meta_line = pblk_should_submit_meta_io(pblk, rqd);  in pblk_submit_io_set()
528 	err = pblk_submit_io(pblk, rqd);  in pblk_submit_io_set()
530 		pblk_err(pblk, "data I/O submission failed: %d\n", err);  in pblk_submit_io_set()
536 		if (pblk_blk_erase_async(pblk, erase_ppa)) {  in pblk_submit_io_set()
537 			struct pblk_line *e_line = pblk_line_get_erase(pblk);  in pblk_submit_io_set()
538 			struct nvm_tgt_dev *dev = pblk->dev;  in pblk_submit_io_set()
550 		err = pblk_submit_meta_io(pblk, meta_line);  in pblk_submit_io_set()
552 			pblk_err(pblk, "metadata I/O submission failed: %d",  in pblk_submit_io_set()
561 static void pblk_free_write_rqd(struct pblk *pblk, struct nvm_rq *rqd)  in pblk_free_write_rqd()  argument
567 		pblk_bio_free_pages(pblk, bio, c_ctx->nr_valid,  in pblk_free_write_rqd()
571 static int pblk_submit_write(struct pblk *pblk)  in pblk_submit_write()  argument
580 	spin_lock(&pblk->resubmit_lock);  in pblk_submit_write()
581 	resubmit = !list_empty(&pblk->resubmit_list);  in pblk_submit_write()
582 	spin_unlock(&pblk->resubmit_lock);  in pblk_submit_write()
588 		spin_lock(&pblk->resubmit_lock);  in pblk_submit_write()
589 		r_ctx = list_first_entry(&pblk->resubmit_list,  in pblk_submit_write()
592 		spin_unlock(&pblk->resubmit_lock);  in pblk_submit_write()
597 		pblk_prepare_resubmit(pblk, pos, secs_avail);  in pblk_submit_write()
598 		secs_to_sync = pblk_calc_secs_to_sync(pblk, secs_avail,  in pblk_submit_write()
607 		secs_avail = pblk_rb_read_count(&pblk->rwb);  in pblk_submit_write()
611 		secs_to_flush = pblk_rb_flush_point_count(&pblk->rwb);  in pblk_submit_write()
612 		if (!secs_to_flush && secs_avail < pblk->min_write_pgs)  in pblk_submit_write()
615 		secs_to_sync = pblk_calc_secs_to_sync(pblk, secs_avail,  in pblk_submit_write()
617 		if (secs_to_sync > pblk->max_write_pgs) {  in pblk_submit_write()
618 			pblk_err(pblk, "bad buffer sync calculation\n");  in pblk_submit_write()
624 		pos = pblk_rb_read_commit(&pblk->rwb, secs_to_com);  in pblk_submit_write()
632 	rqd = pblk_alloc_rqd(pblk, PBLK_WRITE);  in pblk_submit_write()
635 	if (pblk_rb_read_to_bio(&pblk->rwb, rqd, pos, secs_to_sync,  in pblk_submit_write()
637 		pblk_err(pblk, "corrupted write bio\n");  in pblk_submit_write()
641 	if (pblk_submit_io_set(pblk, rqd))  in pblk_submit_write()
645 	atomic_long_add(secs_to_sync, &pblk->sub_writes);  in pblk_submit_write()
651 	pblk_free_write_rqd(pblk, rqd);  in pblk_submit_write()
654 	pblk_free_rqd(pblk, rqd, PBLK_WRITE);  in pblk_submit_write()
661 	struct pblk *pblk = data;  in pblk_write_ts()  local
664 		if (!pblk_submit_write(pblk))  in pblk_write_ts()