20 static unsigned long pblk_end_w_bio(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_end_w_bio()  argument
51 		pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid,  in pblk_end_w_bio()
55 	atomic_long_add(rqd->nr_ppas, &pblk->sync_writes);  in pblk_end_w_bio()
60 	bio_put(rqd->bio);  in pblk_end_w_bio()
61 	pblk_free_rqd(pblk, rqd, PBLK_WRITE);  in pblk_end_w_bio()
67 					   struct nvm_rq *rqd,  in pblk_end_queued_w_bio()  argument
71 	return pblk_end_w_bio(pblk, rqd, c_ctx);  in pblk_end_queued_w_bio()
74 static void pblk_complete_write(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_complete_write()  argument
85 	pblk_up_rq(pblk, rqd->ppa_list, rqd->nr_ppas, c_ctx->lun_bitmap);  in pblk_complete_write()
89 		pos = pblk_end_w_bio(pblk, rqd, c_ctx);  in pblk_complete_write()
93 			rqd = nvm_rq_from_c_ctx(c);  in pblk_complete_write()
95 				pos = pblk_end_queued_w_bio(pblk, rqd, c);  in pblk_complete_write()
100 		WARN_ON(nvm_rq_from_c_ctx(c_ctx) != rqd);  in pblk_complete_write()
209 	struct nvm_rq *rqd = recovery->rqd;  in pblk_submit_rec()  local
210 	struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);  in pblk_submit_rec()
213 	pblk_log_write_err(pblk, rqd);  in pblk_submit_rec()
215 	if (rqd->nr_ppas == 1)  in pblk_submit_rec()
216 		ppa_list = &rqd->ppa_addr;  in pblk_submit_rec()
218 		ppa_list = rqd->ppa_list;  in pblk_submit_rec()
223 	pblk_up_rq(pblk, rqd->ppa_list, rqd->nr_ppas, c_ctx->lun_bitmap);  in pblk_submit_rec()
225 		pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid,  in pblk_submit_rec()
227 	bio_put(rqd->bio);  in pblk_submit_rec()
228 	pblk_free_rqd(pblk, rqd, PBLK_WRITE);  in pblk_submit_rec()
235 static void pblk_end_w_fail(struct pblk *pblk, struct nvm_rq *rqd)  in pblk_end_w_fail()  argument
246 	recovery->rqd = rqd;  in pblk_end_w_fail()
252 static void pblk_end_io_write(struct nvm_rq *rqd)  in pblk_end_io_write()  argument
254 	struct pblk *pblk = rqd->private;  in pblk_end_io_write()
255 	struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);  in pblk_end_io_write()
257 	if (rqd->error) {  in pblk_end_io_write()
258 		pblk_end_w_fail(pblk, rqd);  in pblk_end_io_write()
263 		WARN_ONCE(rqd->bio->bi_status, "pblk: corrupted write error\n");  in pblk_end_io_write()
266 	pblk_complete_write(pblk, rqd, c_ctx);  in pblk_end_io_write()
270 static void pblk_end_io_write_meta(struct nvm_rq *rqd)  in pblk_end_io_write_meta()  argument
272 	struct pblk *pblk = rqd->private;  in pblk_end_io_write_meta()
273 	struct pblk_g_ctx *m_ctx = nvm_rq_to_pdu(rqd);  in pblk_end_io_write_meta()
278 	pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas);  in pblk_end_io_write_meta()
280 	if (rqd->error) {  in pblk_end_io_write_meta()
281 		pblk_log_write_err(pblk, rqd);  in pblk_end_io_write_meta()
286 	sync = atomic_add_return(rqd->nr_ppas, &emeta->sync);  in pblk_end_io_write_meta()
291 	pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT);  in pblk_end_io_write_meta()
296 static int pblk_alloc_w_rq(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_alloc_w_rq()  argument
303 	rqd->opcode = NVM_OP_PWRITE;  in pblk_alloc_w_rq()
304 	rqd->nr_ppas = nr_secs;  in pblk_alloc_w_rq()
305 	rqd->flags = pblk_set_progr_mode(pblk, PBLK_WRITE);  in pblk_alloc_w_rq()
306 	rqd->private = pblk;  in pblk_alloc_w_rq()
307 	rqd->end_io = end_io;  in pblk_alloc_w_rq()
309 	rqd->meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL,  in pblk_alloc_w_rq()
310 							&rqd->dma_meta_list);  in pblk_alloc_w_rq()
311 	if (!rqd->meta_list)  in pblk_alloc_w_rq()
314 	rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size;  in pblk_alloc_w_rq()
315 	rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size;  in pblk_alloc_w_rq()
320 static int pblk_setup_w_rq(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_setup_w_rq()  argument
325 	struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);  in pblk_setup_w_rq()
337 	ret = pblk_alloc_w_rq(pblk, rqd, nr_secs, pblk_end_io_write);  in pblk_setup_w_rq()
344 		pblk_map_rq(pblk, rqd, c_ctx->sentry, lun_bitmap, valid, 0);  in pblk_setup_w_rq()
346 		pblk_map_erase_rq(pblk, rqd, c_ctx->sentry, lun_bitmap,  in pblk_setup_w_rq()
380 	struct nvm_rq *rqd;  in pblk_submit_meta_io()  local
389 	rqd = pblk_alloc_rqd(pblk, PBLK_WRITE_INT);  in pblk_submit_meta_io()
391 	m_ctx = nvm_rq_to_pdu(rqd);  in pblk_submit_meta_io()
406 	rqd->bio = bio;  in pblk_submit_meta_io()
408 	ret = pblk_alloc_w_rq(pblk, rqd, rq_ppas, pblk_end_io_write_meta);  in pblk_submit_meta_io()
412 	for (i = 0; i < rqd->nr_ppas; ) {  in pblk_submit_meta_io()
417 			rqd->ppa_list[i] = addr_to_gen_ppa(pblk, paddr, id);  in pblk_submit_meta_io()
427 	pblk_down_page(pblk, rqd->ppa_list, rqd->nr_ppas);  in pblk_submit_meta_io()
429 	ret = pblk_submit_io(pblk, rqd);  in pblk_submit_meta_io()
438 	pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas);  in pblk_submit_meta_io()
446 	pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT);  in pblk_submit_meta_io()
510 static int pblk_submit_io_set(struct pblk *pblk, struct nvm_rq *rqd)  in pblk_submit_io_set()  argument
519 	err = pblk_setup_w_rq(pblk, rqd, &erase_ppa);  in pblk_submit_io_set()
525 	meta_line = pblk_should_submit_meta_io(pblk, rqd);  in pblk_submit_io_set()
528 	err = pblk_submit_io(pblk, rqd);  in pblk_submit_io_set()
561 static void pblk_free_write_rqd(struct pblk *pblk, struct nvm_rq *rqd)  in pblk_free_write_rqd()  argument
563 	struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);  in pblk_free_write_rqd()
564 	struct bio *bio = rqd->bio;  in pblk_free_write_rqd()
574 	struct nvm_rq *rqd;  in pblk_submit_write()  local
632 	rqd = pblk_alloc_rqd(pblk, PBLK_WRITE);  in pblk_submit_write()
633 	rqd->bio = bio;  in pblk_submit_write()
635 	if (pblk_rb_read_to_bio(&pblk->rwb, rqd, pos, secs_to_sync,  in pblk_submit_write()
641 	if (pblk_submit_io_set(pblk, rqd))  in pblk_submit_write()
651 	pblk_free_write_rqd(pblk, rqd);  in pblk_submit_write()
654 	pblk_free_rqd(pblk, rqd, PBLK_WRITE);  in pblk_submit_write()