22 static unsigned long pblk_end_w_bio(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_end_w_bio()  argument
53 		pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid,  in pblk_end_w_bio()
57 	atomic_long_add(rqd->nr_ppas, &pblk->sync_writes);  in pblk_end_w_bio()
62 	bio_put(rqd->bio);  in pblk_end_w_bio()
63 	pblk_free_rqd(pblk, rqd, PBLK_WRITE);  in pblk_end_w_bio()
69 					   struct nvm_rq *rqd,  in pblk_end_queued_w_bio()  argument
73 	return pblk_end_w_bio(pblk, rqd, c_ctx);  in pblk_end_queued_w_bio()
76 static void pblk_complete_write(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_complete_write()  argument
90 		pos = pblk_end_w_bio(pblk, rqd, c_ctx);  in pblk_complete_write()
94 			rqd = nvm_rq_from_c_ctx(c);  in pblk_complete_write()
96 				pos = pblk_end_queued_w_bio(pblk, rqd, c);  in pblk_complete_write()
101 		WARN_ON(nvm_rq_from_c_ctx(c_ctx) != rqd);  in pblk_complete_write()
213 	struct nvm_rq *rqd = recovery->rqd;  in pblk_submit_rec()  local
214 	struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);  in pblk_submit_rec()
215 	struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);  in pblk_submit_rec()
217 	pblk_log_write_err(pblk, rqd);  in pblk_submit_rec()
219 	pblk_map_remaining(pblk, ppa_list, rqd->nr_ppas);  in pblk_submit_rec()
224 		pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid,  in pblk_submit_rec()
226 	bio_put(rqd->bio);  in pblk_submit_rec()
227 	pblk_free_rqd(pblk, rqd, PBLK_WRITE);  in pblk_submit_rec()
235 static void pblk_end_w_fail(struct pblk *pblk, struct nvm_rq *rqd)  in pblk_end_w_fail()  argument
246 	recovery->rqd = rqd;  in pblk_end_w_fail()
252 static void pblk_end_io_write(struct nvm_rq *rqd)  in pblk_end_io_write()  argument
254 	struct pblk *pblk = rqd->private;  in pblk_end_io_write()
255 	struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);  in pblk_end_io_write()
257 	if (rqd->error) {  in pblk_end_io_write()
258 		pblk_end_w_fail(pblk, rqd);  in pblk_end_io_write()
262 			pblk_check_chunk_state_update(pblk, rqd);  in pblk_end_io_write()
264 		WARN_ONCE(rqd->bio->bi_status, "pblk: corrupted write error\n");  in pblk_end_io_write()
268 	pblk_complete_write(pblk, rqd, c_ctx);  in pblk_end_io_write()
272 static void pblk_end_io_write_meta(struct nvm_rq *rqd)  in pblk_end_io_write_meta()  argument
274 	struct pblk *pblk = rqd->private;  in pblk_end_io_write_meta()
275 	struct pblk_g_ctx *m_ctx = nvm_rq_to_pdu(rqd);  in pblk_end_io_write_meta()
278 	struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);  in pblk_end_io_write_meta()
283 	if (rqd->error) {  in pblk_end_io_write_meta()
284 		pblk_log_write_err(pblk, rqd);  in pblk_end_io_write_meta()
289 			pblk_check_chunk_state_update(pblk, rqd);  in pblk_end_io_write_meta()
292 	sync = atomic_add_return(rqd->nr_ppas, &emeta->sync);  in pblk_end_io_write_meta()
297 	pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT);  in pblk_end_io_write_meta()
302 static int pblk_alloc_w_rq(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_alloc_w_rq()  argument
306 	rqd->opcode = NVM_OP_PWRITE;  in pblk_alloc_w_rq()
307 	rqd->nr_ppas = nr_secs;  in pblk_alloc_w_rq()
308 	rqd->is_seq = 1;  in pblk_alloc_w_rq()
309 	rqd->private = pblk;  in pblk_alloc_w_rq()
310 	rqd->end_io = end_io;  in pblk_alloc_w_rq()
312 	return pblk_alloc_rqd_meta(pblk, rqd);  in pblk_alloc_w_rq()
315 static int pblk_setup_w_rq(struct pblk *pblk, struct nvm_rq *rqd,  in pblk_setup_w_rq()  argument
320 	struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);  in pblk_setup_w_rq()
332 	ret = pblk_alloc_w_rq(pblk, rqd, nr_secs, pblk_end_io_write);  in pblk_setup_w_rq()
339 		ret = pblk_map_rq(pblk, rqd, c_ctx->sentry, lun_bitmap,  in pblk_setup_w_rq()
342 		ret = pblk_map_erase_rq(pblk, rqd, c_ctx->sentry, lun_bitmap,  in pblk_setup_w_rq()
376 	struct nvm_rq *rqd;  in pblk_submit_meta_io()  local
385 	rqd = pblk_alloc_rqd(pblk, PBLK_WRITE_INT);  in pblk_submit_meta_io()
387 	m_ctx = nvm_rq_to_pdu(rqd);  in pblk_submit_meta_io()
393 	ret = pblk_alloc_w_rq(pblk, rqd, rq_ppas, pblk_end_io_write_meta);  in pblk_submit_meta_io()
397 	ppa_list = nvm_rq_to_ppa_list(rqd);  in pblk_submit_meta_io()
398 	for (i = 0; i < rqd->nr_ppas; ) {  in pblk_submit_meta_io()
414 	ret = pblk_submit_io(pblk, rqd, data);  in pblk_submit_meta_io()
429 	pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT);  in pblk_submit_meta_io()
494 static int pblk_submit_io_set(struct pblk *pblk, struct nvm_rq *rqd)  in pblk_submit_io_set()  argument
503 	err = pblk_setup_w_rq(pblk, rqd, &erase_ppa);  in pblk_submit_io_set()
509 	meta_line = pblk_should_submit_meta_io(pblk, rqd);  in pblk_submit_io_set()
512 	err = pblk_submit_io(pblk, rqd, NULL);  in pblk_submit_io_set()
545 static void pblk_free_write_rqd(struct pblk *pblk, struct nvm_rq *rqd)  in pblk_free_write_rqd()  argument
547 	struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);  in pblk_free_write_rqd()
548 	struct bio *bio = rqd->bio;  in pblk_free_write_rqd()
558 	struct nvm_rq *rqd;  in pblk_submit_write()  local
619 	rqd = pblk_alloc_rqd(pblk, PBLK_WRITE);  in pblk_submit_write()
620 	rqd->bio = bio;  in pblk_submit_write()
622 	if (pblk_rb_read_to_bio(&pblk->rwb, rqd, pos, secs_to_sync,  in pblk_submit_write()
628 	if (pblk_submit_io_set(pblk, rqd))  in pblk_submit_write()
639 	pblk_free_write_rqd(pblk, rqd);  in pblk_submit_write()
642 	pblk_free_rqd(pblk, rqd, PBLK_WRITE);  in pblk_submit_write()