Lines Matching refs:rqd
41 static void pblk_read_ppalist_rq(struct pblk *pblk, struct nvm_rq *rqd, in pblk_read_ppalist_rq() argument
45 struct pblk_sec_meta *meta_list = rqd->meta_list; in pblk_read_ppalist_rq()
47 int nr_secs = rqd->nr_ppas; in pblk_read_ppalist_rq()
87 rqd->ppa_list[j++] = p; in pblk_read_ppalist_rq()
96 rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_SEQUENTIAL); in pblk_read_ppalist_rq()
98 rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM); in pblk_read_ppalist_rq()
106 static void pblk_read_check_seq(struct pblk *pblk, struct nvm_rq *rqd, in pblk_read_check_seq() argument
109 struct pblk_sec_meta *meta_lba_list = rqd->meta_list; in pblk_read_check_seq()
110 int nr_lbas = rqd->nr_ppas; in pblk_read_check_seq()
123 p = (nr_lbas == 1) ? &rqd->ppa_list[i] : &rqd->ppa_addr; in pblk_read_check_seq()
136 static void pblk_read_check_rand(struct pblk *pblk, struct nvm_rq *rqd, in pblk_read_check_rand() argument
139 struct pblk_sec_meta *meta_lba_list = rqd->meta_list; in pblk_read_check_rand()
154 int nr_ppas = rqd->nr_ppas; in pblk_read_check_rand()
156 p = (nr_ppas == 1) ? &rqd->ppa_list[j] : &rqd->ppa_addr; in pblk_read_check_rand()
167 WARN_ONCE(j != rqd->nr_ppas, "pblk: corrupted random request\n"); in pblk_read_check_rand()
170 static void pblk_read_put_rqd_kref(struct pblk *pblk, struct nvm_rq *rqd) in pblk_read_put_rqd_kref() argument
175 ppa_list = (rqd->nr_ppas > 1) ? rqd->ppa_list : &rqd->ppa_addr; in pblk_read_put_rqd_kref()
177 for (i = 0; i < rqd->nr_ppas; i++) { in pblk_read_put_rqd_kref()
194 static void __pblk_end_io_read(struct pblk *pblk, struct nvm_rq *rqd, in __pblk_end_io_read() argument
198 struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd); in __pblk_end_io_read()
199 struct bio *int_bio = rqd->bio; in __pblk_end_io_read()
204 if (rqd->error) in __pblk_end_io_read()
205 pblk_log_read_err(pblk, rqd); in __pblk_end_io_read()
207 pblk_read_check_seq(pblk, rqd, r_ctx->lba); in __pblk_end_io_read()
213 pblk_read_put_rqd_kref(pblk, rqd); in __pblk_end_io_read()
216 atomic_long_add(rqd->nr_ppas, &pblk->sync_reads); in __pblk_end_io_read()
217 atomic_long_sub(rqd->nr_ppas, &pblk->inflight_reads); in __pblk_end_io_read()
220 pblk_free_rqd(pblk, rqd, PBLK_READ); in __pblk_end_io_read()
224 static void pblk_end_io_read(struct nvm_rq *rqd) in pblk_end_io_read() argument
226 struct pblk *pblk = rqd->private; in pblk_end_io_read()
227 struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd); in pblk_end_io_read()
231 __pblk_end_io_read(pblk, rqd, true); in pblk_end_io_read()
234 static void pblk_end_partial_read(struct nvm_rq *rqd) in pblk_end_partial_read() argument
236 struct pblk *pblk = rqd->private; in pblk_end_partial_read()
237 struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd); in pblk_end_partial_read()
239 struct bio *new_bio = rqd->bio; in pblk_end_partial_read()
242 struct pblk_sec_meta *meta_list = rqd->meta_list; in pblk_end_partial_read()
254 ppa = rqd->ppa_addr; in pblk_end_partial_read()
255 rqd->ppa_list = pr_ctx->ppa_ptr; in pblk_end_partial_read()
256 rqd->dma_ppa_list = pr_ctx->dma_ppa_list; in pblk_end_partial_read()
257 rqd->ppa_list[0] = ppa; in pblk_end_partial_read()
261 lba_list_mem = (((void *)rqd->ppa_list) + pblk_dma_ppa_size); in pblk_end_partial_read()
262 lba_list_media = (((void *)rqd->ppa_list) + 2 * pblk_dma_ppa_size); in pblk_end_partial_read()
273 int line_id = pblk_ppa_to_line(rqd->ppa_list[i]); in pblk_end_partial_read()
302 rqd->bio = NULL; in pblk_end_partial_read()
303 rqd->nr_ppas = nr_secs; in pblk_end_partial_read()
306 __pblk_end_io_read(pblk, rqd, false); in pblk_end_partial_read()
309 static int pblk_setup_partial_read(struct pblk *pblk, struct nvm_rq *rqd, in pblk_setup_partial_read() argument
314 struct pblk_sec_meta *meta_list = rqd->meta_list; in pblk_setup_partial_read()
315 struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd); in pblk_setup_partial_read()
319 int nr_secs = rqd->nr_ppas; in pblk_setup_partial_read()
323 lba_list_mem = (((void *)rqd->ppa_list) + pblk_dma_ppa_size); in pblk_setup_partial_read()
345 rqd->bio = new_bio; in pblk_setup_partial_read()
346 rqd->nr_ppas = nr_holes; in pblk_setup_partial_read()
347 rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM); in pblk_setup_partial_read()
357 pr_ctx->ppa_ptr = rqd->ppa_list; in pblk_setup_partial_read()
358 pr_ctx->dma_ppa_list = rqd->dma_ppa_list; in pblk_setup_partial_read()
359 rqd->ppa_addr = rqd->ppa_list[0]; in pblk_setup_partial_read()
371 static int pblk_partial_read_bio(struct pblk *pblk, struct nvm_rq *rqd, in pblk_partial_read_bio() argument
380 if (pblk_setup_partial_read(pblk, rqd, bio_init_idx, read_bitmap, in pblk_partial_read_bio()
384 rqd->end_io = pblk_end_partial_read; in pblk_partial_read_bio()
386 ret = pblk_submit_io(pblk, rqd); in pblk_partial_read_bio()
388 bio_put(rqd->bio); in pblk_partial_read_bio()
399 pblk_bio_free_pages(pblk, rqd->bio, 0, rqd->bio->bi_vcnt); in pblk_partial_read_bio()
400 __pblk_end_io_read(pblk, rqd, false); in pblk_partial_read_bio()
404 static void pblk_read_rq(struct pblk *pblk, struct nvm_rq *rqd, struct bio *bio, in pblk_read_rq() argument
407 struct pblk_sec_meta *meta_list = rqd->meta_list; in pblk_read_rq()
439 rqd->ppa_addr = ppa; in pblk_read_rq()
442 rqd->flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM); in pblk_read_rq()
452 struct nvm_rq *rqd; in pblk_submit_read() local
469 rqd = pblk_alloc_rqd(pblk, PBLK_READ); in pblk_submit_read()
471 rqd->opcode = NVM_OP_PREAD; in pblk_submit_read()
472 rqd->nr_ppas = nr_secs; in pblk_submit_read()
473 rqd->bio = NULL; /* cloned bio if needed */ in pblk_submit_read()
474 rqd->private = pblk; in pblk_submit_read()
475 rqd->end_io = pblk_end_io_read; in pblk_submit_read()
477 r_ctx = nvm_rq_to_pdu(rqd); in pblk_submit_read()
487 rqd->meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, in pblk_submit_read()
488 &rqd->dma_meta_list); in pblk_submit_read()
489 if (!rqd->meta_list) { in pblk_submit_read()
495 rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size; in pblk_submit_read()
496 rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size; in pblk_submit_read()
498 pblk_read_ppalist_rq(pblk, rqd, bio, blba, read_bitmap); in pblk_submit_read()
500 pblk_read_rq(pblk, rqd, bio, blba, read_bitmap); in pblk_submit_read()
505 __pblk_end_io_read(pblk, rqd, false); in pblk_submit_read()
510 if (bitmap_empty(read_bitmap, rqd->nr_ppas)) { in pblk_submit_read()
520 rqd->bio = int_bio; in pblk_submit_read()
522 if (pblk_submit_io(pblk, rqd)) { in pblk_submit_read()
534 ret = pblk_partial_read_bio(pblk, rqd, bio_init_idx, read_bitmap, in pblk_submit_read()
542 nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list); in pblk_submit_read()
544 pblk_free_rqd(pblk, rqd, PBLK_READ); in pblk_submit_read()
547 __pblk_end_io_read(pblk, rqd, false); in pblk_submit_read()
551 static int read_ppalist_rq_gc(struct pblk *pblk, struct nvm_rq *rqd, in read_ppalist_rq_gc() argument
572 rqd->ppa_list[valid_secs++] = ppa_list_l2p[i]; in read_ppalist_rq_gc()
582 static int read_rq_gc(struct pblk *pblk, struct nvm_rq *rqd, in read_rq_gc() argument
606 rqd->ppa_addr = ppa_l2p; in read_rq_gc()
622 struct nvm_rq rqd; in pblk_submit_read_gc() local
626 memset(&rqd, 0, sizeof(struct nvm_rq)); in pblk_submit_read_gc()
628 rqd.meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, in pblk_submit_read_gc()
629 &rqd.dma_meta_list); in pblk_submit_read_gc()
630 if (!rqd.meta_list) in pblk_submit_read_gc()
634 rqd.ppa_list = rqd.meta_list + pblk_dma_meta_size; in pblk_submit_read_gc()
635 rqd.dma_ppa_list = rqd.dma_meta_list + pblk_dma_meta_size; in pblk_submit_read_gc()
637 gc_rq->secs_to_gc = read_ppalist_rq_gc(pblk, &rqd, gc_rq->line, in pblk_submit_read_gc()
642 rqd.ppa_addr = rqd.ppa_list[0]; in pblk_submit_read_gc()
644 gc_rq->secs_to_gc = read_rq_gc(pblk, &rqd, gc_rq->line, in pblk_submit_read_gc()
664 rqd.opcode = NVM_OP_PREAD; in pblk_submit_read_gc()
665 rqd.nr_ppas = gc_rq->secs_to_gc; in pblk_submit_read_gc()
666 rqd.flags = pblk_set_read_mode(pblk, PBLK_READ_RANDOM); in pblk_submit_read_gc()
667 rqd.bio = bio; in pblk_submit_read_gc()
669 if (pblk_submit_io_sync(pblk, &rqd)) { in pblk_submit_read_gc()
675 pblk_read_check_rand(pblk, &rqd, gc_rq->lba_list, gc_rq->nr_secs); in pblk_submit_read_gc()
679 if (rqd.error) { in pblk_submit_read_gc()
682 pblk_print_failed_rqd(pblk, &rqd, rqd.error); in pblk_submit_read_gc()
693 nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list); in pblk_submit_read_gc()
699 nvm_dev_dma_free(dev->parent, rqd.meta_list, rqd.dma_meta_list); in pblk_submit_read_gc()