Lines Matching refs:pblk
25 struct pblk *pblk = line_ws->pblk; in pblk_line_mark_bb() local
26 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_mark_bb()
35 line = &pblk->lines[pblk_ppa_to_line(*ppa)]; in pblk_line_mark_bb()
38 pblk_err(pblk, "failed to mark bb, line:%d, pos:%d\n", in pblk_line_mark_bb()
43 mempool_free(line_ws, &pblk->gen_ws_pool); in pblk_line_mark_bb()
46 static void pblk_mark_bb(struct pblk *pblk, struct pblk_line *line, in pblk_mark_bb() argument
49 struct nvm_tgt_dev *dev = pblk->dev; in pblk_mark_bb()
54 pblk_debug(pblk, "erase failed: line:%d, pos:%d\n", line->id, pos); in pblk_mark_bb()
55 atomic_long_inc(&pblk->erase_failed); in pblk_mark_bb()
59 pblk_err(pblk, "attempted to erase bb: line:%d, pos:%d\n", in pblk_mark_bb()
71 pblk_gen_run_ws(pblk, NULL, ppa, pblk_line_mark_bb, in pblk_mark_bb()
72 GFP_ATOMIC, pblk->bb_wq); in pblk_mark_bb()
75 static void __pblk_end_io_erase(struct pblk *pblk, struct nvm_rq *rqd) in __pblk_end_io_erase() argument
77 struct nvm_tgt_dev *dev = pblk->dev; in __pblk_end_io_erase()
83 line = &pblk->lines[pblk_ppa_to_line(rqd->ppa_addr)]; in __pblk_end_io_erase()
91 pblk_mark_bb(pblk, line, rqd->ppa_addr); in __pblk_end_io_erase()
96 atomic_dec(&pblk->inflight_io); in __pblk_end_io_erase()
102 struct pblk *pblk = rqd->private; in pblk_end_io_erase() local
104 __pblk_end_io_erase(pblk, rqd); in pblk_end_io_erase()
105 mempool_free(rqd, &pblk->e_rq_pool); in pblk_end_io_erase()
113 struct nvm_chk_meta *pblk_chunk_get_info(struct pblk *pblk) in pblk_chunk_get_info() argument
115 struct nvm_tgt_dev *dev = pblk->dev; in pblk_chunk_get_info()
138 struct nvm_chk_meta *pblk_chunk_get_off(struct pblk *pblk, in pblk_chunk_get_off() argument
142 struct nvm_tgt_dev *dev = pblk->dev; in pblk_chunk_get_off()
151 void __pblk_map_invalidate(struct pblk *pblk, struct pblk_line *line, in __pblk_map_invalidate() argument
154 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in __pblk_map_invalidate()
172 move_list = pblk_line_gc_list(pblk, line); in __pblk_map_invalidate()
191 void pblk_map_invalidate(struct pblk *pblk, struct ppa_addr ppa) in pblk_map_invalidate() argument
204 line = &pblk->lines[line_id]; in pblk_map_invalidate()
205 paddr = pblk_dev_ppa_to_line_addr(pblk, ppa); in pblk_map_invalidate()
207 __pblk_map_invalidate(pblk, line, paddr); in pblk_map_invalidate()
210 static void pblk_invalidate_range(struct pblk *pblk, sector_t slba, in pblk_invalidate_range() argument
215 spin_lock(&pblk->trans_lock); in pblk_invalidate_range()
219 ppa = pblk_trans_map_get(pblk, lba); in pblk_invalidate_range()
222 pblk_map_invalidate(pblk, ppa); in pblk_invalidate_range()
225 pblk_trans_map_set(pblk, lba, ppa); in pblk_invalidate_range()
227 spin_unlock(&pblk->trans_lock); in pblk_invalidate_range()
231 struct nvm_rq *pblk_alloc_rqd(struct pblk *pblk, int type) in pblk_alloc_rqd() argument
240 pool = &pblk->w_rq_pool; in pblk_alloc_rqd()
244 pool = &pblk->r_rq_pool; in pblk_alloc_rqd()
248 pool = &pblk->e_rq_pool; in pblk_alloc_rqd()
259 void pblk_free_rqd(struct pblk *pblk, struct nvm_rq *rqd, int type) in pblk_free_rqd() argument
261 struct nvm_tgt_dev *dev = pblk->dev; in pblk_free_rqd()
269 pool = &pblk->w_rq_pool; in pblk_free_rqd()
272 pool = &pblk->r_rq_pool; in pblk_free_rqd()
275 pool = &pblk->e_rq_pool; in pblk_free_rqd()
278 pblk_err(pblk, "trying to free unknown rqd type\n"); in pblk_free_rqd()
288 void pblk_bio_free_pages(struct pblk *pblk, struct bio *bio, int off, in pblk_bio_free_pages() argument
298 mempool_free(bv.bv_page, &pblk->page_bio_pool); in pblk_bio_free_pages()
302 int pblk_bio_add_pages(struct pblk *pblk, struct bio *bio, gfp_t flags, in pblk_bio_add_pages() argument
305 struct request_queue *q = pblk->dev->q; in pblk_bio_add_pages()
310 page = mempool_alloc(&pblk->page_bio_pool, flags); in pblk_bio_add_pages()
314 pblk_err(pblk, "could not add page to bio\n"); in pblk_bio_add_pages()
315 mempool_free(page, &pblk->page_bio_pool); in pblk_bio_add_pages()
322 pblk_bio_free_pages(pblk, bio, (bio->bi_vcnt - i), i); in pblk_bio_add_pages()
326 void pblk_write_kick(struct pblk *pblk) in pblk_write_kick() argument
328 wake_up_process(pblk->writer_ts); in pblk_write_kick()
329 mod_timer(&pblk->wtimer, jiffies + msecs_to_jiffies(1000)); in pblk_write_kick()
334 struct pblk *pblk = from_timer(pblk, t, wtimer); in pblk_write_timer_fn() local
337 pblk_write_kick(pblk); in pblk_write_timer_fn()
340 void pblk_write_should_kick(struct pblk *pblk) in pblk_write_should_kick() argument
342 unsigned int secs_avail = pblk_rb_read_count(&pblk->rwb); in pblk_write_should_kick()
344 if (secs_avail >= pblk->min_write_pgs) in pblk_write_should_kick()
345 pblk_write_kick(pblk); in pblk_write_should_kick()
348 static void pblk_wait_for_meta(struct pblk *pblk) in pblk_wait_for_meta() argument
351 if (!atomic_read(&pblk->inflight_io)) in pblk_wait_for_meta()
358 static void pblk_flush_writer(struct pblk *pblk) in pblk_flush_writer() argument
360 pblk_rb_flush(&pblk->rwb); in pblk_flush_writer()
362 if (!pblk_rb_sync_count(&pblk->rwb)) in pblk_flush_writer()
365 pblk_write_kick(pblk); in pblk_flush_writer()
370 struct list_head *pblk_line_gc_list(struct pblk *pblk, struct pblk_line *line) in pblk_line_gc_list() argument
372 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_gc_list()
373 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_gc_list()
383 pblk_rl_werr_line_in(&pblk->rl); in pblk_line_gc_list()
414 pblk_err(pblk, "corrupted vsc for line %d, vsc:%d (%d/%d/%d)\n", in pblk_line_gc_list()
423 void pblk_discard(struct pblk *pblk, struct bio *bio) in pblk_discard() argument
428 pblk_invalidate_range(pblk, slba, nr_secs); in pblk_discard()
431 void pblk_log_write_err(struct pblk *pblk, struct nvm_rq *rqd) in pblk_log_write_err() argument
433 atomic_long_inc(&pblk->write_failed); in pblk_log_write_err()
435 pblk_print_failed_rqd(pblk, rqd, rqd->error); in pblk_log_write_err()
439 void pblk_log_read_err(struct pblk *pblk, struct nvm_rq *rqd) in pblk_log_read_err() argument
443 atomic_long_inc(&pblk->read_empty); in pblk_log_read_err()
449 atomic_long_inc(&pblk->read_high_ecc); in pblk_log_read_err()
453 atomic_long_inc(&pblk->read_failed); in pblk_log_read_err()
456 pblk_err(pblk, "unknown read error:%d\n", rqd->error); in pblk_log_read_err()
459 pblk_print_failed_rqd(pblk, rqd, rqd->error); in pblk_log_read_err()
463 void pblk_set_sec_per_write(struct pblk *pblk, int sec_per_write) in pblk_set_sec_per_write() argument
465 pblk->sec_per_write = sec_per_write; in pblk_set_sec_per_write()
468 int pblk_submit_io(struct pblk *pblk, struct nvm_rq *rqd) in pblk_submit_io() argument
470 struct nvm_tgt_dev *dev = pblk->dev; in pblk_submit_io()
472 atomic_inc(&pblk->inflight_io); in pblk_submit_io()
475 if (pblk_check_io(pblk, rqd)) in pblk_submit_io()
482 int pblk_submit_io_sync(struct pblk *pblk, struct nvm_rq *rqd) in pblk_submit_io_sync() argument
484 struct nvm_tgt_dev *dev = pblk->dev; in pblk_submit_io_sync()
486 atomic_inc(&pblk->inflight_io); in pblk_submit_io_sync()
489 if (pblk_check_io(pblk, rqd)) in pblk_submit_io_sync()
501 struct bio *pblk_bio_map_addr(struct pblk *pblk, void *data, in pblk_bio_map_addr() argument
505 struct nvm_tgt_dev *dev = pblk->dev; in pblk_bio_map_addr()
521 pblk_err(pblk, "could not map vmalloc bio\n"); in pblk_bio_map_addr()
529 pblk_err(pblk, "could not add page to bio\n"); in pblk_bio_map_addr()
543 int pblk_calc_secs(struct pblk *pblk, unsigned long secs_avail, in pblk_calc_secs() argument
546 int max = pblk->sec_per_write; in pblk_calc_secs()
547 int min = pblk->min_write_pgs; in pblk_calc_secs()
560 void pblk_dealloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs) in pblk_dealloc_page() argument
567 pblk->lm.sec_per_line, line->cur_sec); in pblk_dealloc_page()
575 u64 __pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs) in __pblk_alloc_page() argument
583 if (line->cur_sec + nr_secs > pblk->lm.sec_per_line) { in __pblk_alloc_page()
585 nr_secs = pblk->lm.sec_per_line - line->cur_sec; in __pblk_alloc_page()
589 pblk->lm.sec_per_line, line->cur_sec); in __pblk_alloc_page()
596 u64 pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs) in pblk_alloc_page() argument
604 addr = __pblk_alloc_page(pblk, line, nr_secs); in pblk_alloc_page()
612 u64 pblk_lookup_page(struct pblk *pblk, struct pblk_line *line) in pblk_lookup_page() argument
618 pblk->lm.sec_per_line, line->cur_sec); in pblk_lookup_page()
628 static int pblk_line_submit_emeta_io(struct pblk *pblk, struct pblk_line *line, in pblk_line_submit_emeta_io() argument
631 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_submit_emeta_io()
633 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_submit_emeta_io()
634 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_submit_emeta_io()
639 int min = pblk->min_write_pgs; in pblk_line_submit_emeta_io()
667 rq_ppas = pblk_calc_secs(pblk, left_ppas, 0); in pblk_line_submit_emeta_io()
670 bio = pblk_bio_map_addr(pblk, emeta_buf, rq_ppas, rq_len, in pblk_line_submit_emeta_io()
691 rqd.flags = pblk_set_progr_mode(pblk, PBLK_WRITE); in pblk_line_submit_emeta_io()
694 paddr = __pblk_alloc_page(pblk, line, min); in pblk_line_submit_emeta_io()
699 addr_to_gen_ppa(pblk, paddr, id); in pblk_line_submit_emeta_io()
704 struct ppa_addr ppa = addr_to_gen_ppa(pblk, paddr, id); in pblk_line_submit_emeta_io()
708 if (pblk_io_aligned(pblk, rq_ppas)) in pblk_line_submit_emeta_io()
710 rqd.flags = pblk_set_read_mode(pblk, read_type); in pblk_line_submit_emeta_io()
714 if (pblk_boundary_paddr_checks(pblk, paddr)) { in pblk_line_submit_emeta_io()
715 pblk_err(pblk, "corrupt emeta line:%d\n", in pblk_line_submit_emeta_io()
722 ppa = addr_to_gen_ppa(pblk, paddr, id); in pblk_line_submit_emeta_io()
726 if (pblk_boundary_paddr_checks(pblk, paddr + min)) { in pblk_line_submit_emeta_io()
727 pblk_err(pblk, "corrupt emeta line:%d\n", in pblk_line_submit_emeta_io()
736 addr_to_gen_ppa(pblk, paddr, line->id); in pblk_line_submit_emeta_io()
740 ret = pblk_submit_io_sync(pblk, &rqd); in pblk_line_submit_emeta_io()
742 pblk_err(pblk, "emeta I/O submission failed: %d\n", ret); in pblk_line_submit_emeta_io()
747 atomic_dec(&pblk->inflight_io); in pblk_line_submit_emeta_io()
751 pblk_log_write_err(pblk, &rqd); in pblk_line_submit_emeta_io()
753 pblk_log_read_err(pblk, &rqd); in pblk_line_submit_emeta_io()
765 u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line) in pblk_line_smeta_start() argument
767 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_smeta_start()
769 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_smeta_start()
780 static int pblk_line_submit_smeta_io(struct pblk *pblk, struct pblk_line *line, in pblk_line_submit_smeta_io() argument
783 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_submit_smeta_io()
784 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_submit_smeta_io()
795 flags = pblk_set_progr_mode(pblk, PBLK_WRITE); in pblk_line_submit_smeta_io()
796 lba_list = emeta_to_lbas(pblk, line->emeta->buf); in pblk_line_submit_smeta_io()
800 flags = pblk_set_read_mode(pblk, PBLK_READ_SEQUENTIAL); in pblk_line_submit_smeta_io()
831 rqd.ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line->id); in pblk_line_submit_smeta_io()
845 ret = pblk_submit_io_sync(pblk, &rqd); in pblk_line_submit_smeta_io()
847 pblk_err(pblk, "smeta I/O submission failed: %d\n", ret); in pblk_line_submit_smeta_io()
852 atomic_dec(&pblk->inflight_io); in pblk_line_submit_smeta_io()
856 pblk_log_write_err(pblk, &rqd); in pblk_line_submit_smeta_io()
859 pblk_log_read_err(pblk, &rqd); in pblk_line_submit_smeta_io()
868 int pblk_line_read_smeta(struct pblk *pblk, struct pblk_line *line) in pblk_line_read_smeta() argument
870 u64 bpaddr = pblk_line_smeta_start(pblk, line); in pblk_line_read_smeta()
872 return pblk_line_submit_smeta_io(pblk, line, bpaddr, PBLK_READ_RECOV); in pblk_line_read_smeta()
875 int pblk_line_read_emeta(struct pblk *pblk, struct pblk_line *line, in pblk_line_read_emeta() argument
878 return pblk_line_submit_emeta_io(pblk, line, emeta_buf, in pblk_line_read_emeta()
882 static void pblk_setup_e_rq(struct pblk *pblk, struct nvm_rq *rqd, in pblk_setup_e_rq() argument
888 rqd->flags = pblk_set_progr_mode(pblk, PBLK_ERASE); in pblk_setup_e_rq()
892 static int pblk_blk_erase_sync(struct pblk *pblk, struct ppa_addr ppa) in pblk_blk_erase_sync() argument
899 pblk_setup_e_rq(pblk, &rqd, ppa); in pblk_blk_erase_sync()
904 ret = pblk_submit_io_sync(pblk, &rqd); in pblk_blk_erase_sync()
906 struct nvm_tgt_dev *dev = pblk->dev; in pblk_blk_erase_sync()
909 pblk_err(pblk, "could not sync erase line:%d,blk:%d\n", in pblk_blk_erase_sync()
918 rqd.private = pblk; in pblk_blk_erase_sync()
919 __pblk_end_io_erase(pblk, &rqd); in pblk_blk_erase_sync()
924 int pblk_line_erase(struct pblk *pblk, struct pblk_line *line) in pblk_line_erase() argument
926 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_erase()
940 ppa = pblk->luns[bit].bppa; /* set ch and lun */ in pblk_line_erase()
947 ret = pblk_blk_erase_sync(pblk, ppa); in pblk_line_erase()
949 pblk_err(pblk, "failed to erase line %d\n", line->id); in pblk_line_erase()
990 static int pblk_line_init_metadata(struct pblk *pblk, struct pblk_line *line, in pblk_line_init_metadata() argument
993 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_init_metadata()
995 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_init_metadata()
996 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_init_metadata()
1016 pblk_debug(pblk, "line %d is bad\n", line->id); in pblk_line_init_metadata()
1028 memcpy(smeta_buf->header.uuid, pblk->instance_uuid, 16); in pblk_line_init_metadata()
1049 pblk_calc_meta_header_crc(pblk, &smeta_buf->header)); in pblk_line_init_metadata()
1050 smeta_buf->crc = cpu_to_le32(pblk_calc_smeta_crc(pblk, smeta_buf)); in pblk_line_init_metadata()
1059 pblk_calc_meta_header_crc(pblk, &emeta_buf->header)); in pblk_line_init_metadata()
1071 static int pblk_line_alloc_bitmaps(struct pblk *pblk, struct pblk_line *line) in pblk_line_alloc_bitmaps() argument
1073 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_alloc_bitmaps()
1093 static int pblk_line_init_bb(struct pblk *pblk, struct pblk_line *line, in pblk_line_init_bb() argument
1096 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_init_bb()
1098 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_init_bb()
1099 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_init_bb()
1125 if (init && pblk_line_submit_smeta_io(pblk, line, off, PBLK_WRITE)) { in pblk_line_init_bb()
1126 pblk_debug(pblk, "line smeta I/O failed. Retry\n"); in pblk_line_init_bb()
1158 pblk_err(pblk, "unexpected line %d is bad\n", line->id); in pblk_line_init_bb()
1166 static int pblk_prepare_new_line(struct pblk *pblk, struct pblk_line *line) in pblk_prepare_new_line() argument
1168 struct pblk_line_meta *lm = &pblk->lm; in pblk_prepare_new_line()
1169 struct nvm_tgt_dev *dev = pblk->dev; in pblk_prepare_new_line()
1175 struct pblk_lun *rlun = &pblk->luns[i]; in pblk_prepare_new_line()
1190 static int pblk_line_prepare(struct pblk *pblk, struct pblk_line *line) in pblk_line_prepare() argument
1192 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_prepare()
1205 blk_to_erase = pblk_prepare_new_line(pblk, line); in pblk_line_prepare()
1236 int pblk_line_recov_alloc(struct pblk *pblk, struct pblk_line *line) in pblk_line_recov_alloc() argument
1238 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_recov_alloc()
1245 ret = pblk_line_prepare(pblk, line); in pblk_line_recov_alloc()
1253 ret = pblk_line_alloc_bitmaps(pblk, line); in pblk_line_recov_alloc()
1257 if (!pblk_line_init_bb(pblk, line, 0)) { in pblk_line_recov_alloc()
1262 pblk_rl_free_lines_dec(&pblk->rl, line, true); in pblk_line_recov_alloc()
1266 void pblk_line_recov_close(struct pblk *pblk, struct pblk_line *line) in pblk_line_recov_close() argument
1292 struct pblk_line *pblk_line_get(struct pblk *pblk) in pblk_line_get() argument
1294 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_get()
1295 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_get()
1303 pblk_err(pblk, "no free lines\n"); in pblk_line_get()
1319 pblk_debug(pblk, "line %d is bad\n", line->id); in pblk_line_get()
1323 ret = pblk_line_prepare(pblk, line); in pblk_line_get()
1333 pblk_err(pblk, "failed to prepare line %d\n", line->id); in pblk_line_get()
1343 static struct pblk_line *pblk_line_retry(struct pblk *pblk, in pblk_line_retry() argument
1346 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_retry()
1351 retry_line = pblk_line_get(pblk); in pblk_line_retry()
1369 pblk_rl_free_lines_dec(&pblk->rl, line, false); in pblk_line_retry()
1371 if (pblk_line_erase(pblk, retry_line)) in pblk_line_retry()
1377 static void pblk_set_space_limit(struct pblk *pblk) in pblk_set_space_limit() argument
1379 struct pblk_rl *rl = &pblk->rl; in pblk_set_space_limit()
1384 struct pblk_line *pblk_line_get_first_data(struct pblk *pblk) in pblk_line_get_first_data() argument
1386 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_get_first_data()
1390 line = pblk_line_get(pblk); in pblk_line_get_first_data()
1400 pblk_line_setup_metadata(line, l_mg, &pblk->lm); in pblk_line_get_first_data()
1403 l_mg->data_next = pblk_line_get(pblk); in pblk_line_get_first_data()
1409 pblk_set_space_limit(pblk); in pblk_line_get_first_data()
1418 if (pblk_line_alloc_bitmaps(pblk, line)) in pblk_line_get_first_data()
1421 if (pblk_line_erase(pblk, line)) { in pblk_line_get_first_data()
1422 line = pblk_line_retry(pblk, line); in pblk_line_get_first_data()
1428 if (!pblk_line_init_metadata(pblk, line, NULL)) { in pblk_line_get_first_data()
1429 line = pblk_line_retry(pblk, line); in pblk_line_get_first_data()
1436 if (!pblk_line_init_bb(pblk, line, 1)) { in pblk_line_get_first_data()
1437 line = pblk_line_retry(pblk, line); in pblk_line_get_first_data()
1444 pblk_rl_free_lines_dec(&pblk->rl, line, true); in pblk_line_get_first_data()
1449 static void pblk_stop_writes(struct pblk *pblk, struct pblk_line *line) in pblk_stop_writes() argument
1451 lockdep_assert_held(&pblk->l_mg.free_lock); in pblk_stop_writes()
1453 pblk_set_space_limit(pblk); in pblk_stop_writes()
1454 pblk->state = PBLK_STATE_STOPPING; in pblk_stop_writes()
1457 static void pblk_line_close_meta_sync(struct pblk *pblk) in pblk_line_close_meta_sync() argument
1459 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_close_meta_sync()
1460 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_close_meta_sync()
1479 ret = pblk_submit_meta_io(pblk, line); in pblk_line_close_meta_sync()
1481 pblk_err(pblk, "sync meta line %d failed (%d)\n", in pblk_line_close_meta_sync()
1488 pblk_wait_for_meta(pblk); in pblk_line_close_meta_sync()
1489 flush_workqueue(pblk->close_wq); in pblk_line_close_meta_sync()
1492 void __pblk_pipeline_flush(struct pblk *pblk) in __pblk_pipeline_flush() argument
1494 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in __pblk_pipeline_flush()
1498 if (pblk->state == PBLK_STATE_RECOVERING || in __pblk_pipeline_flush()
1499 pblk->state == PBLK_STATE_STOPPED) { in __pblk_pipeline_flush()
1503 pblk->state = PBLK_STATE_RECOVERING; in __pblk_pipeline_flush()
1506 pblk_flush_writer(pblk); in __pblk_pipeline_flush()
1507 pblk_wait_for_meta(pblk); in __pblk_pipeline_flush()
1509 ret = pblk_recov_pad(pblk); in __pblk_pipeline_flush()
1511 pblk_err(pblk, "could not close data on teardown(%d)\n", ret); in __pblk_pipeline_flush()
1515 flush_workqueue(pblk->bb_wq); in __pblk_pipeline_flush()
1516 pblk_line_close_meta_sync(pblk); in __pblk_pipeline_flush()
1519 void __pblk_pipeline_stop(struct pblk *pblk) in __pblk_pipeline_stop() argument
1521 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in __pblk_pipeline_stop()
1524 pblk->state = PBLK_STATE_STOPPED; in __pblk_pipeline_stop()
1530 void pblk_pipeline_stop(struct pblk *pblk) in pblk_pipeline_stop() argument
1532 __pblk_pipeline_flush(pblk); in pblk_pipeline_stop()
1533 __pblk_pipeline_stop(pblk); in pblk_pipeline_stop()
1536 struct pblk_line *pblk_line_replace_data(struct pblk *pblk) in pblk_line_replace_data() argument
1538 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_replace_data()
1549 pblk_line_setup_metadata(new, l_mg, &pblk->lm); in pblk_line_replace_data()
1557 if (pblk_line_erase(pblk, new)) in pblk_line_replace_data()
1565 if (pblk_line_alloc_bitmaps(pblk, new)) in pblk_line_replace_data()
1569 if (!pblk_line_init_metadata(pblk, new, cur)) { in pblk_line_replace_data()
1570 new = pblk_line_retry(pblk, new); in pblk_line_replace_data()
1577 if (!pblk_line_init_bb(pblk, new, 1)) { in pblk_line_replace_data()
1578 new = pblk_line_retry(pblk, new); in pblk_line_replace_data()
1585 pblk_rl_free_lines_dec(&pblk->rl, new, true); in pblk_line_replace_data()
1589 l_mg->data_next = pblk_line_get(pblk); in pblk_line_replace_data()
1595 pblk_stop_writes(pblk, new); in pblk_line_replace_data()
1607 static void __pblk_line_put(struct pblk *pblk, struct pblk_line *line) in __pblk_line_put() argument
1609 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in __pblk_line_put()
1610 struct pblk_gc *gc = &pblk->gc; in __pblk_line_put()
1619 pblk_rl_werr_line_out(&pblk->rl); in __pblk_line_put()
1631 pblk_rl_free_lines_inc(&pblk->rl, line); in __pblk_line_put()
1638 struct pblk *pblk = line_put_ws->pblk; in pblk_line_put_ws() local
1641 __pblk_line_put(pblk, line); in pblk_line_put_ws()
1642 mempool_free(line_put_ws, &pblk->gen_ws_pool); in pblk_line_put_ws()
1648 struct pblk *pblk = line->pblk; in pblk_line_put() local
1650 __pblk_line_put(pblk, line); in pblk_line_put()
1656 struct pblk *pblk = line->pblk; in pblk_line_put_wq() local
1659 line_put_ws = mempool_alloc(&pblk->gen_ws_pool, GFP_ATOMIC); in pblk_line_put_wq()
1663 line_put_ws->pblk = pblk; in pblk_line_put_wq()
1668 queue_work(pblk->r_end_wq, &line_put_ws->ws); in pblk_line_put_wq()
1671 int pblk_blk_erase_async(struct pblk *pblk, struct ppa_addr ppa) in pblk_blk_erase_async() argument
1676 rqd = pblk_alloc_rqd(pblk, PBLK_ERASE); in pblk_blk_erase_async()
1678 pblk_setup_e_rq(pblk, rqd, ppa); in pblk_blk_erase_async()
1681 rqd->private = pblk; in pblk_blk_erase_async()
1686 err = pblk_submit_io(pblk, rqd); in pblk_blk_erase_async()
1688 struct nvm_tgt_dev *dev = pblk->dev; in pblk_blk_erase_async()
1691 pblk_err(pblk, "could not async erase line:%d,blk:%d\n", in pblk_blk_erase_async()
1699 struct pblk_line *pblk_line_get_data(struct pblk *pblk) in pblk_line_get_data() argument
1701 return pblk->l_mg.data_line; in pblk_line_get_data()
1705 struct pblk_line *pblk_line_get_erase(struct pblk *pblk) in pblk_line_get_erase() argument
1707 return pblk->l_mg.data_next; in pblk_line_get_erase()
1715 static void pblk_line_should_sync_meta(struct pblk *pblk) in pblk_line_should_sync_meta() argument
1717 if (pblk_rl_is_limit(&pblk->rl)) in pblk_line_should_sync_meta()
1718 pblk_line_close_meta_sync(pblk); in pblk_line_should_sync_meta()
1721 void pblk_line_close(struct pblk *pblk, struct pblk_line *line) in pblk_line_close() argument
1723 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_close()
1725 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_close()
1726 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_close()
1743 move_list = pblk_line_gc_list(pblk, line); in pblk_line_close()
1753 struct pblk_lun *rlun = &pblk->luns[i]; in pblk_line_close()
1765 void pblk_line_close_meta(struct pblk *pblk, struct pblk_line *line) in pblk_line_close_meta() argument
1767 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_close_meta()
1768 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_close_meta()
1774 memcpy(emeta_to_vsc(pblk, emeta_buf), l_mg->vsc_list, lm->vsc_list_len); in pblk_line_close_meta()
1777 wa->user = cpu_to_le64(atomic64_read(&pblk->user_wa)); in pblk_line_close_meta()
1778 wa->pad = cpu_to_le64(atomic64_read(&pblk->pad_wa)); in pblk_line_close_meta()
1779 wa->gc = cpu_to_le64(atomic64_read(&pblk->gc_wa)); in pblk_line_close_meta()
1782 emeta_buf->crc = cpu_to_le32(pblk_calc_emeta_crc(pblk, emeta_buf)); in pblk_line_close_meta()
1797 pblk_line_should_sync_meta(pblk); in pblk_line_close_meta()
1802 static void pblk_save_lba_list(struct pblk *pblk, struct pblk_line *line) in pblk_save_lba_list() argument
1804 struct pblk_line_meta *lm = &pblk->lm; in pblk_save_lba_list()
1805 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_save_lba_list()
1812 memcpy(w_err_gc->lba_list, emeta_to_lbas(pblk, emeta->buf), in pblk_save_lba_list()
1820 struct pblk *pblk = line_ws->pblk; in pblk_line_close_ws() local
1828 pblk_save_lba_list(pblk, line); in pblk_line_close_ws()
1830 pblk_line_close(pblk, line); in pblk_line_close_ws()
1831 mempool_free(line_ws, &pblk->gen_ws_pool); in pblk_line_close_ws()
1834 void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv, in pblk_gen_run_ws() argument
1840 line_ws = mempool_alloc(&pblk->gen_ws_pool, gfp_mask); in pblk_gen_run_ws()
1842 line_ws->pblk = pblk; in pblk_gen_run_ws()
1850 static void __pblk_down_page(struct pblk *pblk, struct ppa_addr *ppa_list, in __pblk_down_page() argument
1853 struct pblk_lun *rlun = &pblk->luns[pos]; in __pblk_down_page()
1870 pblk_err(pblk, "taking lun semaphore timed out: err %d\n", in __pblk_down_page()
1874 void pblk_down_page(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas) in pblk_down_page() argument
1876 struct nvm_tgt_dev *dev = pblk->dev; in pblk_down_page()
1880 __pblk_down_page(pblk, ppa_list, nr_ppas, pos); in pblk_down_page()
1883 void pblk_down_rq(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas, in pblk_down_rq() argument
1886 struct nvm_tgt_dev *dev = pblk->dev; in pblk_down_rq()
1896 __pblk_down_page(pblk, ppa_list, nr_ppas, pos); in pblk_down_rq()
1899 void pblk_up_page(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas) in pblk_up_page() argument
1901 struct nvm_tgt_dev *dev = pblk->dev; in pblk_up_page()
1914 rlun = &pblk->luns[pos]; in pblk_up_page()
1918 void pblk_up_rq(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas, in pblk_up_rq() argument
1921 struct nvm_tgt_dev *dev = pblk->dev; in pblk_up_rq()
1928 rlun = &pblk->luns[bit]; in pblk_up_rq()
1933 void pblk_update_map(struct pblk *pblk, sector_t lba, struct ppa_addr ppa) in pblk_update_map() argument
1938 if (!(lba < pblk->rl.nr_secs)) { in pblk_update_map()
1943 spin_lock(&pblk->trans_lock); in pblk_update_map()
1944 ppa_l2p = pblk_trans_map_get(pblk, lba); in pblk_update_map()
1947 pblk_map_invalidate(pblk, ppa_l2p); in pblk_update_map()
1949 pblk_trans_map_set(pblk, lba, ppa); in pblk_update_map()
1950 spin_unlock(&pblk->trans_lock); in pblk_update_map()
1953 void pblk_update_map_cache(struct pblk *pblk, sector_t lba, struct ppa_addr ppa) in pblk_update_map_cache() argument
1959 BUG_ON(pblk_rb_pos_oob(&pblk->rwb, pblk_addr_to_cacheline(ppa))); in pblk_update_map_cache()
1962 pblk_update_map(pblk, lba, ppa); in pblk_update_map_cache()
1965 int pblk_update_map_gc(struct pblk *pblk, sector_t lba, struct ppa_addr ppa_new, in pblk_update_map_gc() argument
1974 BUG_ON(pblk_rb_pos_oob(&pblk->rwb, pblk_addr_to_cacheline(ppa_new))); in pblk_update_map_gc()
1978 if (!(lba < pblk->rl.nr_secs)) { in pblk_update_map_gc()
1983 spin_lock(&pblk->trans_lock); in pblk_update_map_gc()
1984 ppa_l2p = pblk_trans_map_get(pblk, lba); in pblk_update_map_gc()
1985 ppa_gc = addr_to_gen_ppa(pblk, paddr_gc, gc_line->id); in pblk_update_map_gc()
1997 pblk_trans_map_set(pblk, lba, ppa_new); in pblk_update_map_gc()
1999 spin_unlock(&pblk->trans_lock); in pblk_update_map_gc()
2003 void pblk_update_map_dev(struct pblk *pblk, sector_t lba, in pblk_update_map_dev() argument
2014 atomic64_inc(&pblk->pad_wa); in pblk_update_map_dev()
2016 atomic_long_inc(&pblk->padded_wb); in pblk_update_map_dev()
2019 pblk_map_invalidate(pblk, ppa_mapped); in pblk_update_map_dev()
2024 if (!(lba < pblk->rl.nr_secs)) { in pblk_update_map_dev()
2029 spin_lock(&pblk->trans_lock); in pblk_update_map_dev()
2030 ppa_l2p = pblk_trans_map_get(pblk, lba); in pblk_update_map_dev()
2037 pblk_map_invalidate(pblk, ppa_mapped); in pblk_update_map_dev()
2045 pblk_trans_map_set(pblk, lba, ppa_mapped); in pblk_update_map_dev()
2047 spin_unlock(&pblk->trans_lock); in pblk_update_map_dev()
2050 void pblk_lookup_l2p_seq(struct pblk *pblk, struct ppa_addr *ppas, in pblk_lookup_l2p_seq() argument
2055 spin_lock(&pblk->trans_lock); in pblk_lookup_l2p_seq()
2059 ppa = ppas[i] = pblk_trans_map_get(pblk, blba + i); in pblk_lookup_l2p_seq()
2064 struct pblk_line *line = &pblk->lines[line_id]; in pblk_lookup_l2p_seq()
2069 spin_unlock(&pblk->trans_lock); in pblk_lookup_l2p_seq()
2072 void pblk_lookup_l2p_rand(struct pblk *pblk, struct ppa_addr *ppas, in pblk_lookup_l2p_rand() argument
2078 spin_lock(&pblk->trans_lock); in pblk_lookup_l2p_rand()
2083 if (!(lba < pblk->rl.nr_secs)) { in pblk_lookup_l2p_rand()
2087 ppas[i] = pblk_trans_map_get(pblk, lba); in pblk_lookup_l2p_rand()
2090 spin_unlock(&pblk->trans_lock); in pblk_lookup_l2p_rand()