Lines Matching refs:pblk
33 static int pblk_rw_io(struct request_queue *q, struct pblk *pblk, in pblk_rw_io() argument
43 ret = pblk_submit_read(pblk, bio); in pblk_rw_io()
54 if (pblk_get_secs(bio) > pblk_rl_max_io(&pblk->rl)) in pblk_rw_io()
57 return pblk_write_to_cache(pblk, bio, PBLK_IOTYPE_USER); in pblk_rw_io()
62 struct pblk *pblk = q->queuedata; in pblk_make_rq() local
65 pblk_discard(pblk, bio); in pblk_make_rq()
72 switch (pblk_rw_io(q, pblk, bio)) { in pblk_make_rq()
84 static size_t pblk_trans_map_size(struct pblk *pblk) in pblk_trans_map_size() argument
88 if (pblk->addrf_len < 32) in pblk_trans_map_size()
91 return entry_size * pblk->rl.nr_secs; in pblk_trans_map_size()
95 static u32 pblk_l2p_crc(struct pblk *pblk) in pblk_l2p_crc() argument
100 map_size = pblk_trans_map_size(pblk); in pblk_l2p_crc()
101 crc = crc32_le(crc, pblk->trans_map, map_size); in pblk_l2p_crc()
106 static void pblk_l2p_free(struct pblk *pblk) in pblk_l2p_free() argument
108 vfree(pblk->trans_map); in pblk_l2p_free()
111 static int pblk_l2p_recover(struct pblk *pblk, bool factory_init) in pblk_l2p_recover() argument
116 pblk_setup_uuid(pblk); in pblk_l2p_recover()
118 line = pblk_recov_l2p(pblk); in pblk_l2p_recover()
120 pblk_err(pblk, "could not recover l2p table\n"); in pblk_l2p_recover()
126 pblk_info(pblk, "init: L2P CRC: %x\n", pblk_l2p_crc(pblk)); in pblk_l2p_recover()
130 pblk_gc_free_full_lines(pblk); in pblk_l2p_recover()
134 line = pblk_line_get_first_data(pblk); in pblk_l2p_recover()
142 static int pblk_l2p_init(struct pblk *pblk, bool factory_init) in pblk_l2p_init() argument
149 map_size = pblk_trans_map_size(pblk); in pblk_l2p_init()
150 pblk->trans_map = vmalloc(map_size); in pblk_l2p_init()
151 if (!pblk->trans_map) in pblk_l2p_init()
156 for (i = 0; i < pblk->rl.nr_secs; i++) in pblk_l2p_init()
157 pblk_trans_map_set(pblk, i, ppa); in pblk_l2p_init()
159 ret = pblk_l2p_recover(pblk, factory_init); in pblk_l2p_init()
161 vfree(pblk->trans_map); in pblk_l2p_init()
166 static void pblk_rwb_free(struct pblk *pblk) in pblk_rwb_free() argument
168 if (pblk_rb_tear_down_check(&pblk->rwb)) in pblk_rwb_free()
169 pblk_err(pblk, "write buffer error on tear down\n"); in pblk_rwb_free()
171 pblk_rb_data_free(&pblk->rwb); in pblk_rwb_free()
172 vfree(pblk_rb_entries_ref(&pblk->rwb)); in pblk_rwb_free()
175 static int pblk_rwb_init(struct pblk *pblk) in pblk_rwb_init() argument
177 struct nvm_tgt_dev *dev = pblk->dev; in pblk_rwb_init()
200 return pblk_rb_init(&pblk->rwb, entries, power_size, power_seg_sz); in pblk_rwb_init()
206 static int pblk_set_addrf_12(struct pblk *pblk, struct nvm_geo *geo, in pblk_set_addrf_12() argument
215 pblk_err(pblk, "supports only power-of-two channel config.\n"); in pblk_set_addrf_12()
222 pblk_err(pblk, "supports only power-of-two LUN config.\n"); in pblk_set_addrf_12()
279 static int pblk_set_addrf(struct pblk *pblk) in pblk_set_addrf() argument
281 struct nvm_tgt_dev *dev = pblk->dev; in pblk_set_addrf()
287 div_u64_rem(geo->clba, pblk->min_write_pgs, &mod); in pblk_set_addrf()
289 pblk_err(pblk, "bad configuration of sectors/pages\n"); in pblk_set_addrf()
293 pblk->addrf_len = pblk_set_addrf_12(pblk, geo, in pblk_set_addrf()
294 (void *)&pblk->addrf); in pblk_set_addrf()
297 pblk->addrf_len = pblk_set_addrf_20(geo, (void *)&pblk->addrf, in pblk_set_addrf()
298 &pblk->uaddrf); in pblk_set_addrf()
301 pblk_err(pblk, "OCSSD revision not supported (%d)\n", in pblk_set_addrf()
309 static int pblk_init_global_caches(struct pblk *pblk) in pblk_init_global_caches() argument
350 static void pblk_free_global_caches(struct pblk *pblk) in pblk_free_global_caches() argument
358 static int pblk_core_init(struct pblk *pblk) in pblk_core_init() argument
360 struct nvm_tgt_dev *dev = pblk->dev; in pblk_core_init()
364 atomic64_set(&pblk->user_wa, 0); in pblk_core_init()
365 atomic64_set(&pblk->pad_wa, 0); in pblk_core_init()
366 atomic64_set(&pblk->gc_wa, 0); in pblk_core_init()
367 pblk->user_rst_wa = 0; in pblk_core_init()
368 pblk->pad_rst_wa = 0; in pblk_core_init()
369 pblk->gc_rst_wa = 0; in pblk_core_init()
371 atomic64_set(&pblk->nr_flush, 0); in pblk_core_init()
372 pblk->nr_flush_rst = 0; in pblk_core_init()
374 pblk->min_write_pgs = geo->ws_opt * (geo->csecs / PAGE_SIZE); in pblk_core_init()
375 max_write_ppas = pblk->min_write_pgs * geo->all_luns; in pblk_core_init()
376 pblk->max_write_pgs = min_t(int, max_write_ppas, NVM_MAX_VLBA); in pblk_core_init()
377 pblk_set_sec_per_write(pblk, pblk->min_write_pgs); in pblk_core_init()
379 if (pblk->max_write_pgs > PBLK_MAX_REQ_ADDRS) { in pblk_core_init()
380 pblk_err(pblk, "vector list too big(%u > %u)\n", in pblk_core_init()
381 pblk->max_write_pgs, PBLK_MAX_REQ_ADDRS); in pblk_core_init()
385 pblk->pad_dist = kcalloc(pblk->min_write_pgs - 1, sizeof(atomic64_t), in pblk_core_init()
387 if (!pblk->pad_dist) in pblk_core_init()
390 if (pblk_init_global_caches(pblk)) in pblk_core_init()
394 ret = mempool_init_page_pool(&pblk->page_bio_pool, NVM_MAX_VLBA, 0); in pblk_core_init()
398 ret = mempool_init_slab_pool(&pblk->gen_ws_pool, PBLK_GEN_WS_POOL_SIZE, in pblk_core_init()
403 ret = mempool_init_slab_pool(&pblk->rec_pool, geo->all_luns, in pblk_core_init()
408 ret = mempool_init_slab_pool(&pblk->r_rq_pool, geo->all_luns, in pblk_core_init()
413 ret = mempool_init_slab_pool(&pblk->e_rq_pool, geo->all_luns, in pblk_core_init()
418 ret = mempool_init_slab_pool(&pblk->w_rq_pool, geo->all_luns, in pblk_core_init()
423 pblk->close_wq = alloc_workqueue("pblk-close-wq", in pblk_core_init()
425 if (!pblk->close_wq) in pblk_core_init()
428 pblk->bb_wq = alloc_workqueue("pblk-bb-wq", in pblk_core_init()
430 if (!pblk->bb_wq) in pblk_core_init()
433 pblk->r_end_wq = alloc_workqueue("pblk-read-end-wq", in pblk_core_init()
435 if (!pblk->r_end_wq) in pblk_core_init()
438 if (pblk_set_addrf(pblk)) in pblk_core_init()
441 INIT_LIST_HEAD(&pblk->compl_list); in pblk_core_init()
442 INIT_LIST_HEAD(&pblk->resubmit_list); in pblk_core_init()
447 destroy_workqueue(pblk->r_end_wq); in pblk_core_init()
449 destroy_workqueue(pblk->bb_wq); in pblk_core_init()
451 destroy_workqueue(pblk->close_wq); in pblk_core_init()
453 mempool_exit(&pblk->w_rq_pool); in pblk_core_init()
455 mempool_exit(&pblk->e_rq_pool); in pblk_core_init()
457 mempool_exit(&pblk->r_rq_pool); in pblk_core_init()
459 mempool_exit(&pblk->rec_pool); in pblk_core_init()
461 mempool_exit(&pblk->gen_ws_pool); in pblk_core_init()
463 mempool_exit(&pblk->page_bio_pool); in pblk_core_init()
465 pblk_free_global_caches(pblk); in pblk_core_init()
467 kfree(pblk->pad_dist); in pblk_core_init()
471 static void pblk_core_free(struct pblk *pblk) in pblk_core_free() argument
473 if (pblk->close_wq) in pblk_core_free()
474 destroy_workqueue(pblk->close_wq); in pblk_core_free()
476 if (pblk->r_end_wq) in pblk_core_free()
477 destroy_workqueue(pblk->r_end_wq); in pblk_core_free()
479 if (pblk->bb_wq) in pblk_core_free()
480 destroy_workqueue(pblk->bb_wq); in pblk_core_free()
482 mempool_exit(&pblk->page_bio_pool); in pblk_core_free()
483 mempool_exit(&pblk->gen_ws_pool); in pblk_core_free()
484 mempool_exit(&pblk->rec_pool); in pblk_core_free()
485 mempool_exit(&pblk->r_rq_pool); in pblk_core_free()
486 mempool_exit(&pblk->e_rq_pool); in pblk_core_free()
487 mempool_exit(&pblk->w_rq_pool); in pblk_core_free()
489 pblk_free_global_caches(pblk); in pblk_core_free()
490 kfree(pblk->pad_dist); in pblk_core_free()
493 static void pblk_line_mg_free(struct pblk *pblk) in pblk_line_mg_free() argument
495 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_mg_free()
522 static void pblk_lines_free(struct pblk *pblk) in pblk_lines_free() argument
524 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_lines_free()
530 line = &pblk->lines[i]; in pblk_lines_free()
537 pblk_line_mg_free(pblk); in pblk_lines_free()
539 kfree(pblk->luns); in pblk_lines_free()
540 kfree(pblk->lines); in pblk_lines_free()
564 static void *pblk_bb_get_meta(struct pblk *pblk) in pblk_bb_get_meta() argument
566 struct nvm_tgt_dev *dev = pblk->dev; in pblk_bb_get_meta()
580 struct pblk_lun *rlun = &pblk->luns[i]; in pblk_bb_get_meta()
593 static void *pblk_chunk_get_meta(struct pblk *pblk) in pblk_chunk_get_meta() argument
595 struct nvm_tgt_dev *dev = pblk->dev; in pblk_chunk_get_meta()
599 return pblk_bb_get_meta(pblk); in pblk_chunk_get_meta()
601 return pblk_chunk_get_info(pblk); in pblk_chunk_get_meta()
604 static int pblk_luns_init(struct pblk *pblk) in pblk_luns_init() argument
606 struct nvm_tgt_dev *dev = pblk->dev; in pblk_luns_init()
613 pblk_err(pblk, "unbalanced LUN config.\n"); in pblk_luns_init()
617 pblk->luns = kcalloc(geo->all_luns, sizeof(struct pblk_lun), in pblk_luns_init()
619 if (!pblk->luns) in pblk_luns_init()
628 rlun = &pblk->luns[i]; in pblk_luns_init()
638 static unsigned int calc_emeta_len(struct pblk *pblk) in calc_emeta_len() argument
640 struct pblk_line_meta *lm = &pblk->lm; in calc_emeta_len()
641 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in calc_emeta_len()
642 struct nvm_tgt_dev *dev = pblk->dev; in calc_emeta_len()
666 static void pblk_set_provision(struct pblk *pblk, long nr_free_blks) in pblk_set_provision() argument
668 struct nvm_tgt_dev *dev = pblk->dev; in pblk_set_provision()
669 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_set_provision()
670 struct pblk_line_meta *lm = &pblk->lm; in pblk_set_provision()
676 pblk->op = PBLK_DEFAULT_OP; in pblk_set_provision()
678 pblk->op = geo->op; in pblk_set_provision()
681 provisioned *= (100 - pblk->op); in pblk_set_provision()
684 pblk->op_blks = nr_free_blks - provisioned; in pblk_set_provision()
689 pblk->rl.total_blocks = nr_free_blks; in pblk_set_provision()
690 pblk->rl.nr_secs = nr_free_blks * geo->clba; in pblk_set_provision()
696 pblk->capacity = (provisioned - blk_meta) * geo->clba; in pblk_set_provision()
698 atomic_set(&pblk->rl.free_blocks, nr_free_blks); in pblk_set_provision()
699 atomic_set(&pblk->rl.free_user_blocks, nr_free_blks); in pblk_set_provision()
702 static int pblk_setup_line_meta_12(struct pblk *pblk, struct pblk_line *line, in pblk_setup_line_meta_12() argument
705 struct nvm_tgt_dev *dev = pblk->dev; in pblk_setup_line_meta_12()
707 struct pblk_line_meta *lm = &pblk->lm; in pblk_setup_line_meta_12()
713 struct pblk_lun *rlun = &pblk->luns[i]; in pblk_setup_line_meta_12()
746 static int pblk_setup_line_meta_20(struct pblk *pblk, struct pblk_line *line, in pblk_setup_line_meta_20() argument
749 struct nvm_tgt_dev *dev = pblk->dev; in pblk_setup_line_meta_20()
751 struct pblk_line_meta *lm = &pblk->lm; in pblk_setup_line_meta_20()
755 struct pblk_lun *rlun = &pblk->luns[i]; in pblk_setup_line_meta_20()
766 chunk_meta = pblk_chunk_get_off(pblk, meta, ppa); in pblk_setup_line_meta_20()
790 static long pblk_setup_line_meta(struct pblk *pblk, struct pblk_line *line, in pblk_setup_line_meta() argument
793 struct nvm_tgt_dev *dev = pblk->dev; in pblk_setup_line_meta()
795 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_setup_line_meta()
796 struct pblk_line_meta *lm = &pblk->lm; in pblk_setup_line_meta()
799 line->pblk = pblk; in pblk_setup_line_meta()
808 nr_bad_chks = pblk_setup_line_meta_12(pblk, line, chunk_meta); in pblk_setup_line_meta()
810 nr_bad_chks = pblk_setup_line_meta_20(pblk, line, chunk_meta); in pblk_setup_line_meta()
827 static int pblk_alloc_line_meta(struct pblk *pblk, struct pblk_line *line) in pblk_alloc_line_meta() argument
829 struct pblk_line_meta *lm = &pblk->lm; in pblk_alloc_line_meta()
860 static int pblk_line_mg_init(struct pblk *pblk) in pblk_line_mg_init() argument
862 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_mg_init()
864 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_mg_init()
865 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_mg_init()
980 static int pblk_line_meta_init(struct pblk *pblk) in pblk_line_meta_init() argument
982 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_meta_init()
984 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_meta_init()
995 lm->meta_distance = (geo->all_luns / 2) * pblk->min_write_pgs; in pblk_line_meta_init()
1019 emeta_len = calc_emeta_len(pblk); in pblk_line_meta_init()
1033 pblk_err(pblk, "config. not supported. Min. LUN in line:%d\n", in pblk_line_meta_init()
1041 static int pblk_lines_init(struct pblk *pblk) in pblk_lines_init() argument
1043 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_lines_init()
1049 ret = pblk_line_meta_init(pblk); in pblk_lines_init()
1053 ret = pblk_line_mg_init(pblk); in pblk_lines_init()
1057 ret = pblk_luns_init(pblk); in pblk_lines_init()
1061 chunk_meta = pblk_chunk_get_meta(pblk); in pblk_lines_init()
1067 pblk->lines = kcalloc(l_mg->nr_lines, sizeof(struct pblk_line), in pblk_lines_init()
1069 if (!pblk->lines) { in pblk_lines_init()
1075 line = &pblk->lines[i]; in pblk_lines_init()
1077 ret = pblk_alloc_line_meta(pblk, line); in pblk_lines_init()
1081 nr_free_chks += pblk_setup_line_meta(pblk, line, chunk_meta, i); in pblk_lines_init()
1085 pblk_err(pblk, "too many bad blocks prevent for sane instance\n"); in pblk_lines_init()
1089 pblk_set_provision(pblk, nr_free_chks); in pblk_lines_init()
1096 pblk_line_meta_free(l_mg, &pblk->lines[i]); in pblk_lines_init()
1097 kfree(pblk->lines); in pblk_lines_init()
1101 kfree(pblk->luns); in pblk_lines_init()
1103 pblk_line_mg_free(pblk); in pblk_lines_init()
1108 static int pblk_writer_init(struct pblk *pblk) in pblk_writer_init() argument
1110 pblk->writer_ts = kthread_create(pblk_write_ts, pblk, "pblk-writer-t"); in pblk_writer_init()
1111 if (IS_ERR(pblk->writer_ts)) { in pblk_writer_init()
1112 int err = PTR_ERR(pblk->writer_ts); in pblk_writer_init()
1115 pblk_err(pblk, "could not allocate writer kthread (%d)\n", in pblk_writer_init()
1120 timer_setup(&pblk->wtimer, pblk_write_timer_fn, 0); in pblk_writer_init()
1121 mod_timer(&pblk->wtimer, jiffies + msecs_to_jiffies(100)); in pblk_writer_init()
1126 static void pblk_writer_stop(struct pblk *pblk) in pblk_writer_stop() argument
1131 WARN(pblk_rb_read_count(&pblk->rwb), in pblk_writer_stop()
1134 WARN(pblk_rb_sync_count(&pblk->rwb), in pblk_writer_stop()
1137 del_timer_sync(&pblk->wtimer); in pblk_writer_stop()
1138 if (pblk->writer_ts) in pblk_writer_stop()
1139 kthread_stop(pblk->writer_ts); in pblk_writer_stop()
1142 static void pblk_free(struct pblk *pblk) in pblk_free() argument
1144 pblk_lines_free(pblk); in pblk_free()
1145 pblk_l2p_free(pblk); in pblk_free()
1146 pblk_rwb_free(pblk); in pblk_free()
1147 pblk_core_free(pblk); in pblk_free()
1149 kfree(pblk); in pblk_free()
1152 static void pblk_tear_down(struct pblk *pblk, bool graceful) in pblk_tear_down() argument
1155 __pblk_pipeline_flush(pblk); in pblk_tear_down()
1156 __pblk_pipeline_stop(pblk); in pblk_tear_down()
1157 pblk_writer_stop(pblk); in pblk_tear_down()
1158 pblk_rb_sync_l2p(&pblk->rwb); in pblk_tear_down()
1159 pblk_rl_free(&pblk->rl); in pblk_tear_down()
1161 pblk_debug(pblk, "consistent tear down (graceful:%d)\n", graceful); in pblk_tear_down()
1166 struct pblk *pblk = private; in pblk_exit() local
1169 pblk_gc_exit(pblk, graceful); in pblk_exit()
1170 pblk_tear_down(pblk, graceful); in pblk_exit()
1173 pblk_info(pblk, "exit: L2P CRC: %x\n", pblk_l2p_crc(pblk)); in pblk_exit()
1176 pblk_free(pblk); in pblk_exit()
1182 struct pblk *pblk = private; in pblk_capacity() local
1184 return pblk->capacity * NR_PHY_IN_LOG; in pblk_capacity()
1193 struct pblk *pblk; in pblk_init() local
1196 pblk = kzalloc(sizeof(struct pblk), GFP_KERNEL); in pblk_init()
1197 if (!pblk) in pblk_init()
1200 pblk->dev = dev; in pblk_init()
1201 pblk->disk = tdisk; in pblk_init()
1202 pblk->state = PBLK_STATE_RUNNING; in pblk_init()
1203 pblk->gc.gc_enabled = 0; in pblk_init()
1207 pblk_err(pblk, "OCSSD version not supported (%u)\n", in pblk_init()
1209 kfree(pblk); in pblk_init()
1214 pblk_err(pblk, "host-side L2P table not supported. (%x)\n", in pblk_init()
1216 kfree(pblk); in pblk_init()
1220 spin_lock_init(&pblk->resubmit_lock); in pblk_init()
1221 spin_lock_init(&pblk->trans_lock); in pblk_init()
1222 spin_lock_init(&pblk->lock); in pblk_init()
1225 atomic_long_set(&pblk->inflight_writes, 0); in pblk_init()
1226 atomic_long_set(&pblk->padded_writes, 0); in pblk_init()
1227 atomic_long_set(&pblk->padded_wb, 0); in pblk_init()
1228 atomic_long_set(&pblk->req_writes, 0); in pblk_init()
1229 atomic_long_set(&pblk->sub_writes, 0); in pblk_init()
1230 atomic_long_set(&pblk->sync_writes, 0); in pblk_init()
1231 atomic_long_set(&pblk->inflight_reads, 0); in pblk_init()
1232 atomic_long_set(&pblk->cache_reads, 0); in pblk_init()
1233 atomic_long_set(&pblk->sync_reads, 0); in pblk_init()
1234 atomic_long_set(&pblk->recov_writes, 0); in pblk_init()
1235 atomic_long_set(&pblk->recov_writes, 0); in pblk_init()
1236 atomic_long_set(&pblk->recov_gc_writes, 0); in pblk_init()
1237 atomic_long_set(&pblk->recov_gc_reads, 0); in pblk_init()
1240 atomic_long_set(&pblk->read_failed, 0); in pblk_init()
1241 atomic_long_set(&pblk->read_empty, 0); in pblk_init()
1242 atomic_long_set(&pblk->read_high_ecc, 0); in pblk_init()
1243 atomic_long_set(&pblk->read_failed_gc, 0); in pblk_init()
1244 atomic_long_set(&pblk->write_failed, 0); in pblk_init()
1245 atomic_long_set(&pblk->erase_failed, 0); in pblk_init()
1247 ret = pblk_core_init(pblk); in pblk_init()
1249 pblk_err(pblk, "could not initialize core\n"); in pblk_init()
1253 ret = pblk_lines_init(pblk); in pblk_init()
1255 pblk_err(pblk, "could not initialize lines\n"); in pblk_init()
1259 ret = pblk_rwb_init(pblk); in pblk_init()
1261 pblk_err(pblk, "could not initialize write buffer\n"); in pblk_init()
1265 ret = pblk_l2p_init(pblk, flags & NVM_TARGET_FACTORY); in pblk_init()
1267 pblk_err(pblk, "could not initialize maps\n"); in pblk_init()
1271 ret = pblk_writer_init(pblk); in pblk_init()
1274 pblk_err(pblk, "could not initialize write thread\n"); in pblk_init()
1278 ret = pblk_gc_init(pblk); in pblk_init()
1280 pblk_err(pblk, "could not initialize gc\n"); in pblk_init()
1295 pblk_info(pblk, "luns:%u, lines:%d, secs:%llu, buf entries:%u\n", in pblk_init()
1296 geo->all_luns, pblk->l_mg.nr_lines, in pblk_init()
1297 (unsigned long long)pblk->rl.nr_secs, in pblk_init()
1298 pblk->rwb.nr_entries); in pblk_init()
1300 wake_up_process(pblk->writer_ts); in pblk_init()
1303 pblk_gc_should_kick(pblk); in pblk_init()
1305 return pblk; in pblk_init()
1308 pblk_writer_stop(pblk); in pblk_init()
1310 pblk_l2p_free(pblk); in pblk_init()
1312 pblk_rwb_free(pblk); in pblk_init()
1314 pblk_lines_free(pblk); in pblk_init()
1316 pblk_core_free(pblk); in pblk_init()
1318 kfree(pblk); in pblk_init()