Lines Matching refs:pblk
52 struct pblk *pblk = q->queuedata; in pblk_make_rq() local
55 pblk_discard(pblk, bio); in pblk_make_rq()
67 pblk_submit_read(pblk, bio); in pblk_make_rq()
73 if (pblk_get_secs(bio) > pblk_rl_max_io(&pblk->rl)) in pblk_make_rq()
76 pblk_write_to_cache(pblk, bio, PBLK_IOTYPE_USER); in pblk_make_rq()
82 static size_t pblk_trans_map_size(struct pblk *pblk) in pblk_trans_map_size() argument
86 if (pblk->addrf_len < 32) in pblk_trans_map_size()
89 return entry_size * pblk->capacity; in pblk_trans_map_size()
93 static u32 pblk_l2p_crc(struct pblk *pblk) in pblk_l2p_crc() argument
98 map_size = pblk_trans_map_size(pblk); in pblk_l2p_crc()
99 crc = crc32_le(crc, pblk->trans_map, map_size); in pblk_l2p_crc()
104 static void pblk_l2p_free(struct pblk *pblk) in pblk_l2p_free() argument
106 vfree(pblk->trans_map); in pblk_l2p_free()
109 static int pblk_l2p_recover(struct pblk *pblk, bool factory_init) in pblk_l2p_recover() argument
114 guid_gen(&pblk->instance_uuid); in pblk_l2p_recover()
116 line = pblk_recov_l2p(pblk); in pblk_l2p_recover()
118 pblk_err(pblk, "could not recover l2p table\n"); in pblk_l2p_recover()
124 pblk_info(pblk, "init: L2P CRC: %x\n", pblk_l2p_crc(pblk)); in pblk_l2p_recover()
128 pblk_gc_free_full_lines(pblk); in pblk_l2p_recover()
132 line = pblk_line_get_first_data(pblk); in pblk_l2p_recover()
140 static int pblk_l2p_init(struct pblk *pblk, bool factory_init) in pblk_l2p_init() argument
147 map_size = pblk_trans_map_size(pblk); in pblk_l2p_init()
148 pblk->trans_map = __vmalloc(map_size, GFP_KERNEL | __GFP_NOWARN in pblk_l2p_init()
151 if (!pblk->trans_map) { in pblk_l2p_init()
152 pblk_err(pblk, "failed to allocate L2P (need %zu of memory)\n", in pblk_l2p_init()
159 for (i = 0; i < pblk->capacity; i++) in pblk_l2p_init()
160 pblk_trans_map_set(pblk, i, ppa); in pblk_l2p_init()
162 ret = pblk_l2p_recover(pblk, factory_init); in pblk_l2p_init()
164 vfree(pblk->trans_map); in pblk_l2p_init()
169 static void pblk_rwb_free(struct pblk *pblk) in pblk_rwb_free() argument
171 if (pblk_rb_tear_down_check(&pblk->rwb)) in pblk_rwb_free()
172 pblk_err(pblk, "write buffer error on tear down\n"); in pblk_rwb_free()
174 pblk_rb_free(&pblk->rwb); in pblk_rwb_free()
177 static int pblk_rwb_init(struct pblk *pblk) in pblk_rwb_init() argument
179 struct nvm_tgt_dev *dev = pblk->dev; in pblk_rwb_init()
193 return pblk_rb_init(&pblk->rwb, buffer_size, threshold, geo->csecs); in pblk_rwb_init()
196 static int pblk_set_addrf_12(struct pblk *pblk, struct nvm_geo *geo, in pblk_set_addrf_12() argument
205 pblk_err(pblk, "supports only power-of-two channel config.\n"); in pblk_set_addrf_12()
212 pblk_err(pblk, "supports only power-of-two LUN config.\n"); in pblk_set_addrf_12()
269 static int pblk_set_addrf(struct pblk *pblk) in pblk_set_addrf() argument
271 struct nvm_tgt_dev *dev = pblk->dev; in pblk_set_addrf()
277 div_u64_rem(geo->clba, pblk->min_write_pgs, &mod); in pblk_set_addrf()
279 pblk_err(pblk, "bad configuration of sectors/pages\n"); in pblk_set_addrf()
283 pblk->addrf_len = pblk_set_addrf_12(pblk, geo, in pblk_set_addrf()
284 (void *)&pblk->addrf); in pblk_set_addrf()
287 pblk->addrf_len = pblk_set_addrf_20(geo, (void *)&pblk->addrf, in pblk_set_addrf()
288 &pblk->uaddrf); in pblk_set_addrf()
291 pblk_err(pblk, "OCSSD revision not supported (%d)\n", in pblk_set_addrf()
371 static int pblk_core_init(struct pblk *pblk) in pblk_core_init() argument
373 struct nvm_tgt_dev *dev = pblk->dev; in pblk_core_init()
377 atomic64_set(&pblk->user_wa, 0); in pblk_core_init()
378 atomic64_set(&pblk->pad_wa, 0); in pblk_core_init()
379 atomic64_set(&pblk->gc_wa, 0); in pblk_core_init()
380 pblk->user_rst_wa = 0; in pblk_core_init()
381 pblk->pad_rst_wa = 0; in pblk_core_init()
382 pblk->gc_rst_wa = 0; in pblk_core_init()
384 atomic64_set(&pblk->nr_flush, 0); in pblk_core_init()
385 pblk->nr_flush_rst = 0; in pblk_core_init()
387 pblk->min_write_pgs = geo->ws_opt; in pblk_core_init()
388 pblk->min_write_pgs_data = pblk->min_write_pgs; in pblk_core_init()
389 max_write_ppas = pblk->min_write_pgs * geo->all_luns; in pblk_core_init()
390 pblk->max_write_pgs = min_t(int, max_write_ppas, NVM_MAX_VLBA); in pblk_core_init()
391 pblk->max_write_pgs = min_t(int, pblk->max_write_pgs, in pblk_core_init()
393 pblk_set_sec_per_write(pblk, pblk->min_write_pgs); in pblk_core_init()
395 pblk->oob_meta_size = geo->sos; in pblk_core_init()
396 if (!pblk_is_oob_meta_supported(pblk)) { in pblk_core_init()
404 if (pblk->min_write_pgs in pblk_core_init()
414 pblk_err(pblk, "Not supported min write size\n"); in pblk_core_init()
423 pblk->max_write_pgs = pblk->min_write_pgs; in pblk_core_init()
424 pblk->min_write_pgs_data = pblk->min_write_pgs - 1; in pblk_core_init()
427 pblk->pad_dist = kcalloc(pblk->min_write_pgs - 1, sizeof(atomic64_t), in pblk_core_init()
429 if (!pblk->pad_dist) in pblk_core_init()
436 ret = mempool_init_page_pool(&pblk->page_bio_pool, NVM_MAX_VLBA, 0); in pblk_core_init()
440 ret = mempool_init_slab_pool(&pblk->gen_ws_pool, PBLK_GEN_WS_POOL_SIZE, in pblk_core_init()
445 ret = mempool_init_slab_pool(&pblk->rec_pool, geo->all_luns, in pblk_core_init()
450 ret = mempool_init_slab_pool(&pblk->r_rq_pool, geo->all_luns, in pblk_core_init()
455 ret = mempool_init_slab_pool(&pblk->e_rq_pool, geo->all_luns, in pblk_core_init()
460 ret = mempool_init_slab_pool(&pblk->w_rq_pool, geo->all_luns, in pblk_core_init()
465 pblk->close_wq = alloc_workqueue("pblk-close-wq", in pblk_core_init()
467 if (!pblk->close_wq) in pblk_core_init()
470 pblk->bb_wq = alloc_workqueue("pblk-bb-wq", in pblk_core_init()
472 if (!pblk->bb_wq) in pblk_core_init()
475 pblk->r_end_wq = alloc_workqueue("pblk-read-end-wq", in pblk_core_init()
477 if (!pblk->r_end_wq) in pblk_core_init()
480 if (pblk_set_addrf(pblk)) in pblk_core_init()
483 INIT_LIST_HEAD(&pblk->compl_list); in pblk_core_init()
484 INIT_LIST_HEAD(&pblk->resubmit_list); in pblk_core_init()
489 destroy_workqueue(pblk->r_end_wq); in pblk_core_init()
491 destroy_workqueue(pblk->bb_wq); in pblk_core_init()
493 destroy_workqueue(pblk->close_wq); in pblk_core_init()
495 mempool_exit(&pblk->w_rq_pool); in pblk_core_init()
497 mempool_exit(&pblk->e_rq_pool); in pblk_core_init()
499 mempool_exit(&pblk->r_rq_pool); in pblk_core_init()
501 mempool_exit(&pblk->rec_pool); in pblk_core_init()
503 mempool_exit(&pblk->gen_ws_pool); in pblk_core_init()
505 mempool_exit(&pblk->page_bio_pool); in pblk_core_init()
509 kfree(pblk->pad_dist); in pblk_core_init()
513 static void pblk_core_free(struct pblk *pblk) in pblk_core_free() argument
515 if (pblk->close_wq) in pblk_core_free()
516 destroy_workqueue(pblk->close_wq); in pblk_core_free()
518 if (pblk->r_end_wq) in pblk_core_free()
519 destroy_workqueue(pblk->r_end_wq); in pblk_core_free()
521 if (pblk->bb_wq) in pblk_core_free()
522 destroy_workqueue(pblk->bb_wq); in pblk_core_free()
524 mempool_exit(&pblk->page_bio_pool); in pblk_core_free()
525 mempool_exit(&pblk->gen_ws_pool); in pblk_core_free()
526 mempool_exit(&pblk->rec_pool); in pblk_core_free()
527 mempool_exit(&pblk->r_rq_pool); in pblk_core_free()
528 mempool_exit(&pblk->e_rq_pool); in pblk_core_free()
529 mempool_exit(&pblk->w_rq_pool); in pblk_core_free()
532 kfree(pblk->pad_dist); in pblk_core_free()
535 static void pblk_line_mg_free(struct pblk *pblk) in pblk_line_mg_free() argument
537 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_mg_free()
567 static void pblk_lines_free(struct pblk *pblk) in pblk_lines_free() argument
569 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_lines_free()
574 line = &pblk->lines[i]; in pblk_lines_free()
580 pblk_line_mg_free(pblk); in pblk_lines_free()
582 kfree(pblk->luns); in pblk_lines_free()
583 kfree(pblk->lines); in pblk_lines_free()
586 static int pblk_luns_init(struct pblk *pblk) in pblk_luns_init() argument
588 struct nvm_tgt_dev *dev = pblk->dev; in pblk_luns_init()
595 pblk_err(pblk, "unbalanced LUN config.\n"); in pblk_luns_init()
599 pblk->luns = kcalloc(geo->all_luns, sizeof(struct pblk_lun), in pblk_luns_init()
601 if (!pblk->luns) in pblk_luns_init()
610 rlun = &pblk->luns[i]; in pblk_luns_init()
620 static unsigned int calc_emeta_len(struct pblk *pblk) in calc_emeta_len() argument
622 struct pblk_line_meta *lm = &pblk->lm; in calc_emeta_len()
623 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in calc_emeta_len()
624 struct nvm_tgt_dev *dev = pblk->dev; in calc_emeta_len()
648 static int pblk_set_provision(struct pblk *pblk, int nr_free_chks) in pblk_set_provision() argument
650 struct nvm_tgt_dev *dev = pblk->dev; in pblk_set_provision()
651 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_set_provision()
652 struct pblk_line_meta *lm = &pblk->lm; in pblk_set_provision()
659 pblk->op = PBLK_DEFAULT_OP; in pblk_set_provision()
661 pblk->op = geo->op; in pblk_set_provision()
663 minimum = pblk_get_min_chks(pblk); in pblk_set_provision()
665 provisioned *= (100 - pblk->op); in pblk_set_provision()
670 pblk_err(pblk, "OP too small to create a sane instance\n"); in pblk_set_provision()
679 pblk->op = (100 * minimum) / nr_free_chks; in pblk_set_provision()
680 pblk_info(pblk, "Default OP insufficient, adjusting OP to %d\n", in pblk_set_provision()
681 pblk->op); in pblk_set_provision()
684 pblk->op_blks = nr_free_chks - provisioned; in pblk_set_provision()
689 pblk->rl.total_blocks = nr_free_chks; in pblk_set_provision()
695 clba = (geo->clba / pblk->min_write_pgs) * pblk->min_write_pgs_data; in pblk_set_provision()
696 pblk->capacity = (provisioned - blk_meta) * clba; in pblk_set_provision()
698 atomic_set(&pblk->rl.free_blocks, nr_free_chks); in pblk_set_provision()
699 atomic_set(&pblk->rl.free_user_blocks, nr_free_chks); in pblk_set_provision()
704 static int pblk_setup_line_meta_chk(struct pblk *pblk, struct pblk_line *line, in pblk_setup_line_meta_chk() argument
707 struct nvm_tgt_dev *dev = pblk->dev; in pblk_setup_line_meta_chk()
709 struct pblk_line_meta *lm = &pblk->lm; in pblk_setup_line_meta_chk()
713 struct pblk_lun *rlun = &pblk->luns[i]; in pblk_setup_line_meta_chk()
724 chunk_meta = pblk_chunk_get_off(pblk, meta, ppa); in pblk_setup_line_meta_chk()
733 trace_pblk_chunk_state(pblk_disk_name(pblk), &ppa, in pblk_setup_line_meta_chk()
751 static long pblk_setup_line_meta(struct pblk *pblk, struct pblk_line *line, in pblk_setup_line_meta() argument
754 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_setup_line_meta()
755 struct pblk_line_meta *lm = &pblk->lm; in pblk_setup_line_meta()
758 line->pblk = pblk; in pblk_setup_line_meta()
766 nr_bad_chks = pblk_setup_line_meta_chk(pblk, line, chunk_meta); in pblk_setup_line_meta()
783 static int pblk_alloc_line_meta(struct pblk *pblk, struct pblk_line *line) in pblk_alloc_line_meta() argument
785 struct pblk_line_meta *lm = &pblk->lm; in pblk_alloc_line_meta()
816 static int pblk_line_mg_init(struct pblk *pblk) in pblk_line_mg_init() argument
818 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_mg_init()
820 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_line_mg_init()
821 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_mg_init()
933 static int pblk_line_meta_init(struct pblk *pblk) in pblk_line_meta_init() argument
935 struct nvm_tgt_dev *dev = pblk->dev; in pblk_line_meta_init()
937 struct pblk_line_meta *lm = &pblk->lm; in pblk_line_meta_init()
948 lm->meta_distance = (geo->all_luns / 2) * pblk->min_write_pgs; in pblk_line_meta_init()
972 emeta_len = calc_emeta_len(pblk); in pblk_line_meta_init()
986 pblk_err(pblk, "config. not supported. Min. LUN in line:%d\n", in pblk_line_meta_init()
994 static int pblk_lines_init(struct pblk *pblk) in pblk_lines_init() argument
996 struct pblk_line_mgmt *l_mg = &pblk->l_mg; in pblk_lines_init()
1002 ret = pblk_line_meta_init(pblk); in pblk_lines_init()
1006 ret = pblk_line_mg_init(pblk); in pblk_lines_init()
1010 ret = pblk_luns_init(pblk); in pblk_lines_init()
1014 chunk_meta = pblk_get_chunk_meta(pblk); in pblk_lines_init()
1020 pblk->lines = kcalloc(l_mg->nr_lines, sizeof(struct pblk_line), in pblk_lines_init()
1022 if (!pblk->lines) { in pblk_lines_init()
1028 line = &pblk->lines[i]; in pblk_lines_init()
1030 ret = pblk_alloc_line_meta(pblk, line); in pblk_lines_init()
1034 nr_free_chks += pblk_setup_line_meta(pblk, line, chunk_meta, i); in pblk_lines_init()
1036 trace_pblk_line_state(pblk_disk_name(pblk), line->id, in pblk_lines_init()
1041 pblk_err(pblk, "too many bad blocks prevent for sane instance\n"); in pblk_lines_init()
1046 ret = pblk_set_provision(pblk, nr_free_chks); in pblk_lines_init()
1055 pblk_line_meta_free(l_mg, &pblk->lines[i]); in pblk_lines_init()
1056 kfree(pblk->lines); in pblk_lines_init()
1060 kfree(pblk->luns); in pblk_lines_init()
1062 pblk_line_mg_free(pblk); in pblk_lines_init()
1067 static int pblk_writer_init(struct pblk *pblk) in pblk_writer_init() argument
1069 pblk->writer_ts = kthread_create(pblk_write_ts, pblk, "pblk-writer-t"); in pblk_writer_init()
1070 if (IS_ERR(pblk->writer_ts)) { in pblk_writer_init()
1071 int err = PTR_ERR(pblk->writer_ts); in pblk_writer_init()
1074 pblk_err(pblk, "could not allocate writer kthread (%d)\n", in pblk_writer_init()
1079 timer_setup(&pblk->wtimer, pblk_write_timer_fn, 0); in pblk_writer_init()
1080 mod_timer(&pblk->wtimer, jiffies + msecs_to_jiffies(100)); in pblk_writer_init()
1085 static void pblk_writer_stop(struct pblk *pblk) in pblk_writer_stop() argument
1090 WARN(pblk_rb_read_count(&pblk->rwb), in pblk_writer_stop()
1093 WARN(pblk_rb_sync_count(&pblk->rwb), in pblk_writer_stop()
1096 del_timer_sync(&pblk->wtimer); in pblk_writer_stop()
1097 if (pblk->writer_ts) in pblk_writer_stop()
1098 kthread_stop(pblk->writer_ts); in pblk_writer_stop()
1101 static void pblk_free(struct pblk *pblk) in pblk_free() argument
1103 pblk_lines_free(pblk); in pblk_free()
1104 pblk_l2p_free(pblk); in pblk_free()
1105 pblk_rwb_free(pblk); in pblk_free()
1106 pblk_core_free(pblk); in pblk_free()
1108 kfree(pblk); in pblk_free()
1111 static void pblk_tear_down(struct pblk *pblk, bool graceful) in pblk_tear_down() argument
1114 __pblk_pipeline_flush(pblk); in pblk_tear_down()
1115 __pblk_pipeline_stop(pblk); in pblk_tear_down()
1116 pblk_writer_stop(pblk); in pblk_tear_down()
1117 pblk_rb_sync_l2p(&pblk->rwb); in pblk_tear_down()
1118 pblk_rl_free(&pblk->rl); in pblk_tear_down()
1120 pblk_debug(pblk, "consistent tear down (graceful:%d)\n", graceful); in pblk_tear_down()
1125 struct pblk *pblk = private; in pblk_exit() local
1127 pblk_gc_exit(pblk, graceful); in pblk_exit()
1128 pblk_tear_down(pblk, graceful); in pblk_exit()
1131 pblk_info(pblk, "exit: L2P CRC: %x\n", pblk_l2p_crc(pblk)); in pblk_exit()
1134 pblk_free(pblk); in pblk_exit()
1139 struct pblk *pblk = private; in pblk_capacity() local
1141 return pblk->capacity * NR_PHY_IN_LOG; in pblk_capacity()
1150 struct pblk *pblk; in pblk_init() local
1153 pblk = kzalloc(sizeof(struct pblk), GFP_KERNEL); in pblk_init()
1154 if (!pblk) in pblk_init()
1157 pblk->dev = dev; in pblk_init()
1158 pblk->disk = tdisk; in pblk_init()
1159 pblk->state = PBLK_STATE_RUNNING; in pblk_init()
1160 trace_pblk_state(pblk_disk_name(pblk), pblk->state); in pblk_init()
1161 pblk->gc.gc_enabled = 0; in pblk_init()
1165 pblk_err(pblk, "OCSSD version not supported (%u)\n", in pblk_init()
1167 kfree(pblk); in pblk_init()
1172 pblk_err(pblk, "extended metadata not supported\n"); in pblk_init()
1173 kfree(pblk); in pblk_init()
1177 spin_lock_init(&pblk->resubmit_lock); in pblk_init()
1178 spin_lock_init(&pblk->trans_lock); in pblk_init()
1179 spin_lock_init(&pblk->lock); in pblk_init()
1182 atomic_long_set(&pblk->inflight_writes, 0); in pblk_init()
1183 atomic_long_set(&pblk->padded_writes, 0); in pblk_init()
1184 atomic_long_set(&pblk->padded_wb, 0); in pblk_init()
1185 atomic_long_set(&pblk->req_writes, 0); in pblk_init()
1186 atomic_long_set(&pblk->sub_writes, 0); in pblk_init()
1187 atomic_long_set(&pblk->sync_writes, 0); in pblk_init()
1188 atomic_long_set(&pblk->inflight_reads, 0); in pblk_init()
1189 atomic_long_set(&pblk->cache_reads, 0); in pblk_init()
1190 atomic_long_set(&pblk->sync_reads, 0); in pblk_init()
1191 atomic_long_set(&pblk->recov_writes, 0); in pblk_init()
1192 atomic_long_set(&pblk->recov_writes, 0); in pblk_init()
1193 atomic_long_set(&pblk->recov_gc_writes, 0); in pblk_init()
1194 atomic_long_set(&pblk->recov_gc_reads, 0); in pblk_init()
1197 atomic_long_set(&pblk->read_failed, 0); in pblk_init()
1198 atomic_long_set(&pblk->read_empty, 0); in pblk_init()
1199 atomic_long_set(&pblk->read_high_ecc, 0); in pblk_init()
1200 atomic_long_set(&pblk->read_failed_gc, 0); in pblk_init()
1201 atomic_long_set(&pblk->write_failed, 0); in pblk_init()
1202 atomic_long_set(&pblk->erase_failed, 0); in pblk_init()
1204 ret = pblk_core_init(pblk); in pblk_init()
1206 pblk_err(pblk, "could not initialize core\n"); in pblk_init()
1210 ret = pblk_lines_init(pblk); in pblk_init()
1212 pblk_err(pblk, "could not initialize lines\n"); in pblk_init()
1216 ret = pblk_rwb_init(pblk); in pblk_init()
1218 pblk_err(pblk, "could not initialize write buffer\n"); in pblk_init()
1222 ret = pblk_l2p_init(pblk, flags & NVM_TARGET_FACTORY); in pblk_init()
1224 pblk_err(pblk, "could not initialize maps\n"); in pblk_init()
1228 ret = pblk_writer_init(pblk); in pblk_init()
1231 pblk_err(pblk, "could not initialize write thread\n"); in pblk_init()
1235 ret = pblk_gc_init(pblk); in pblk_init()
1237 pblk_err(pblk, "could not initialize gc\n"); in pblk_init()
1252 pblk_info(pblk, "luns:%u, lines:%d, secs:%llu, buf entries:%u\n", in pblk_init()
1253 geo->all_luns, pblk->l_mg.nr_lines, in pblk_init()
1254 (unsigned long long)pblk->capacity, in pblk_init()
1255 pblk->rwb.nr_entries); in pblk_init()
1257 wake_up_process(pblk->writer_ts); in pblk_init()
1260 pblk_gc_should_kick(pblk); in pblk_init()
1262 return pblk; in pblk_init()
1265 pblk_writer_stop(pblk); in pblk_init()
1267 pblk_l2p_free(pblk); in pblk_init()
1269 pblk_rwb_free(pblk); in pblk_init()
1271 pblk_lines_free(pblk); in pblk_init()
1273 pblk_core_free(pblk); in pblk_init()
1275 kfree(pblk); in pblk_init()