122 	struct mem_ctl_info *mci = dimm->mci;  in edac_dimm_info_location()  local
126 	for (i = 0; i < mci->n_layers; i++) {  in edac_dimm_info_location()
128 			      edac_layer_name[mci->layers[i].type],  in edac_dimm_info_location()
157 		 dimm->mci->csbased ? "rank" : "dimm",  in edac_mc_dump_dimm()
175 	edac_dbg(4, "  csrow->mci = %p\n", csrow->mci);  in edac_mc_dump_csrow()
178 static void edac_mc_dump_mci(struct mem_ctl_info *mci)  in edac_mc_dump_mci()  argument
180 	edac_dbg(3, "\tmci = %p\n", mci);  in edac_mc_dump_mci()
181 	edac_dbg(3, "\tmci->mtype_cap = %lx\n", mci->mtype_cap);  in edac_mc_dump_mci()
182 	edac_dbg(3, "\tmci->edac_ctl_cap = %lx\n", mci->edac_ctl_cap);  in edac_mc_dump_mci()
183 	edac_dbg(3, "\tmci->edac_cap = %lx\n", mci->edac_cap);  in edac_mc_dump_mci()
184 	edac_dbg(4, "\tmci->edac_check = %p\n", mci->edac_check);  in edac_mc_dump_mci()
186 		 mci->nr_csrows, mci->csrows);  in edac_mc_dump_mci()
188 		 mci->tot_dimms, mci->dimms);  in edac_mc_dump_mci()
189 	edac_dbg(3, "\tdev = %p\n", mci->pdev);  in edac_mc_dump_mci()
191 		 mci->mod_name, mci->ctl_name);  in edac_mc_dump_mci()
192 	edac_dbg(3, "\tpvt_info = %p\n\n", mci->pvt_info);  in edac_mc_dump_mci()
278 static void _edac_mc_free(struct mem_ctl_info *mci)  in _edac_mc_free()  argument
282 	const unsigned int tot_dimms = mci->tot_dimms;  in _edac_mc_free()
283 	const unsigned int tot_channels = mci->num_cschannel;  in _edac_mc_free()
284 	const unsigned int tot_csrows = mci->nr_csrows;  in _edac_mc_free()
286 	if (mci->dimms) {  in _edac_mc_free()
288 			kfree(mci->dimms[i]);  in _edac_mc_free()
289 		kfree(mci->dimms);  in _edac_mc_free()
291 	if (mci->csrows) {  in _edac_mc_free()
293 			csr = mci->csrows[row];  in _edac_mc_free()
303 		kfree(mci->csrows);  in _edac_mc_free()
305 	kfree(mci);  in _edac_mc_free()
313 	struct mem_ctl_info *mci;  in edac_mc_alloc()  local
347 	mci = edac_align_ptr(&ptr, sizeof(*mci), 1);  in edac_mc_alloc()
367 	mci = kzalloc(size, GFP_KERNEL);  in edac_mc_alloc()
368 	if (mci == NULL)  in edac_mc_alloc()
374 	layer = (struct edac_mc_layer *)(((char *)mci) + ((unsigned long)layer));  in edac_mc_alloc()
376 		mci->ce_per_layer[i] = (u32 *)((char *)mci + ((unsigned long)ce_per_layer[i]));  in edac_mc_alloc()
377 		mci->ue_per_layer[i] = (u32 *)((char *)mci + ((unsigned long)ue_per_layer[i]));  in edac_mc_alloc()
379 	pvt = sz_pvt ? (((char *)mci) + ((unsigned long)pvt)) : NULL;  in edac_mc_alloc()
382 	mci->mc_idx = mc_num;  in edac_mc_alloc()
383 	mci->tot_dimms = tot_dimms;  in edac_mc_alloc()
384 	mci->pvt_info = pvt;  in edac_mc_alloc()
385 	mci->n_layers = n_layers;  in edac_mc_alloc()
386 	mci->layers = layer;  in edac_mc_alloc()
387 	memcpy(mci->layers, layers, sizeof(*layer) * n_layers);  in edac_mc_alloc()
388 	mci->nr_csrows = tot_csrows;  in edac_mc_alloc()
389 	mci->num_cschannel = tot_channels;  in edac_mc_alloc()
390 	mci->csbased = per_rank;  in edac_mc_alloc()
395 	mci->csrows = kcalloc(tot_csrows, sizeof(*mci->csrows), GFP_KERNEL);  in edac_mc_alloc()
396 	if (!mci->csrows)  in edac_mc_alloc()
399 		csr = kzalloc(sizeof(**mci->csrows), GFP_KERNEL);  in edac_mc_alloc()
402 		mci->csrows[row] = csr;  in edac_mc_alloc()
404 		csr->mci = mci;  in edac_mc_alloc()
424 	mci->dimms  = kcalloc(tot_dimms, sizeof(*mci->dimms), GFP_KERNEL);  in edac_mc_alloc()
425 	if (!mci->dimms)  in edac_mc_alloc()
432 		chan = mci->csrows[row]->channels[chn];  in edac_mc_alloc()
435 …edac_mc_printk(mci, KERN_ERR, "EDAC core bug: EDAC_DIMM_OFF is trying to do an illegal data access…  in edac_mc_alloc()
439 		dimm = kzalloc(sizeof(**mci->dimms), GFP_KERNEL);  in edac_mc_alloc()
442 		mci->dimms[off] = dimm;  in edac_mc_alloc()
443 		dimm->mci = mci;  in edac_mc_alloc()
494 	mci->op_state = OP_ALLOC;  in edac_mc_alloc()
496 	return mci;  in edac_mc_alloc()
499 	_edac_mc_free(mci);  in edac_mc_alloc()
505 void edac_mc_free(struct mem_ctl_info *mci)  in edac_mc_free()  argument
512 	if (!device_is_registered(&mci->dev)) {  in edac_mc_free()
513 		_edac_mc_free(mci);  in edac_mc_free()
518 	edac_unregister_sysfs(mci);  in edac_mc_free()
539 	struct mem_ctl_info *mci;  in __find_mci_by_dev()  local
545 		mci = list_entry(item, struct mem_ctl_info, link);  in __find_mci_by_dev()
547 		if (mci->pdev == dev)  in __find_mci_by_dev()
548 			return mci;  in __find_mci_by_dev()
580 	struct mem_ctl_info *mci = to_edac_mem_ctl_work(d_work);  in edac_mc_workq_function()  local
584 	if (mci->op_state != OP_RUNNING_POLL) {  in edac_mc_workq_function()
590 		mci->edac_check(mci);  in edac_mc_workq_function()
595 	edac_queue_work(&mci->work, msecs_to_jiffies(edac_mc_get_poll_msec()));  in edac_mc_workq_function()
606 	struct mem_ctl_info *mci;  in edac_mc_reset_delay_period()  local
612 		mci = list_entry(item, struct mem_ctl_info, link);  in edac_mc_reset_delay_period()
614 		if (mci->op_state == OP_RUNNING_POLL)  in edac_mc_reset_delay_period()
615 			edac_mod_work(&mci->work, value);  in edac_mc_reset_delay_period()
630 static int add_mc_to_global_list(struct mem_ctl_info *mci)  in add_mc_to_global_list()  argument
637 	p = __find_mci_by_dev(mci->pdev);  in add_mc_to_global_list()
644 		if (p->mc_idx >= mci->mc_idx) {  in add_mc_to_global_list()
645 			if (unlikely(p->mc_idx == mci->mc_idx))  in add_mc_to_global_list()
653 	list_add_tail_rcu(&mci->link, insert_before);  in add_mc_to_global_list()
659 		edac_dev_name(mci), p->mod_name, p->ctl_name, p->mc_idx);  in add_mc_to_global_list()
669 static int del_mc_from_global_list(struct mem_ctl_info *mci)  in del_mc_from_global_list()  argument
671 	list_del_rcu(&mci->link);  in del_mc_from_global_list()
677 	INIT_LIST_HEAD(&mci->link);  in del_mc_from_global_list()
684 	struct mem_ctl_info *mci = NULL;  in edac_mc_find()  local
690 		mci = list_entry(item, struct mem_ctl_info, link);  in edac_mc_find()
692 		if (mci->mc_idx >= idx) {  in edac_mc_find()
693 			if (mci->mc_idx == idx) {  in edac_mc_find()
702 	return mci;  in edac_mc_find()
713 int edac_mc_add_mc_with_groups(struct mem_ctl_info *mci,  in edac_mc_add_mc_with_groups()  argument
719 	if (mci->mc_idx >= EDAC_MAX_MCS) {  in edac_mc_add_mc_with_groups()
720 		pr_warn_once("Too many memory controllers: %d\n", mci->mc_idx);  in edac_mc_add_mc_with_groups()
726 		edac_mc_dump_mci(mci);  in edac_mc_add_mc_with_groups()
731 		for (i = 0; i < mci->nr_csrows; i++) {  in edac_mc_add_mc_with_groups()
732 			struct csrow_info *csrow = mci->csrows[i];  in edac_mc_add_mc_with_groups()
745 		for (i = 0; i < mci->tot_dimms; i++)  in edac_mc_add_mc_with_groups()
746 			if (mci->dimms[i]->nr_pages)  in edac_mc_add_mc_with_groups()
747 				edac_mc_dump_dimm(mci->dimms[i], i);  in edac_mc_add_mc_with_groups()
752 	if (edac_mc_owner && edac_mc_owner != mci->mod_name) {  in edac_mc_add_mc_with_groups()
757 	if (add_mc_to_global_list(mci))  in edac_mc_add_mc_with_groups()
761 	mci->start_time = jiffies;  in edac_mc_add_mc_with_groups()
763 	mci->bus = &mc_bus[mci->mc_idx];  in edac_mc_add_mc_with_groups()
765 	if (edac_create_sysfs_mci_device(mci, groups)) {  in edac_mc_add_mc_with_groups()
766 		edac_mc_printk(mci, KERN_WARNING,  in edac_mc_add_mc_with_groups()
771 	if (mci->edac_check) {  in edac_mc_add_mc_with_groups()
772 		mci->op_state = OP_RUNNING_POLL;  in edac_mc_add_mc_with_groups()
774 		INIT_DELAYED_WORK(&mci->work, edac_mc_workq_function);  in edac_mc_add_mc_with_groups()
775 		edac_queue_work(&mci->work, msecs_to_jiffies(edac_mc_get_poll_msec()));  in edac_mc_add_mc_with_groups()
778 		mci->op_state = OP_RUNNING_INTERRUPT;  in edac_mc_add_mc_with_groups()
782 	edac_mc_printk(mci, KERN_INFO,  in edac_mc_add_mc_with_groups()
784 		mci->mod_name, mci->ctl_name, mci->dev_name,  in edac_mc_add_mc_with_groups()
785 		edac_op_state_to_string(mci->op_state));  in edac_mc_add_mc_with_groups()
787 	edac_mc_owner = mci->mod_name;  in edac_mc_add_mc_with_groups()
793 	del_mc_from_global_list(mci);  in edac_mc_add_mc_with_groups()
803 	struct mem_ctl_info *mci;  in edac_mc_del_mc()  local
810 	mci = __find_mci_by_dev(dev);  in edac_mc_del_mc()
811 	if (mci == NULL) {  in edac_mc_del_mc()
817 	mci->op_state = OP_OFFLINE;  in edac_mc_del_mc()
819 	if (del_mc_from_global_list(mci))  in edac_mc_del_mc()
824 	if (mci->edac_check)  in edac_mc_del_mc()
825 		edac_stop_work(&mci->work);  in edac_mc_del_mc()
828 	edac_remove_sysfs_mci_device(mci);  in edac_mc_del_mc()
831 		"Removed device %d for %s %s: DEV %s\n", mci->mc_idx,  in edac_mc_del_mc()
832 		mci->mod_name, mci->ctl_name, edac_dev_name(mci));  in edac_mc_del_mc()
834 	return mci;  in edac_mc_del_mc()
870 int edac_mc_find_csrow_by_page(struct mem_ctl_info *mci, unsigned long page)  in edac_mc_find_csrow_by_page()  argument
872 	struct csrow_info **csrows = mci->csrows;  in edac_mc_find_csrow_by_page()
875 	edac_dbg(1, "MC%d: 0x%lx\n", mci->mc_idx, page);  in edac_mc_find_csrow_by_page()
878 	for (i = 0; i < mci->nr_csrows; i++) {  in edac_mc_find_csrow_by_page()
889 			 mci->mc_idx,  in edac_mc_find_csrow_by_page()
903 		edac_mc_printk(mci, KERN_ERR,  in edac_mc_find_csrow_by_page()
920 static void edac_inc_ce_error(struct mem_ctl_info *mci,  in edac_inc_ce_error()  argument
927 	mci->ce_mc += count;  in edac_inc_ce_error()
930 		mci->ce_noinfo_count += count;  in edac_inc_ce_error()
934 	for (i = 0; i < mci->n_layers; i++) {  in edac_inc_ce_error()
938 		mci->ce_per_layer[i][index] += count;  in edac_inc_ce_error()
940 		if (i < mci->n_layers - 1)  in edac_inc_ce_error()
941 			index *= mci->layers[i + 1].size;  in edac_inc_ce_error()
945 static void edac_inc_ue_error(struct mem_ctl_info *mci,  in edac_inc_ue_error()  argument
952 	mci->ue_mc += count;  in edac_inc_ue_error()
955 		mci->ue_noinfo_count += count;  in edac_inc_ue_error()
959 	for (i = 0; i < mci->n_layers; i++) {  in edac_inc_ue_error()
963 		mci->ue_per_layer[i][index] += count;  in edac_inc_ue_error()
965 		if (i < mci->n_layers - 1)  in edac_inc_ue_error()
966 			index *= mci->layers[i + 1].size;  in edac_inc_ue_error()
970 static void edac_ce_error(struct mem_ctl_info *mci,  in edac_ce_error()  argument
991 			edac_mc_printk(mci, KERN_WARNING,  in edac_ce_error()
996 			edac_mc_printk(mci, KERN_WARNING,  in edac_ce_error()
1001 	edac_inc_ce_error(mci, enable_per_layer_report, pos, error_count);  in edac_ce_error()
1003 	if (mci->scrub_mode == SCRUB_SW_SRC) {  in edac_ce_error()
1015 		remapped_page = mci->ctl_page_to_phys ?  in edac_ce_error()
1016 			mci->ctl_page_to_phys(mci, page_frame_number) :  in edac_ce_error()
1024 static void edac_ue_error(struct mem_ctl_info *mci,  in edac_ue_error()  argument
1041 			edac_mc_printk(mci, KERN_WARNING,  in edac_ue_error()
1046 			edac_mc_printk(mci, KERN_WARNING,  in edac_ue_error()
1061 	edac_inc_ue_error(mci, enable_per_layer_report, pos, error_count);  in edac_ue_error()
1065 			      struct mem_ctl_info *mci,  in edac_raw_mc_handle_error()  argument
1077 		edac_ce_error(mci, e->error_count, pos, e->msg, e->location, e->label,  in edac_raw_mc_handle_error()
1085 		edac_ue_error(mci, e->error_count, pos, e->msg, e->location, e->label,  in edac_raw_mc_handle_error()
1094 			  struct mem_ctl_info *mci,  in edac_mc_handle_error()  argument
1110 	struct edac_raw_error_desc *e = &mci->error_desc;  in edac_mc_handle_error()
1112 	edac_dbg(3, "MC%d\n", mci->mc_idx);  in edac_mc_handle_error()
1132 	for (i = 0; i < mci->n_layers; i++) {  in edac_mc_handle_error()
1133 		if (pos[i] >= (int)mci->layers[i].size) {  in edac_mc_handle_error()
1135 			edac_mc_printk(mci, KERN_ERR,  in edac_mc_handle_error()
1137 				       edac_layer_name[mci->layers[i].type],  in edac_mc_handle_error()
1138 				       pos[i], mci->layers[i].size);  in edac_mc_handle_error()
1165 	for (i = 0; i < mci->tot_dimms; i++) {  in edac_mc_handle_error()
1166 		struct dimm_info *dimm = mci->dimms[i];  in edac_mc_handle_error()
1204 				 mci->csbased ? "rank" : "dimm",  in edac_mc_handle_error()
1226 				mci->csrows[row]->ce_count += error_count;  in edac_mc_handle_error()
1228 					mci->csrows[row]->channels[chan]->ce_count += error_count;  in edac_mc_handle_error()
1232 				mci->csrows[row]->ue_count += error_count;  in edac_mc_handle_error()
1238 	for (i = 0; i < mci->n_layers; i++) {  in edac_mc_handle_error()
1243 			     edac_layer_name[mci->layers[i].type],  in edac_mc_handle_error()
1254 			       mci->mc_idx, e->top_layer, e->mid_layer,  in edac_mc_handle_error()
1259 	edac_raw_mc_handle_error(type, mci, e);  in edac_mc_handle_error()