98 static int switchtec_ntb_part_op(struct switchtec_ntb *sndev,  in switchtec_ntb_part_op()  argument
144 		dev_err(&sndev->stdev->dev,  in switchtec_ntb_part_op()
155 static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx,  in switchtec_ntb_send_msg()  argument
158 	if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg))  in switchtec_ntb_send_msg()
161 	iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg);  in switchtec_ntb_send_msg()
168 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_mw_count()  local
169 	int nr_direct_mw = sndev->peer_nr_direct_mw;  in switchtec_ntb_mw_count()
170 	int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts;  in switchtec_ntb_mw_count()
181 static int lut_index(struct switchtec_ntb *sndev, int mw_idx)  in lut_index()  argument
183 	return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts;  in lut_index()
186 static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx)  in peer_lut_index()  argument
188 	return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts;  in peer_lut_index()
196 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_mw_get_align()  local
203 	lut = widx >= sndev->peer_nr_direct_mw;  in switchtec_ntb_mw_get_align()
204 	size = ioread64(&sndev->peer_shared->mw_sizes[widx]);  in switchtec_ntb_mw_get_align()
221 static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx)  in switchtec_ntb_mw_clr_direct()  argument
223 	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;  in switchtec_ntb_mw_clr_direct()
224 	int bar = sndev->peer_direct_mw_to_bar[idx];  in switchtec_ntb_mw_clr_direct()
232 	iowrite64(sndev->self_partition, &ctl->bar_entry[bar].xlate_addr);  in switchtec_ntb_mw_clr_direct()
235 static void switchtec_ntb_mw_clr_lut(struct switchtec_ntb *sndev, int idx)  in switchtec_ntb_mw_clr_lut()  argument
237 	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;  in switchtec_ntb_mw_clr_lut()
239 	iowrite64(0, &ctl->lut_entry[peer_lut_index(sndev, idx)]);  in switchtec_ntb_mw_clr_lut()
242 static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx,  in switchtec_ntb_mw_set_direct()  argument
246 	int bar = sndev->peer_direct_mw_to_bar[idx];  in switchtec_ntb_mw_set_direct()
247 	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;  in switchtec_ntb_mw_set_direct()
257 	iowrite64(sndev->self_partition | addr,  in switchtec_ntb_mw_set_direct()
261 static void switchtec_ntb_mw_set_lut(struct switchtec_ntb *sndev, int idx,  in switchtec_ntb_mw_set_lut()  argument
264 	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;  in switchtec_ntb_mw_set_lut()
266 	iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | addr),  in switchtec_ntb_mw_set_lut()
267 		  &ctl->lut_entry[peer_lut_index(sndev, idx)]);  in switchtec_ntb_mw_set_lut()
273 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_mw_set_trans()  local
274 	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;  in switchtec_ntb_mw_set_trans()
276 	int nr_direct_mw = sndev->peer_nr_direct_mw;  in switchtec_ntb_mw_set_trans()
282 	dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n",  in switchtec_ntb_mw_set_trans()
299 		dev_err(&sndev->stdev->dev,  in switchtec_ntb_mw_set_trans()
304 	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,  in switchtec_ntb_mw_set_trans()
311 			switchtec_ntb_mw_clr_direct(sndev, widx);  in switchtec_ntb_mw_set_trans()
313 			switchtec_ntb_mw_clr_lut(sndev, widx);  in switchtec_ntb_mw_set_trans()
316 			switchtec_ntb_mw_set_direct(sndev, widx, addr, size);  in switchtec_ntb_mw_set_trans()
318 			switchtec_ntb_mw_set_lut(sndev, widx, addr, size);  in switchtec_ntb_mw_set_trans()
321 	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,  in switchtec_ntb_mw_set_trans()
325 		dev_err(&sndev->stdev->dev,  in switchtec_ntb_mw_set_trans()
330 			switchtec_ntb_mw_clr_direct(sndev, widx);  in switchtec_ntb_mw_set_trans()
332 			switchtec_ntb_mw_clr_lut(sndev, widx);  in switchtec_ntb_mw_set_trans()
334 		switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,  in switchtec_ntb_mw_set_trans()
343 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_peer_mw_count()  local
344 	int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts;  in switchtec_ntb_peer_mw_count()
346 	return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0);  in switchtec_ntb_peer_mw_count()
349 static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev,  in switchtec_ntb_direct_get_addr()  argument
353 	int bar = sndev->direct_mw_to_bar[idx];  in switchtec_ntb_direct_get_addr()
366 		offset = LUT_SIZE * sndev->nr_lut_mw;  in switchtec_ntb_direct_get_addr()
370 		*base = pci_resource_start(sndev->ntb.pdev, bar) + offset;  in switchtec_ntb_direct_get_addr()
373 		*size = pci_resource_len(sndev->ntb.pdev, bar) - offset;  in switchtec_ntb_direct_get_addr()
384 static int switchtec_ntb_lut_get_addr(struct switchtec_ntb *sndev,  in switchtec_ntb_lut_get_addr()  argument
388 	int bar = sndev->direct_mw_to_bar[0];  in switchtec_ntb_lut_get_addr()
391 	offset = LUT_SIZE * lut_index(sndev, idx);  in switchtec_ntb_lut_get_addr()
394 		*base = pci_resource_start(sndev->ntb.pdev, bar) + offset;  in switchtec_ntb_lut_get_addr()
406 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_peer_mw_get_addr()  local
408 	if (idx < sndev->nr_direct_mw)  in switchtec_ntb_peer_mw_get_addr()
409 		return switchtec_ntb_direct_get_addr(sndev, idx, base, size);  in switchtec_ntb_peer_mw_get_addr()
411 		return switchtec_ntb_lut_get_addr(sndev, idx, base, size);  in switchtec_ntb_peer_mw_get_addr()
416 static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev,  in switchtec_ntb_part_link_speed()  argument
421 	struct switchtec_dev *stdev = sndev->stdev;  in switchtec_ntb_part_link_speed()
433 static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev)  in switchtec_ntb_set_link_speed()  argument
438 	if (!sndev->link_is_up) {  in switchtec_ntb_set_link_speed()
439 		sndev->link_speed = NTB_SPEED_NONE;  in switchtec_ntb_set_link_speed()
440 		sndev->link_width = NTB_WIDTH_NONE;  in switchtec_ntb_set_link_speed()
444 	switchtec_ntb_part_link_speed(sndev, sndev->self_partition,  in switchtec_ntb_set_link_speed()
446 	switchtec_ntb_part_link_speed(sndev, sndev->peer_partition,  in switchtec_ntb_set_link_speed()
449 	sndev->link_speed = min(self_speed, peer_speed);  in switchtec_ntb_set_link_speed()
450 	sndev->link_width = min(self_width, peer_width);  in switchtec_ntb_set_link_speed()
453 static int crosslink_is_enabled(struct switchtec_ntb *sndev)  in crosslink_is_enabled()  argument
455 	struct ntb_info_regs __iomem *inf = sndev->mmio_ntb;  in crosslink_is_enabled()
457 	return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled);  in crosslink_is_enabled()
460 static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev)  in crosslink_init_dbmsgs()  argument
465 	if (!crosslink_is_enabled(sndev))  in crosslink_init_dbmsgs()
468 	for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) {  in crosslink_init_dbmsgs()
469 		int m = i | sndev->self_partition << 2;  in crosslink_init_dbmsgs()
474 	iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map);  in crosslink_init_dbmsgs()
475 	iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,  in crosslink_init_dbmsgs()
476 		  &sndev->mmio_peer_dbmsg->odb_mask);  in crosslink_init_dbmsgs()
487 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev);
489 static void switchtec_ntb_link_status_update(struct switchtec_ntb *sndev)  in switchtec_ntb_link_status_update()  argument
492 	int old = sndev->link_is_up;  in switchtec_ntb_link_status_update()
494 	link_sta = sndev->self_shared->link_sta;  in switchtec_ntb_link_status_update()
496 		u64 peer = ioread64(&sndev->peer_shared->magic);  in switchtec_ntb_link_status_update()
504 	sndev->link_is_up = link_sta;  in switchtec_ntb_link_status_update()
505 	switchtec_ntb_set_link_speed(sndev);  in switchtec_ntb_link_status_update()
508 		switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK);  in switchtec_ntb_link_status_update()
509 		ntb_link_event(&sndev->ntb);  in switchtec_ntb_link_status_update()
510 		dev_info(&sndev->stdev->dev, "ntb link %s\n",  in switchtec_ntb_link_status_update()
514 			crosslink_init_dbmsgs(sndev);  in switchtec_ntb_link_status_update()
520 	struct switchtec_ntb *sndev;  in check_link_status_work()  local
522 	sndev = container_of(work, struct switchtec_ntb,  in check_link_status_work()
525 	if (sndev->link_force_down) {  in check_link_status_work()
526 		sndev->link_force_down = false;  in check_link_status_work()
527 		switchtec_ntb_reinit_peer(sndev);  in check_link_status_work()
529 		if (sndev->link_is_up) {  in check_link_status_work()
530 			sndev->link_is_up = 0;  in check_link_status_work()
531 			ntb_link_event(&sndev->ntb);  in check_link_status_work()
532 			dev_info(&sndev->stdev->dev, "ntb link forced down\n");  in check_link_status_work()
538 	switchtec_ntb_link_status_update(sndev);  in check_link_status_work()
541 static void switchtec_ntb_check_link(struct switchtec_ntb *sndev,  in switchtec_ntb_check_link()  argument
545 		sndev->link_force_down = true;  in switchtec_ntb_check_link()
547 	schedule_work(&sndev->check_link_status_work);  in switchtec_ntb_check_link()
552 	struct switchtec_ntb *sndev = stdev->sndev;  in switchtec_ntb_link_notification()  local
554 	switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);  in switchtec_ntb_link_notification()
561 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_link_is_up()  local
564 		*speed = sndev->link_speed;  in switchtec_ntb_link_is_up()
566 		*width = sndev->link_width;  in switchtec_ntb_link_is_up()
568 	return sndev->link_is_up;  in switchtec_ntb_link_is_up()
575 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_link_enable()  local
577 	dev_dbg(&sndev->stdev->dev, "enabling link\n");  in switchtec_ntb_link_enable()
579 	sndev->self_shared->link_sta = 1;  in switchtec_ntb_link_enable()
580 	switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP);  in switchtec_ntb_link_enable()
582 	switchtec_ntb_link_status_update(sndev);  in switchtec_ntb_link_enable()
589 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_link_disable()  local
591 	dev_dbg(&sndev->stdev->dev, "disabling link\n");  in switchtec_ntb_link_disable()
593 	sndev->self_shared->link_sta = 0;  in switchtec_ntb_link_disable()
594 	switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN);  in switchtec_ntb_link_disable()
596 	switchtec_ntb_link_status_update(sndev);  in switchtec_ntb_link_disable()
603 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_db_valid_mask()  local
605 	return sndev->db_valid_mask;  in switchtec_ntb_db_valid_mask()
615 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_db_vector_mask()  local
620 	return sndev->db_valid_mask;  in switchtec_ntb_db_vector_mask()
626 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_db_read()  local
628 	ret = ioread64(&sndev->mmio_self_dbmsg->idb) >> sndev->db_shift;  in switchtec_ntb_db_read()
630 	return ret & sndev->db_valid_mask;  in switchtec_ntb_db_read()
635 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_db_clear()  local
637 	iowrite64(db_bits << sndev->db_shift, &sndev->mmio_self_dbmsg->idb);  in switchtec_ntb_db_clear()
645 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_db_set_mask()  local
647 	if (db_bits & ~sndev->db_valid_mask)  in switchtec_ntb_db_set_mask()
650 	spin_lock_irqsave(&sndev->db_mask_lock, irqflags);  in switchtec_ntb_db_set_mask()
652 	sndev->db_mask |= db_bits << sndev->db_shift;  in switchtec_ntb_db_set_mask()
653 	iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);  in switchtec_ntb_db_set_mask()
655 	spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags);  in switchtec_ntb_db_set_mask()
663 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_db_clear_mask()  local
665 	if (db_bits & ~sndev->db_valid_mask)  in switchtec_ntb_db_clear_mask()
668 	spin_lock_irqsave(&sndev->db_mask_lock, irqflags);  in switchtec_ntb_db_clear_mask()
670 	sndev->db_mask &= ~(db_bits << sndev->db_shift);  in switchtec_ntb_db_clear_mask()
671 	iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);  in switchtec_ntb_db_clear_mask()
673 	spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags);  in switchtec_ntb_db_clear_mask()
680 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_db_read_mask()  local
682 	return (sndev->db_mask >> sndev->db_shift) & sndev->db_valid_mask;  in switchtec_ntb_db_read_mask()
691 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_peer_db_addr()  local
697 	offset = (unsigned long)sndev->mmio_peer_dbmsg->odb -  in switchtec_ntb_peer_db_addr()
698 		(unsigned long)sndev->stdev->mmio;  in switchtec_ntb_peer_db_addr()
700 	offset += sndev->db_shift / 8;  in switchtec_ntb_peer_db_addr()
707 		*db_data = BIT_ULL(db_bit) << sndev->db_peer_shift;  in switchtec_ntb_peer_db_addr()
714 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_peer_db_set()  local
716 	iowrite64(db_bits << sndev->db_peer_shift,  in switchtec_ntb_peer_db_set()
717 		  &sndev->mmio_peer_dbmsg->odb);  in switchtec_ntb_peer_db_set()
724 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_spad_count()  local
726 	return ARRAY_SIZE(sndev->self_shared->spad);  in switchtec_ntb_spad_count()
731 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_spad_read()  local
733 	if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))  in switchtec_ntb_spad_read()
736 	if (!sndev->self_shared)  in switchtec_ntb_spad_read()
739 	return sndev->self_shared->spad[idx];  in switchtec_ntb_spad_read()
744 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_spad_write()  local
746 	if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))  in switchtec_ntb_spad_write()
749 	if (!sndev->self_shared)  in switchtec_ntb_spad_write()
752 	sndev->self_shared->spad[idx] = val;  in switchtec_ntb_spad_write()
760 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_peer_spad_read()  local
765 	if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))  in switchtec_ntb_peer_spad_read()
768 	if (!sndev->peer_shared)  in switchtec_ntb_peer_spad_read()
771 	return ioread32(&sndev->peer_shared->spad[sidx]);  in switchtec_ntb_peer_spad_read()
777 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_peer_spad_write()  local
782 	if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))  in switchtec_ntb_peer_spad_write()
785 	if (!sndev->peer_shared)  in switchtec_ntb_peer_spad_write()
788 	iowrite32(val, &sndev->peer_shared->spad[sidx]);  in switchtec_ntb_peer_spad_write()
796 	struct switchtec_ntb *sndev = ntb_sndev(ntb);  in switchtec_ntb_peer_spad_addr()  local
802 	offset = (unsigned long)&sndev->peer_shared->spad[sidx] -  in switchtec_ntb_peer_spad_addr()
803 		(unsigned long)sndev->stdev->mmio;  in switchtec_ntb_peer_spad_addr()
838 static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)  in switchtec_ntb_init_sndev()  argument
845 	sndev->ntb.pdev = sndev->stdev->pdev;  in switchtec_ntb_init_sndev()
846 	sndev->ntb.topo = NTB_TOPO_SWITCH;  in switchtec_ntb_init_sndev()
847 	sndev->ntb.ops = &switchtec_ntb_ops;  in switchtec_ntb_init_sndev()
849 	INIT_WORK(&sndev->check_link_status_work, check_link_status_work);  in switchtec_ntb_init_sndev()
850 	sndev->link_force_down = false;  in switchtec_ntb_init_sndev()
852 	sndev->self_partition = sndev->stdev->partition;  in switchtec_ntb_init_sndev()
854 	sndev->mmio_ntb = sndev->stdev->mmio_ntb;  in switchtec_ntb_init_sndev()
856 	self = sndev->self_partition;  in switchtec_ntb_init_sndev()
857 	tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high);  in switchtec_ntb_init_sndev()
859 	tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low);  in switchtec_ntb_init_sndev()
861 	part_map = ioread64(&sndev->mmio_ntb->ep_map);  in switchtec_ntb_init_sndev()
862 	part_map &= ~(1 << sndev->self_partition);  in switchtec_ntb_init_sndev()
865 		if (sndev->stdev->partition_count != 2) {  in switchtec_ntb_init_sndev()
866 			dev_err(&sndev->stdev->dev,  in switchtec_ntb_init_sndev()
873 			dev_err(&sndev->stdev->dev,  in switchtec_ntb_init_sndev()
878 		sndev->peer_partition = bit - 1;  in switchtec_ntb_init_sndev()
881 			dev_err(&sndev->stdev->dev,  in switchtec_ntb_init_sndev()
886 		sndev->peer_partition = ffs(tpart_vec) - 1;  in switchtec_ntb_init_sndev()
887 		if (!(part_map & (1ULL << sndev->peer_partition))) {  in switchtec_ntb_init_sndev()
888 			dev_err(&sndev->stdev->dev,  in switchtec_ntb_init_sndev()
894 	dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n",  in switchtec_ntb_init_sndev()
895 		sndev->self_partition, sndev->stdev->partition_count);  in switchtec_ntb_init_sndev()
897 	sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb +  in switchtec_ntb_init_sndev()
899 	sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb +  in switchtec_ntb_init_sndev()
902 	sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition];  in switchtec_ntb_init_sndev()
903 	sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition];  in switchtec_ntb_init_sndev()
904 	sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition];  in switchtec_ntb_init_sndev()
905 	sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg;  in switchtec_ntb_init_sndev()
910 static int config_rsvd_lut_win(struct switchtec_ntb *sndev,  in config_rsvd_lut_win()  argument
914 	int peer_bar = sndev->peer_direct_mw_to_bar[0];  in config_rsvd_lut_win()
918 	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,  in config_rsvd_lut_win()
927 	ctl_val |= (sndev->nr_lut_mw - 1) << 14;  in config_rsvd_lut_win()
933 	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,  in config_rsvd_lut_win()
940 		dev_err(&sndev->stdev->dev,  in config_rsvd_lut_win()
949 static int config_req_id_table(struct switchtec_ntb *sndev,  in config_req_id_table()  argument
958 		dev_err(&sndev->stdev->dev,  in config_req_id_table()
963 	rc = switchtec_ntb_part_op(sndev, mmio_ctrl,  in config_req_id_table()
977 		dev_dbg(&sndev->stdev->dev,  in config_req_id_table()
984 	rc = switchtec_ntb_part_op(sndev, mmio_ctrl,  in config_req_id_table()
990 		dev_err(&sndev->stdev->dev,  in config_req_id_table()
998 static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx,  in crosslink_setup_mws()  argument
1002 	struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl;  in crosslink_setup_mws()
1009 	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,  in crosslink_setup_mws()
1014 	for (i = 0; i < sndev->nr_lut_mw; i++) {  in crosslink_setup_mws()
1020 		iowrite64((NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) |  in crosslink_setup_mws()
1025 	sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count);  in crosslink_setup_mws()
1027 	for (i = 0; i < sndev->nr_direct_mw; i++) {  in crosslink_setup_mws()
1028 		bar = sndev->direct_mw_to_bar[i];  in crosslink_setup_mws()
1029 		offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0;  in crosslink_setup_mws()
1031 		size = pci_resource_len(sndev->ntb.pdev, bar) - offset;  in crosslink_setup_mws()
1044 		iowrite64(sndev->peer_partition | addr,  in crosslink_setup_mws()
1048 	rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,  in crosslink_setup_mws()
1055 		dev_err(&sndev->stdev->dev,  in crosslink_setup_mws()
1064 static int crosslink_setup_req_ids(struct switchtec_ntb *sndev,  in crosslink_setup_req_ids()  argument
1072 		proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]);  in crosslink_setup_req_ids()
1080 	return config_req_id_table(sndev, mmio_ctrl, req_ids, i);  in crosslink_setup_req_ids()
1088 static int crosslink_enum_partition(struct switchtec_ntb *sndev,  in crosslink_enum_partition()  argument
1092 		&sndev->stdev->mmio_part_cfg_all[sndev->peer_partition];  in crosslink_enum_partition()
1095 		&sndev->stdev->mmio_pff_csr[pff];  in crosslink_enum_partition()
1108 		dev_dbg(&sndev->stdev->dev,  in crosslink_enum_partition()
1121 static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev)  in switchtec_ntb_init_crosslink()  argument
1124 	int bar = sndev->direct_mw_to_bar[0];  in switchtec_ntb_init_crosslink()
1131 	if (!crosslink_is_enabled(sndev))  in switchtec_ntb_init_crosslink()
1134 	dev_info(&sndev->stdev->dev, "Using crosslink configuration\n");  in switchtec_ntb_init_crosslink()
1135 	sndev->ntb.topo = NTB_TOPO_CROSSLINK;  in switchtec_ntb_init_crosslink()
1137 	bar_cnt = crosslink_enum_partition(sndev, bar_addrs);  in switchtec_ntb_init_crosslink()
1138 	if (bar_cnt < sndev->nr_direct_mw + 1) {  in switchtec_ntb_init_crosslink()
1139 		dev_err(&sndev->stdev->dev,  in switchtec_ntb_init_crosslink()
1146 		sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition);  in switchtec_ntb_init_crosslink()
1151 	rc = config_rsvd_lut_win(sndev, sndev->mmio_self_ctrl, ntb_lut_idx,  in switchtec_ntb_init_crosslink()
1152 				 sndev->peer_partition, addr);  in switchtec_ntb_init_crosslink()
1156 	rc = crosslink_setup_mws(sndev, ntb_lut_idx, &bar_addrs[1],  in switchtec_ntb_init_crosslink()
1161 	rc = crosslink_setup_req_ids(sndev, sndev->mmio_peer_ctrl);  in switchtec_ntb_init_crosslink()
1165 	sndev->mmio_xlink_win = pci_iomap_range(sndev->stdev->pdev, bar,  in switchtec_ntb_init_crosslink()
1167 	if (!sndev->mmio_xlink_win) {  in switchtec_ntb_init_crosslink()
1172 	sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset;  in switchtec_ntb_init_crosslink()
1173 	sndev->nr_rsvd_luts++;  in switchtec_ntb_init_crosslink()
1175 	crosslink_init_dbmsgs(sndev);  in switchtec_ntb_init_crosslink()
1180 static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev)  in switchtec_ntb_deinit_crosslink()  argument
1182 	if (sndev->mmio_xlink_win)  in switchtec_ntb_deinit_crosslink()
1183 		pci_iounmap(sndev->stdev->pdev, sndev->mmio_xlink_win);  in switchtec_ntb_deinit_crosslink()
1201 static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)  in switchtec_ntb_init_mw()  argument
1203 	sndev->nr_direct_mw = map_bars(sndev->direct_mw_to_bar,  in switchtec_ntb_init_mw()
1204 				       sndev->mmio_self_ctrl);  in switchtec_ntb_init_mw()
1206 	sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries);  in switchtec_ntb_init_mw()
1207 	sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw);  in switchtec_ntb_init_mw()
1209 	dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n",  in switchtec_ntb_init_mw()
1210 		sndev->nr_direct_mw, sndev->nr_lut_mw);  in switchtec_ntb_init_mw()
1212 	sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar,  in switchtec_ntb_init_mw()
1213 					    sndev->mmio_peer_ctrl);  in switchtec_ntb_init_mw()
1215 	sndev->peer_nr_lut_mw =  in switchtec_ntb_init_mw()
1216 		ioread16(&sndev->mmio_peer_ctrl->lut_table_entries);  in switchtec_ntb_init_mw()
1217 	sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw);  in switchtec_ntb_init_mw()
1219 	dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n",  in switchtec_ntb_init_mw()
1220 		sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw);  in switchtec_ntb_init_mw()
1234 static void switchtec_ntb_init_db(struct switchtec_ntb *sndev)  in switchtec_ntb_init_db()  argument
1236 	sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL;  in switchtec_ntb_init_db()
1238 	if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) {  in switchtec_ntb_init_db()
1239 		sndev->db_shift = 0;  in switchtec_ntb_init_db()
1240 		sndev->db_peer_shift = 0;  in switchtec_ntb_init_db()
1241 		sndev->db_valid_mask = sndev->db_mask;  in switchtec_ntb_init_db()
1242 	} else if (sndev->self_partition < sndev->peer_partition) {  in switchtec_ntb_init_db()
1243 		sndev->db_shift = 0;  in switchtec_ntb_init_db()
1244 		sndev->db_peer_shift = 32;  in switchtec_ntb_init_db()
1245 		sndev->db_valid_mask = 0x0FFFFFFF;  in switchtec_ntb_init_db()
1247 		sndev->db_shift = 32;  in switchtec_ntb_init_db()
1248 		sndev->db_peer_shift = 0;  in switchtec_ntb_init_db()
1249 		sndev->db_valid_mask = 0x0FFFFFFF;  in switchtec_ntb_init_db()
1252 	iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);  in switchtec_ntb_init_db()
1253 	iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,  in switchtec_ntb_init_db()
1254 		  &sndev->mmio_peer_dbmsg->odb_mask);  in switchtec_ntb_init_db()
1256 	dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n",  in switchtec_ntb_init_db()
1257 		sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask);  in switchtec_ntb_init_db()
1260 static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev)  in switchtec_ntb_init_msgs()  argument
1265 	for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {  in switchtec_ntb_init_msgs()
1266 		int m = i | sndev->peer_partition << 2;  in switchtec_ntb_init_msgs()
1271 	iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map);  in switchtec_ntb_init_msgs()
1273 	for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++)  in switchtec_ntb_init_msgs()
1275 			  &sndev->mmio_self_dbmsg->imsg[i]);  in switchtec_ntb_init_msgs()
1279 switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev)  in switchtec_ntb_init_req_id_table()  argument
1291 	req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id);  in switchtec_ntb_init_req_id_table()
1293 	return config_req_id_table(sndev, sndev->mmio_self_ctrl, req_ids,  in switchtec_ntb_init_req_id_table()
1297 static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev)  in switchtec_ntb_init_shared()  argument
1301 	memset(sndev->self_shared, 0, LUT_SIZE);  in switchtec_ntb_init_shared()
1302 	sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC;  in switchtec_ntb_init_shared()
1303 	sndev->self_shared->partition_id = sndev->stdev->partition;  in switchtec_ntb_init_shared()
1305 	for (i = 0; i < sndev->nr_direct_mw; i++) {  in switchtec_ntb_init_shared()
1306 		int bar = sndev->direct_mw_to_bar[i];  in switchtec_ntb_init_shared()
1307 		resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar);  in switchtec_ntb_init_shared()
1311 				   LUT_SIZE * sndev->nr_lut_mw);  in switchtec_ntb_init_shared()
1313 		sndev->self_shared->mw_sizes[i] = sz;  in switchtec_ntb_init_shared()
1316 	for (i = 0; i < sndev->nr_lut_mw; i++) {  in switchtec_ntb_init_shared()
1317 		int idx = sndev->nr_direct_mw + i;  in switchtec_ntb_init_shared()
1319 		sndev->self_shared->mw_sizes[idx] = LUT_SIZE;  in switchtec_ntb_init_shared()
1323 static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev)  in switchtec_ntb_init_shared_mw()  argument
1325 	int self_bar = sndev->direct_mw_to_bar[0];  in switchtec_ntb_init_shared_mw()
1328 	sndev->nr_rsvd_luts++;  in switchtec_ntb_init_shared_mw()
1329 	sndev->self_shared = dma_alloc_coherent(&sndev->stdev->pdev->dev,  in switchtec_ntb_init_shared_mw()
1331 						&sndev->self_shared_dma,  in switchtec_ntb_init_shared_mw()
1333 	if (!sndev->self_shared) {  in switchtec_ntb_init_shared_mw()
1334 		dev_err(&sndev->stdev->dev,  in switchtec_ntb_init_shared_mw()
1339 	switchtec_ntb_init_shared(sndev);  in switchtec_ntb_init_shared_mw()
1341 	rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0,  in switchtec_ntb_init_shared_mw()
1342 				 sndev->self_partition,  in switchtec_ntb_init_shared_mw()
1343 				 sndev->self_shared_dma);  in switchtec_ntb_init_shared_mw()
1347 	sndev->peer_shared = pci_iomap(sndev->stdev->pdev, self_bar, LUT_SIZE);  in switchtec_ntb_init_shared_mw()
1348 	if (!sndev->peer_shared) {  in switchtec_ntb_init_shared_mw()
1353 	dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n");  in switchtec_ntb_init_shared_mw()
1357 	dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,  in switchtec_ntb_init_shared_mw()
1358 			  sndev->self_shared, sndev->self_shared_dma);  in switchtec_ntb_init_shared_mw()
1363 static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev)  in switchtec_ntb_deinit_shared_mw()  argument
1365 	if (sndev->peer_shared)  in switchtec_ntb_deinit_shared_mw()
1366 		pci_iounmap(sndev->stdev->pdev, sndev->peer_shared);  in switchtec_ntb_deinit_shared_mw()
1368 	if (sndev->self_shared)  in switchtec_ntb_deinit_shared_mw()
1369 		dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,  in switchtec_ntb_deinit_shared_mw()
1370 				  sndev->self_shared,  in switchtec_ntb_deinit_shared_mw()
1371 				  sndev->self_shared_dma);  in switchtec_ntb_deinit_shared_mw()
1372 	sndev->nr_rsvd_luts--;  in switchtec_ntb_deinit_shared_mw()
1377 	struct switchtec_ntb *sndev = dev;  in switchtec_ntb_doorbell_isr()  local
1379 	dev_dbg(&sndev->stdev->dev, "doorbell\n");  in switchtec_ntb_doorbell_isr()
1381 	ntb_db_event(&sndev->ntb, 0);  in switchtec_ntb_doorbell_isr()
1389 	struct switchtec_ntb *sndev = dev;  in switchtec_ntb_message_isr()  local
1391 	for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {  in switchtec_ntb_message_isr()
1392 		u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]);  in switchtec_ntb_message_isr()
1395 			dev_dbg(&sndev->stdev->dev, "message: %d %08x\n",  in switchtec_ntb_message_isr()
1397 			iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status);  in switchtec_ntb_message_isr()
1400 				switchtec_ntb_check_link(sndev, msg);  in switchtec_ntb_message_isr()
1407 static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev)  in switchtec_ntb_init_db_msg_irq()  argument
1414 	int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map);  in switchtec_ntb_init_db_msg_irq()
1416 	event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number);  in switchtec_ntb_init_db_msg_irq()
1424 	dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n",  in switchtec_ntb_init_db_msg_irq()
1429 			 &sndev->mmio_self_dbmsg->idb_vec_map[i]);  in switchtec_ntb_init_db_msg_irq()
1433 			 &sndev->mmio_self_dbmsg->idb_vec_map[i]);  in switchtec_ntb_init_db_msg_irq()
1435 	sndev->doorbell_irq = pci_irq_vector(sndev->stdev->pdev, doorbell_irq);  in switchtec_ntb_init_db_msg_irq()
1436 	sndev->message_irq = pci_irq_vector(sndev->stdev->pdev, message_irq);  in switchtec_ntb_init_db_msg_irq()
1438 	rc = request_irq(sndev->doorbell_irq,  in switchtec_ntb_init_db_msg_irq()
1440 			 "switchtec_ntb_doorbell", sndev);  in switchtec_ntb_init_db_msg_irq()
1444 	rc = request_irq(sndev->message_irq,  in switchtec_ntb_init_db_msg_irq()
1446 			 "switchtec_ntb_message", sndev);  in switchtec_ntb_init_db_msg_irq()
1448 		free_irq(sndev->doorbell_irq, sndev);  in switchtec_ntb_init_db_msg_irq()
1455 static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev)  in switchtec_ntb_deinit_db_msg_irq()  argument
1457 	free_irq(sndev->doorbell_irq, sndev);  in switchtec_ntb_deinit_db_msg_irq()
1458 	free_irq(sndev->message_irq, sndev);  in switchtec_ntb_deinit_db_msg_irq()
1461 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev)  in switchtec_ntb_reinit_peer()  argument
1465 	if (crosslink_is_enabled(sndev))  in switchtec_ntb_reinit_peer()
1468 	dev_info(&sndev->stdev->dev, "reinitialize shared memory window\n");  in switchtec_ntb_reinit_peer()
1469 	rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0,  in switchtec_ntb_reinit_peer()
1470 				 sndev->self_partition,  in switchtec_ntb_reinit_peer()
1471 				 sndev->self_shared_dma);  in switchtec_ntb_reinit_peer()
1479 	struct switchtec_ntb *sndev;  in switchtec_ntb_add()  local
1482 	stdev->sndev = NULL;  in switchtec_ntb_add()
1487 	sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev));  in switchtec_ntb_add()
1488 	if (!sndev)  in switchtec_ntb_add()
1491 	sndev->stdev = stdev;  in switchtec_ntb_add()
1492 	rc = switchtec_ntb_init_sndev(sndev);  in switchtec_ntb_add()
1496 	switchtec_ntb_init_mw(sndev);  in switchtec_ntb_add()
1498 	rc = switchtec_ntb_init_req_id_table(sndev);  in switchtec_ntb_add()
1502 	rc = switchtec_ntb_init_crosslink(sndev);  in switchtec_ntb_add()
1506 	switchtec_ntb_init_db(sndev);  in switchtec_ntb_add()
1507 	switchtec_ntb_init_msgs(sndev);  in switchtec_ntb_add()
1509 	rc = switchtec_ntb_init_shared_mw(sndev);  in switchtec_ntb_add()
1513 	rc = switchtec_ntb_init_db_msg_irq(sndev);  in switchtec_ntb_add()
1522 	switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_FORCE_DOWN);  in switchtec_ntb_add()
1524 	rc = ntb_register_device(&sndev->ntb);  in switchtec_ntb_add()
1528 	stdev->sndev = sndev;  in switchtec_ntb_add()
1535 	switchtec_ntb_deinit_db_msg_irq(sndev);  in switchtec_ntb_add()
1537 	switchtec_ntb_deinit_shared_mw(sndev);  in switchtec_ntb_add()
1539 	switchtec_ntb_deinit_crosslink(sndev);  in switchtec_ntb_add()
1541 	kfree(sndev);  in switchtec_ntb_add()
1550 	struct switchtec_ntb *sndev = stdev->sndev;  in switchtec_ntb_remove()  local
1552 	if (!sndev)  in switchtec_ntb_remove()
1556 	stdev->sndev = NULL;  in switchtec_ntb_remove()
1557 	ntb_unregister_device(&sndev->ntb);  in switchtec_ntb_remove()
1558 	switchtec_ntb_deinit_db_msg_irq(sndev);  in switchtec_ntb_remove()
1559 	switchtec_ntb_deinit_shared_mw(sndev);  in switchtec_ntb_remove()
1560 	switchtec_ntb_deinit_crosslink(sndev);  in switchtec_ntb_remove()
1561 	kfree(sndev);  in switchtec_ntb_remove()