185 	struct ntb_dev *ntb;  member
250 	link = ntb_link_is_up(peer->perf->ntb, NULL, NULL);  in perf_link_is_up()
261 	dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);  in perf_spad_cmd_send()
274 		sts = ntb_peer_spad_read(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
281 		ntb_peer_spad_write(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
284 		ntb_peer_spad_write(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
287 		ntb_peer_spad_write(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
290 		ntb_peer_db_set(perf->ntb, PERF_SPAD_NOTIFY(peer->gidx));  in perf_spad_cmd_send()
292 		dev_dbg(&perf->ntb->dev, "DB ring peer %#llx\n",  in perf_spad_cmd_send()
307 	ntb_db_clear(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));  in perf_spad_cmd_recv()
321 		val = ntb_spad_read(perf->ntb, PERF_SPAD_CMD(peer->gidx));  in perf_spad_cmd_recv()
327 		val = ntb_spad_read(perf->ntb, PERF_SPAD_LDATA(peer->gidx));  in perf_spad_cmd_recv()
330 		val = ntb_spad_read(perf->ntb, PERF_SPAD_HDATA(peer->gidx));  in perf_spad_cmd_recv()
334 		ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx),  in perf_spad_cmd_recv()
337 		dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);  in perf_spad_cmd_recv()
352 	dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);  in perf_msg_cmd_send()
361 	outbits = ntb_msg_outbits(perf->ntb);  in perf_msg_cmd_send()
366 		ret = ntb_msg_clear_sts(perf->ntb, outbits);  in perf_msg_cmd_send()
370 		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_LDATA,  in perf_msg_cmd_send()
373 		if (ntb_msg_read_sts(perf->ntb) & outbits) {  in perf_msg_cmd_send()
378 		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_HDATA,  in perf_msg_cmd_send()
382 		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_CMD, cmd);  in perf_msg_cmd_send()
396 	inbits = ntb_msg_inbits(perf->ntb);  in perf_msg_cmd_recv()
398 	if (hweight64(ntb_msg_read_sts(perf->ntb) & inbits) < 3)  in perf_msg_cmd_recv()
401 	val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_CMD);  in perf_msg_cmd_recv()
404 	val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_LDATA);  in perf_msg_cmd_recv()
407 	val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_HDATA);  in perf_msg_cmd_recv()
411 	ntb_msg_clear_sts(perf->ntb, inbits);  in perf_msg_cmd_recv()
413 	dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);  in perf_msg_cmd_recv()
425 	dev_err(&perf->ntb->dev, "Send invalid command\n");  in perf_cmd_send()
439 		dev_err(&peer->perf->ntb->dev, "Exec invalid command\n");  in perf_cmd_exec()
446 	dev_dbg(&peer->perf->ntb->dev, "CMD exec: %d\n", cmd);  in perf_cmd_exec()
470 			dev_err(&perf->ntb->dev, "Recv invalid command\n");  in perf_cmd_recv()
505 	dev_dbg(&perf->ntb->dev, "DB vec %d mask %#llx bits %#llx\n", vec,  in perf_db_event()
506 		ntb_db_vector_mask(perf->ntb, vec), ntb_db_read(perf->ntb));  in perf_db_event()
516 	dev_dbg(&perf->ntb->dev, "Msg status bits %#llx\n",  in perf_msg_event()
517 		ntb_msg_read_sts(perf->ntb));  in perf_msg_event()
531 	(void)ntb_peer_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);  in perf_free_outbuf()
540 	ret = ntb_peer_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,  in perf_setup_outbuf()
543 		dev_err(&perf->ntb->dev, "Failed to set outbuf translation\n");  in perf_setup_outbuf()
558 	(void)ntb_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);  in perf_free_inbuf()
559 	dma_free_coherent(&peer->perf->ntb->dev, peer->inbuf_size,  in perf_free_inbuf()
571 	ret = ntb_mw_get_align(perf->ntb, peer->pidx, perf->gidx,  in perf_setup_inbuf()
574 		dev_err(&perf->ntb->dev, "Couldn't get inbuf restrictions\n");  in perf_setup_inbuf()
579 		dev_err(&perf->ntb->dev, "Too big inbuf size %pa > %pa\n",  in perf_setup_inbuf()
588 	peer->inbuf = dma_alloc_coherent(&perf->ntb->dev, peer->inbuf_size,  in perf_setup_inbuf()
591 		dev_err(&perf->ntb->dev, "Failed to alloc inbuf of %pa\n",  in perf_setup_inbuf()
596 		dev_err(&perf->ntb->dev, "Unaligned inbuf allocated\n");  in perf_setup_inbuf()
600 	ret = ntb_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,  in perf_setup_inbuf()
603 		dev_err(&perf->ntb->dev, "Failed to set inbuf translation\n");  in perf_setup_inbuf()
642 			dev_warn(&peer->perf->ntb->dev,  in perf_service_work()
655 	if (ntb_peer_mw_count(perf->ntb) < perf->pcnt + 1) {  in perf_init_service()
656 		dev_err(&perf->ntb->dev, "Not enough memory windows\n");  in perf_init_service()
660 	if (ntb_msg_count(perf->ntb) >= PERF_MSG_CNT) {  in perf_init_service()
664 		dev_dbg(&perf->ntb->dev, "Message service initialized\n");  in perf_init_service()
669 	dev_dbg(&perf->ntb->dev, "Message service unsupported\n");  in perf_init_service()
672 	if (ntb_spad_count(perf->ntb) >= PERF_SPAD_CNT(perf->pcnt) &&  in perf_init_service()
673 	    (ntb_db_valid_mask(perf->ntb) & mask) == mask) {  in perf_init_service()
677 		dev_dbg(&perf->ntb->dev, "Scratchpad service initialized\n");  in perf_init_service()
682 	dev_dbg(&perf->ntb->dev, "Scratchpad service unsupported\n");  in perf_init_service()
684 	dev_err(&perf->ntb->dev, "Command services unsupported\n");  in perf_init_service()
694 	mask = ntb_db_valid_mask(perf->ntb);  in perf_enable_service()
695 	(void)ntb_db_set_mask(perf->ntb, mask);  in perf_enable_service()
697 	ret = ntb_set_ctx(perf->ntb, perf, &perf_ops);  in perf_enable_service()
704 		inbits = ntb_msg_inbits(perf->ntb);  in perf_enable_service()
705 		outbits = ntb_msg_outbits(perf->ntb);  in perf_enable_service()
706 		(void)ntb_msg_set_mask(perf->ntb, inbits | outbits);  in perf_enable_service()
709 		ret = ntb_msg_clear_mask(perf->ntb, incmd_bit);  in perf_enable_service()
711 		dev_dbg(&perf->ntb->dev, "MSG sts unmasked %#llx\n", incmd_bit);  in perf_enable_service()
713 		scnt = ntb_spad_count(perf->ntb);  in perf_enable_service()
715 			ntb_spad_write(perf->ntb, sidx, PERF_CMD_INVAL);  in perf_enable_service()
717 		ret = ntb_db_clear_mask(perf->ntb, incmd_bit);  in perf_enable_service()
719 		dev_dbg(&perf->ntb->dev, "DB bits unmasked %#llx\n", incmd_bit);  in perf_enable_service()
722 		ntb_clear_ctx(perf->ntb);  in perf_enable_service()
726 	ntb_link_enable(perf->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);  in perf_enable_service()
728 	ntb_link_event(perf->ntb);  in perf_enable_service()
740 		inbits = ntb_msg_inbits(perf->ntb);  in perf_disable_service()
741 		(void)ntb_msg_set_mask(perf->ntb, inbits);  in perf_disable_service()
743 		(void)ntb_db_set_mask(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));  in perf_disable_service()
746 	ntb_clear_ctx(perf->ntb);  in perf_disable_service()
757 		ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx), 0);  in perf_disable_service()
760 	ntb_db_clear(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));  in perf_disable_service()
762 	ntb_link_disable(perf->ntb);  in perf_disable_service()
859 	node = dev_to_node(&perf->ntb->dev);  in perf_dma_filter()
870 				 dev_to_node(&perf->ntb->dev));  in perf_init_test()
883 		dev_err(&perf->ntb->dev, "%d: Failed to get DMA channel\n",  in perf_init_test()
919 			dev_err(&perf->ntb->dev, "%d: Got error %d on test\n",  in perf_run_test()
957 	dev_dbg(&perf->ntb->dev, "%d: copied %llu bytes\n",  in perf_sync_test()
960 	dev_dbg(&perf->ntb->dev, "%d: lasted %llu usecs\n",  in perf_sync_test()
963 	dev_dbg(&perf->ntb->dev, "%d: %llu MBytes/s\n", pthr->tidx,  in perf_sync_test()
1170 		"Local port %d, Global index %d\n", ntb_port_number(perf->ntb),  in perf_dbgfs_read_info()
1176 			ntb_peer_port_number(perf->ntb, perf->test_peer->pidx),  in perf_dbgfs_read_info()
1187 			ntb_peer_port_number(perf->ntb, peer->pidx), peer->pidx,  in perf_dbgfs_read_info()
1320 	struct pci_dev *pdev = perf->ntb->pdev;  in perf_setup_dbgfs()
1324 		dev_warn(&perf->ntb->dev, "DebugFS unsupported\n");  in perf_setup_dbgfs()
1355 static struct perf_ctx *perf_create_data(struct ntb_dev *ntb)  in perf_create_data()  argument
1359 	perf = devm_kzalloc(&ntb->dev, sizeof(*perf), GFP_KERNEL);  in perf_create_data()
1363 	perf->pcnt = ntb_peer_port_count(ntb);  in perf_create_data()
1364 	perf->peers = devm_kcalloc(&ntb->dev, perf->pcnt, sizeof(*perf->peers),  in perf_create_data()
1369 	perf->ntb = ntb;  in perf_create_data()
1381 	ret = ntb_peer_mw_get_addr(perf->ntb, perf->gidx, &phys_addr,  in perf_setup_peer_mw()
1386 	peer->outbuf = devm_ioremap_wc(&perf->ntb->dev, phys_addr,  in perf_setup_peer_mw()
1393 		dev_warn(&peer->perf->ntb->dev,  in perf_setup_peer_mw()
1406 	lport = ntb_port_number(perf->ntb);  in perf_init_peers()
1413 		if (lport < ntb_peer_port_number(perf->ntb, pidx)) {  in perf_init_peers()
1431 	dev_dbg(&perf->ntb->dev, "Global port index %d\n", perf->gidx);  in perf_init_peers()
1436 static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb)  in perf_probe()  argument
1441 	perf = perf_create_data(ntb);  in perf_probe()
1464 static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb)  in perf_remove()  argument
1466 	struct perf_ctx *perf = ntb->ctx;  in perf_remove()