201 	int (*cmd_send)(struct perf_peer *peer, enum perf_cmd cmd, u64 data);
246 static inline bool perf_link_is_up(struct perf_peer *peer)  in perf_link_is_up()  argument
250 	link = ntb_link_is_up(peer->perf->ntb, NULL, NULL);  in perf_link_is_up()
251 	return !!(link & BIT_ULL_MASK(peer->pidx));  in perf_link_is_up()
254 static int perf_spad_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,  in perf_spad_cmd_send()  argument
257 	struct perf_ctx *perf = peer->perf;  in perf_spad_cmd_send()
271 		if (!perf_link_is_up(peer))  in perf_spad_cmd_send()
274 		sts = ntb_peer_spad_read(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
281 		ntb_peer_spad_write(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
284 		ntb_peer_spad_write(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
287 		ntb_peer_spad_write(perf->ntb, peer->pidx,  in perf_spad_cmd_send()
290 		ntb_peer_db_set(perf->ntb, PERF_SPAD_NOTIFY(peer->gidx));  in perf_spad_cmd_send()
293 			PERF_SPAD_NOTIFY(peer->gidx));  in perf_spad_cmd_send()
304 	struct perf_peer *peer;  in perf_spad_cmd_recv()  local
316 		peer = &perf->peers[*pidx];  in perf_spad_cmd_recv()
318 		if (!perf_link_is_up(peer))  in perf_spad_cmd_recv()
321 		val = ntb_spad_read(perf->ntb, PERF_SPAD_CMD(peer->gidx));  in perf_spad_cmd_recv()
327 		val = ntb_spad_read(perf->ntb, PERF_SPAD_LDATA(peer->gidx));  in perf_spad_cmd_recv()
330 		val = ntb_spad_read(perf->ntb, PERF_SPAD_HDATA(peer->gidx));  in perf_spad_cmd_recv()
334 		ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx),  in perf_spad_cmd_recv()
345 static int perf_msg_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,  in perf_msg_cmd_send()  argument
348 	struct perf_ctx *perf = peer->perf;  in perf_msg_cmd_send()
363 		if (!perf_link_is_up(peer))  in perf_msg_cmd_send()
370 		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_LDATA,  in perf_msg_cmd_send()
378 		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_HDATA,  in perf_msg_cmd_send()
382 		ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_CMD, cmd);  in perf_msg_cmd_send()
418 static int perf_cmd_send(struct perf_peer *peer, enum perf_cmd cmd, u64 data)  in perf_cmd_send()  argument
420 	struct perf_ctx *perf = peer->perf;  in perf_cmd_send()
423 		return perf->cmd_send(peer, cmd, data);  in perf_cmd_send()
429 static int perf_cmd_exec(struct perf_peer *peer, enum perf_cmd cmd)  in perf_cmd_exec()  argument
439 		dev_err(&peer->perf->ntb->dev, "Exec invalid command\n");  in perf_cmd_exec()
444 	set_bit(cmd, &peer->sts);  in perf_cmd_exec()
446 	dev_dbg(&peer->perf->ntb->dev, "CMD exec: %d\n", cmd);  in perf_cmd_exec()
448 	(void)queue_work(system_highpri_wq, &peer->service);  in perf_cmd_exec()
455 	struct perf_peer *peer;  in perf_cmd_recv()  local
460 		peer = &perf->peers[pidx];  in perf_cmd_recv()
464 			peer->inbuf_size = data;  in perf_cmd_recv()
465 			return perf_cmd_exec(peer, PERF_CMD_RSIZE);  in perf_cmd_recv()
467 			peer->outbuf_xlat = data;  in perf_cmd_recv()
468 			return perf_cmd_exec(peer, PERF_CMD_RXLAT);  in perf_cmd_recv()
482 	struct perf_peer *peer;  in perf_link_event()  local
487 		peer = &perf->peers[pidx];  in perf_link_event()
489 		lnk_up = perf_link_is_up(peer);  in perf_link_event()
492 		    !test_and_set_bit(PERF_STS_LNKUP, &peer->sts)) {  in perf_link_event()
493 			perf_cmd_exec(peer, PERF_CMD_SSIZE);  in perf_link_event()
495 			   test_and_clear_bit(PERF_STS_LNKUP, &peer->sts)) {  in perf_link_event()
496 			perf_cmd_exec(peer, PERF_CMD_CLEAR);  in perf_link_event()
529 static void perf_free_outbuf(struct perf_peer *peer)  in perf_free_outbuf()  argument
531 	(void)ntb_peer_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);  in perf_free_outbuf()
534 static int perf_setup_outbuf(struct perf_peer *peer)  in perf_setup_outbuf()  argument
536 	struct perf_ctx *perf = peer->perf;  in perf_setup_outbuf()
540 	ret = ntb_peer_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,  in perf_setup_outbuf()
541 				    peer->outbuf_xlat, peer->outbuf_size);  in perf_setup_outbuf()
548 	set_bit(PERF_STS_DONE, &peer->sts);  in perf_setup_outbuf()
553 static void perf_free_inbuf(struct perf_peer *peer)  in perf_free_inbuf()  argument
555 	if (!peer->inbuf)  in perf_free_inbuf()
558 	(void)ntb_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);  in perf_free_inbuf()
559 	dma_free_coherent(&peer->perf->ntb->dev, peer->inbuf_size,  in perf_free_inbuf()
560 			  peer->inbuf, peer->inbuf_xlat);  in perf_free_inbuf()
561 	peer->inbuf = NULL;  in perf_free_inbuf()
564 static int perf_setup_inbuf(struct perf_peer *peer)  in perf_setup_inbuf()  argument
567 	struct perf_ctx *perf = peer->perf;  in perf_setup_inbuf()
571 	ret = ntb_mw_get_align(perf->ntb, peer->pidx, perf->gidx,  in perf_setup_inbuf()
578 	if (peer->inbuf_size > size_max) {  in perf_setup_inbuf()
580 			&peer->inbuf_size, &size_max);  in perf_setup_inbuf()
584 	peer->inbuf_size = round_up(peer->inbuf_size, size_align);  in perf_setup_inbuf()
586 	perf_free_inbuf(peer);  in perf_setup_inbuf()
588 	peer->inbuf = dma_alloc_coherent(&perf->ntb->dev, peer->inbuf_size,  in perf_setup_inbuf()
589 					 &peer->inbuf_xlat, GFP_KERNEL);  in perf_setup_inbuf()
590 	if (!peer->inbuf) {  in perf_setup_inbuf()
592 			&peer->inbuf_size);  in perf_setup_inbuf()
595 	if (!IS_ALIGNED(peer->inbuf_xlat, xlat_align)) {  in perf_setup_inbuf()
600 	ret = ntb_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,  in perf_setup_inbuf()
601 			       peer->inbuf_xlat, peer->inbuf_size);  in perf_setup_inbuf()
612 	(void)perf_cmd_exec(peer, PERF_CMD_SXLAT);  in perf_setup_inbuf()
617 	perf_free_inbuf(peer);  in perf_setup_inbuf()
624 	struct perf_peer *peer = to_peer_service(work);  in perf_service_work()  local
626 	if (test_and_clear_bit(PERF_CMD_SSIZE, &peer->sts))  in perf_service_work()
627 		perf_cmd_send(peer, PERF_CMD_SSIZE, peer->outbuf_size);  in perf_service_work()
629 	if (test_and_clear_bit(PERF_CMD_RSIZE, &peer->sts))  in perf_service_work()
630 		perf_setup_inbuf(peer);  in perf_service_work()
632 	if (test_and_clear_bit(PERF_CMD_SXLAT, &peer->sts))  in perf_service_work()
633 		perf_cmd_send(peer, PERF_CMD_SXLAT, peer->inbuf_xlat);  in perf_service_work()
635 	if (test_and_clear_bit(PERF_CMD_RXLAT, &peer->sts))  in perf_service_work()
636 		perf_setup_outbuf(peer);  in perf_service_work()
638 	if (test_and_clear_bit(PERF_CMD_CLEAR, &peer->sts)) {  in perf_service_work()
639 		clear_bit(PERF_STS_DONE, &peer->sts);  in perf_service_work()
640 		if (test_bit(0, &peer->perf->busy_flag) &&  in perf_service_work()
641 		    peer == peer->perf->test_peer) {  in perf_service_work()
642 			dev_warn(&peer->perf->ntb->dev,  in perf_service_work()
644 			perf_terminate_test(peer->perf);  in perf_service_work()
646 		perf_free_outbuf(peer);  in perf_service_work()
647 		perf_free_inbuf(peer);  in perf_service_work()
755 		struct perf_peer *peer = &perf->peers[pidx];  in perf_disable_service()  local
757 		ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx), 0);  in perf_disable_service()
898 	struct perf_peer *peer = pthr->perf->test_peer;  in perf_run_test()  local
907 	chunk_size = min_t(u64, peer->outbuf_size, chunk_size);  in perf_run_test()
910 	bnd_dst = peer->outbuf + peer->outbuf_size;  in perf_run_test()
911 	flt_dst = peer->outbuf;  in perf_run_test()
928 		if (flt_dst >= bnd_dst || flt_dst < peer->outbuf) {  in perf_run_test()
929 			flt_dst = peer->outbuf;  in perf_run_test()
1048 static int perf_submit_test(struct perf_peer *peer)  in perf_submit_test()  argument
1050 	struct perf_ctx *perf = peer->perf;  in perf_submit_test()
1054 	if (!test_bit(PERF_STS_DONE, &peer->sts))  in perf_submit_test()
1060 	perf->test_peer = peer;  in perf_submit_test()
1154 	struct perf_peer *peer;  in perf_dbgfs_read_info()  local
1183 		peer = &perf->peers[pidx];  in perf_dbgfs_read_info()
1187 			ntb_peer_port_number(perf->ntb, peer->pidx), peer->pidx,  in perf_dbgfs_read_info()
1188 			peer->gidx);  in perf_dbgfs_read_info()
1192 			test_bit(PERF_STS_LNKUP, &peer->sts) ? "up" : "down");  in perf_dbgfs_read_info()
1195 			"\tOut buffer addr 0x%pK\n", peer->outbuf);  in perf_dbgfs_read_info()
1198 			"\tOut buffer size %pa\n", &peer->outbuf_size);  in perf_dbgfs_read_info()
1201 			"\tOut buffer xlat 0x%016llx[p]\n", peer->outbuf_xlat);  in perf_dbgfs_read_info()
1203 		if (!peer->inbuf) {  in perf_dbgfs_read_info()
1210 			"\tIn buffer addr 0x%pK\n", peer->inbuf);  in perf_dbgfs_read_info()
1213 			"\tIn buffer size %pa\n", &peer->inbuf_size);  in perf_dbgfs_read_info()
1216 			"\tIn buffer xlat %pad[p]\n", &peer->inbuf_xlat);  in perf_dbgfs_read_info()
1256 	struct perf_peer *peer;  in perf_dbgfs_write_run()  local
1266 	peer = &perf->peers[pidx];  in perf_dbgfs_write_run()
1268 	ret = perf_submit_test(peer);  in perf_dbgfs_write_run()
1374 static int perf_setup_peer_mw(struct perf_peer *peer)  in perf_setup_peer_mw()  argument
1376 	struct perf_ctx *perf = peer->perf;  in perf_setup_peer_mw()
1382 				   &peer->outbuf_size);  in perf_setup_peer_mw()
1386 	peer->outbuf = devm_ioremap_wc(&perf->ntb->dev, phys_addr,  in perf_setup_peer_mw()
1387 					peer->outbuf_size);  in perf_setup_peer_mw()
1388 	if (!peer->outbuf)  in perf_setup_peer_mw()
1391 	if (max_mw_size && peer->outbuf_size > max_mw_size) {  in perf_setup_peer_mw()
1392 		peer->outbuf_size = max_mw_size;  in perf_setup_peer_mw()
1393 		dev_warn(&peer->perf->ntb->dev,  in perf_setup_peer_mw()
1394 			"Peer %d outbuf reduced to %pa\n", peer->pidx,  in perf_setup_peer_mw()
1395 			&peer->outbuf_size);  in perf_setup_peer_mw()
1403 	struct perf_peer *peer;  in perf_init_peers()  local
1409 		peer = &perf->peers[pidx];  in perf_init_peers()
1411 		peer->perf = perf;  in perf_init_peers()
1412 		peer->pidx = pidx;  in perf_init_peers()
1416 			peer->gidx = pidx + 1;  in perf_init_peers()
1418 			peer->gidx = pidx;  in perf_init_peers()
1420 		INIT_WORK(&peer->service, perf_service_work);  in perf_init_peers()