116 static int pp_find_next_peer(struct pp_ctx *pp)  in pp_find_next_peer()  argument
121 	link = ntb_link_is_up(pp->ntb, NULL, NULL);  in pp_find_next_peer()
124 	if (link & pp->nmask) {  in pp_find_next_peer()
125 		pidx = __ffs64(link & pp->nmask);  in pp_find_next_peer()
127 	} else if (link & pp->pmask) {  in pp_find_next_peer()
128 		pidx = __ffs64(link & pp->pmask);  in pp_find_next_peer()
134 	spin_lock(&pp->lock);  in pp_find_next_peer()
135 	pp->out_pidx = pidx;  in pp_find_next_peer()
136 	pp->out_db = out_db;  in pp_find_next_peer()
137 	spin_unlock(&pp->lock);  in pp_find_next_peer()
142 static void pp_setup(struct pp_ctx *pp)  in pp_setup()  argument
146 	ntb_db_set_mask(pp->ntb, pp->in_db);  in pp_setup()
148 	hrtimer_cancel(&pp->timer);  in pp_setup()
150 	ret = pp_find_next_peer(pp);  in pp_setup()
152 		dev_dbg(&pp->ntb->dev, "Got no peers, so cancel\n");  in pp_setup()
156 	dev_dbg(&pp->ntb->dev, "Ping-pong started with port %d, db %#llx\n",  in pp_setup()
157 		ntb_peer_port_number(pp->ntb, pp->out_pidx), pp->out_db);  in pp_setup()
159 	hrtimer_start(&pp->timer, ms_to_ktime(delay_ms), HRTIMER_MODE_REL);  in pp_setup()
162 static void pp_clear(struct pp_ctx *pp)  in pp_clear()  argument
164 	hrtimer_cancel(&pp->timer);  in pp_clear()
166 	ntb_db_set_mask(pp->ntb, pp->in_db);  in pp_clear()
168 	dev_dbg(&pp->ntb->dev, "Ping-pong cancelled\n");  in pp_clear()
171 static void pp_ping(struct pp_ctx *pp)  in pp_ping()  argument
175 	count = atomic_read(&pp->count);  in pp_ping()
177 	spin_lock(&pp->lock);  in pp_ping()
178 	ntb_peer_spad_write(pp->ntb, pp->out_pidx, 0, count);  in pp_ping()
179 	ntb_peer_msg_write(pp->ntb, pp->out_pidx, 0, count);  in pp_ping()
181 	dev_dbg(&pp->ntb->dev, "Ping port %d spad %#x, msg %#x\n",  in pp_ping()
182 		ntb_peer_port_number(pp->ntb, pp->out_pidx), count, count);  in pp_ping()
184 	ntb_peer_db_set(pp->ntb, pp->out_db);  in pp_ping()
185 	ntb_db_clear_mask(pp->ntb, pp->in_db);  in pp_ping()
186 	spin_unlock(&pp->lock);  in pp_ping()
189 static void pp_pong(struct pp_ctx *pp)  in pp_pong()  argument
195 	spad_data = ntb_spad_read(pp->ntb, 0);  in pp_pong()
196 	msg_data = ntb_msg_read(pp->ntb, &pidx, 0);  in pp_pong()
197 	ntb_msg_clear_sts(pp->ntb, -1);  in pp_pong()
204 	dev_dbg(&pp->ntb->dev, "Pong spad %#x, msg %#x (port %d)\n",  in pp_pong()
205 		spad_data, msg_data, ntb_peer_port_number(pp->ntb, pidx));  in pp_pong()
207 	atomic_inc(&pp->count);  in pp_pong()
209 	ntb_db_set_mask(pp->ntb, pp->in_db);  in pp_pong()
210 	ntb_db_clear(pp->ntb, pp->in_db);  in pp_pong()
212 	hrtimer_start(&pp->timer, ms_to_ktime(delay_ms), HRTIMER_MODE_REL);  in pp_pong()
217 	struct pp_ctx *pp = to_pp_timer(t);  in pp_timer_func()  local
219 	pp_ping(pp);  in pp_timer_func()
226 	struct pp_ctx *pp = ctx;  in pp_link_event()  local
228 	pp_setup(pp);  in pp_link_event()
233 	struct pp_ctx *pp = ctx;  in pp_db_event()  local
235 	pp_pong(pp);  in pp_db_event()
279 	struct pp_ctx *pp;  in pp_create_data()  local
281 	pp = devm_kzalloc(&ntb->dev, sizeof(*pp), GFP_KERNEL);  in pp_create_data()
282 	if (!pp)  in pp_create_data()
285 	pp->ntb = ntb;  in pp_create_data()
286 	atomic_set(&pp->count, 0);  in pp_create_data()
287 	spin_lock_init(&pp->lock);  in pp_create_data()
288 	hrtimer_init(&pp->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in pp_create_data()
289 	pp->timer.function = pp_timer_func;  in pp_create_data()
291 	return pp;  in pp_create_data()
294 static void pp_init_flds(struct pp_ctx *pp)  in pp_init_flds()  argument
299 	lport = ntb_port_number(pp->ntb);  in pp_init_flds()
300 	pcnt = ntb_peer_port_count(pp->ntb);  in pp_init_flds()
302 		if (lport < ntb_peer_port_number(pp->ntb, pidx))  in pp_init_flds()
306 	pp->in_db = BIT_ULL(pidx);  in pp_init_flds()
307 	pp->pmask = GENMASK_ULL(pidx, 0) >> 1;  in pp_init_flds()
308 	pp->nmask = GENMASK_ULL(pcnt - 1, pidx);  in pp_init_flds()
310 	dev_dbg(&pp->ntb->dev, "Inbound db %#llx, prev %#llx, next %#llx\n",  in pp_init_flds()
311 		pp->in_db, pp->pmask, pp->nmask);  in pp_init_flds()
314 static int pp_mask_events(struct pp_ctx *pp)  in pp_mask_events()  argument
319 	db_mask = ntb_db_valid_mask(pp->ntb);  in pp_mask_events()
320 	ret = ntb_db_set_mask(pp->ntb, db_mask);  in pp_mask_events()
325 	if (ntb_msg_count(pp->ntb) < 1)  in pp_mask_events()
328 	msg_mask = ntb_msg_outbits(pp->ntb) | ntb_msg_inbits(pp->ntb);  in pp_mask_events()
329 	return ntb_msg_set_mask(pp->ntb, msg_mask);  in pp_mask_events()
332 static int pp_setup_ctx(struct pp_ctx *pp)  in pp_setup_ctx()  argument
336 	ret = ntb_set_ctx(pp->ntb, pp, &pp_ops);  in pp_setup_ctx()
340 	ntb_link_enable(pp->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);  in pp_setup_ctx()
342 	ntb_link_event(pp->ntb);  in pp_setup_ctx()
347 static void pp_clear_ctx(struct pp_ctx *pp)  in pp_clear_ctx()  argument
349 	ntb_link_disable(pp->ntb);  in pp_clear_ctx()
351 	ntb_clear_ctx(pp->ntb);  in pp_clear_ctx()
354 static void pp_setup_dbgfs(struct pp_ctx *pp)  in pp_setup_dbgfs()  argument
356 	struct pci_dev *pdev = pp->ntb->pdev;  in pp_setup_dbgfs()
359 	pp->dbgfs_dir = debugfs_create_dir(pci_name(pdev), pp_dbgfs_topdir);  in pp_setup_dbgfs()
361 	ret = debugfs_create_atomic_t("count", 0600, pp->dbgfs_dir, &pp->count);  in pp_setup_dbgfs()
363 		dev_warn(&pp->ntb->dev, "DebugFS unsupported\n");  in pp_setup_dbgfs()
366 static void pp_clear_dbgfs(struct pp_ctx *pp)  in pp_clear_dbgfs()  argument
368 	debugfs_remove_recursive(pp->dbgfs_dir);  in pp_clear_dbgfs()
373 	struct pp_ctx *pp;  in pp_probe()  local
380 	pp = pp_create_data(ntb);  in pp_probe()
381 	if (IS_ERR(pp))  in pp_probe()
382 		return PTR_ERR(pp);  in pp_probe()
384 	pp_init_flds(pp);  in pp_probe()
386 	ret = pp_mask_events(pp);  in pp_probe()
390 	ret = pp_setup_ctx(pp);  in pp_probe()
394 	pp_setup_dbgfs(pp);  in pp_probe()
401 	struct pp_ctx *pp = ntb->ctx;  in pp_remove()  local
403 	pp_clear_dbgfs(pp);  in pp_remove()
405 	pp_clear_ctx(pp);  in pp_remove()
407 	pp_clear(pp);  in pp_remove()