1 /* Copyright 2009 - 2016 Freescale Semiconductor, Inc.
33 #include <linux/dma-mapping.h>
44  * is to allow enough handlers/FQs to truly test the significance of caching -
45  * ie. when cache-expiries are occurring.)
50  * 32-bit "mixer", that is produced using a 32-bit LFSR. When a frame is
55  * expected path, this also provides some quasi-realistic overheads to each
56  * forwarding action - dereferencing *all* the frame data, computation, and
65  *    handlers and link-list them (but do no other handler setup).
94 	atomic_inc(&bstrap->started);  in bstrap_fn()
95 	err = bstrap->fn();  in bstrap_fn()
116 			return -ENOMEM;  in on_all_cpus()
121 		 * effect, then the thread may exit with -EINTR without ever  in on_all_cpus()
137 	/* -------------- */
142 	/* The value we XOR post-dequeue, prior to validating */
144 	/* The value we XOR pre-enqueue, after validating */
151 	/* -------------- */
162 	/* root node for the per-cpu list of handlers */
168 	 * handlers together, this is used as per-cpu iterator state
176 /* links together the hp_cpu structs, in first-come first-serve order. */
195 static const struct qm_portal_config *pcfg;  variable
204 /* First word of the LFSR-based frame data */
209 	return (prev >> 1) ^ (-(prev & 1u) & 0xd0000001u);  in do_lfsr()
219 		return -EIO;  in allocate_frame_data()
222 	pcfg = qman_get_qm_portal_config(qman_dma_portal);  in allocate_frame_data()
226 		return -ENOMEM;  in allocate_frame_data()
234 	frame_dma = dma_map_single(pcfg->dev, frame_ptr, 4 * HP_NUM_WORDS,  in allocate_frame_data()
236 	if (dma_mapping_error(pcfg->dev, frame_dma)) {  in allocate_frame_data()
239 		return -EIO;  in allocate_frame_data()
247 	dma_unmap_single(pcfg->dev, frame_dma, 4 * HP_NUM_WORDS,  in deallocate_frame_data()
255 	u32 *p = handler->frame_ptr;  in process_frame_data()
259 	if (qm_fd_addr_get64(fd) != handler->addr) {  in process_frame_data()
261 			qm_fd_addr_get64(fd), handler->addr);  in process_frame_data()
262 		return -EIO;  in process_frame_data()
265 		*p ^= handler->rx_mixer;  in process_frame_data()
268 			return -EIO;  in process_frame_data()
270 		*p ^= handler->tx_mixer;  in process_frame_data()
283 	if (process_frame_data(handler, &dqrr->fd)) {  in normal_dqrr()
287 	if (qman_enqueue(&handler->tx, &dqrr->fd)) {  in normal_dqrr()
302 	process_frame_data(handler, &dqrr->fd);  in special_dqrr()
304 		if (qman_enqueue(&handler->tx, &dqrr->fd)) {  in special_dqrr()
323 	hp_cpu->processor_id = smp_processor_id();  in create_per_cpu_handlers()
325 	list_add_tail(&hp_cpu->node, &hp_cpu_list);  in create_per_cpu_handlers()
328 	INIT_LIST_HEAD(&hp_cpu->handlers);  in create_per_cpu_handlers()
334 			return -EIO;  in create_per_cpu_handlers()
336 		handler->processor_id = hp_cpu->processor_id;  in create_per_cpu_handlers()
337 		handler->addr = frame_dma;  in create_per_cpu_handlers()
338 		handler->frame_ptr = frame_ptr;  in create_per_cpu_handlers()
339 		list_add_tail(&handler->node, &hp_cpu->handlers);  in create_per_cpu_handlers()
350 	list_del(&hp_cpu->node);  in destroy_per_cpu_handlers()
352 	list_for_each_safe(loop, tmp, &hp_cpu->handlers) {  in destroy_per_cpu_handlers()
356 		if (qman_retire_fq(&handler->rx, &flags) ||  in destroy_per_cpu_handlers()
360 			return -EIO;  in destroy_per_cpu_handlers()
362 		if (qman_oos_fq(&handler->rx)) {  in destroy_per_cpu_handlers()
365 			return -EIO;  in destroy_per_cpu_handlers()
367 		qman_destroy_fq(&handler->rx);  in destroy_per_cpu_handlers()
368 		qman_destroy_fq(&handler->tx);  in destroy_per_cpu_handlers()
369 		qman_release_fqid(handler->fqid_rx);  in destroy_per_cpu_handlers()
370 		list_del(&handler->node);  in destroy_per_cpu_handlers()
378 	u8 res = (offset + (L1_CACHE_BYTES - 1))  in num_cachelines()
395 	if (handler->processor_id != smp_processor_id()) {  in init_handler()
396 		err = -EIO;  in init_handler()
400 	memset(&handler->rx, 0, sizeof(handler->rx));  in init_handler()
402 		handler->rx.cb.dqrr = special_dqrr;  in init_handler()
404 		handler->rx.cb.dqrr = normal_dqrr;  in init_handler()
405 	err = qman_create_fq(handler->fqid_rx, 0, &handler->rx);  in init_handler()
415 	err = qman_init_fq(&handler->rx, QMAN_INITFQ_FLAG_SCHED |  in init_handler()
422 	memset(&handler->tx, 0, sizeof(handler->tx));  in init_handler()
423 	err = qman_create_fq(handler->fqid_tx, QMAN_FQ_FLAG_NO_MODIFY,  in init_handler()
424 			     &handler->tx);  in init_handler()
454 				hp_cpu->iterator = list_first_entry(  in init_phase2()
455 						&hp_cpu->handlers,  in init_phase2()
458 				hp_cpu->iterator = list_entry(  in init_phase2()
459 						hp_cpu->iterator->node.next,  in init_phase2()
462 			hp_cpu->iterator->fqid_rx = fqid;  in init_phase2()
469 			hp_cpu->iterator->fqid_tx = fqid;  in init_phase2()
471 			hp_cpu->iterator->rx_mixer = lfsr;  in init_phase2()
474 			hp_cpu->iterator->tx_mixer = lfsr;  in init_phase2()
479 	handler = list_first_entry(&hp_cpu->handlers, struct hp_handler, node);  in init_phase2()
480 	if (handler->fqid_rx != 0 || handler->rx_mixer != 0xdeadbeef)  in init_phase2()
482 	handler->fqid_rx = fqid;  in init_phase2()
483 	handler->rx_mixer = lfsr;  in init_phase2()
497 				hp_cpu->iterator = list_first_entry(  in init_phase3()
498 						&hp_cpu->handlers,  in init_phase3()
501 				hp_cpu->iterator = list_entry(  in init_phase3()
502 						hp_cpu->iterator->node.next,  in init_phase3()
505 			if (hp_cpu->processor_id == smp_processor_id()) {  in init_phase3()
506 				err = init_handler(hp_cpu->iterator);  in init_phase3()
510 				smp_call_function_single(hp_cpu->processor_id,  in init_phase3()
511 					init_handler_cb, hp_cpu->iterator, 1);  in init_phase3()
521 	u32 *p = special_handler->frame_ptr;  in send_first_frame()
526 	if (special_handler->processor_id != smp_processor_id()) {  in send_first_frame()
527 		err = -EIO;  in send_first_frame()
531 	qm_fd_addr_set64(&fd, special_handler->addr);  in send_first_frame()
535 			err = -EIO;  in send_first_frame()
539 		*p ^= special_handler->tx_mixer;  in send_first_frame()
543 	err = qman_enqueue(&special_handler->tx, &fd);  in send_first_frame()
565 		pr_info("%s(): skip - only 1 CPU\n", __func__);  in qman_test_stash()
577 		err = -EIO;  in qman_test_stash()
589 		err = -EIO;  in qman_test_stash()
605 	if (special_handler->processor_id == smp_processor_id()) {  in qman_test_stash()
610 		smp_call_function_single(special_handler->processor_id,  in qman_test_stash()
618 		err = -EIO;  in qman_test_stash()