281 qed_chain_advance_page(struct qed_chain *p_chain,  in qed_chain_advance_page()  argument
287 	switch (p_chain->mode) {  in qed_chain_advance_page()
291 		if (is_chain_u16(p_chain))  in qed_chain_advance_page()
292 			*(u16 *)idx_to_inc += p_chain->elem_unusable;  in qed_chain_advance_page()
294 			*(u32 *)idx_to_inc += p_chain->elem_unusable;  in qed_chain_advance_page()
297 		*p_next_elem = p_chain->p_virt_addr;  in qed_chain_advance_page()
301 		if (is_chain_u16(p_chain)) {  in qed_chain_advance_page()
302 			if (++(*(u16 *)page_to_inc) == p_chain->page_cnt)  in qed_chain_advance_page()
306 			if (++(*(u32 *)page_to_inc) == p_chain->page_cnt)  in qed_chain_advance_page()
310 		*p_next_elem = p_chain->pbl.pp_addr_tbl[page_index].virt_addr;  in qed_chain_advance_page()
346 static inline void qed_chain_return_produced(struct qed_chain *p_chain)  in qed_chain_return_produced()  argument
348 	if (is_chain_u16(p_chain))  in qed_chain_return_produced()
349 		p_chain->u.chain16.cons_idx++;  in qed_chain_return_produced()
351 		p_chain->u.chain32.cons_idx++;  in qed_chain_return_produced()
352 	test_and_skip(p_chain, cons_idx);  in qed_chain_return_produced()
366 static inline void *qed_chain_produce(struct qed_chain *p_chain)  in qed_chain_produce()  argument
370 	if (is_chain_u16(p_chain)) {  in qed_chain_produce()
371 		if ((p_chain->u.chain16.prod_idx &  in qed_chain_produce()
372 		     p_chain->elem_per_page_mask) == p_chain->next_page_mask) {  in qed_chain_produce()
373 			p_prod_idx = &p_chain->u.chain16.prod_idx;  in qed_chain_produce()
374 			p_prod_page_idx = &p_chain->pbl.c.u16.prod_page_idx;  in qed_chain_produce()
375 			qed_chain_advance_page(p_chain, &p_chain->p_prod_elem,  in qed_chain_produce()
378 		p_chain->u.chain16.prod_idx++;  in qed_chain_produce()
380 		if ((p_chain->u.chain32.prod_idx &  in qed_chain_produce()
381 		     p_chain->elem_per_page_mask) == p_chain->next_page_mask) {  in qed_chain_produce()
382 			p_prod_idx = &p_chain->u.chain32.prod_idx;  in qed_chain_produce()
383 			p_prod_page_idx = &p_chain->pbl.c.u32.prod_page_idx;  in qed_chain_produce()
384 			qed_chain_advance_page(p_chain, &p_chain->p_prod_elem,  in qed_chain_produce()
387 		p_chain->u.chain32.prod_idx++;  in qed_chain_produce()
390 	p_ret = p_chain->p_prod_elem;  in qed_chain_produce()
391 	p_chain->p_prod_elem = (void *)(((u8 *)p_chain->p_prod_elem) +  in qed_chain_produce()
392 					p_chain->elem_size);  in qed_chain_produce()
407 static inline u32 qed_chain_get_capacity(struct qed_chain *p_chain)  in qed_chain_get_capacity()  argument
409 	return p_chain->capacity;  in qed_chain_get_capacity()
420 static inline void qed_chain_recycle_consumed(struct qed_chain *p_chain)  in qed_chain_recycle_consumed()  argument
422 	test_and_skip(p_chain, prod_idx);  in qed_chain_recycle_consumed()
423 	if (is_chain_u16(p_chain))  in qed_chain_recycle_consumed()
424 		p_chain->u.chain16.prod_idx++;  in qed_chain_recycle_consumed()
426 		p_chain->u.chain32.prod_idx++;  in qed_chain_recycle_consumed()
439 static inline void *qed_chain_consume(struct qed_chain *p_chain)  in qed_chain_consume()  argument
443 	if (is_chain_u16(p_chain)) {  in qed_chain_consume()
444 		if ((p_chain->u.chain16.cons_idx &  in qed_chain_consume()
445 		     p_chain->elem_per_page_mask) == p_chain->next_page_mask) {  in qed_chain_consume()
446 			p_cons_idx = &p_chain->u.chain16.cons_idx;  in qed_chain_consume()
447 			p_cons_page_idx = &p_chain->pbl.c.u16.cons_page_idx;  in qed_chain_consume()
448 			qed_chain_advance_page(p_chain, &p_chain->p_cons_elem,  in qed_chain_consume()
451 		p_chain->u.chain16.cons_idx++;  in qed_chain_consume()
453 		if ((p_chain->u.chain32.cons_idx &  in qed_chain_consume()
454 		     p_chain->elem_per_page_mask) == p_chain->next_page_mask) {  in qed_chain_consume()
455 			p_cons_idx = &p_chain->u.chain32.cons_idx;  in qed_chain_consume()
456 			p_cons_page_idx = &p_chain->pbl.c.u32.cons_page_idx;  in qed_chain_consume()
457 			qed_chain_advance_page(p_chain, &p_chain->p_cons_elem,  in qed_chain_consume()
460 		p_chain->u.chain32.cons_idx++;  in qed_chain_consume()
463 	p_ret = p_chain->p_cons_elem;  in qed_chain_consume()
464 	p_chain->p_cons_elem = (void *)(((u8 *)p_chain->p_cons_elem) +  in qed_chain_consume()
465 					p_chain->elem_size);  in qed_chain_consume()
475 static inline void qed_chain_reset(struct qed_chain *p_chain)  in qed_chain_reset()  argument
479 	if (is_chain_u16(p_chain)) {  in qed_chain_reset()
480 		p_chain->u.chain16.prod_idx = 0;  in qed_chain_reset()
481 		p_chain->u.chain16.cons_idx = 0;  in qed_chain_reset()
483 		p_chain->u.chain32.prod_idx = 0;  in qed_chain_reset()
484 		p_chain->u.chain32.cons_idx = 0;  in qed_chain_reset()
486 	p_chain->p_cons_elem = p_chain->p_virt_addr;  in qed_chain_reset()
487 	p_chain->p_prod_elem = p_chain->p_virt_addr;  in qed_chain_reset()
489 	if (p_chain->mode == QED_CHAIN_MODE_PBL) {  in qed_chain_reset()
495 		u32 reset_val = p_chain->page_cnt - 1;  in qed_chain_reset()
497 		if (is_chain_u16(p_chain)) {  in qed_chain_reset()
498 			p_chain->pbl.c.u16.prod_page_idx = (u16)reset_val;  in qed_chain_reset()
499 			p_chain->pbl.c.u16.cons_page_idx = (u16)reset_val;  in qed_chain_reset()
501 			p_chain->pbl.c.u32.prod_page_idx = reset_val;  in qed_chain_reset()
502 			p_chain->pbl.c.u32.cons_page_idx = reset_val;  in qed_chain_reset()
506 	switch (p_chain->intended_use) {  in qed_chain_reset()
509 		for (i = 0; i < p_chain->capacity; i++)  in qed_chain_reset()
510 			qed_chain_recycle_consumed(p_chain);  in qed_chain_reset()
530 static inline void *qed_chain_get_last_elem(struct qed_chain *p_chain)  in qed_chain_get_last_elem()  argument
536 	if (!p_chain->p_virt_addr)  in qed_chain_get_last_elem()
539 	switch (p_chain->mode) {  in qed_chain_get_last_elem()
541 		size = p_chain->elem_size * p_chain->usable_per_page;  in qed_chain_get_last_elem()
542 		p_virt_addr = p_chain->p_virt_addr;  in qed_chain_get_last_elem()
544 		while (p_next->next_virt != p_chain->p_virt_addr) {  in qed_chain_get_last_elem()
551 		p_virt_addr = p_chain->p_virt_addr;  in qed_chain_get_last_elem()
554 		last_page_idx = p_chain->page_cnt - 1;  in qed_chain_get_last_elem()
555 		p_virt_addr = p_chain->pbl.pp_addr_tbl[last_page_idx].virt_addr;  in qed_chain_get_last_elem()
559 	size = p_chain->elem_size * (p_chain->usable_per_page - 1);  in qed_chain_get_last_elem()
571 static inline void qed_chain_set_prod(struct qed_chain *p_chain,  in qed_chain_set_prod()  argument
574 	if (p_chain->mode == QED_CHAIN_MODE_PBL) {  in qed_chain_set_prod()
577 		cur_prod = is_chain_u16(p_chain) ? p_chain->u.chain16.prod_idx :  in qed_chain_set_prod()
578 			   p_chain->u.chain32.prod_idx;  in qed_chain_set_prod()
581 		page_mask = ~p_chain->elem_per_page_mask;  in qed_chain_set_prod()
592 			    p_chain->elem_per_page;  in qed_chain_set_prod()
594 		page_cnt = qed_chain_get_page_cnt(p_chain);  in qed_chain_set_prod()
595 		if (is_chain_u16(p_chain))  in qed_chain_set_prod()
596 			p_chain->pbl.c.u16.prod_page_idx =  in qed_chain_set_prod()
597 				(p_chain->pbl.c.u16.prod_page_idx -  in qed_chain_set_prod()
600 			p_chain->pbl.c.u32.prod_page_idx =  in qed_chain_set_prod()
601 				(p_chain->pbl.c.u32.prod_page_idx -  in qed_chain_set_prod()
605 	if (is_chain_u16(p_chain))  in qed_chain_set_prod()
606 		p_chain->u.chain16.prod_idx = (u16) prod_idx;  in qed_chain_set_prod()
608 		p_chain->u.chain32.prod_idx = prod_idx;  in qed_chain_set_prod()
609 	p_chain->p_prod_elem = p_prod_elem;  in qed_chain_set_prod()
617 static inline void qed_chain_pbl_zero_mem(struct qed_chain *p_chain)  in qed_chain_pbl_zero_mem()  argument
621 	if (p_chain->mode != QED_CHAIN_MODE_PBL)  in qed_chain_pbl_zero_mem()
624 	page_cnt = qed_chain_get_page_cnt(p_chain);  in qed_chain_pbl_zero_mem()
627 		memset(p_chain->pbl.pp_addr_tbl[i].virt_addr, 0,  in qed_chain_pbl_zero_mem()
628 		       p_chain->page_size);  in qed_chain_pbl_zero_mem()