421 static inline int qm_eqcr_init(struct qm_portal *portal,  in qm_eqcr_init()  argument
426 	struct qm_eqcr *eqcr = &portal->eqcr;  in qm_eqcr_init()
430 	eqcr->ring = portal->addr.ce + QM_CL_EQCR;  in qm_eqcr_init()
431 	eqcr->ci = qm_in(portal, QM_REG_EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);  in qm_eqcr_init()
432 	qm_cl_invalidate(portal, QM_CL_EQCR_CI_CENA);  in qm_eqcr_init()
433 	pi = qm_in(portal, QM_REG_EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);  in qm_eqcr_init()
435 	eqcr->vbit = (qm_in(portal, QM_REG_EQCR_PI_CINH) & QM_EQCR_SIZE) ?  in qm_eqcr_init()
439 	eqcr->ithresh = qm_in(portal, QM_REG_EQCR_ITR);  in qm_eqcr_init()
444 	cfg = (qm_in(portal, QM_REG_CFG) & 0x00ffffff) |  in qm_eqcr_init()
448 	qm_out(portal, QM_REG_CFG, cfg);  in qm_eqcr_init()
452 static inline void qm_eqcr_finish(struct qm_portal *portal)  in qm_eqcr_finish()  argument
454 	struct qm_eqcr *eqcr = &portal->eqcr;  in qm_eqcr_finish()
455 	u8 pi = qm_in(portal, QM_REG_EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);  in qm_eqcr_finish()
456 	u8 ci = qm_in(portal, QM_REG_EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);  in qm_eqcr_finish()
468 								 *portal)  in qm_eqcr_start_no_stash()
470 	struct qm_eqcr *eqcr = &portal->eqcr;  in qm_eqcr_start_no_stash()
484 								*portal)  in qm_eqcr_start_stash()
486 	struct qm_eqcr *eqcr = &portal->eqcr;  in qm_eqcr_start_stash()
492 		eqcr->ci = qm_ce_in(portal, QM_CL_EQCR_CI_CENA) &  in qm_eqcr_start_stash()
513 static inline void qm_eqcr_pvb_commit(struct qm_portal *portal, u8 myverb)  in qm_eqcr_pvb_commit()  argument
515 	struct qm_eqcr *eqcr = &portal->eqcr;  in qm_eqcr_pvb_commit()
531 static inline void qm_eqcr_cce_prefetch(struct qm_portal *portal)  in qm_eqcr_cce_prefetch()  argument
533 	qm_cl_touch_ro(portal, QM_CL_EQCR_CI_CENA);  in qm_eqcr_cce_prefetch()
536 static inline u8 qm_eqcr_cce_update(struct qm_portal *portal)  in qm_eqcr_cce_update()  argument
538 	struct qm_eqcr *eqcr = &portal->eqcr;  in qm_eqcr_cce_update()
541 	eqcr->ci = qm_ce_in(portal, QM_CL_EQCR_CI_CENA) & (QM_EQCR_SIZE - 1);  in qm_eqcr_cce_update()
542 	qm_cl_invalidate(portal, QM_CL_EQCR_CI_CENA);  in qm_eqcr_cce_update()
548 static inline void qm_eqcr_set_ithresh(struct qm_portal *portal, u8 ithresh)  in qm_eqcr_set_ithresh()  argument
550 	struct qm_eqcr *eqcr = &portal->eqcr;  in qm_eqcr_set_ithresh()
553 	qm_out(portal, QM_REG_EQCR_ITR, ithresh);  in qm_eqcr_set_ithresh()
556 static inline u8 qm_eqcr_get_avail(struct qm_portal *portal)  in qm_eqcr_get_avail()  argument
558 	struct qm_eqcr *eqcr = &portal->eqcr;  in qm_eqcr_get_avail()
563 static inline u8 qm_eqcr_get_fill(struct qm_portal *portal)  in qm_eqcr_get_fill()  argument
565 	struct qm_eqcr *eqcr = &portal->eqcr;  in qm_eqcr_get_fill()
595 static inline void qm_dqrr_set_maxfill(struct qm_portal *portal, u8 mf)  in qm_dqrr_set_maxfill()  argument
597 	qm_out(portal, QM_REG_CFG, (qm_in(portal, QM_REG_CFG) & 0xff0fffff) |  in qm_dqrr_set_maxfill()
601 static inline int qm_dqrr_init(struct qm_portal *portal,  in qm_dqrr_init()  argument
607 	struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_init()
611 	qm_out(portal, QM_REG_DQRR_SDQCR, 0);  in qm_dqrr_init()
612 	qm_out(portal, QM_REG_DQRR_VDQCR, 0);  in qm_dqrr_init()
613 	qm_out(portal, QM_REG_DQRR_PDQCR, 0);  in qm_dqrr_init()
614 	dqrr->ring = portal->addr.ce + QM_CL_DQRR;  in qm_dqrr_init()
615 	dqrr->pi = qm_in(portal, QM_REG_DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);  in qm_dqrr_init()
616 	dqrr->ci = qm_in(portal, QM_REG_DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);  in qm_dqrr_init()
619 	dqrr->vbit = (qm_in(portal, QM_REG_DQRR_PI_CINH) & QM_DQRR_SIZE) ?  in qm_dqrr_init()
621 	dqrr->ithresh = qm_in(portal, QM_REG_DQRR_ITR);  in qm_dqrr_init()
630 	cfg = (qm_in(portal, QM_REG_CFG) & 0xff000f00) |  in qm_dqrr_init()
637 	qm_out(portal, QM_REG_CFG, cfg);  in qm_dqrr_init()
638 	qm_dqrr_set_maxfill(portal, max_fill);  in qm_dqrr_init()
642 static inline void qm_dqrr_finish(struct qm_portal *portal)  in qm_dqrr_finish()  argument
645 	struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_finish()
654 						struct qm_portal *portal)  in qm_dqrr_current()  argument
656 	struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_current()
663 static inline u8 qm_dqrr_next(struct qm_portal *portal)  in qm_dqrr_next()  argument
665 	struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_next()
672 static inline void qm_dqrr_pvb_update(struct qm_portal *portal)  in qm_dqrr_pvb_update()  argument
674 	struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_pvb_update()
693 static inline void qm_dqrr_cdc_consume_1ptr(struct qm_portal *portal,  in qm_dqrr_cdc_consume_1ptr()  argument
697 	__maybe_unused struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_cdc_consume_1ptr()
703 	qm_out(portal, QM_REG_DQRR_DCAP, (0 << 8) | /* DQRR_DCAP::S */  in qm_dqrr_cdc_consume_1ptr()
708 static inline void qm_dqrr_cdc_consume_n(struct qm_portal *portal, u32 bitmask)  in qm_dqrr_cdc_consume_n()  argument
710 	__maybe_unused struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_cdc_consume_n()
713 	qm_out(portal, QM_REG_DQRR_DCAP, (1 << 8) | /* DQRR_DCAP::S */  in qm_dqrr_cdc_consume_n()
717 static inline void qm_dqrr_sdqcr_set(struct qm_portal *portal, u32 sdqcr)  in qm_dqrr_sdqcr_set()  argument
719 	qm_out(portal, QM_REG_DQRR_SDQCR, sdqcr);  in qm_dqrr_sdqcr_set()
722 static inline void qm_dqrr_vdqcr_set(struct qm_portal *portal, u32 vdqcr)  in qm_dqrr_vdqcr_set()  argument
724 	qm_out(portal, QM_REG_DQRR_VDQCR, vdqcr);  in qm_dqrr_vdqcr_set()
727 static inline int qm_dqrr_set_ithresh(struct qm_portal *portal, u8 ithresh)  in qm_dqrr_set_ithresh()  argument
733 	qm_out(portal, QM_REG_DQRR_ITR, ithresh);  in qm_dqrr_set_ithresh()
762 static inline int qm_mr_init(struct qm_portal *portal, enum qm_mr_pmode pmode,  in qm_mr_init()  argument
765 	struct qm_mr *mr = &portal->mr;  in qm_mr_init()
768 	mr->ring = portal->addr.ce + QM_CL_MR;  in qm_mr_init()
769 	mr->pi = qm_in(portal, QM_REG_MR_PI_CINH) & (QM_MR_SIZE - 1);  in qm_mr_init()
770 	mr->ci = qm_in(portal, QM_REG_MR_CI_CINH) & (QM_MR_SIZE - 1);  in qm_mr_init()
773 	mr->vbit = (qm_in(portal, QM_REG_MR_PI_CINH) & QM_MR_SIZE)  in qm_mr_init()
775 	mr->ithresh = qm_in(portal, QM_REG_MR_ITR);  in qm_mr_init()
780 	cfg = (qm_in(portal, QM_REG_CFG) & 0xfffff0ff) |  in qm_mr_init()
782 	qm_out(portal, QM_REG_CFG, cfg);  in qm_mr_init()
786 static inline void qm_mr_finish(struct qm_portal *portal)  in qm_mr_finish()  argument
788 	struct qm_mr *mr = &portal->mr;  in qm_mr_finish()
794 static inline const union qm_mr_entry *qm_mr_current(struct qm_portal *portal)  in qm_mr_current()  argument
796 	struct qm_mr *mr = &portal->mr;  in qm_mr_current()
803 static inline int qm_mr_next(struct qm_portal *portal)  in qm_mr_next()  argument
805 	struct qm_mr *mr = &portal->mr;  in qm_mr_next()
812 static inline void qm_mr_pvb_update(struct qm_portal *portal)  in qm_mr_pvb_update()  argument
814 	struct qm_mr *mr = &portal->mr;  in qm_mr_pvb_update()
829 static inline void qm_mr_cci_consume(struct qm_portal *portal, u8 num)  in qm_mr_cci_consume()  argument
831 	struct qm_mr *mr = &portal->mr;  in qm_mr_cci_consume()
835 	qm_out(portal, QM_REG_MR_CI_CINH, mr->ci);  in qm_mr_cci_consume()
838 static inline void qm_mr_cci_consume_to_current(struct qm_portal *portal)  in qm_mr_cci_consume_to_current()  argument
840 	struct qm_mr *mr = &portal->mr;  in qm_mr_cci_consume_to_current()
844 	qm_out(portal, QM_REG_MR_CI_CINH, mr->ci);  in qm_mr_cci_consume_to_current()
847 static inline void qm_mr_set_ithresh(struct qm_portal *portal, u8 ithresh)  in qm_mr_set_ithresh()  argument
849 	qm_out(portal, QM_REG_MR_ITR, ithresh);  in qm_mr_set_ithresh()
854 static inline int qm_mc_init(struct qm_portal *portal)  in qm_mc_init()  argument
857 	struct qm_mc *mc = &portal->mc;  in qm_mc_init()
859 	mc->cr = portal->addr.ce + QM_CL_CR;  in qm_mc_init()
860 	mc->rr = portal->addr.ce + QM_CL_RR0;  in qm_mc_init()
881 static inline void qm_mc_finish(struct qm_portal *portal)  in qm_mc_finish()  argument
884 	struct qm_mc *mc = &portal->mc;  in qm_mc_finish()
892 static inline union qm_mc_command *qm_mc_start(struct qm_portal *portal)  in qm_mc_start()  argument
894 	struct qm_mc *mc = &portal->mc;  in qm_mc_start()
904 static inline void qm_mc_commit(struct qm_portal *portal, u8 myverb)  in qm_mc_commit()  argument
906 	struct qm_mc *mc = &portal->mc;  in qm_mc_commit()
919 static inline union qm_mc_result *qm_mc_result(struct qm_portal *portal)  in qm_mc_result()  argument
921 	struct qm_mc *mc = &portal->mc;  in qm_mc_result()
942 static inline int qm_mc_result_timeout(struct qm_portal *portal,  in qm_mc_result_timeout()  argument
948 		*mcr = qm_mc_result(portal);  in qm_mc_result_timeout()
1032 int qman_dqrr_set_ithresh(struct qman_portal *portal, u8 ithresh)  in qman_dqrr_set_ithresh()  argument
1036 	if (!portal)  in qman_dqrr_set_ithresh()
1039 	res = qm_dqrr_set_ithresh(&portal->p, ithresh);  in qman_dqrr_set_ithresh()
1043 	portal->p.dqrr.ithresh = ithresh;  in qman_dqrr_set_ithresh()
1049 void qman_dqrr_get_ithresh(struct qman_portal *portal, u8 *ithresh)  in qman_dqrr_get_ithresh()  argument
1051 	if (portal && ithresh)  in qman_dqrr_get_ithresh()
1052 		*ithresh = qm_in(&portal->p, QM_REG_DQRR_ITR);  in qman_dqrr_get_ithresh()
1056 void qman_portal_get_iperiod(struct qman_portal *portal, u32 *iperiod)  in qman_portal_get_iperiod()  argument
1058 	if (portal && iperiod)  in qman_portal_get_iperiod()
1059 		*iperiod = qm_in(&portal->p, QM_REG_ITPR);  in qman_portal_get_iperiod()
1063 int qman_portal_set_iperiod(struct qman_portal *portal, u32 iperiod)  in qman_portal_set_iperiod()  argument
1065 	if (!portal || iperiod > QMAN_ITP_MAX)  in qman_portal_set_iperiod()
1068 	qm_out(&portal->p, QM_REG_ITPR, iperiod);  in qman_portal_set_iperiod()
1223 static int qman_create_portal(struct qman_portal *portal,  in qman_create_portal()  argument
1231 	p = &portal->p;  in qman_create_portal()
1235 	portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ? 1 : 0);  in qman_create_portal()
1237 	portal->use_eqcr_ci_stashing = 0;  in qman_create_portal()
1252 			portal->use_eqcr_ci_stashing ? 3 : 0, 1)) {  in qman_create_portal()
1273 	portal->cgrs = kmalloc_array(2, sizeof(*cgrs), GFP_KERNEL);  in qman_create_portal()
1274 	if (!portal->cgrs)  in qman_create_portal()
1277 	qman_cgrs_init(&portal->cgrs[1]);  in qman_create_portal()
1279 		portal->cgrs[0] = *cgrs;  in qman_create_portal()
1282 		qman_cgrs_fill(&portal->cgrs[0]);  in qman_create_portal()
1283 	INIT_LIST_HEAD(&portal->cgr_cbs);  in qman_create_portal()
1284 	spin_lock_init(&portal->cgr_lock);  in qman_create_portal()
1285 	INIT_WORK(&portal->congestion_work, qm_congestion_task);  in qman_create_portal()
1286 	INIT_WORK(&portal->mr_work, qm_mr_process_task);  in qman_create_portal()
1287 	portal->bits = 0;  in qman_create_portal()
1288 	portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 |  in qman_create_portal()
1293 	portal->irq_sources = 0;  in qman_create_portal()
1295 	snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu);  in qman_create_portal()
1297 	if (request_irq(c->irq, portal_isr, 0, portal->irqname,	portal)) {  in qman_create_portal()
1328 	portal->config = c;  in qman_create_portal()
1334 	qm_dqrr_sdqcr_set(p, portal->sdqcr);  in qman_create_portal()
1340 	free_irq(c->irq, portal);  in qman_create_portal()
1342 	kfree(portal->cgrs);  in qman_create_portal()
1358 	struct qman_portal *portal;  in qman_create_affine_portal()  local
1361 	portal = &per_cpu(qman_affine_portal, c->cpu);  in qman_create_affine_portal()
1362 	err = qman_create_portal(portal, c, cgrs);  in qman_create_affine_portal()
1369 	affine_portals[c->cpu] = portal;  in qman_create_affine_portal()
1372 	return portal;  in qman_create_affine_portal()
1731 		struct qman_portal *portal = get_affine_portal();  in qman_affine_channel()  local
1733 		cpu = portal->config->cpu;  in qman_affine_channel()
2830 						struct qman_portal *portal)  in qman_get_qm_portal_config()  argument
2832 	return portal->config;  in qman_get_qm_portal_config()