40 #define QMAN_ITP_MAX 0xFFF
48 #define QM_REG_EQCR_PI_CINH	0x3000
49 #define QM_REG_EQCR_CI_CINH	0x3040
50 #define QM_REG_EQCR_ITR		0x3080
51 #define QM_REG_DQRR_PI_CINH	0x3100
52 #define QM_REG_DQRR_CI_CINH	0x3140
53 #define QM_REG_DQRR_ITR		0x3180
54 #define QM_REG_DQRR_DCAP	0x31C0
55 #define QM_REG_DQRR_SDQCR	0x3200
56 #define QM_REG_DQRR_VDQCR	0x3240
57 #define QM_REG_DQRR_PDQCR	0x3280
58 #define QM_REG_MR_PI_CINH	0x3300
59 #define QM_REG_MR_CI_CINH	0x3340
60 #define QM_REG_MR_ITR		0x3380
61 #define QM_REG_CFG		0x3500
62 #define QM_REG_ISR		0x3600
63 #define QM_REG_IER		0x3640
64 #define QM_REG_ISDR		0x3680
65 #define QM_REG_IIR		0x36C0
66 #define QM_REG_ITPR		0x3740
69 #define QM_CL_EQCR		0x0000
70 #define QM_CL_DQRR		0x1000
71 #define QM_CL_MR		0x2000
72 #define QM_CL_EQCR_PI_CENA	0x3000
73 #define QM_CL_EQCR_CI_CENA	0x3040
74 #define QM_CL_DQRR_PI_CENA	0x3100
75 #define QM_CL_DQRR_CI_CENA	0x3140
76 #define QM_CL_MR_PI_CENA	0x3300
77 #define QM_CL_MR_CI_CENA	0x3340
78 #define QM_CL_CR		0x3800
79 #define QM_CL_RR0		0x3900
80 #define QM_CL_RR1		0x3940
84 #define QM_REG_EQCR_PI_CINH	0x0000
85 #define QM_REG_EQCR_CI_CINH	0x0004
86 #define QM_REG_EQCR_ITR		0x0008
87 #define QM_REG_DQRR_PI_CINH	0x0040
88 #define QM_REG_DQRR_CI_CINH	0x0044
89 #define QM_REG_DQRR_ITR		0x0048
90 #define QM_REG_DQRR_DCAP	0x0050
91 #define QM_REG_DQRR_SDQCR	0x0054
92 #define QM_REG_DQRR_VDQCR	0x0058
93 #define QM_REG_DQRR_PDQCR	0x005c
94 #define QM_REG_MR_PI_CINH	0x0080
95 #define QM_REG_MR_CI_CINH	0x0084
96 #define QM_REG_MR_ITR		0x0088
97 #define QM_REG_CFG		0x0100
98 #define QM_REG_ISR		0x0e00
99 #define QM_REG_IER		0x0e04
100 #define QM_REG_ISDR		0x0e08
101 #define QM_REG_IIR		0x0e0c
102 #define QM_REG_ITPR		0x0e14
105 #define QM_CL_EQCR		0x0000
106 #define QM_CL_DQRR		0x1000
107 #define QM_CL_MR		0x2000
108 #define QM_CL_EQCR_PI_CENA	0x3000
109 #define QM_CL_EQCR_CI_CENA	0x3100
110 #define QM_CL_DQRR_PI_CENA	0x3200
111 #define QM_CL_DQRR_CI_CENA	0x3300
112 #define QM_CL_MR_PI_CENA	0x3400
113 #define QM_CL_MR_CI_CENA	0x3500
114 #define QM_CL_CR		0x3800
115 #define QM_CL_RR0		0x3900
116 #define QM_CL_RR1		0x3940
145 	qm_eqcr_pci = 0,	/* PI index, cache-inhibited */
150 	qm_dqrr_dpush = 0,	/* SDQCR  + VDQCR */
159 	qm_dqrr_cci = 0,	/* CI index, cache-inhibited */
169 	qm_mr_cci = 0,		/* CI index, cache-inhibited */
190 #define QM_EQCR_VERB_VBIT		0x80
191 #define QM_EQCR_VERB_CMD_MASK		0x61	/* but only one value; */
192 #define QM_EQCR_VERB_CMD_ENQUEUE	0x01
193 #define QM_EQCR_SEQNUM_NESN		0x8000	/* Advance NESN */
194 #define QM_EQCR_SEQNUM_NLIS		0x4000	/* More fragments to come */
195 #define QM_EQCR_SEQNUM_SEQMASK		0x3fff	/* sequence number goes here */
242 #define QM_MCC_VERB_VBIT		0x80
243 #define QM_MCC_VERB_MASK		0x7f	/* where the verb contains; */
244 #define QM_MCC_VERB_INITFQ_PARKED	0x40
245 #define QM_MCC_VERB_INITFQ_SCHED	0x41
246 #define QM_MCC_VERB_QUERYFQ		0x44
247 #define QM_MCC_VERB_QUERYFQ_NP		0x45	/* "non-programmable" fields */
248 #define QM_MCC_VERB_QUERYWQ		0x46
249 #define QM_MCC_VERB_QUERYWQ_DEDICATED	0x47
250 #define QM_MCC_VERB_ALTER_SCHED		0x48	/* Schedule FQ */
251 #define QM_MCC_VERB_ALTER_FE		0x49	/* Force Eligible FQ */
252 #define QM_MCC_VERB_ALTER_RETIRE	0x4a	/* Retire FQ */
253 #define QM_MCC_VERB_ALTER_OOS		0x4b	/* Take FQ out of service */
254 #define QM_MCC_VERB_ALTER_FQXON		0x4d	/* FQ XON */
255 #define QM_MCC_VERB_ALTER_FQXOFF	0x4e	/* FQ XOFF */
256 #define QM_MCC_VERB_INITCGR		0x50
257 #define QM_MCC_VERB_MODIFYCGR		0x51
258 #define QM_MCC_VERB_CGRTESTWRITE	0x52
259 #define QM_MCC_VERB_QUERYCGR		0x58
260 #define QM_MCC_VERB_QUERYCONGESTION	0x59
289 #define QM_MCR_VERB_RRID		0x80
301 #define QM_MCR_RESULT_NULL		0x00
302 #define QM_MCR_RESULT_OK		0xf0
303 #define QM_MCR_RESULT_ERR_FQID		0xf1
304 #define QM_MCR_RESULT_ERR_FQSTATE	0xf2
305 #define QM_MCR_RESULT_ERR_NOTEMPTY	0xf3	/* OOS fails if FQ is !empty */
306 #define QM_MCR_RESULT_ERR_BADCHANNEL	0xf4
307 #define QM_MCR_RESULT_PENDING		0xf8
308 #define QM_MCR_RESULT_ERR_BADCOMMAND	0xff
309 #define QM_MCR_FQS_ORLPRESENT		0x02	/* ORL fragments to come */
310 #define QM_MCR_FQS_NOTEMPTY		0x01	/* FQ has enqueued frames */
436 		     QM_EQCR_VERB_VBIT : 0;  in qm_eqcr_init()
441 	eqcr->busy = 0;  in qm_eqcr_init()
444 	cfg = (qm_in(portal, QM_REG_CFG) & 0x00ffffff) |  in qm_eqcr_init()
447 	      ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */  in qm_eqcr_init()
449 	return 0;  in qm_eqcr_init()
527 	eqcr->busy = 0;  in qm_eqcr_pvb_commit()
597 	qm_out(portal, QM_REG_CFG, (qm_in(portal, QM_REG_CFG) & 0xff0fffff) |  in qm_dqrr_set_maxfill()
611 	qm_out(portal, QM_REG_DQRR_SDQCR, 0);  in qm_dqrr_init()
612 	qm_out(portal, QM_REG_DQRR_VDQCR, 0);  in qm_dqrr_init()
613 	qm_out(portal, QM_REG_DQRR_PDQCR, 0);  in qm_dqrr_init()
620 			QM_DQRR_VERB_VBIT : 0;  in qm_dqrr_init()
628 	for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++)  in qm_dqrr_init()
630 	cfg = (qm_in(portal, QM_REG_CFG) & 0xff000f00) |  in qm_dqrr_init()
634 		0xa0 |					/* RE+SE */  in qm_dqrr_init()
635 		(0 ? 0x40 : 0) |			/* Ignore RP */  in qm_dqrr_init()
636 		(0 ? 0x10 : 0);				/* Ignore SP */  in qm_dqrr_init()
639 	return 0;  in qm_dqrr_init()
703 	qm_out(portal, QM_REG_DQRR_DCAP, (0 << 8) | /* DQRR_DCAP::S */  in qm_dqrr_cdc_consume_1ptr()
704 	       ((park ? 1 : 0) << 6) |		    /* DQRR_DCAP::PK */  in qm_dqrr_cdc_consume_1ptr()
735 	return 0;  in qm_dqrr_set_ithresh()
774 		? QM_MR_VERB_VBIT : 0;  in qm_mr_init()
780 	cfg = (qm_in(portal, QM_REG_CFG) & 0xfffff0ff) |  in qm_mr_init()
783 	return 0;  in qm_mr_init()
862 	 * The expected valid bit polarity for the next CR command is 0  in qm_mc_init()
864 	 * valid response. If both RR contain all 0, this indicates either  in qm_mc_init()
870 	if ((rr0 == 0 && rr1 == 0) || rr0 != 0)  in qm_mc_init()
873 		mc->rridx = 0;  in qm_mc_init()
874 	mc->vbit = mc->rridx ? QM_MCC_VERB_VBIT : 0;  in qm_mc_init()
878 	return 0;  in qm_mc_init()
989 	/* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */
1021 	for (i = 0; i < num_possible_cpus(); i++) {  in get_portal_for_channel()
1045 	return 0;  in qman_dqrr_set_ithresh()
1070 	return 0;  in qman_portal_set_iperiod()
1076 	qm_portal_wq = alloc_workqueue("qman_portal_wq", 0, 1);  in qman_wq_alloc()
1079 	return 0;  in qman_wq_alloc()
1087 	for (i = 0; i < num_possible_cpus(); i++) {  in qman_enable_irqs()
1089 			qm_out(&affine_portals[i]->p, QM_REG_ISR, 0xffffffff);  in qman_enable_irqs()
1090 			qm_out(&affine_portals[i]->p, QM_REG_IIR, 0);  in qman_enable_irqs()
1116 	return 0;  in qman_alloc_fq_table()
1170 	u32 clear = 0;  in portal_isr()
1211 			return 0;  in drain_mr_fqrni()
1215 		pr_err("Found verb 0x%x in MR\n", msg->verb);  in drain_mr_fqrni()
1235 	portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ? 1 : 0);  in qman_create_portal()
1237 	portal->use_eqcr_ci_stashing = 0;  in qman_create_portal()
1252 			portal->use_eqcr_ci_stashing ? 3 : 0, 1)) {  in qman_create_portal()
1279 		portal->cgrs[0] = *cgrs;  in qman_create_portal()
1282 		qman_cgrs_fill(&portal->cgrs[0]);  in qman_create_portal()
1287 	portal->bits = 0;  in qman_create_portal()
1290 			QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED;  in qman_create_portal()
1291 	isdr = 0xffffffff;  in qman_create_portal()
1293 	portal->irq_sources = 0;  in qman_create_portal()
1294 	qm_out(p, QM_REG_IER, 0);  in qman_create_portal()
1297 	if (request_irq(c->irq, portal_isr, 0, portal->irqname,	portal)) {  in qman_create_portal()
1317 		qm_dqrr_cdc_consume_n(p, 0xffff);  in qman_create_portal()
1323 		dev_err(c->dev, "MR dirty, VB 0x%x, rc 0x%x, addr 0x%llx\n",  in qman_create_portal()
1329 	qm_out(p, QM_REG_ISR, 0xffffffff);  in qman_create_portal()
1330 	qm_out(p, QM_REG_ISDR, 0);  in qman_create_portal()
1332 		qm_out(p, QM_REG_IIR, 0);  in qman_create_portal()
1335 	return 0;  in qman_create_portal()
1380 	qm_dqrr_sdqcr_set(&qm->p, 0);  in qman_destroy_portal()
1470 		      &p->cgrs[0]);  in qm_congestion_task()
1489 	u8 verb, num = 0;  in qm_mr_process_task()
1500 		/* The message is a software ERN iff the 0x20 bit is clear */  in qm_mr_process_task()
1501 		if (verb & 0x20) {  in qm_mr_process_task()
1528 				pr_crit("Invalid MR verb 0x%02x\n", verb);  in qm_mr_process_task()
1554 		qm_eqcr_set_ithresh(&p->p, 0);  in __poll_portal_slow()
1610 	unsigned int limit = 0;  in __poll_portal_fast()
1730 	if (cpu < 0) {  in qman_affine_channel()
1750 				 DL_FLAG_AUTOREMOVE_CONSUMER)) ? -EINVAL : 0;  in qman_start_using_portal()
1806 	fq->cgr_groupid = 0;  in qman_create_fq()
1808 	/* A context_b of 0 is allegedly special, so don't use that fqid */  in qman_create_fq()
1809 	if (fqid == 0 || fqid >= num_fqids) {  in qman_create_fq()
1821 	return 0;  in qman_create_fq()
1859 	int ret = 0;  in qman_init_fq()
1888 	mcc->initfq.count = 0;  in qman_init_fq()
1907 			memset(&mcc->initfq.fqd.context_a, 0,  in qman_init_fq()
1924 		int wq = 0;  in qman_init_fq()
1971 	int ret = 0;  in qman_schedule_fq()
2050 		ret = 0;  in qman_retire_fq()
2094 	int ret = 0;  in qman_oos_fq()
2132 	int ret = 0;  in qman_query_fq()
2157 	int ret = 0;  in qman_query_fq_np()
2186 	int ret = 0;  in qman_query_cgr()
2218 	return 0;  in qman_query_cgr_congested()
2237 	ret = 0;  in set_p_vdqcr()
2256 	int ret = 0;  in wait_vdqcr_start()
2300 	return 0;  in qman_volatile_dequeue()
2350 	return 0;  in qman_enqueue()
2361 	int ret = 0;  in qm_modify_cgr()
2409 	int err_cnt = 0;  in qman_init_cgr_all()
2411 	for (cgr.cgrid = 0; cgr.cgrid < CGR_NUM; cgr.cgrid++) {  in qman_init_cgr_all()
2461 			ret = qm_modify_cgr(cgr, 0, &local_opts);  in qman_create_cgr()
2473 		ret = 0;  in qman_create_cgr()
2493 		dev_dbg(p->config->dev, " create 0x%x, delete 0x%x\n",  in qman_cgr_get_affine_portal()
2507 	int ret = 0;  in qman_delete_cgr()
2514 	memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));  in qman_delete_cgr()
2535 	ret = qm_modify_cgr(cgr, 0, &local_opts);  in qman_delete_cgr()
2581 	ret = qm_modify_cgr(cgr, 0, opts);  in qman_update_cgr()
2624 	int found = 0;  in _qm_mr_consume_and_match_verb()
2643 	int found = 0;  in _qm_dqrr_consume_and_match()
2655 		qm_dqrr_cdc_consume_1ptr(p, dqrr, 0);  in _qm_dqrr_consume_and_match()
2673 	_qm_dqrr_consume_and_match(p, 0, 0, false)
2681 	int orl_empty, drain = 0, ret = 0;  in qman_shutdown_fq()
2720 			dev_err(dev, "Can't find portal for dedicated channel 0x%x\n",  in qman_shutdown_fq()
2733 		orl_empty = 0;  in qman_shutdown_fq()
2756 			int found_fqrn = 0;  in qman_shutdown_fq()
2767 				dev_err(dev, "Can't recover FQ 0x%x, ch: 0x%x",  in qman_shutdown_fq()
2797 			dev_err(dev, "retire_fq failed: FQ 0x%x, res=0x%x\n",  in qman_shutdown_fq()
2842 			dev_err(dev, "OOS after drain fail: FQ 0x%x (0x%x)\n",  in qman_shutdown_fq()
2862 			dev_err(dev, "OOS fail: FQ 0x%x (0x%x)\n",  in qman_shutdown_fq()
2906 	return 0;  in qman_alloc_range()
2937 	return 0;  in qman_release_fqid()
2960 			return 0;  in qpool_cleanup()
2997 	return 0;  in qman_release_pool()
3018 			return 0;  in cgr_cleanup()
3030 				pr_err("CRGID 0x%x is being used by FQID 0x%x, CGR will be leaked\n",  in cgr_cleanup()
3051 	return 0;  in qman_release_cgrid()