1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
3 #include "osdep.h"
4 #include "status.h"
5 #include "hmc.h"
6 #include "defs.h"
7 #include "type.h"
8 #include "ws.h"
9 #include "protos.h"
10
11 /**
12 * irdma_get_qp_from_list - get next qp from a list
13 * @head: Listhead of qp's
14 * @qp: current qp
15 */
irdma_get_qp_from_list(struct list_head * head,struct irdma_sc_qp * qp)16 struct irdma_sc_qp *irdma_get_qp_from_list(struct list_head *head,
17 struct irdma_sc_qp *qp)
18 {
19 struct list_head *lastentry;
20 struct list_head *entry = NULL;
21
22 if (list_empty(head))
23 return NULL;
24
25 if (!qp) {
26 entry = head->next;
27 } else {
28 lastentry = &qp->list;
29 entry = lastentry->next;
30 if (entry == head)
31 return NULL;
32 }
33
34 return container_of(entry, struct irdma_sc_qp, list);
35 }
36
37 /**
38 * irdma_sc_suspend_resume_qps - suspend/resume all qp's on VSI
39 * @vsi: the VSI struct pointer
40 * @op: Set to IRDMA_OP_RESUME or IRDMA_OP_SUSPEND
41 */
irdma_sc_suspend_resume_qps(struct irdma_sc_vsi * vsi,u8 op)42 void irdma_sc_suspend_resume_qps(struct irdma_sc_vsi *vsi, u8 op)
43 {
44 struct irdma_sc_qp *qp = NULL;
45 u8 i;
46
47 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
48 mutex_lock(&vsi->qos[i].qos_mutex);
49 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
50 while (qp) {
51 if (op == IRDMA_OP_RESUME) {
52 if (!qp->dev->ws_add(vsi, i)) {
53 qp->qs_handle =
54 vsi->qos[qp->user_pri].qs_handle;
55 irdma_cqp_qp_suspend_resume(qp, op);
56 } else {
57 irdma_cqp_qp_suspend_resume(qp, op);
58 irdma_modify_qp_to_err(qp);
59 }
60 } else if (op == IRDMA_OP_SUSPEND) {
61 /* issue cqp suspend command */
62 if (!irdma_cqp_qp_suspend_resume(qp, op))
63 atomic_inc(&vsi->qp_suspend_reqs);
64 }
65 qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
66 }
67 mutex_unlock(&vsi->qos[i].qos_mutex);
68 }
69 }
70
71 /**
72 * irdma_change_l2params - given the new l2 parameters, change all qp
73 * @vsi: RDMA VSI pointer
74 * @l2params: New parameters from l2
75 */
irdma_change_l2params(struct irdma_sc_vsi * vsi,struct irdma_l2params * l2params)76 void irdma_change_l2params(struct irdma_sc_vsi *vsi,
77 struct irdma_l2params *l2params)
78 {
79 if (l2params->mtu_changed) {
80 vsi->mtu = l2params->mtu;
81 if (vsi->ieq)
82 irdma_reinitialize_ieq(vsi);
83 }
84
85 if (!l2params->tc_changed)
86 return;
87
88 vsi->tc_change_pending = false;
89 irdma_sc_suspend_resume_qps(vsi, IRDMA_OP_RESUME);
90 }
91
92 /**
93 * irdma_qp_rem_qos - remove qp from qos lists during destroy qp
94 * @qp: qp to be removed from qos
95 */
irdma_qp_rem_qos(struct irdma_sc_qp * qp)96 void irdma_qp_rem_qos(struct irdma_sc_qp *qp)
97 {
98 struct irdma_sc_vsi *vsi = qp->vsi;
99
100 ibdev_dbg(to_ibdev(qp->dev),
101 "DCB: DCB: Remove qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n",
102 qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle,
103 qp->on_qoslist);
104 mutex_lock(&vsi->qos[qp->user_pri].qos_mutex);
105 if (qp->on_qoslist) {
106 qp->on_qoslist = false;
107 list_del(&qp->list);
108 }
109 mutex_unlock(&vsi->qos[qp->user_pri].qos_mutex);
110 }
111
112 /**
113 * irdma_qp_add_qos - called during setctx for qp to be added to qos
114 * @qp: qp to be added to qos
115 */
irdma_qp_add_qos(struct irdma_sc_qp * qp)116 void irdma_qp_add_qos(struct irdma_sc_qp *qp)
117 {
118 struct irdma_sc_vsi *vsi = qp->vsi;
119
120 ibdev_dbg(to_ibdev(qp->dev),
121 "DCB: DCB: Add qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n",
122 qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle,
123 qp->on_qoslist);
124 mutex_lock(&vsi->qos[qp->user_pri].qos_mutex);
125 if (!qp->on_qoslist) {
126 list_add(&qp->list, &vsi->qos[qp->user_pri].qplist);
127 qp->on_qoslist = true;
128 qp->qs_handle = vsi->qos[qp->user_pri].qs_handle;
129 }
130 mutex_unlock(&vsi->qos[qp->user_pri].qos_mutex);
131 }
132
133 /**
134 * irdma_sc_pd_init - initialize sc pd struct
135 * @dev: sc device struct
136 * @pd: sc pd ptr
137 * @pd_id: pd_id for allocated pd
138 * @abi_ver: User/Kernel ABI version
139 */
irdma_sc_pd_init(struct irdma_sc_dev * dev,struct irdma_sc_pd * pd,u32 pd_id,int abi_ver)140 void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id,
141 int abi_ver)
142 {
143 pd->pd_id = pd_id;
144 pd->abi_ver = abi_ver;
145 pd->dev = dev;
146 }
147
148 /**
149 * irdma_sc_add_arp_cache_entry - cqp wqe add arp cache entry
150 * @cqp: struct for cqp hw
151 * @info: arp entry information
152 * @scratch: u64 saved to be used during cqp completion
153 * @post_sq: flag for cqp db to ring
154 */
155 static enum irdma_status_code
irdma_sc_add_arp_cache_entry(struct irdma_sc_cqp * cqp,struct irdma_add_arp_cache_entry_info * info,u64 scratch,bool post_sq)156 irdma_sc_add_arp_cache_entry(struct irdma_sc_cqp *cqp,
157 struct irdma_add_arp_cache_entry_info *info,
158 u64 scratch, bool post_sq)
159 {
160 __le64 *wqe;
161 u64 hdr;
162
163 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
164 if (!wqe)
165 return IRDMA_ERR_RING_FULL;
166 set_64bit_val(wqe, 8, info->reach_max);
167 set_64bit_val(wqe, 16, ether_addr_to_u64(info->mac_addr));
168
169 hdr = info->arp_index |
170 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) |
171 FIELD_PREP(IRDMA_CQPSQ_MAT_PERMANENT, (info->permanent ? 1 : 0)) |
172 FIELD_PREP(IRDMA_CQPSQ_MAT_ENTRYVALID, 1) |
173 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
174 dma_wmb(); /* make sure WQE is written before valid bit is set */
175
176 set_64bit_val(wqe, 24, hdr);
177
178 print_hex_dump_debug("WQE: ARP_CACHE_ENTRY WQE", DUMP_PREFIX_OFFSET,
179 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
180 if (post_sq)
181 irdma_sc_cqp_post_sq(cqp);
182
183 return 0;
184 }
185
186 /**
187 * irdma_sc_del_arp_cache_entry - dele arp cache entry
188 * @cqp: struct for cqp hw
189 * @scratch: u64 saved to be used during cqp completion
190 * @arp_index: arp index to delete arp entry
191 * @post_sq: flag for cqp db to ring
192 */
193 static enum irdma_status_code
irdma_sc_del_arp_cache_entry(struct irdma_sc_cqp * cqp,u64 scratch,u16 arp_index,bool post_sq)194 irdma_sc_del_arp_cache_entry(struct irdma_sc_cqp *cqp, u64 scratch,
195 u16 arp_index, bool post_sq)
196 {
197 __le64 *wqe;
198 u64 hdr;
199
200 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
201 if (!wqe)
202 return IRDMA_ERR_RING_FULL;
203
204 hdr = arp_index |
205 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) |
206 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
207 dma_wmb(); /* make sure WQE is written before valid bit is set */
208
209 set_64bit_val(wqe, 24, hdr);
210
211 print_hex_dump_debug("WQE: ARP_CACHE_DEL_ENTRY WQE",
212 DUMP_PREFIX_OFFSET, 16, 8, wqe,
213 IRDMA_CQP_WQE_SIZE * 8, false);
214 if (post_sq)
215 irdma_sc_cqp_post_sq(cqp);
216
217 return 0;
218 }
219
220 /**
221 * irdma_sc_manage_apbvt_entry - for adding and deleting apbvt entries
222 * @cqp: struct for cqp hw
223 * @info: info for apbvt entry to add or delete
224 * @scratch: u64 saved to be used during cqp completion
225 * @post_sq: flag for cqp db to ring
226 */
227 static enum irdma_status_code
irdma_sc_manage_apbvt_entry(struct irdma_sc_cqp * cqp,struct irdma_apbvt_info * info,u64 scratch,bool post_sq)228 irdma_sc_manage_apbvt_entry(struct irdma_sc_cqp *cqp,
229 struct irdma_apbvt_info *info, u64 scratch,
230 bool post_sq)
231 {
232 __le64 *wqe;
233 u64 hdr;
234
235 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
236 if (!wqe)
237 return IRDMA_ERR_RING_FULL;
238
239 set_64bit_val(wqe, 16, info->port);
240
241 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_APBVT) |
242 FIELD_PREP(IRDMA_CQPSQ_MAPT_ADDPORT, info->add) |
243 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
244 dma_wmb(); /* make sure WQE is written before valid bit is set */
245
246 set_64bit_val(wqe, 24, hdr);
247
248 print_hex_dump_debug("WQE: MANAGE_APBVT WQE", DUMP_PREFIX_OFFSET, 16,
249 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
250 if (post_sq)
251 irdma_sc_cqp_post_sq(cqp);
252
253 return 0;
254 }
255
256 /**
257 * irdma_sc_manage_qhash_table_entry - manage quad hash entries
258 * @cqp: struct for cqp hw
259 * @info: info for quad hash to manage
260 * @scratch: u64 saved to be used during cqp completion
261 * @post_sq: flag for cqp db to ring
262 *
263 * This is called before connection establishment is started.
264 * For passive connections, when listener is created, it will
265 * call with entry type of IRDMA_QHASH_TYPE_TCP_SYN with local
266 * ip address and tcp port. When SYN is received (passive
267 * connections) or sent (active connections), this routine is
268 * called with entry type of IRDMA_QHASH_TYPE_TCP_ESTABLISHED
269 * and quad is passed in info.
270 *
271 * When iwarp connection is done and its state moves to RTS, the
272 * quad hash entry in the hardware will point to iwarp's qp
273 * number and requires no calls from the driver.
274 */
275 static enum irdma_status_code
irdma_sc_manage_qhash_table_entry(struct irdma_sc_cqp * cqp,struct irdma_qhash_table_info * info,u64 scratch,bool post_sq)276 irdma_sc_manage_qhash_table_entry(struct irdma_sc_cqp *cqp,
277 struct irdma_qhash_table_info *info,
278 u64 scratch, bool post_sq)
279 {
280 __le64 *wqe;
281 u64 qw1 = 0;
282 u64 qw2 = 0;
283 u64 temp;
284 struct irdma_sc_vsi *vsi = info->vsi;
285
286 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
287 if (!wqe)
288 return IRDMA_ERR_RING_FULL;
289
290 set_64bit_val(wqe, 0, ether_addr_to_u64(info->mac_addr));
291
292 qw1 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QPN, info->qp_num) |
293 FIELD_PREP(IRDMA_CQPSQ_QHASH_DEST_PORT, info->dest_port);
294 if (info->ipv4_valid) {
295 set_64bit_val(wqe, 48,
296 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[0]));
297 } else {
298 set_64bit_val(wqe, 56,
299 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->dest_ip[0]) |
300 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->dest_ip[1]));
301
302 set_64bit_val(wqe, 48,
303 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->dest_ip[2]) |
304 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[3]));
305 }
306 qw2 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QS_HANDLE,
307 vsi->qos[info->user_pri].qs_handle);
308 if (info->vlan_valid)
309 qw2 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANID, info->vlan_id);
310 set_64bit_val(wqe, 16, qw2);
311 if (info->entry_type == IRDMA_QHASH_TYPE_TCP_ESTABLISHED) {
312 qw1 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_SRC_PORT, info->src_port);
313 if (!info->ipv4_valid) {
314 set_64bit_val(wqe, 40,
315 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->src_ip[0]) |
316 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->src_ip[1]));
317 set_64bit_val(wqe, 32,
318 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->src_ip[2]) |
319 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[3]));
320 } else {
321 set_64bit_val(wqe, 32,
322 FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[0]));
323 }
324 }
325
326 set_64bit_val(wqe, 8, qw1);
327 temp = FIELD_PREP(IRDMA_CQPSQ_QHASH_WQEVALID, cqp->polarity) |
328 FIELD_PREP(IRDMA_CQPSQ_QHASH_OPCODE,
329 IRDMA_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY) |
330 FIELD_PREP(IRDMA_CQPSQ_QHASH_MANAGE, info->manage) |
331 FIELD_PREP(IRDMA_CQPSQ_QHASH_IPV4VALID, info->ipv4_valid) |
332 FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANVALID, info->vlan_valid) |
333 FIELD_PREP(IRDMA_CQPSQ_QHASH_ENTRYTYPE, info->entry_type);
334 dma_wmb(); /* make sure WQE is written before valid bit is set */
335
336 set_64bit_val(wqe, 24, temp);
337
338 print_hex_dump_debug("WQE: MANAGE_QHASH WQE", DUMP_PREFIX_OFFSET, 16,
339 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
340 if (post_sq)
341 irdma_sc_cqp_post_sq(cqp);
342
343 return 0;
344 }
345
346 /**
347 * irdma_sc_qp_init - initialize qp
348 * @qp: sc qp
349 * @info: initialization qp info
350 */
irdma_sc_qp_init(struct irdma_sc_qp * qp,struct irdma_qp_init_info * info)351 enum irdma_status_code irdma_sc_qp_init(struct irdma_sc_qp *qp,
352 struct irdma_qp_init_info *info)
353 {
354 enum irdma_status_code ret_code;
355 u32 pble_obj_cnt;
356 u16 wqe_size;
357
358 if (info->qp_uk_init_info.max_sq_frag_cnt >
359 info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags ||
360 info->qp_uk_init_info.max_rq_frag_cnt >
361 info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags)
362 return IRDMA_ERR_INVALID_FRAG_COUNT;
363
364 qp->dev = info->pd->dev;
365 qp->vsi = info->vsi;
366 qp->ieq_qp = info->vsi->exception_lan_q;
367 qp->sq_pa = info->sq_pa;
368 qp->rq_pa = info->rq_pa;
369 qp->hw_host_ctx_pa = info->host_ctx_pa;
370 qp->q2_pa = info->q2_pa;
371 qp->shadow_area_pa = info->shadow_area_pa;
372 qp->q2_buf = info->q2;
373 qp->pd = info->pd;
374 qp->hw_host_ctx = info->host_ctx;
375 info->qp_uk_init_info.wqe_alloc_db = qp->pd->dev->wqe_alloc_db;
376 ret_code = irdma_uk_qp_init(&qp->qp_uk, &info->qp_uk_init_info);
377 if (ret_code)
378 return ret_code;
379
380 qp->virtual_map = info->virtual_map;
381 pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
382
383 if ((info->virtual_map && info->sq_pa >= pble_obj_cnt) ||
384 (info->virtual_map && info->rq_pa >= pble_obj_cnt))
385 return IRDMA_ERR_INVALID_PBLE_INDEX;
386
387 qp->llp_stream_handle = (void *)(-1);
388 qp->hw_sq_size = irdma_get_encoded_wqe_size(qp->qp_uk.sq_ring.size,
389 IRDMA_QUEUE_TYPE_SQ_RQ);
390 ibdev_dbg(to_ibdev(qp->dev),
391 "WQE: hw_sq_size[%04d] sq_ring.size[%04d]\n",
392 qp->hw_sq_size, qp->qp_uk.sq_ring.size);
393 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1 && qp->pd->abi_ver > 4)
394 wqe_size = IRDMA_WQE_SIZE_128;
395 else
396 ret_code = irdma_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt,
397 &wqe_size);
398 if (ret_code)
399 return ret_code;
400
401 qp->hw_rq_size = irdma_get_encoded_wqe_size(qp->qp_uk.rq_size *
402 (wqe_size / IRDMA_QP_WQE_MIN_SIZE), IRDMA_QUEUE_TYPE_SQ_RQ);
403 ibdev_dbg(to_ibdev(qp->dev),
404 "WQE: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n",
405 qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size);
406 qp->sq_tph_val = info->sq_tph_val;
407 qp->rq_tph_val = info->rq_tph_val;
408 qp->sq_tph_en = info->sq_tph_en;
409 qp->rq_tph_en = info->rq_tph_en;
410 qp->rcv_tph_en = info->rcv_tph_en;
411 qp->xmit_tph_en = info->xmit_tph_en;
412 qp->qp_uk.first_sq_wq = info->qp_uk_init_info.first_sq_wq;
413 qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle;
414
415 return 0;
416 }
417
418 /**
419 * irdma_sc_qp_create - create qp
420 * @qp: sc qp
421 * @info: qp create info
422 * @scratch: u64 saved to be used during cqp completion
423 * @post_sq: flag for cqp db to ring
424 */
irdma_sc_qp_create(struct irdma_sc_qp * qp,struct irdma_create_qp_info * info,u64 scratch,bool post_sq)425 enum irdma_status_code irdma_sc_qp_create(struct irdma_sc_qp *qp, struct irdma_create_qp_info *info,
426 u64 scratch, bool post_sq)
427 {
428 struct irdma_sc_cqp *cqp;
429 __le64 *wqe;
430 u64 hdr;
431
432 cqp = qp->dev->cqp;
433 if (qp->qp_uk.qp_id < cqp->dev->hw_attrs.min_hw_qp_id ||
434 qp->qp_uk.qp_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt - 1))
435 return IRDMA_ERR_INVALID_QP_ID;
436
437 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
438 if (!wqe)
439 return IRDMA_ERR_RING_FULL;
440
441 set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
442 set_64bit_val(wqe, 40, qp->shadow_area_pa);
443
444 hdr = qp->qp_uk.qp_id |
445 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_QP) |
446 FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, (info->ord_valid ? 1 : 0)) |
447 FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) |
448 FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) |
449 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) |
450 FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) |
451 FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) |
452 FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) |
453 FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID,
454 info->arp_cache_idx_valid) |
455 FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) |
456 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
457 dma_wmb(); /* make sure WQE is written before valid bit is set */
458
459 set_64bit_val(wqe, 24, hdr);
460
461 print_hex_dump_debug("WQE: QP_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,
462 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
463 if (post_sq)
464 irdma_sc_cqp_post_sq(cqp);
465
466 return 0;
467 }
468
469 /**
470 * irdma_sc_qp_modify - modify qp cqp wqe
471 * @qp: sc qp
472 * @info: modify qp info
473 * @scratch: u64 saved to be used during cqp completion
474 * @post_sq: flag for cqp db to ring
475 */
irdma_sc_qp_modify(struct irdma_sc_qp * qp,struct irdma_modify_qp_info * info,u64 scratch,bool post_sq)476 enum irdma_status_code irdma_sc_qp_modify(struct irdma_sc_qp *qp,
477 struct irdma_modify_qp_info *info,
478 u64 scratch, bool post_sq)
479 {
480 __le64 *wqe;
481 struct irdma_sc_cqp *cqp;
482 u64 hdr;
483 u8 term_actions = 0;
484 u8 term_len = 0;
485
486 cqp = qp->dev->cqp;
487 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
488 if (!wqe)
489 return IRDMA_ERR_RING_FULL;
490
491 if (info->next_iwarp_state == IRDMA_QP_STATE_TERMINATE) {
492 if (info->dont_send_fin)
493 term_actions += IRDMAQP_TERM_SEND_TERM_ONLY;
494 if (info->dont_send_term)
495 term_actions += IRDMAQP_TERM_SEND_FIN_ONLY;
496 if (term_actions == IRDMAQP_TERM_SEND_TERM_AND_FIN ||
497 term_actions == IRDMAQP_TERM_SEND_TERM_ONLY)
498 term_len = info->termlen;
499 }
500
501 set_64bit_val(wqe, 8,
502 FIELD_PREP(IRDMA_CQPSQ_QP_NEWMSS, info->new_mss) |
503 FIELD_PREP(IRDMA_CQPSQ_QP_TERMLEN, term_len));
504 set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
505 set_64bit_val(wqe, 40, qp->shadow_area_pa);
506
507 hdr = qp->qp_uk.qp_id |
508 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_QP) |
509 FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, info->ord_valid) |
510 FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) |
511 FIELD_PREP(IRDMA_CQPSQ_QP_CACHEDVARVALID,
512 info->cached_var_valid) |
513 FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) |
514 FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) |
515 FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) |
516 FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) |
517 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) |
518 FIELD_PREP(IRDMA_CQPSQ_QP_MSSCHANGE, info->mss_change) |
519 FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY,
520 info->remove_hash_idx) |
521 FIELD_PREP(IRDMA_CQPSQ_QP_TERMACT, term_actions) |
522 FIELD_PREP(IRDMA_CQPSQ_QP_RESETCON, info->reset_tcp_conn) |
523 FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID,
524 info->arp_cache_idx_valid) |
525 FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) |
526 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
527 dma_wmb(); /* make sure WQE is written before valid bit is set */
528
529 set_64bit_val(wqe, 24, hdr);
530
531 print_hex_dump_debug("WQE: QP_MODIFY WQE", DUMP_PREFIX_OFFSET, 16, 8,
532 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
533 if (post_sq)
534 irdma_sc_cqp_post_sq(cqp);
535
536 return 0;
537 }
538
539 /**
540 * irdma_sc_qp_destroy - cqp destroy qp
541 * @qp: sc qp
542 * @scratch: u64 saved to be used during cqp completion
543 * @remove_hash_idx: flag if to remove hash idx
544 * @ignore_mw_bnd: memory window bind flag
545 * @post_sq: flag for cqp db to ring
546 */
irdma_sc_qp_destroy(struct irdma_sc_qp * qp,u64 scratch,bool remove_hash_idx,bool ignore_mw_bnd,bool post_sq)547 enum irdma_status_code irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch,
548 bool remove_hash_idx, bool ignore_mw_bnd,
549 bool post_sq)
550 {
551 __le64 *wqe;
552 struct irdma_sc_cqp *cqp;
553 u64 hdr;
554
555 cqp = qp->dev->cqp;
556 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
557 if (!wqe)
558 return IRDMA_ERR_RING_FULL;
559
560 set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
561 set_64bit_val(wqe, 40, qp->shadow_area_pa);
562
563 hdr = qp->qp_uk.qp_id |
564 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_QP) |
565 FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) |
566 FIELD_PREP(IRDMA_CQPSQ_QP_IGNOREMWBOUND, ignore_mw_bnd) |
567 FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY, remove_hash_idx) |
568 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
569 dma_wmb(); /* make sure WQE is written before valid bit is set */
570
571 set_64bit_val(wqe, 24, hdr);
572
573 print_hex_dump_debug("WQE: QP_DESTROY WQE", DUMP_PREFIX_OFFSET, 16, 8,
574 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
575 if (post_sq)
576 irdma_sc_cqp_post_sq(cqp);
577
578 return 0;
579 }
580
581 /**
582 * irdma_sc_get_encoded_ird_size -
583 * @ird_size: IRD size
584 * The ird from the connection is rounded to a supported HW setting and then encoded
585 * for ird_size field of qp_ctx. Consumers are expected to provide valid ird size based
586 * on hardware attributes. IRD size defaults to a value of 4 in case of invalid input
587 */
irdma_sc_get_encoded_ird_size(u16 ird_size)588 static u8 irdma_sc_get_encoded_ird_size(u16 ird_size)
589 {
590 switch (ird_size ?
591 roundup_pow_of_two(2 * ird_size) : 4) {
592 case 256:
593 return IRDMA_IRD_HW_SIZE_256;
594 case 128:
595 return IRDMA_IRD_HW_SIZE_128;
596 case 64:
597 case 32:
598 return IRDMA_IRD_HW_SIZE_64;
599 case 16:
600 case 8:
601 return IRDMA_IRD_HW_SIZE_16;
602 case 4:
603 default:
604 break;
605 }
606
607 return IRDMA_IRD_HW_SIZE_4;
608 }
609
610 /**
611 * irdma_sc_qp_setctx_roce - set qp's context
612 * @qp: sc qp
613 * @qp_ctx: context ptr
614 * @info: ctx info
615 */
irdma_sc_qp_setctx_roce(struct irdma_sc_qp * qp,__le64 * qp_ctx,struct irdma_qp_host_ctx_info * info)616 void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx,
617 struct irdma_qp_host_ctx_info *info)
618 {
619 struct irdma_roce_offload_info *roce_info;
620 struct irdma_udp_offload_info *udp;
621 u8 push_mode_en;
622 u32 push_idx;
623
624 roce_info = info->roce_info;
625 udp = info->udp_info;
626 qp->user_pri = info->user_pri;
627 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {
628 push_mode_en = 0;
629 push_idx = 0;
630 } else {
631 push_mode_en = 1;
632 push_idx = qp->push_idx;
633 }
634 set_64bit_val(qp_ctx, 0,
635 FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) |
636 FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) |
637 FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) |
638 FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) |
639 FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) |
640 FIELD_PREP(IRDMAQPC_PPIDX, push_idx) |
641 FIELD_PREP(IRDMAQPC_PMENA, push_mode_en) |
642 FIELD_PREP(IRDMAQPC_PDIDXHI, roce_info->pd_id >> 16) |
643 FIELD_PREP(IRDMAQPC_DC_TCP_EN, roce_info->dctcp_en) |
644 FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID, roce_info->err_rq_idx_valid) |
645 FIELD_PREP(IRDMAQPC_ISQP1, roce_info->is_qp1) |
646 FIELD_PREP(IRDMAQPC_ROCE_TVER, roce_info->roce_tver) |
647 FIELD_PREP(IRDMAQPC_IPV4, udp->ipv4) |
648 FIELD_PREP(IRDMAQPC_INSERTVLANTAG, udp->insert_vlan_tag));
649 set_64bit_val(qp_ctx, 8, qp->sq_pa);
650 set_64bit_val(qp_ctx, 16, qp->rq_pa);
651 if ((roce_info->dcqcn_en || roce_info->dctcp_en) &&
652 !(udp->tos & 0x03))
653 udp->tos |= ECN_CODE_PT_VAL;
654 set_64bit_val(qp_ctx, 24,
655 FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) |
656 FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size) |
657 FIELD_PREP(IRDMAQPC_TTL, udp->ttl) | FIELD_PREP(IRDMAQPC_TOS, udp->tos) |
658 FIELD_PREP(IRDMAQPC_SRCPORTNUM, udp->src_port) |
659 FIELD_PREP(IRDMAQPC_DESTPORTNUM, udp->dst_port));
660 set_64bit_val(qp_ctx, 32,
661 FIELD_PREP(IRDMAQPC_DESTIPADDR2, udp->dest_ip_addr[2]) |
662 FIELD_PREP(IRDMAQPC_DESTIPADDR3, udp->dest_ip_addr[3]));
663 set_64bit_val(qp_ctx, 40,
664 FIELD_PREP(IRDMAQPC_DESTIPADDR0, udp->dest_ip_addr[0]) |
665 FIELD_PREP(IRDMAQPC_DESTIPADDR1, udp->dest_ip_addr[1]));
666 set_64bit_val(qp_ctx, 48,
667 FIELD_PREP(IRDMAQPC_SNDMSS, udp->snd_mss) |
668 FIELD_PREP(IRDMAQPC_VLANTAG, udp->vlan_tag) |
669 FIELD_PREP(IRDMAQPC_ARPIDX, udp->arp_idx));
670 set_64bit_val(qp_ctx, 56,
671 FIELD_PREP(IRDMAQPC_PKEY, roce_info->p_key) |
672 FIELD_PREP(IRDMAQPC_PDIDX, roce_info->pd_id) |
673 FIELD_PREP(IRDMAQPC_ACKCREDITS, roce_info->ack_credits) |
674 FIELD_PREP(IRDMAQPC_FLOWLABEL, udp->flow_label));
675 set_64bit_val(qp_ctx, 64,
676 FIELD_PREP(IRDMAQPC_QKEY, roce_info->qkey) |
677 FIELD_PREP(IRDMAQPC_DESTQP, roce_info->dest_qp));
678 set_64bit_val(qp_ctx, 80,
679 FIELD_PREP(IRDMAQPC_PSNNXT, udp->psn_nxt) |
680 FIELD_PREP(IRDMAQPC_LSN, udp->lsn));
681 set_64bit_val(qp_ctx, 88,
682 FIELD_PREP(IRDMAQPC_EPSN, udp->epsn));
683 set_64bit_val(qp_ctx, 96,
684 FIELD_PREP(IRDMAQPC_PSNMAX, udp->psn_max) |
685 FIELD_PREP(IRDMAQPC_PSNUNA, udp->psn_una));
686 set_64bit_val(qp_ctx, 112,
687 FIELD_PREP(IRDMAQPC_CWNDROCE, udp->cwnd));
688 set_64bit_val(qp_ctx, 128,
689 FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, roce_info->err_rq_idx) |
690 FIELD_PREP(IRDMAQPC_RNRNAK_THRESH, udp->rnr_nak_thresh) |
691 FIELD_PREP(IRDMAQPC_REXMIT_THRESH, udp->rexmit_thresh) |
692 FIELD_PREP(IRDMAQPC_RTOMIN, roce_info->rtomin));
693 set_64bit_val(qp_ctx, 136,
694 FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) |
695 FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num));
696 set_64bit_val(qp_ctx, 144,
697 FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx));
698 set_64bit_val(qp_ctx, 152, ether_addr_to_u64(roce_info->mac_addr) << 16);
699 set_64bit_val(qp_ctx, 160,
700 FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info->ord_size) |
701 FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info->ird_size)) |
702 FIELD_PREP(IRDMAQPC_WRRDRSPOK, roce_info->wr_rdresp_en) |
703 FIELD_PREP(IRDMAQPC_RDOK, roce_info->rd_en) |
704 FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |
705 FIELD_PREP(IRDMAQPC_BINDEN, roce_info->bind_en) |
706 FIELD_PREP(IRDMAQPC_FASTREGEN, roce_info->fast_reg_en) |
707 FIELD_PREP(IRDMAQPC_DCQCNENABLE, roce_info->dcqcn_en) |
708 FIELD_PREP(IRDMAQPC_RCVNOICRC, roce_info->rcv_no_icrc) |
709 FIELD_PREP(IRDMAQPC_FW_CC_ENABLE, roce_info->fw_cc_enable) |
710 FIELD_PREP(IRDMAQPC_UDPRIVCQENABLE, roce_info->udprivcq_en) |
711 FIELD_PREP(IRDMAQPC_PRIVEN, roce_info->priv_mode_en) |
712 FIELD_PREP(IRDMAQPC_TIMELYENABLE, roce_info->timely_en));
713 set_64bit_val(qp_ctx, 168,
714 FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx));
715 set_64bit_val(qp_ctx, 176,
716 FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) |
717 FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) |
718 FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle));
719 set_64bit_val(qp_ctx, 184,
720 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, udp->local_ipaddr[3]) |
721 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, udp->local_ipaddr[2]));
722 set_64bit_val(qp_ctx, 192,
723 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, udp->local_ipaddr[1]) |
724 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, udp->local_ipaddr[0]));
725 set_64bit_val(qp_ctx, 200,
726 FIELD_PREP(IRDMAQPC_THIGH, roce_info->t_high) |
727 FIELD_PREP(IRDMAQPC_TLOW, roce_info->t_low));
728 set_64bit_val(qp_ctx, 208,
729 FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx));
730
731 print_hex_dump_debug("WQE: QP_HOST CTX WQE", DUMP_PREFIX_OFFSET, 16,
732 8, qp_ctx, IRDMA_QP_CTX_SIZE, false);
733 }
734
735 /* irdma_sc_alloc_local_mac_entry - allocate a mac entry
736 * @cqp: struct for cqp hw
737 * @scratch: u64 saved to be used during cqp completion
738 * @post_sq: flag for cqp db to ring
739 */
740 static enum irdma_status_code
irdma_sc_alloc_local_mac_entry(struct irdma_sc_cqp * cqp,u64 scratch,bool post_sq)741 irdma_sc_alloc_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch,
742 bool post_sq)
743 {
744 __le64 *wqe;
745 u64 hdr;
746
747 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
748 if (!wqe)
749 return IRDMA_ERR_RING_FULL;
750
751 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE,
752 IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY) |
753 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
754
755 dma_wmb(); /* make sure WQE is written before valid bit is set */
756
757 set_64bit_val(wqe, 24, hdr);
758
759 print_hex_dump_debug("WQE: ALLOCATE_LOCAL_MAC WQE",
760 DUMP_PREFIX_OFFSET, 16, 8, wqe,
761 IRDMA_CQP_WQE_SIZE * 8, false);
762
763 if (post_sq)
764 irdma_sc_cqp_post_sq(cqp);
765 return 0;
766 }
767
768 /**
769 * irdma_sc_add_local_mac_entry - add mac enry
770 * @cqp: struct for cqp hw
771 * @info:mac addr info
772 * @scratch: u64 saved to be used during cqp completion
773 * @post_sq: flag for cqp db to ring
774 */
775 static enum irdma_status_code
irdma_sc_add_local_mac_entry(struct irdma_sc_cqp * cqp,struct irdma_local_mac_entry_info * info,u64 scratch,bool post_sq)776 irdma_sc_add_local_mac_entry(struct irdma_sc_cqp *cqp,
777 struct irdma_local_mac_entry_info *info,
778 u64 scratch, bool post_sq)
779 {
780 __le64 *wqe;
781 u64 header;
782
783 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
784 if (!wqe)
785 return IRDMA_ERR_RING_FULL;
786
787 set_64bit_val(wqe, 32, ether_addr_to_u64(info->mac_addr));
788
789 header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, info->entry_idx) |
790 FIELD_PREP(IRDMA_CQPSQ_OPCODE,
791 IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) |
792 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
793
794 dma_wmb(); /* make sure WQE is written before valid bit is set */
795
796 set_64bit_val(wqe, 24, header);
797
798 print_hex_dump_debug("WQE: ADD_LOCAL_MAC WQE", DUMP_PREFIX_OFFSET, 16,
799 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
800
801 if (post_sq)
802 irdma_sc_cqp_post_sq(cqp);
803 return 0;
804 }
805
806 /**
807 * irdma_sc_del_local_mac_entry - cqp wqe to dele local mac
808 * @cqp: struct for cqp hw
809 * @scratch: u64 saved to be used during cqp completion
810 * @entry_idx: index of mac entry
811 * @ignore_ref_count: to force mac adde delete
812 * @post_sq: flag for cqp db to ring
813 */
814 static enum irdma_status_code
irdma_sc_del_local_mac_entry(struct irdma_sc_cqp * cqp,u64 scratch,u16 entry_idx,u8 ignore_ref_count,bool post_sq)815 irdma_sc_del_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch,
816 u16 entry_idx, u8 ignore_ref_count, bool post_sq)
817 {
818 __le64 *wqe;
819 u64 header;
820
821 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
822 if (!wqe)
823 return IRDMA_ERR_RING_FULL;
824 header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, entry_idx) |
825 FIELD_PREP(IRDMA_CQPSQ_OPCODE,
826 IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) |
827 FIELD_PREP(IRDMA_CQPSQ_MLM_FREEENTRY, 1) |
828 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) |
829 FIELD_PREP(IRDMA_CQPSQ_MLM_IGNORE_REF_CNT, ignore_ref_count);
830
831 dma_wmb(); /* make sure WQE is written before valid bit is set */
832
833 set_64bit_val(wqe, 24, header);
834
835 print_hex_dump_debug("WQE: DEL_LOCAL_MAC_IPADDR WQE",
836 DUMP_PREFIX_OFFSET, 16, 8, wqe,
837 IRDMA_CQP_WQE_SIZE * 8, false);
838
839 if (post_sq)
840 irdma_sc_cqp_post_sq(cqp);
841 return 0;
842 }
843
844 /**
845 * irdma_sc_qp_setctx - set qp's context
846 * @qp: sc qp
847 * @qp_ctx: context ptr
848 * @info: ctx info
849 */
irdma_sc_qp_setctx(struct irdma_sc_qp * qp,__le64 * qp_ctx,struct irdma_qp_host_ctx_info * info)850 void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,
851 struct irdma_qp_host_ctx_info *info)
852 {
853 struct irdma_iwarp_offload_info *iw;
854 struct irdma_tcp_offload_info *tcp;
855 struct irdma_sc_dev *dev;
856 u8 push_mode_en;
857 u32 push_idx;
858 u64 qw0, qw3, qw7 = 0, qw16 = 0;
859 u64 mac = 0;
860
861 iw = info->iwarp_info;
862 tcp = info->tcp_info;
863 dev = qp->dev;
864 if (iw->rcv_mark_en) {
865 qp->pfpdu.marker_len = 4;
866 qp->pfpdu.rcv_start_seq = tcp->rcv_nxt;
867 }
868 qp->user_pri = info->user_pri;
869 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {
870 push_mode_en = 0;
871 push_idx = 0;
872 } else {
873 push_mode_en = 1;
874 push_idx = qp->push_idx;
875 }
876 qw0 = FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) |
877 FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) |
878 FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) |
879 FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) |
880 FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) |
881 FIELD_PREP(IRDMAQPC_PPIDX, push_idx) |
882 FIELD_PREP(IRDMAQPC_PMENA, push_mode_en);
883
884 set_64bit_val(qp_ctx, 8, qp->sq_pa);
885 set_64bit_val(qp_ctx, 16, qp->rq_pa);
886
887 qw3 = FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) |
888 FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size);
889 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
890 qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX,
891 qp->src_mac_addr_idx);
892 set_64bit_val(qp_ctx, 136,
893 FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) |
894 FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num));
895 set_64bit_val(qp_ctx, 168,
896 FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx));
897 set_64bit_val(qp_ctx, 176,
898 FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) |
899 FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) |
900 FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle) |
901 FIELD_PREP(IRDMAQPC_EXCEPTION_LAN_QUEUE, qp->ieq_qp));
902 if (info->iwarp_info_valid) {
903 qw0 |= FIELD_PREP(IRDMAQPC_DDP_VER, iw->ddp_ver) |
904 FIELD_PREP(IRDMAQPC_RDMAP_VER, iw->rdmap_ver) |
905 FIELD_PREP(IRDMAQPC_DC_TCP_EN, iw->dctcp_en) |
906 FIELD_PREP(IRDMAQPC_ECN_EN, iw->ecn_en) |
907 FIELD_PREP(IRDMAQPC_IBRDENABLE, iw->ib_rd_en) |
908 FIELD_PREP(IRDMAQPC_PDIDXHI, iw->pd_id >> 16) |
909 FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID,
910 iw->err_rq_idx_valid);
911 qw7 |= FIELD_PREP(IRDMAQPC_PDIDX, iw->pd_id);
912 qw16 |= FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, iw->err_rq_idx) |
913 FIELD_PREP(IRDMAQPC_RTOMIN, iw->rtomin);
914 set_64bit_val(qp_ctx, 144,
915 FIELD_PREP(IRDMAQPC_Q2ADDR, qp->q2_pa >> 8) |
916 FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx));
917
918 if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
919 mac = ether_addr_to_u64(iw->mac_addr);
920
921 set_64bit_val(qp_ctx, 152,
922 mac << 16 | FIELD_PREP(IRDMAQPC_LASTBYTESENT, iw->last_byte_sent));
923 set_64bit_val(qp_ctx, 160,
924 FIELD_PREP(IRDMAQPC_ORDSIZE, iw->ord_size) |
925 FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(iw->ird_size)) |
926 FIELD_PREP(IRDMAQPC_WRRDRSPOK, iw->wr_rdresp_en) |
927 FIELD_PREP(IRDMAQPC_RDOK, iw->rd_en) |
928 FIELD_PREP(IRDMAQPC_SNDMARKERS, iw->snd_mark_en) |
929 FIELD_PREP(IRDMAQPC_BINDEN, iw->bind_en) |
930 FIELD_PREP(IRDMAQPC_FASTREGEN, iw->fast_reg_en) |
931 FIELD_PREP(IRDMAQPC_PRIVEN, iw->priv_mode_en) |
932 FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |
933 FIELD_PREP(IRDMAQPC_IWARPMODE, 1) |
934 FIELD_PREP(IRDMAQPC_RCVMARKERS, iw->rcv_mark_en) |
935 FIELD_PREP(IRDMAQPC_ALIGNHDRS, iw->align_hdrs) |
936 FIELD_PREP(IRDMAQPC_RCVNOMPACRC, iw->rcv_no_mpa_crc) |
937 FIELD_PREP(IRDMAQPC_RCVMARKOFFSET, iw->rcv_mark_offset || !tcp ? iw->rcv_mark_offset : tcp->rcv_nxt) |
938 FIELD_PREP(IRDMAQPC_SNDMARKOFFSET, iw->snd_mark_offset || !tcp ? iw->snd_mark_offset : tcp->snd_nxt) |
939 FIELD_PREP(IRDMAQPC_TIMELYENABLE, iw->timely_en));
940 }
941 if (info->tcp_info_valid) {
942 qw0 |= FIELD_PREP(IRDMAQPC_IPV4, tcp->ipv4) |
943 FIELD_PREP(IRDMAQPC_NONAGLE, tcp->no_nagle) |
944 FIELD_PREP(IRDMAQPC_INSERTVLANTAG,
945 tcp->insert_vlan_tag) |
946 FIELD_PREP(IRDMAQPC_TIMESTAMP, tcp->time_stamp) |
947 FIELD_PREP(IRDMAQPC_LIMIT, tcp->cwnd_inc_limit) |
948 FIELD_PREP(IRDMAQPC_DROPOOOSEG, tcp->drop_ooo_seg) |
949 FIELD_PREP(IRDMAQPC_DUPACK_THRESH, tcp->dup_ack_thresh);
950
951 if ((iw->ecn_en || iw->dctcp_en) && !(tcp->tos & 0x03))
952 tcp->tos |= ECN_CODE_PT_VAL;
953
954 qw3 |= FIELD_PREP(IRDMAQPC_TTL, tcp->ttl) |
955 FIELD_PREP(IRDMAQPC_AVOIDSTRETCHACK, tcp->avoid_stretch_ack) |
956 FIELD_PREP(IRDMAQPC_TOS, tcp->tos) |
957 FIELD_PREP(IRDMAQPC_SRCPORTNUM, tcp->src_port) |
958 FIELD_PREP(IRDMAQPC_DESTPORTNUM, tcp->dst_port);
959 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {
960 qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX, tcp->src_mac_addr_idx);
961
962 qp->src_mac_addr_idx = tcp->src_mac_addr_idx;
963 }
964 set_64bit_val(qp_ctx, 32,
965 FIELD_PREP(IRDMAQPC_DESTIPADDR2, tcp->dest_ip_addr[2]) |
966 FIELD_PREP(IRDMAQPC_DESTIPADDR3, tcp->dest_ip_addr[3]));
967 set_64bit_val(qp_ctx, 40,
968 FIELD_PREP(IRDMAQPC_DESTIPADDR0, tcp->dest_ip_addr[0]) |
969 FIELD_PREP(IRDMAQPC_DESTIPADDR1, tcp->dest_ip_addr[1]));
970 set_64bit_val(qp_ctx, 48,
971 FIELD_PREP(IRDMAQPC_SNDMSS, tcp->snd_mss) |
972 FIELD_PREP(IRDMAQPC_SYN_RST_HANDLING, tcp->syn_rst_handling) |
973 FIELD_PREP(IRDMAQPC_VLANTAG, tcp->vlan_tag) |
974 FIELD_PREP(IRDMAQPC_ARPIDX, tcp->arp_idx));
975 qw7 |= FIELD_PREP(IRDMAQPC_FLOWLABEL, tcp->flow_label) |
976 FIELD_PREP(IRDMAQPC_WSCALE, tcp->wscale) |
977 FIELD_PREP(IRDMAQPC_IGNORE_TCP_OPT,
978 tcp->ignore_tcp_opt) |
979 FIELD_PREP(IRDMAQPC_IGNORE_TCP_UNS_OPT,
980 tcp->ignore_tcp_uns_opt) |
981 FIELD_PREP(IRDMAQPC_TCPSTATE, tcp->tcp_state) |
982 FIELD_PREP(IRDMAQPC_RCVSCALE, tcp->rcv_wscale) |
983 FIELD_PREP(IRDMAQPC_SNDSCALE, tcp->snd_wscale);
984 set_64bit_val(qp_ctx, 72,
985 FIELD_PREP(IRDMAQPC_TIMESTAMP_RECENT, tcp->time_stamp_recent) |
986 FIELD_PREP(IRDMAQPC_TIMESTAMP_AGE, tcp->time_stamp_age));
987 set_64bit_val(qp_ctx, 80,
988 FIELD_PREP(IRDMAQPC_SNDNXT, tcp->snd_nxt) |
989 FIELD_PREP(IRDMAQPC_SNDWND, tcp->snd_wnd));
990 set_64bit_val(qp_ctx, 88,
991 FIELD_PREP(IRDMAQPC_RCVNXT, tcp->rcv_nxt) |
992 FIELD_PREP(IRDMAQPC_RCVWND, tcp->rcv_wnd));
993 set_64bit_val(qp_ctx, 96,
994 FIELD_PREP(IRDMAQPC_SNDMAX, tcp->snd_max) |
995 FIELD_PREP(IRDMAQPC_SNDUNA, tcp->snd_una));
996 set_64bit_val(qp_ctx, 104,
997 FIELD_PREP(IRDMAQPC_SRTT, tcp->srtt) |
998 FIELD_PREP(IRDMAQPC_RTTVAR, tcp->rtt_var));
999 set_64bit_val(qp_ctx, 112,
1000 FIELD_PREP(IRDMAQPC_SSTHRESH, tcp->ss_thresh) |
1001 FIELD_PREP(IRDMAQPC_CWND, tcp->cwnd));
1002 set_64bit_val(qp_ctx, 120,
1003 FIELD_PREP(IRDMAQPC_SNDWL1, tcp->snd_wl1) |
1004 FIELD_PREP(IRDMAQPC_SNDWL2, tcp->snd_wl2));
1005 qw16 |= FIELD_PREP(IRDMAQPC_MAXSNDWND, tcp->max_snd_window) |
1006 FIELD_PREP(IRDMAQPC_REXMIT_THRESH, tcp->rexmit_thresh);
1007 set_64bit_val(qp_ctx, 184,
1008 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, tcp->local_ipaddr[3]) |
1009 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, tcp->local_ipaddr[2]));
1010 set_64bit_val(qp_ctx, 192,
1011 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, tcp->local_ipaddr[1]) |
1012 FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, tcp->local_ipaddr[0]));
1013 set_64bit_val(qp_ctx, 200,
1014 FIELD_PREP(IRDMAQPC_THIGH, iw->t_high) |
1015 FIELD_PREP(IRDMAQPC_TLOW, iw->t_low));
1016 set_64bit_val(qp_ctx, 208,
1017 FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx));
1018 }
1019
1020 set_64bit_val(qp_ctx, 0, qw0);
1021 set_64bit_val(qp_ctx, 24, qw3);
1022 set_64bit_val(qp_ctx, 56, qw7);
1023 set_64bit_val(qp_ctx, 128, qw16);
1024
1025 print_hex_dump_debug("WQE: QP_HOST CTX", DUMP_PREFIX_OFFSET, 16, 8,
1026 qp_ctx, IRDMA_QP_CTX_SIZE, false);
1027 }
1028
1029 /**
1030 * irdma_sc_alloc_stag - mr stag alloc
1031 * @dev: sc device struct
1032 * @info: stag info
1033 * @scratch: u64 saved to be used during cqp completion
1034 * @post_sq: flag for cqp db to ring
1035 */
1036 static enum irdma_status_code
irdma_sc_alloc_stag(struct irdma_sc_dev * dev,struct irdma_allocate_stag_info * info,u64 scratch,bool post_sq)1037 irdma_sc_alloc_stag(struct irdma_sc_dev *dev,
1038 struct irdma_allocate_stag_info *info, u64 scratch,
1039 bool post_sq)
1040 {
1041 __le64 *wqe;
1042 struct irdma_sc_cqp *cqp;
1043 u64 hdr;
1044 enum irdma_page_size page_size;
1045
1046 if (info->page_size == 0x40000000)
1047 page_size = IRDMA_PAGE_SIZE_1G;
1048 else if (info->page_size == 0x200000)
1049 page_size = IRDMA_PAGE_SIZE_2M;
1050 else
1051 page_size = IRDMA_PAGE_SIZE_4K;
1052
1053 cqp = dev->cqp;
1054 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
1055 if (!wqe)
1056 return IRDMA_ERR_RING_FULL;
1057
1058 set_64bit_val(wqe, 8,
1059 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID) |
1060 FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len));
1061 set_64bit_val(wqe, 16,
1062 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx));
1063 set_64bit_val(wqe, 40,
1064 FIELD_PREP(IRDMA_CQPSQ_STAG_HMCFNIDX, info->hmc_fcn_index));
1065
1066 if (info->chunk_size)
1067 set_64bit_val(wqe, 48,
1068 FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_idx));
1069
1070 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) |
1071 FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) |
1072 FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) |
1073 FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) |
1074 FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) |
1075 FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, info->remote_access) |
1076 FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) |
1077 FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) |
1078 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
1079 dma_wmb(); /* make sure WQE is written before valid bit is set */
1080
1081 set_64bit_val(wqe, 24, hdr);
1082
1083 print_hex_dump_debug("WQE: ALLOC_STAG WQE", DUMP_PREFIX_OFFSET, 16, 8,
1084 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
1085 if (post_sq)
1086 irdma_sc_cqp_post_sq(cqp);
1087
1088 return 0;
1089 }
1090
1091 /**
1092 * irdma_sc_mr_reg_non_shared - non-shared mr registration
1093 * @dev: sc device struct
1094 * @info: mr info
1095 * @scratch: u64 saved to be used during cqp completion
1096 * @post_sq: flag for cqp db to ring
1097 */
1098 static enum irdma_status_code
irdma_sc_mr_reg_non_shared(struct irdma_sc_dev * dev,struct irdma_reg_ns_stag_info * info,u64 scratch,bool post_sq)1099 irdma_sc_mr_reg_non_shared(struct irdma_sc_dev *dev,
1100 struct irdma_reg_ns_stag_info *info, u64 scratch,
1101 bool post_sq)
1102 {
1103 __le64 *wqe;
1104 u64 fbo;
1105 struct irdma_sc_cqp *cqp;
1106 u64 hdr;
1107 u32 pble_obj_cnt;
1108 bool remote_access;
1109 u8 addr_type;
1110 enum irdma_page_size page_size;
1111
1112 if (info->page_size == 0x40000000)
1113 page_size = IRDMA_PAGE_SIZE_1G;
1114 else if (info->page_size == 0x200000)
1115 page_size = IRDMA_PAGE_SIZE_2M;
1116 else if (info->page_size == 0x1000)
1117 page_size = IRDMA_PAGE_SIZE_4K;
1118 else
1119 return IRDMA_ERR_PARAM;
1120
1121 if (info->access_rights & (IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY |
1122 IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY))
1123 remote_access = true;
1124 else
1125 remote_access = false;
1126
1127 pble_obj_cnt = dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
1128 if (info->chunk_size && info->first_pm_pbl_index >= pble_obj_cnt)
1129 return IRDMA_ERR_INVALID_PBLE_INDEX;
1130
1131 cqp = dev->cqp;
1132 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
1133 if (!wqe)
1134 return IRDMA_ERR_RING_FULL;
1135 fbo = info->va & (info->page_size - 1);
1136
1137 set_64bit_val(wqe, 0,
1138 (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED ?
1139 info->va : fbo));
1140 set_64bit_val(wqe, 8,
1141 FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len) |
1142 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
1143 set_64bit_val(wqe, 16,
1144 FIELD_PREP(IRDMA_CQPSQ_STAG_KEY, info->stag_key) |
1145 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx));
1146 if (!info->chunk_size) {
1147 set_64bit_val(wqe, 32, info->reg_addr_pa);
1148 set_64bit_val(wqe, 48, 0);
1149 } else {
1150 set_64bit_val(wqe, 32, 0);
1151 set_64bit_val(wqe, 48,
1152 FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_index));
1153 }
1154 set_64bit_val(wqe, 40, info->hmc_fcn_index);
1155 set_64bit_val(wqe, 56, 0);
1156
1157 addr_type = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0;
1158 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_REG_MR) |
1159 FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) |
1160 FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) |
1161 FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) |
1162 FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) |
1163 FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, remote_access) |
1164 FIELD_PREP(IRDMA_CQPSQ_STAG_VABASEDTO, addr_type) |
1165 FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) |
1166 FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) |
1167 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
1168 dma_wmb(); /* make sure WQE is written before valid bit is set */
1169
1170 set_64bit_val(wqe, 24, hdr);
1171
1172 print_hex_dump_debug("WQE: MR_REG_NS WQE", DUMP_PREFIX_OFFSET, 16, 8,
1173 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
1174 if (post_sq)
1175 irdma_sc_cqp_post_sq(cqp);
1176
1177 return 0;
1178 }
1179
1180 /**
1181 * irdma_sc_dealloc_stag - deallocate stag
1182 * @dev: sc device struct
1183 * @info: dealloc stag info
1184 * @scratch: u64 saved to be used during cqp completion
1185 * @post_sq: flag for cqp db to ring
1186 */
1187 static enum irdma_status_code
irdma_sc_dealloc_stag(struct irdma_sc_dev * dev,struct irdma_dealloc_stag_info * info,u64 scratch,bool post_sq)1188 irdma_sc_dealloc_stag(struct irdma_sc_dev *dev,
1189 struct irdma_dealloc_stag_info *info, u64 scratch,
1190 bool post_sq)
1191 {
1192 u64 hdr;
1193 __le64 *wqe;
1194 struct irdma_sc_cqp *cqp;
1195
1196 cqp = dev->cqp;
1197 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
1198 if (!wqe)
1199 return IRDMA_ERR_RING_FULL;
1200
1201 set_64bit_val(wqe, 8,
1202 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
1203 set_64bit_val(wqe, 16,
1204 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx));
1205
1206 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DEALLOC_STAG) |
1207 FIELD_PREP(IRDMA_CQPSQ_STAG_MR, info->mr) |
1208 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
1209 dma_wmb(); /* make sure WQE is written before valid bit is set */
1210
1211 set_64bit_val(wqe, 24, hdr);
1212
1213 print_hex_dump_debug("WQE: DEALLOC_STAG WQE", DUMP_PREFIX_OFFSET, 16,
1214 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
1215 if (post_sq)
1216 irdma_sc_cqp_post_sq(cqp);
1217
1218 return 0;
1219 }
1220
1221 /**
1222 * irdma_sc_mw_alloc - mw allocate
1223 * @dev: sc device struct
1224 * @info: memory window allocation information
1225 * @scratch: u64 saved to be used during cqp completion
1226 * @post_sq: flag for cqp db to ring
1227 */
1228 static enum irdma_status_code
irdma_sc_mw_alloc(struct irdma_sc_dev * dev,struct irdma_mw_alloc_info * info,u64 scratch,bool post_sq)1229 irdma_sc_mw_alloc(struct irdma_sc_dev *dev, struct irdma_mw_alloc_info *info,
1230 u64 scratch, bool post_sq)
1231 {
1232 u64 hdr;
1233 struct irdma_sc_cqp *cqp;
1234 __le64 *wqe;
1235
1236 cqp = dev->cqp;
1237 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
1238 if (!wqe)
1239 return IRDMA_ERR_RING_FULL;
1240
1241 set_64bit_val(wqe, 8,
1242 FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
1243 set_64bit_val(wqe, 16,
1244 FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->mw_stag_index));
1245
1246 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) |
1247 FIELD_PREP(IRDMA_CQPSQ_STAG_MWTYPE, info->mw_wide) |
1248 FIELD_PREP(IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY,
1249 info->mw1_bind_dont_vldt_key) |
1250 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
1251 dma_wmb(); /* make sure WQE is written before valid bit is set */
1252
1253 set_64bit_val(wqe, 24, hdr);
1254
1255 print_hex_dump_debug("WQE: MW_ALLOC WQE", DUMP_PREFIX_OFFSET, 16, 8,
1256 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
1257 if (post_sq)
1258 irdma_sc_cqp_post_sq(cqp);
1259
1260 return 0;
1261 }
1262
1263 /**
1264 * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
1265 * @qp: sc qp struct
1266 * @info: fast mr info
1267 * @post_sq: flag for cqp db to ring
1268 */
1269 enum irdma_status_code
irdma_sc_mr_fast_register(struct irdma_sc_qp * qp,struct irdma_fast_reg_stag_info * info,bool post_sq)1270 irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
1271 struct irdma_fast_reg_stag_info *info, bool post_sq)
1272 {
1273 u64 temp, hdr;
1274 __le64 *wqe;
1275 u32 wqe_idx;
1276 enum irdma_page_size page_size;
1277 struct irdma_post_sq_info sq_info = {};
1278
1279 if (info->page_size == 0x40000000)
1280 page_size = IRDMA_PAGE_SIZE_1G;
1281 else if (info->page_size == 0x200000)
1282 page_size = IRDMA_PAGE_SIZE_2M;
1283 else
1284 page_size = IRDMA_PAGE_SIZE_4K;
1285
1286 sq_info.wr_id = info->wr_id;
1287 sq_info.signaled = info->signaled;
1288 sq_info.push_wqe = info->push_wqe;
1289
1290 wqe = irdma_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx,
1291 IRDMA_QP_WQE_MIN_QUANTA, 0, &sq_info);
1292 if (!wqe)
1293 return IRDMA_ERR_QP_TOOMANY_WRS_POSTED;
1294
1295 irdma_clr_wqes(&qp->qp_uk, wqe_idx);
1296
1297 ibdev_dbg(to_ibdev(qp->dev),
1298 "MR: wr_id[%llxh] wqe_idx[%04d] location[%p]\n",
1299 info->wr_id, wqe_idx,
1300 &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
1301
1302 temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ?
1303 (uintptr_t)info->va : info->fbo;
1304 set_64bit_val(wqe, 0, temp);
1305
1306 temp = FIELD_GET(IRDMAQPSQ_FIRSTPMPBLIDXHI,
1307 info->first_pm_pbl_index >> 16);
1308 set_64bit_val(wqe, 8,
1309 FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXHI, temp) |
1310 FIELD_PREP(IRDMAQPSQ_PBLADDR >> IRDMA_HW_PAGE_SHIFT, info->reg_addr_pa));
1311 set_64bit_val(wqe, 16,
1312 info->total_len |
1313 FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXLO, info->first_pm_pbl_index));
1314
1315 hdr = FIELD_PREP(IRDMAQPSQ_STAGKEY, info->stag_key) |
1316 FIELD_PREP(IRDMAQPSQ_STAGINDEX, info->stag_idx) |
1317 FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_FAST_REGISTER) |
1318 FIELD_PREP(IRDMAQPSQ_LPBLSIZE, info->chunk_size) |
1319 FIELD_PREP(IRDMAQPSQ_HPAGESIZE, page_size) |
1320 FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, info->access_rights) |
1321 FIELD_PREP(IRDMAQPSQ_VABASEDTO, info->addr_type) |
1322 FIELD_PREP(IRDMAQPSQ_PUSHWQE, (sq_info.push_wqe ? 1 : 0)) |
1323 FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
1324 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
1325 FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
1326 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1327 dma_wmb(); /* make sure WQE is written before valid bit is set */
1328
1329 set_64bit_val(wqe, 24, hdr);
1330
1331 print_hex_dump_debug("WQE: FAST_REG WQE", DUMP_PREFIX_OFFSET, 16, 8,
1332 wqe, IRDMA_QP_WQE_MIN_SIZE, false);
1333 if (sq_info.push_wqe) {
1334 irdma_qp_push_wqe(&qp->qp_uk, wqe, IRDMA_QP_WQE_MIN_QUANTA,
1335 wqe_idx, post_sq);
1336 } else {
1337 if (post_sq)
1338 irdma_uk_qp_post_wr(&qp->qp_uk);
1339 }
1340
1341 return 0;
1342 }
1343
1344 /**
1345 * irdma_sc_gen_rts_ae - request AE generated after RTS
1346 * @qp: sc qp struct
1347 */
irdma_sc_gen_rts_ae(struct irdma_sc_qp * qp)1348 static void irdma_sc_gen_rts_ae(struct irdma_sc_qp *qp)
1349 {
1350 __le64 *wqe;
1351 u64 hdr;
1352 struct irdma_qp_uk *qp_uk;
1353
1354 qp_uk = &qp->qp_uk;
1355
1356 wqe = qp_uk->sq_base[1].elem;
1357
1358 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
1359 FIELD_PREP(IRDMAQPSQ_LOCALFENCE, 1) |
1360 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1361 dma_wmb(); /* make sure WQE is written before valid bit is set */
1362
1363 set_64bit_val(wqe, 24, hdr);
1364 print_hex_dump_debug("QP: NOP W/LOCAL FENCE WQE", DUMP_PREFIX_OFFSET,
1365 16, 8, wqe, IRDMA_QP_WQE_MIN_SIZE, false);
1366
1367 wqe = qp_uk->sq_base[2].elem;
1368 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_GEN_RTS_AE) |
1369 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1370 dma_wmb(); /* make sure WQE is written before valid bit is set */
1371
1372 set_64bit_val(wqe, 24, hdr);
1373 print_hex_dump_debug("QP: CONN EST WQE", DUMP_PREFIX_OFFSET, 16, 8,
1374 wqe, IRDMA_QP_WQE_MIN_SIZE, false);
1375 }
1376
1377 /**
1378 * irdma_sc_send_lsmm - send last streaming mode message
1379 * @qp: sc qp struct
1380 * @lsmm_buf: buffer with lsmm message
1381 * @size: size of lsmm buffer
1382 * @stag: stag of lsmm buffer
1383 */
irdma_sc_send_lsmm(struct irdma_sc_qp * qp,void * lsmm_buf,u32 size,irdma_stag stag)1384 void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,
1385 irdma_stag stag)
1386 {
1387 __le64 *wqe;
1388 u64 hdr;
1389 struct irdma_qp_uk *qp_uk;
1390
1391 qp_uk = &qp->qp_uk;
1392 wqe = qp_uk->sq_base->elem;
1393
1394 set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
1395 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
1396 set_64bit_val(wqe, 8,
1397 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, size) |
1398 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, stag));
1399 } else {
1400 set_64bit_val(wqe, 8,
1401 FIELD_PREP(IRDMAQPSQ_FRAG_LEN, size) |
1402 FIELD_PREP(IRDMAQPSQ_FRAG_STAG, stag) |
1403 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));
1404 }
1405 set_64bit_val(wqe, 16, 0);
1406
1407 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_SEND) |
1408 FIELD_PREP(IRDMAQPSQ_STREAMMODE, 1) |
1409 FIELD_PREP(IRDMAQPSQ_WAITFORRCVPDU, 1) |
1410 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1411 dma_wmb(); /* make sure WQE is written before valid bit is set */
1412
1413 set_64bit_val(wqe, 24, hdr);
1414
1415 print_hex_dump_debug("WQE: SEND_LSMM WQE", DUMP_PREFIX_OFFSET, 16, 8,
1416 wqe, IRDMA_QP_WQE_MIN_SIZE, false);
1417
1418 if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)
1419 irdma_sc_gen_rts_ae(qp);
1420 }
1421
1422 /**
1423 * irdma_sc_send_lsmm_nostag - for privilege qp
1424 * @qp: sc qp struct
1425 * @lsmm_buf: buffer with lsmm message
1426 * @size: size of lsmm buffer
1427 */
irdma_sc_send_lsmm_nostag(struct irdma_sc_qp * qp,void * lsmm_buf,u32 size)1428 void irdma_sc_send_lsmm_nostag(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size)
1429 {
1430 __le64 *wqe;
1431 u64 hdr;
1432 struct irdma_qp_uk *qp_uk;
1433
1434 qp_uk = &qp->qp_uk;
1435 wqe = qp_uk->sq_base->elem;
1436
1437 set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
1438
1439 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1)
1440 set_64bit_val(wqe, 8,
1441 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, size));
1442 else
1443 set_64bit_val(wqe, 8,
1444 FIELD_PREP(IRDMAQPSQ_FRAG_LEN, size) |
1445 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));
1446 set_64bit_val(wqe, 16, 0);
1447
1448 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_SEND) |
1449 FIELD_PREP(IRDMAQPSQ_STREAMMODE, 1) |
1450 FIELD_PREP(IRDMAQPSQ_WAITFORRCVPDU, 1) |
1451 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1452 dma_wmb(); /* make sure WQE is written before valid bit is set */
1453
1454 set_64bit_val(wqe, 24, hdr);
1455
1456 print_hex_dump_debug("WQE: SEND_LSMM_NOSTAG WQE", DUMP_PREFIX_OFFSET,
1457 16, 8, wqe, IRDMA_QP_WQE_MIN_SIZE, false);
1458 }
1459
1460 /**
1461 * irdma_sc_send_rtt - send last read0 or write0
1462 * @qp: sc qp struct
1463 * @read: Do read0 or write0
1464 */
irdma_sc_send_rtt(struct irdma_sc_qp * qp,bool read)1465 void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read)
1466 {
1467 __le64 *wqe;
1468 u64 hdr;
1469 struct irdma_qp_uk *qp_uk;
1470
1471 qp_uk = &qp->qp_uk;
1472 wqe = qp_uk->sq_base->elem;
1473
1474 set_64bit_val(wqe, 0, 0);
1475 set_64bit_val(wqe, 16, 0);
1476 if (read) {
1477 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
1478 set_64bit_val(wqe, 8,
1479 FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, 0xabcd));
1480 } else {
1481 set_64bit_val(wqe, 8,
1482 (u64)0xabcd | FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));
1483 }
1484 hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, 0x1234) |
1485 FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_READ) |
1486 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1487
1488 } else {
1489 if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
1490 set_64bit_val(wqe, 8, 0);
1491 } else {
1492 set_64bit_val(wqe, 8,
1493 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));
1494 }
1495 hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_WRITE) |
1496 FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1497 }
1498
1499 dma_wmb(); /* make sure WQE is written before valid bit is set */
1500
1501 set_64bit_val(wqe, 24, hdr);
1502
1503 print_hex_dump_debug("WQE: RTR WQE", DUMP_PREFIX_OFFSET, 16, 8, wqe,
1504 IRDMA_QP_WQE_MIN_SIZE, false);
1505
1506 if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)
1507 irdma_sc_gen_rts_ae(qp);
1508 }
1509
1510 /**
1511 * irdma_iwarp_opcode - determine if incoming is rdma layer
1512 * @info: aeq info for the packet
1513 * @pkt: packet for error
1514 */
irdma_iwarp_opcode(struct irdma_aeqe_info * info,u8 * pkt)1515 static u32 irdma_iwarp_opcode(struct irdma_aeqe_info *info, u8 *pkt)
1516 {
1517 __be16 *mpa;
1518 u32 opcode = 0xffffffff;
1519
1520 if (info->q2_data_written) {
1521 mpa = (__be16 *)pkt;
1522 opcode = ntohs(mpa[1]) & 0xf;
1523 }
1524
1525 return opcode;
1526 }
1527
1528 /**
1529 * irdma_locate_mpa - return pointer to mpa in the pkt
1530 * @pkt: packet with data
1531 */
irdma_locate_mpa(u8 * pkt)1532 static u8 *irdma_locate_mpa(u8 *pkt)
1533 {
1534 /* skip over ethernet header */
1535 pkt += IRDMA_MAC_HLEN;
1536
1537 /* Skip over IP and TCP headers */
1538 pkt += 4 * (pkt[0] & 0x0f);
1539 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
1540
1541 return pkt;
1542 }
1543
1544 /**
1545 * irdma_bld_termhdr_ctrl - setup terminate hdr control fields
1546 * @qp: sc qp ptr for pkt
1547 * @hdr: term hdr
1548 * @opcode: flush opcode for termhdr
1549 * @layer_etype: error layer + error type
1550 * @err: error cod ein the header
1551 */
irdma_bld_termhdr_ctrl(struct irdma_sc_qp * qp,struct irdma_terminate_hdr * hdr,enum irdma_flush_opcode opcode,u8 layer_etype,u8 err)1552 static void irdma_bld_termhdr_ctrl(struct irdma_sc_qp *qp,
1553 struct irdma_terminate_hdr *hdr,
1554 enum irdma_flush_opcode opcode,
1555 u8 layer_etype, u8 err)
1556 {
1557 qp->flush_code = opcode;
1558 hdr->layer_etype = layer_etype;
1559 hdr->error_code = err;
1560 }
1561
1562 /**
1563 * irdma_bld_termhdr_ddp_rdma - setup ddp and rdma hdrs in terminate hdr
1564 * @pkt: ptr to mpa in offending pkt
1565 * @hdr: term hdr
1566 * @copy_len: offending pkt length to be copied to term hdr
1567 * @is_tagged: DDP tagged or untagged
1568 */
irdma_bld_termhdr_ddp_rdma(u8 * pkt,struct irdma_terminate_hdr * hdr,int * copy_len,u8 * is_tagged)1569 static void irdma_bld_termhdr_ddp_rdma(u8 *pkt, struct irdma_terminate_hdr *hdr,
1570 int *copy_len, u8 *is_tagged)
1571 {
1572 u16 ddp_seg_len;
1573
1574 ddp_seg_len = ntohs(*(__be16 *)pkt);
1575 if (ddp_seg_len) {
1576 *copy_len = 2;
1577 hdr->hdrct = DDP_LEN_FLAG;
1578 if (pkt[2] & 0x80) {
1579 *is_tagged = 1;
1580 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
1581 *copy_len += TERM_DDP_LEN_TAGGED;
1582 hdr->hdrct |= DDP_HDR_FLAG;
1583 }
1584 } else {
1585 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
1586 *copy_len += TERM_DDP_LEN_UNTAGGED;
1587 hdr->hdrct |= DDP_HDR_FLAG;
1588 }
1589 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN) &&
1590 ((pkt[3] & RDMA_OPCODE_M) == RDMA_READ_REQ_OPCODE)) {
1591 *copy_len += TERM_RDMA_LEN;
1592 hdr->hdrct |= RDMA_HDR_FLAG;
1593 }
1594 }
1595 }
1596 }
1597
1598 /**
1599 * irdma_bld_terminate_hdr - build terminate message header
1600 * @qp: qp associated with received terminate AE
1601 * @info: the struct contiaing AE information
1602 */
irdma_bld_terminate_hdr(struct irdma_sc_qp * qp,struct irdma_aeqe_info * info)1603 static int irdma_bld_terminate_hdr(struct irdma_sc_qp *qp,
1604 struct irdma_aeqe_info *info)
1605 {
1606 u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
1607 int copy_len = 0;
1608 u8 is_tagged = 0;
1609 u32 opcode;
1610 struct irdma_terminate_hdr *termhdr;
1611
1612 termhdr = (struct irdma_terminate_hdr *)qp->q2_buf;
1613 memset(termhdr, 0, Q2_BAD_FRAME_OFFSET);
1614
1615 if (info->q2_data_written) {
1616 pkt = irdma_locate_mpa(pkt);
1617 irdma_bld_termhdr_ddp_rdma(pkt, termhdr, ©_len, &is_tagged);
1618 }
1619
1620 opcode = irdma_iwarp_opcode(info, pkt);
1621 qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC;
1622 qp->sq_flush_code = info->sq;
1623 qp->rq_flush_code = info->rq;
1624
1625 switch (info->ae_id) {
1626 case IRDMA_AE_AMP_UNALLOCATED_STAG:
1627 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1628 if (opcode == IRDMA_OP_TYPE_RDMA_WRITE)
1629 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,
1630 (LAYER_DDP << 4) | DDP_TAGGED_BUF,
1631 DDP_TAGGED_INV_STAG);
1632 else
1633 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1634 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1635 RDMAP_INV_STAG);
1636 break;
1637 case IRDMA_AE_AMP_BOUNDS_VIOLATION:
1638 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1639 if (info->q2_data_written)
1640 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,
1641 (LAYER_DDP << 4) | DDP_TAGGED_BUF,
1642 DDP_TAGGED_BOUNDS);
1643 else
1644 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1645 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1646 RDMAP_INV_BOUNDS);
1647 break;
1648 case IRDMA_AE_AMP_BAD_PD:
1649 switch (opcode) {
1650 case IRDMA_OP_TYPE_RDMA_WRITE:
1651 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,
1652 (LAYER_DDP << 4) | DDP_TAGGED_BUF,
1653 DDP_TAGGED_UNASSOC_STAG);
1654 break;
1655 case IRDMA_OP_TYPE_SEND_INV:
1656 case IRDMA_OP_TYPE_SEND_SOL_INV:
1657 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1658 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1659 RDMAP_CANT_INV_STAG);
1660 break;
1661 default:
1662 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1663 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1664 RDMAP_UNASSOC_STAG);
1665 }
1666 break;
1667 case IRDMA_AE_AMP_INVALID_STAG:
1668 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1669 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1670 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1671 RDMAP_INV_STAG);
1672 break;
1673 case IRDMA_AE_AMP_BAD_QP:
1674 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
1675 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1676 DDP_UNTAGGED_INV_QN);
1677 break;
1678 case IRDMA_AE_AMP_BAD_STAG_KEY:
1679 case IRDMA_AE_AMP_BAD_STAG_INDEX:
1680 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1681 switch (opcode) {
1682 case IRDMA_OP_TYPE_SEND_INV:
1683 case IRDMA_OP_TYPE_SEND_SOL_INV:
1684 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR,
1685 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
1686 RDMAP_CANT_INV_STAG);
1687 break;
1688 default:
1689 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1690 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
1691 RDMAP_INV_STAG);
1692 }
1693 break;
1694 case IRDMA_AE_AMP_RIGHTS_VIOLATION:
1695 case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
1696 case IRDMA_AE_PRIV_OPERATION_DENIED:
1697 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1698 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1699 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1700 RDMAP_ACCESS);
1701 break;
1702 case IRDMA_AE_AMP_TO_WRAP:
1703 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1704 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1705 (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1706 RDMAP_TO_WRAP);
1707 break;
1708 case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
1709 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1710 (LAYER_MPA << 4) | DDP_LLP, MPA_CRC);
1711 break;
1712 case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
1713 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR,
1714 (LAYER_DDP << 4) | DDP_CATASTROPHIC,
1715 DDP_CATASTROPHIC_LOCAL);
1716 break;
1717 case IRDMA_AE_LCE_QP_CATASTROPHIC:
1718 case IRDMA_AE_DDP_NO_L_BIT:
1719 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR,
1720 (LAYER_DDP << 4) | DDP_CATASTROPHIC,
1721 DDP_CATASTROPHIC_LOCAL);
1722 break;
1723 case IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN:
1724 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1725 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1726 DDP_UNTAGGED_INV_MSN_RANGE);
1727 break;
1728 case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
1729 qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1730 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR,
1731 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1732 DDP_UNTAGGED_INV_TOO_LONG);
1733 break;
1734 case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:
1735 if (is_tagged)
1736 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1737 (LAYER_DDP << 4) | DDP_TAGGED_BUF,
1738 DDP_TAGGED_INV_DDP_VER);
1739 else
1740 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1741 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1742 DDP_UNTAGGED_INV_DDP_VER);
1743 break;
1744 case IRDMA_AE_DDP_UBE_INVALID_MO:
1745 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1746 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1747 DDP_UNTAGGED_INV_MO);
1748 break;
1749 case IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
1750 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR,
1751 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1752 DDP_UNTAGGED_INV_MSN_NO_BUF);
1753 break;
1754 case IRDMA_AE_DDP_UBE_INVALID_QN:
1755 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1756 (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1757 DDP_UNTAGGED_INV_QN);
1758 break;
1759 case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
1760 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1761 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
1762 RDMAP_INV_RDMAP_VER);
1763 break;
1764 default:
1765 irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR,
1766 (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
1767 RDMAP_UNSPECIFIED);
1768 break;
1769 }
1770
1771 if (copy_len)
1772 memcpy(termhdr + 1, pkt, copy_len);
1773
1774 return sizeof(struct irdma_terminate_hdr) + copy_len;
1775 }
1776
1777 /**
1778 * irdma_terminate_send_fin() - Send fin for terminate message
1779 * @qp: qp associated with received terminate AE
1780 */
irdma_terminate_send_fin(struct irdma_sc_qp * qp)1781 void irdma_terminate_send_fin(struct irdma_sc_qp *qp)
1782 {
1783 irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE,
1784 IRDMAQP_TERM_SEND_FIN_ONLY, 0);
1785 }
1786
1787 /**
1788 * irdma_terminate_connection() - Bad AE and send terminate to remote QP
1789 * @qp: qp associated with received terminate AE
1790 * @info: the struct contiaing AE information
1791 */
irdma_terminate_connection(struct irdma_sc_qp * qp,struct irdma_aeqe_info * info)1792 void irdma_terminate_connection(struct irdma_sc_qp *qp,
1793 struct irdma_aeqe_info *info)
1794 {
1795 u8 termlen = 0;
1796
1797 if (qp->term_flags & IRDMA_TERM_SENT)
1798 return;
1799
1800 termlen = irdma_bld_terminate_hdr(qp, info);
1801 irdma_terminate_start_timer(qp);
1802 qp->term_flags |= IRDMA_TERM_SENT;
1803 irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE,
1804 IRDMAQP_TERM_SEND_TERM_ONLY, termlen);
1805 }
1806
1807 /**
1808 * irdma_terminate_received - handle terminate received AE
1809 * @qp: qp associated with received terminate AE
1810 * @info: the struct contiaing AE information
1811 */
irdma_terminate_received(struct irdma_sc_qp * qp,struct irdma_aeqe_info * info)1812 void irdma_terminate_received(struct irdma_sc_qp *qp,
1813 struct irdma_aeqe_info *info)
1814 {
1815 u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
1816 __be32 *mpa;
1817 u8 ddp_ctl;
1818 u8 rdma_ctl;
1819 u16 aeq_id = 0;
1820 struct irdma_terminate_hdr *termhdr;
1821
1822 mpa = (__be32 *)irdma_locate_mpa(pkt);
1823 if (info->q2_data_written) {
1824 /* did not validate the frame - do it now */
1825 ddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff;
1826 rdma_ctl = ntohl(mpa[0]) & 0xff;
1827 if ((ddp_ctl & 0xc0) != 0x40)
1828 aeq_id = IRDMA_AE_LCE_QP_CATASTROPHIC;
1829 else if ((ddp_ctl & 0x03) != 1)
1830 aeq_id = IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION;
1831 else if (ntohl(mpa[2]) != 2)
1832 aeq_id = IRDMA_AE_DDP_UBE_INVALID_QN;
1833 else if (ntohl(mpa[3]) != 1)
1834 aeq_id = IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN;
1835 else if (ntohl(mpa[4]) != 0)
1836 aeq_id = IRDMA_AE_DDP_UBE_INVALID_MO;
1837 else if ((rdma_ctl & 0xc0) != 0x40)
1838 aeq_id = IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION;
1839
1840 info->ae_id = aeq_id;
1841 if (info->ae_id) {
1842 /* Bad terminate recvd - send back a terminate */
1843 irdma_terminate_connection(qp, info);
1844 return;
1845 }
1846 }
1847
1848 qp->term_flags |= IRDMA_TERM_RCVD;
1849 qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC;
1850 termhdr = (struct irdma_terminate_hdr *)&mpa[5];
1851 if (termhdr->layer_etype == RDMAP_REMOTE_PROT ||
1852 termhdr->layer_etype == RDMAP_REMOTE_OP) {
1853 irdma_terminate_done(qp, 0);
1854 } else {
1855 irdma_terminate_start_timer(qp);
1856 irdma_terminate_send_fin(qp);
1857 }
1858 }
1859
irdma_null_ws_add(struct irdma_sc_vsi * vsi,u8 user_pri)1860 static enum irdma_status_code irdma_null_ws_add(struct irdma_sc_vsi *vsi,
1861 u8 user_pri)
1862 {
1863 return 0;
1864 }
1865
irdma_null_ws_remove(struct irdma_sc_vsi * vsi,u8 user_pri)1866 static void irdma_null_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri)
1867 {
1868 /* do nothing */
1869 }
1870
irdma_null_ws_reset(struct irdma_sc_vsi * vsi)1871 static void irdma_null_ws_reset(struct irdma_sc_vsi *vsi)
1872 {
1873 /* do nothing */
1874 }
1875
1876 /**
1877 * irdma_sc_vsi_init - Init the vsi structure
1878 * @vsi: pointer to vsi structure to initialize
1879 * @info: the info used to initialize the vsi struct
1880 */
irdma_sc_vsi_init(struct irdma_sc_vsi * vsi,struct irdma_vsi_init_info * info)1881 void irdma_sc_vsi_init(struct irdma_sc_vsi *vsi,
1882 struct irdma_vsi_init_info *info)
1883 {
1884 struct irdma_l2params *l2p;
1885 int i;
1886
1887 vsi->dev = info->dev;
1888 vsi->back_vsi = info->back_vsi;
1889 vsi->register_qset = info->register_qset;
1890 vsi->unregister_qset = info->unregister_qset;
1891 vsi->mtu = info->params->mtu;
1892 vsi->exception_lan_q = info->exception_lan_q;
1893 vsi->vsi_idx = info->pf_data_vsi_num;
1894 if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1895 vsi->fcn_id = info->dev->hmc_fn_id;
1896
1897 l2p = info->params;
1898 vsi->qos_rel_bw = l2p->vsi_rel_bw;
1899 vsi->qos_prio_type = l2p->vsi_prio_type;
1900 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
1901 if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1902 vsi->qos[i].qs_handle = l2p->qs_handle_list[i];
1903 vsi->qos[i].traffic_class = info->params->up2tc[i];
1904 vsi->qos[i].rel_bw =
1905 l2p->tc_info[vsi->qos[i].traffic_class].rel_bw;
1906 vsi->qos[i].prio_type =
1907 l2p->tc_info[vsi->qos[i].traffic_class].prio_type;
1908 vsi->qos[i].valid = false;
1909 mutex_init(&vsi->qos[i].qos_mutex);
1910 INIT_LIST_HEAD(&vsi->qos[i].qplist);
1911 }
1912 if (vsi->register_qset) {
1913 vsi->dev->ws_add = irdma_ws_add;
1914 vsi->dev->ws_remove = irdma_ws_remove;
1915 vsi->dev->ws_reset = irdma_ws_reset;
1916 } else {
1917 vsi->dev->ws_add = irdma_null_ws_add;
1918 vsi->dev->ws_remove = irdma_null_ws_remove;
1919 vsi->dev->ws_reset = irdma_null_ws_reset;
1920 }
1921 }
1922
1923 /**
1924 * irdma_get_fcn_id - Return the function id
1925 * @vsi: pointer to the vsi
1926 */
irdma_get_fcn_id(struct irdma_sc_vsi * vsi)1927 static u8 irdma_get_fcn_id(struct irdma_sc_vsi *vsi)
1928 {
1929 struct irdma_stats_inst_info stats_info = {};
1930 struct irdma_sc_dev *dev = vsi->dev;
1931 u8 fcn_id = IRDMA_INVALID_FCN_ID;
1932 u8 start_idx, max_stats, i;
1933
1934 if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {
1935 if (!irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_ALLOCATE,
1936 &stats_info))
1937 return stats_info.stats_idx;
1938 }
1939
1940 start_idx = 1;
1941 max_stats = 16;
1942 for (i = start_idx; i < max_stats; i++)
1943 if (!dev->fcn_id_array[i]) {
1944 fcn_id = i;
1945 dev->fcn_id_array[i] = true;
1946 break;
1947 }
1948
1949 return fcn_id;
1950 }
1951
1952 /**
1953 * irdma_vsi_stats_init - Initialize the vsi statistics
1954 * @vsi: pointer to the vsi structure
1955 * @info: The info structure used for initialization
1956 */
irdma_vsi_stats_init(struct irdma_sc_vsi * vsi,struct irdma_vsi_stats_info * info)1957 enum irdma_status_code irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,
1958 struct irdma_vsi_stats_info *info)
1959 {
1960 u8 fcn_id = info->fcn_id;
1961 struct irdma_dma_mem *stats_buff_mem;
1962
1963 vsi->pestat = info->pestat;
1964 vsi->pestat->hw = vsi->dev->hw;
1965 vsi->pestat->vsi = vsi;
1966 stats_buff_mem = &vsi->pestat->gather_info.stats_buff_mem;
1967 stats_buff_mem->size = ALIGN(IRDMA_GATHER_STATS_BUF_SIZE * 2, 1);
1968 stats_buff_mem->va = dma_alloc_coherent(vsi->pestat->hw->device,
1969 stats_buff_mem->size,
1970 &stats_buff_mem->pa,
1971 GFP_KERNEL);
1972 if (!stats_buff_mem->va)
1973 return IRDMA_ERR_NO_MEMORY;
1974
1975 vsi->pestat->gather_info.gather_stats_va = stats_buff_mem->va;
1976 vsi->pestat->gather_info.last_gather_stats_va =
1977 (void *)((uintptr_t)stats_buff_mem->va +
1978 IRDMA_GATHER_STATS_BUF_SIZE);
1979
1980 irdma_hw_stats_start_timer(vsi);
1981 if (info->alloc_fcn_id)
1982 fcn_id = irdma_get_fcn_id(vsi);
1983 if (fcn_id == IRDMA_INVALID_FCN_ID)
1984 goto stats_error;
1985
1986 vsi->stats_fcn_id_alloc = info->alloc_fcn_id;
1987 vsi->fcn_id = fcn_id;
1988 if (info->alloc_fcn_id) {
1989 vsi->pestat->gather_info.use_stats_inst = true;
1990 vsi->pestat->gather_info.stats_inst_index = fcn_id;
1991 }
1992
1993 return 0;
1994
1995 stats_error:
1996 dma_free_coherent(vsi->pestat->hw->device, stats_buff_mem->size,
1997 stats_buff_mem->va, stats_buff_mem->pa);
1998 stats_buff_mem->va = NULL;
1999
2000 return IRDMA_ERR_CQP_COMPL_ERROR;
2001 }
2002
2003 /**
2004 * irdma_vsi_stats_free - Free the vsi stats
2005 * @vsi: pointer to the vsi structure
2006 */
irdma_vsi_stats_free(struct irdma_sc_vsi * vsi)2007 void irdma_vsi_stats_free(struct irdma_sc_vsi *vsi)
2008 {
2009 struct irdma_stats_inst_info stats_info = {};
2010 u8 fcn_id = vsi->fcn_id;
2011 struct irdma_sc_dev *dev = vsi->dev;
2012
2013 if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {
2014 if (vsi->stats_fcn_id_alloc) {
2015 stats_info.stats_idx = vsi->fcn_id;
2016 irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_FREE,
2017 &stats_info);
2018 }
2019 } else {
2020 if (vsi->stats_fcn_id_alloc &&
2021 fcn_id < vsi->dev->hw_attrs.max_stat_inst)
2022 vsi->dev->fcn_id_array[fcn_id] = false;
2023 }
2024
2025 if (!vsi->pestat)
2026 return;
2027 irdma_hw_stats_stop_timer(vsi);
2028 dma_free_coherent(vsi->pestat->hw->device,
2029 vsi->pestat->gather_info.stats_buff_mem.size,
2030 vsi->pestat->gather_info.stats_buff_mem.va,
2031 vsi->pestat->gather_info.stats_buff_mem.pa);
2032 vsi->pestat->gather_info.stats_buff_mem.va = NULL;
2033 }
2034
2035 /**
2036 * irdma_get_encoded_wqe_size - given wq size, returns hardware encoded size
2037 * @wqsize: size of the wq (sq, rq) to encoded_size
2038 * @queue_type: queue type selected for the calculation algorithm
2039 */
irdma_get_encoded_wqe_size(u32 wqsize,enum irdma_queue_type queue_type)2040 u8 irdma_get_encoded_wqe_size(u32 wqsize, enum irdma_queue_type queue_type)
2041 {
2042 u8 encoded_size = 0;
2043
2044 /* cqp sq's hw coded value starts from 1 for size of 4
2045 * while it starts from 0 for qp' wq's.
2046 */
2047 if (queue_type == IRDMA_QUEUE_TYPE_CQP)
2048 encoded_size = 1;
2049 wqsize >>= 2;
2050 while (wqsize >>= 1)
2051 encoded_size++;
2052
2053 return encoded_size;
2054 }
2055
2056 /**
2057 * irdma_sc_gather_stats - collect the statistics
2058 * @cqp: struct for cqp hw
2059 * @info: gather stats info structure
2060 * @scratch: u64 saved to be used during cqp completion
2061 */
2062 static enum irdma_status_code
irdma_sc_gather_stats(struct irdma_sc_cqp * cqp,struct irdma_stats_gather_info * info,u64 scratch)2063 irdma_sc_gather_stats(struct irdma_sc_cqp *cqp,
2064 struct irdma_stats_gather_info *info, u64 scratch)
2065 {
2066 __le64 *wqe;
2067 u64 temp;
2068
2069 if (info->stats_buff_mem.size < IRDMA_GATHER_STATS_BUF_SIZE)
2070 return IRDMA_ERR_BUF_TOO_SHORT;
2071
2072 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2073 if (!wqe)
2074 return IRDMA_ERR_RING_FULL;
2075
2076 set_64bit_val(wqe, 40,
2077 FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fcn_index));
2078 set_64bit_val(wqe, 32, info->stats_buff_mem.pa);
2079
2080 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) |
2081 FIELD_PREP(IRDMA_CQPSQ_STATS_USE_INST, info->use_stats_inst) |
2082 FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX,
2083 info->stats_inst_index) |
2084 FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX,
2085 info->use_hmc_fcn_index) |
2086 FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_GATHER_STATS);
2087 dma_wmb(); /* make sure WQE is written before valid bit is set */
2088
2089 set_64bit_val(wqe, 24, temp);
2090
2091 print_hex_dump_debug("STATS: GATHER_STATS WQE", DUMP_PREFIX_OFFSET,
2092 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2093
2094 irdma_sc_cqp_post_sq(cqp);
2095 ibdev_dbg(to_ibdev(cqp->dev),
2096 "STATS: CQP SQ head 0x%x tail 0x%x size 0x%x\n",
2097 cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size);
2098
2099 return 0;
2100 }
2101
2102 /**
2103 * irdma_sc_manage_stats_inst - allocate or free stats instance
2104 * @cqp: struct for cqp hw
2105 * @info: stats info structure
2106 * @alloc: alloc vs. delete flag
2107 * @scratch: u64 saved to be used during cqp completion
2108 */
2109 static enum irdma_status_code
irdma_sc_manage_stats_inst(struct irdma_sc_cqp * cqp,struct irdma_stats_inst_info * info,bool alloc,u64 scratch)2110 irdma_sc_manage_stats_inst(struct irdma_sc_cqp *cqp,
2111 struct irdma_stats_inst_info *info, bool alloc,
2112 u64 scratch)
2113 {
2114 __le64 *wqe;
2115 u64 temp;
2116
2117 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2118 if (!wqe)
2119 return IRDMA_ERR_RING_FULL;
2120
2121 set_64bit_val(wqe, 40,
2122 FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fn_id));
2123 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) |
2124 FIELD_PREP(IRDMA_CQPSQ_STATS_ALLOC_INST, alloc) |
2125 FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX,
2126 info->use_hmc_fcn_index) |
2127 FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX, info->stats_idx) |
2128 FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_MANAGE_STATS);
2129
2130 dma_wmb(); /* make sure WQE is written before valid bit is set */
2131
2132 set_64bit_val(wqe, 24, temp);
2133
2134 print_hex_dump_debug("WQE: MANAGE_STATS WQE", DUMP_PREFIX_OFFSET, 16,
2135 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2136
2137 irdma_sc_cqp_post_sq(cqp);
2138 return 0;
2139 }
2140
2141 /**
2142 * irdma_sc_set_up_map - set the up map table
2143 * @cqp: struct for cqp hw
2144 * @info: User priority map info
2145 * @scratch: u64 saved to be used during cqp completion
2146 */
irdma_sc_set_up_map(struct irdma_sc_cqp * cqp,struct irdma_up_info * info,u64 scratch)2147 static enum irdma_status_code irdma_sc_set_up_map(struct irdma_sc_cqp *cqp,
2148 struct irdma_up_info *info,
2149 u64 scratch)
2150 {
2151 __le64 *wqe;
2152 u64 temp = 0;
2153 int i;
2154
2155 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2156 if (!wqe)
2157 return IRDMA_ERR_RING_FULL;
2158
2159 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++)
2160 temp |= (u64)info->map[i] << (i * 8);
2161
2162 set_64bit_val(wqe, 0, temp);
2163 set_64bit_val(wqe, 40,
2164 FIELD_PREP(IRDMA_CQPSQ_UP_CNPOVERRIDE, info->cnp_up_override) |
2165 FIELD_PREP(IRDMA_CQPSQ_UP_HMCFCNIDX, info->hmc_fcn_idx));
2166
2167 temp = FIELD_PREP(IRDMA_CQPSQ_UP_WQEVALID, cqp->polarity) |
2168 FIELD_PREP(IRDMA_CQPSQ_UP_USEVLAN, info->use_vlan) |
2169 FIELD_PREP(IRDMA_CQPSQ_UP_USEOVERRIDE,
2170 info->use_cnp_up_override) |
2171 FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_UP_MAP);
2172 dma_wmb(); /* make sure WQE is written before valid bit is set */
2173
2174 set_64bit_val(wqe, 24, temp);
2175
2176 print_hex_dump_debug("WQE: UPMAP WQE", DUMP_PREFIX_OFFSET, 16, 8, wqe,
2177 IRDMA_CQP_WQE_SIZE * 8, false);
2178 irdma_sc_cqp_post_sq(cqp);
2179
2180 return 0;
2181 }
2182
2183 /**
2184 * irdma_sc_manage_ws_node - create/modify/destroy WS node
2185 * @cqp: struct for cqp hw
2186 * @info: node info structure
2187 * @node_op: 0 for add 1 for modify, 2 for delete
2188 * @scratch: u64 saved to be used during cqp completion
2189 */
2190 static enum irdma_status_code
irdma_sc_manage_ws_node(struct irdma_sc_cqp * cqp,struct irdma_ws_node_info * info,enum irdma_ws_node_op node_op,u64 scratch)2191 irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp,
2192 struct irdma_ws_node_info *info,
2193 enum irdma_ws_node_op node_op, u64 scratch)
2194 {
2195 __le64 *wqe;
2196 u64 temp = 0;
2197
2198 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2199 if (!wqe)
2200 return IRDMA_ERR_RING_FULL;
2201
2202 set_64bit_val(wqe, 32,
2203 FIELD_PREP(IRDMA_CQPSQ_WS_VSI, info->vsi) |
2204 FIELD_PREP(IRDMA_CQPSQ_WS_WEIGHT, info->weight));
2205
2206 temp = FIELD_PREP(IRDMA_CQPSQ_WS_WQEVALID, cqp->polarity) |
2207 FIELD_PREP(IRDMA_CQPSQ_WS_NODEOP, node_op) |
2208 FIELD_PREP(IRDMA_CQPSQ_WS_ENABLENODE, info->enable) |
2209 FIELD_PREP(IRDMA_CQPSQ_WS_NODETYPE, info->type_leaf) |
2210 FIELD_PREP(IRDMA_CQPSQ_WS_PRIOTYPE, info->prio_type) |
2211 FIELD_PREP(IRDMA_CQPSQ_WS_TC, info->tc) |
2212 FIELD_PREP(IRDMA_CQPSQ_WS_OP, IRDMA_CQP_OP_WORK_SCHED_NODE) |
2213 FIELD_PREP(IRDMA_CQPSQ_WS_PARENTID, info->parent_id) |
2214 FIELD_PREP(IRDMA_CQPSQ_WS_NODEID, info->id);
2215 dma_wmb(); /* make sure WQE is written before valid bit is set */
2216
2217 set_64bit_val(wqe, 24, temp);
2218
2219 print_hex_dump_debug("WQE: MANAGE_WS WQE", DUMP_PREFIX_OFFSET, 16, 8,
2220 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2221 irdma_sc_cqp_post_sq(cqp);
2222
2223 return 0;
2224 }
2225
2226 /**
2227 * irdma_sc_qp_flush_wqes - flush qp's wqe
2228 * @qp: sc qp
2229 * @info: dlush information
2230 * @scratch: u64 saved to be used during cqp completion
2231 * @post_sq: flag for cqp db to ring
2232 */
irdma_sc_qp_flush_wqes(struct irdma_sc_qp * qp,struct irdma_qp_flush_info * info,u64 scratch,bool post_sq)2233 enum irdma_status_code irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
2234 struct irdma_qp_flush_info *info,
2235 u64 scratch, bool post_sq)
2236 {
2237 u64 temp = 0;
2238 __le64 *wqe;
2239 struct irdma_sc_cqp *cqp;
2240 u64 hdr;
2241 bool flush_sq = false, flush_rq = false;
2242
2243 if (info->rq && !qp->flush_rq)
2244 flush_rq = true;
2245 if (info->sq && !qp->flush_sq)
2246 flush_sq = true;
2247 qp->flush_sq |= flush_sq;
2248 qp->flush_rq |= flush_rq;
2249
2250 if (!flush_sq && !flush_rq) {
2251 ibdev_dbg(to_ibdev(qp->dev),
2252 "CQP: Additional flush request ignored for qp %x\n",
2253 qp->qp_uk.qp_id);
2254 return IRDMA_ERR_FLUSHED_Q;
2255 }
2256
2257 cqp = qp->pd->dev->cqp;
2258 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2259 if (!wqe)
2260 return IRDMA_ERR_RING_FULL;
2261
2262 if (info->userflushcode) {
2263 if (flush_rq)
2264 temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMNERR,
2265 info->rq_minor_code) |
2266 FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMJERR,
2267 info->rq_major_code);
2268 if (flush_sq)
2269 temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMNERR,
2270 info->sq_minor_code) |
2271 FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMJERR,
2272 info->sq_major_code);
2273 }
2274 set_64bit_val(wqe, 16, temp);
2275
2276 temp = (info->generate_ae) ?
2277 info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE,
2278 info->ae_src) : 0;
2279 set_64bit_val(wqe, 8, temp);
2280
2281 hdr = qp->qp_uk.qp_id |
2282 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_FLUSH_WQES) |
2283 FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, info->generate_ae) |
2284 FIELD_PREP(IRDMA_CQPSQ_FWQE_USERFLCODE, info->userflushcode) |
2285 FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHSQ, flush_sq) |
2286 FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHRQ, flush_rq) |
2287 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2288 dma_wmb(); /* make sure WQE is written before valid bit is set */
2289
2290 set_64bit_val(wqe, 24, hdr);
2291
2292 print_hex_dump_debug("WQE: QP_FLUSH WQE", DUMP_PREFIX_OFFSET, 16, 8,
2293 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2294 if (post_sq)
2295 irdma_sc_cqp_post_sq(cqp);
2296
2297 return 0;
2298 }
2299
2300 /**
2301 * irdma_sc_gen_ae - generate AE, uses flush WQE CQP OP
2302 * @qp: sc qp
2303 * @info: gen ae information
2304 * @scratch: u64 saved to be used during cqp completion
2305 * @post_sq: flag for cqp db to ring
2306 */
irdma_sc_gen_ae(struct irdma_sc_qp * qp,struct irdma_gen_ae_info * info,u64 scratch,bool post_sq)2307 static enum irdma_status_code irdma_sc_gen_ae(struct irdma_sc_qp *qp,
2308 struct irdma_gen_ae_info *info,
2309 u64 scratch, bool post_sq)
2310 {
2311 u64 temp;
2312 __le64 *wqe;
2313 struct irdma_sc_cqp *cqp;
2314 u64 hdr;
2315
2316 cqp = qp->pd->dev->cqp;
2317 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2318 if (!wqe)
2319 return IRDMA_ERR_RING_FULL;
2320
2321 temp = info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE,
2322 info->ae_src);
2323 set_64bit_val(wqe, 8, temp);
2324
2325 hdr = qp->qp_uk.qp_id | FIELD_PREP(IRDMA_CQPSQ_OPCODE,
2326 IRDMA_CQP_OP_GEN_AE) |
2327 FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, 1) |
2328 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2329 dma_wmb(); /* make sure WQE is written before valid bit is set */
2330
2331 set_64bit_val(wqe, 24, hdr);
2332
2333 print_hex_dump_debug("WQE: GEN_AE WQE", DUMP_PREFIX_OFFSET, 16, 8,
2334 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2335 if (post_sq)
2336 irdma_sc_cqp_post_sq(cqp);
2337
2338 return 0;
2339 }
2340
2341 /*** irdma_sc_qp_upload_context - upload qp's context
2342 * @dev: sc device struct
2343 * @info: upload context info ptr for return
2344 * @scratch: u64 saved to be used during cqp completion
2345 * @post_sq: flag for cqp db to ring
2346 */
2347 static enum irdma_status_code
irdma_sc_qp_upload_context(struct irdma_sc_dev * dev,struct irdma_upload_context_info * info,u64 scratch,bool post_sq)2348 irdma_sc_qp_upload_context(struct irdma_sc_dev *dev,
2349 struct irdma_upload_context_info *info, u64 scratch,
2350 bool post_sq)
2351 {
2352 __le64 *wqe;
2353 struct irdma_sc_cqp *cqp;
2354 u64 hdr;
2355
2356 cqp = dev->cqp;
2357 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2358 if (!wqe)
2359 return IRDMA_ERR_RING_FULL;
2360
2361 set_64bit_val(wqe, 16, info->buf_pa);
2362
2363 hdr = FIELD_PREP(IRDMA_CQPSQ_UCTX_QPID, info->qp_id) |
2364 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPLOAD_CONTEXT) |
2365 FIELD_PREP(IRDMA_CQPSQ_UCTX_QPTYPE, info->qp_type) |
2366 FIELD_PREP(IRDMA_CQPSQ_UCTX_RAWFORMAT, info->raw_format) |
2367 FIELD_PREP(IRDMA_CQPSQ_UCTX_FREEZEQP, info->freeze_qp) |
2368 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2369 dma_wmb(); /* make sure WQE is written before valid bit is set */
2370
2371 set_64bit_val(wqe, 24, hdr);
2372
2373 print_hex_dump_debug("WQE: QP_UPLOAD_CTX WQE", DUMP_PREFIX_OFFSET, 16,
2374 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2375 if (post_sq)
2376 irdma_sc_cqp_post_sq(cqp);
2377
2378 return 0;
2379 }
2380
2381 /**
2382 * irdma_sc_manage_push_page - Handle push page
2383 * @cqp: struct for cqp hw
2384 * @info: push page info
2385 * @scratch: u64 saved to be used during cqp completion
2386 * @post_sq: flag for cqp db to ring
2387 */
2388 static enum irdma_status_code
irdma_sc_manage_push_page(struct irdma_sc_cqp * cqp,struct irdma_cqp_manage_push_page_info * info,u64 scratch,bool post_sq)2389 irdma_sc_manage_push_page(struct irdma_sc_cqp *cqp,
2390 struct irdma_cqp_manage_push_page_info *info,
2391 u64 scratch, bool post_sq)
2392 {
2393 __le64 *wqe;
2394 u64 hdr;
2395
2396 if (info->free_page &&
2397 info->push_idx >= cqp->dev->hw_attrs.max_hw_device_pages)
2398 return IRDMA_ERR_INVALID_PUSH_PAGE_INDEX;
2399
2400 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2401 if (!wqe)
2402 return IRDMA_ERR_RING_FULL;
2403
2404 set_64bit_val(wqe, 16, info->qs_handle);
2405 hdr = FIELD_PREP(IRDMA_CQPSQ_MPP_PPIDX, info->push_idx) |
2406 FIELD_PREP(IRDMA_CQPSQ_MPP_PPTYPE, info->push_page_type) |
2407 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_PUSH_PAGES) |
2408 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) |
2409 FIELD_PREP(IRDMA_CQPSQ_MPP_FREE_PAGE, info->free_page);
2410 dma_wmb(); /* make sure WQE is written before valid bit is set */
2411
2412 set_64bit_val(wqe, 24, hdr);
2413
2414 print_hex_dump_debug("WQE: MANAGE_PUSH_PAGES WQE", DUMP_PREFIX_OFFSET,
2415 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2416 if (post_sq)
2417 irdma_sc_cqp_post_sq(cqp);
2418
2419 return 0;
2420 }
2421
2422 /**
2423 * irdma_sc_suspend_qp - suspend qp for param change
2424 * @cqp: struct for cqp hw
2425 * @qp: sc qp struct
2426 * @scratch: u64 saved to be used during cqp completion
2427 */
irdma_sc_suspend_qp(struct irdma_sc_cqp * cqp,struct irdma_sc_qp * qp,u64 scratch)2428 static enum irdma_status_code irdma_sc_suspend_qp(struct irdma_sc_cqp *cqp,
2429 struct irdma_sc_qp *qp,
2430 u64 scratch)
2431 {
2432 u64 hdr;
2433 __le64 *wqe;
2434
2435 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2436 if (!wqe)
2437 return IRDMA_ERR_RING_FULL;
2438
2439 hdr = FIELD_PREP(IRDMA_CQPSQ_SUSPENDQP_QPID, qp->qp_uk.qp_id) |
2440 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_SUSPEND_QP) |
2441 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2442 dma_wmb(); /* make sure WQE is written before valid bit is set */
2443
2444 set_64bit_val(wqe, 24, hdr);
2445
2446 print_hex_dump_debug("WQE: SUSPEND_QP WQE", DUMP_PREFIX_OFFSET, 16, 8,
2447 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2448 irdma_sc_cqp_post_sq(cqp);
2449
2450 return 0;
2451 }
2452
2453 /**
2454 * irdma_sc_resume_qp - resume qp after suspend
2455 * @cqp: struct for cqp hw
2456 * @qp: sc qp struct
2457 * @scratch: u64 saved to be used during cqp completion
2458 */
irdma_sc_resume_qp(struct irdma_sc_cqp * cqp,struct irdma_sc_qp * qp,u64 scratch)2459 static enum irdma_status_code irdma_sc_resume_qp(struct irdma_sc_cqp *cqp,
2460 struct irdma_sc_qp *qp,
2461 u64 scratch)
2462 {
2463 u64 hdr;
2464 __le64 *wqe;
2465
2466 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2467 if (!wqe)
2468 return IRDMA_ERR_RING_FULL;
2469
2470 set_64bit_val(wqe, 16,
2471 FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QSHANDLE, qp->qs_handle));
2472
2473 hdr = FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QPID, qp->qp_uk.qp_id) |
2474 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_RESUME_QP) |
2475 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2476 dma_wmb(); /* make sure WQE is written before valid bit is set */
2477
2478 set_64bit_val(wqe, 24, hdr);
2479
2480 print_hex_dump_debug("WQE: RESUME_QP WQE", DUMP_PREFIX_OFFSET, 16, 8,
2481 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2482 irdma_sc_cqp_post_sq(cqp);
2483
2484 return 0;
2485 }
2486
2487 /**
2488 * irdma_sc_cq_ack - acknowledge completion q
2489 * @cq: cq struct
2490 */
irdma_sc_cq_ack(struct irdma_sc_cq * cq)2491 static inline void irdma_sc_cq_ack(struct irdma_sc_cq *cq)
2492 {
2493 writel(cq->cq_uk.cq_id, cq->cq_uk.cq_ack_db);
2494 }
2495
2496 /**
2497 * irdma_sc_cq_init - initialize completion q
2498 * @cq: cq struct
2499 * @info: cq initialization info
2500 */
irdma_sc_cq_init(struct irdma_sc_cq * cq,struct irdma_cq_init_info * info)2501 enum irdma_status_code irdma_sc_cq_init(struct irdma_sc_cq *cq,
2502 struct irdma_cq_init_info *info)
2503 {
2504 enum irdma_status_code ret_code;
2505 u32 pble_obj_cnt;
2506
2507 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
2508 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
2509 return IRDMA_ERR_INVALID_PBLE_INDEX;
2510
2511 cq->cq_pa = info->cq_base_pa;
2512 cq->dev = info->dev;
2513 cq->ceq_id = info->ceq_id;
2514 info->cq_uk_init_info.cqe_alloc_db = cq->dev->cq_arm_db;
2515 info->cq_uk_init_info.cq_ack_db = cq->dev->cq_ack_db;
2516 ret_code = irdma_uk_cq_init(&cq->cq_uk, &info->cq_uk_init_info);
2517 if (ret_code)
2518 return ret_code;
2519
2520 cq->virtual_map = info->virtual_map;
2521 cq->pbl_chunk_size = info->pbl_chunk_size;
2522 cq->ceqe_mask = info->ceqe_mask;
2523 cq->cq_type = (info->type) ? info->type : IRDMA_CQ_TYPE_IWARP;
2524 cq->shadow_area_pa = info->shadow_area_pa;
2525 cq->shadow_read_threshold = info->shadow_read_threshold;
2526 cq->ceq_id_valid = info->ceq_id_valid;
2527 cq->tph_en = info->tph_en;
2528 cq->tph_val = info->tph_val;
2529 cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2530 cq->vsi = info->vsi;
2531
2532 return 0;
2533 }
2534
2535 /**
2536 * irdma_sc_cq_create - create completion q
2537 * @cq: cq struct
2538 * @scratch: u64 saved to be used during cqp completion
2539 * @check_overflow: flag for overflow check
2540 * @post_sq: flag for cqp db to ring
2541 */
irdma_sc_cq_create(struct irdma_sc_cq * cq,u64 scratch,bool check_overflow,bool post_sq)2542 static enum irdma_status_code irdma_sc_cq_create(struct irdma_sc_cq *cq,
2543 u64 scratch,
2544 bool check_overflow,
2545 bool post_sq)
2546 {
2547 __le64 *wqe;
2548 struct irdma_sc_cqp *cqp;
2549 u64 hdr;
2550 struct irdma_sc_ceq *ceq;
2551 enum irdma_status_code ret_code = 0;
2552
2553 cqp = cq->dev->cqp;
2554 if (cq->cq_uk.cq_id > (cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt - 1))
2555 return IRDMA_ERR_INVALID_CQ_ID;
2556
2557 if (cq->ceq_id > (cq->dev->hmc_fpm_misc.max_ceqs - 1))
2558 return IRDMA_ERR_INVALID_CEQ_ID;
2559
2560 ceq = cq->dev->ceq[cq->ceq_id];
2561 if (ceq && ceq->reg_cq)
2562 ret_code = irdma_sc_add_cq_ctx(ceq, cq);
2563
2564 if (ret_code)
2565 return ret_code;
2566
2567 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2568 if (!wqe) {
2569 if (ceq && ceq->reg_cq)
2570 irdma_sc_remove_cq_ctx(ceq, cq);
2571 return IRDMA_ERR_RING_FULL;
2572 }
2573
2574 set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
2575 set_64bit_val(wqe, 8, (uintptr_t)cq >> 1);
2576 set_64bit_val(wqe, 16,
2577 FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold));
2578 set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
2579 set_64bit_val(wqe, 40, cq->shadow_area_pa);
2580 set_64bit_val(wqe, 48,
2581 FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX, (cq->virtual_map ? cq->first_pm_pbl_idx : 0)));
2582 set_64bit_val(wqe, 56,
2583 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
2584 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
2585
2586 hdr = FLD_LS_64(cq->dev, cq->cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) |
2587 FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),
2588 IRDMA_CQPSQ_CQ_CEQID) |
2589 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) |
2590 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) |
2591 FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, check_overflow) |
2592 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) |
2593 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) |
2594 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) |
2595 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) |
2596 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT,
2597 cq->cq_uk.avoid_mem_cflct) |
2598 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2599 dma_wmb(); /* make sure WQE is written before valid bit is set */
2600
2601 set_64bit_val(wqe, 24, hdr);
2602
2603 print_hex_dump_debug("WQE: CQ_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,
2604 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2605 if (post_sq)
2606 irdma_sc_cqp_post_sq(cqp);
2607
2608 return 0;
2609 }
2610
2611 /**
2612 * irdma_sc_cq_destroy - destroy completion q
2613 * @cq: cq struct
2614 * @scratch: u64 saved to be used during cqp completion
2615 * @post_sq: flag for cqp db to ring
2616 */
irdma_sc_cq_destroy(struct irdma_sc_cq * cq,u64 scratch,bool post_sq)2617 enum irdma_status_code irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch,
2618 bool post_sq)
2619 {
2620 struct irdma_sc_cqp *cqp;
2621 __le64 *wqe;
2622 u64 hdr;
2623 struct irdma_sc_ceq *ceq;
2624
2625 cqp = cq->dev->cqp;
2626 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2627 if (!wqe)
2628 return IRDMA_ERR_RING_FULL;
2629
2630 ceq = cq->dev->ceq[cq->ceq_id];
2631 if (ceq && ceq->reg_cq)
2632 irdma_sc_remove_cq_ctx(ceq, cq);
2633
2634 set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
2635 set_64bit_val(wqe, 8, (uintptr_t)cq >> 1);
2636 set_64bit_val(wqe, 40, cq->shadow_area_pa);
2637 set_64bit_val(wqe, 48,
2638 (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
2639
2640 hdr = cq->cq_uk.cq_id |
2641 FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),
2642 IRDMA_CQPSQ_CQ_CEQID) |
2643 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) |
2644 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) |
2645 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) |
2646 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) |
2647 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) |
2648 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) |
2649 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, cq->cq_uk.avoid_mem_cflct) |
2650 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2651 dma_wmb(); /* make sure WQE is written before valid bit is set */
2652
2653 set_64bit_val(wqe, 24, hdr);
2654
2655 print_hex_dump_debug("WQE: CQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16, 8,
2656 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2657 if (post_sq)
2658 irdma_sc_cqp_post_sq(cqp);
2659
2660 return 0;
2661 }
2662
2663 /**
2664 * irdma_sc_cq_resize - set resized cq buffer info
2665 * @cq: resized cq
2666 * @info: resized cq buffer info
2667 */
irdma_sc_cq_resize(struct irdma_sc_cq * cq,struct irdma_modify_cq_info * info)2668 void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info)
2669 {
2670 cq->virtual_map = info->virtual_map;
2671 cq->cq_pa = info->cq_pa;
2672 cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2673 cq->pbl_chunk_size = info->pbl_chunk_size;
2674 irdma_uk_cq_resize(&cq->cq_uk, info->cq_base, info->cq_size);
2675 }
2676
2677 /**
2678 * irdma_sc_cq_modify - modify a Completion Queue
2679 * @cq: cq struct
2680 * @info: modification info struct
2681 * @scratch: u64 saved to be used during cqp completion
2682 * @post_sq: flag to post to sq
2683 */
2684 static enum irdma_status_code
irdma_sc_cq_modify(struct irdma_sc_cq * cq,struct irdma_modify_cq_info * info,u64 scratch,bool post_sq)2685 irdma_sc_cq_modify(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info,
2686 u64 scratch, bool post_sq)
2687 {
2688 struct irdma_sc_cqp *cqp;
2689 __le64 *wqe;
2690 u64 hdr;
2691 u32 pble_obj_cnt;
2692
2693 pble_obj_cnt = cq->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
2694 if (info->cq_resize && info->virtual_map &&
2695 info->first_pm_pbl_idx >= pble_obj_cnt)
2696 return IRDMA_ERR_INVALID_PBLE_INDEX;
2697
2698 cqp = cq->dev->cqp;
2699 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2700 if (!wqe)
2701 return IRDMA_ERR_RING_FULL;
2702
2703 set_64bit_val(wqe, 0, info->cq_size);
2704 set_64bit_val(wqe, 8, (uintptr_t)cq >> 1);
2705 set_64bit_val(wqe, 16,
2706 FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, info->shadow_read_threshold));
2707 set_64bit_val(wqe, 32, info->cq_pa);
2708 set_64bit_val(wqe, 40, cq->shadow_area_pa);
2709 set_64bit_val(wqe, 48, info->first_pm_pbl_idx);
2710 set_64bit_val(wqe, 56,
2711 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
2712 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
2713
2714 hdr = cq->cq_uk.cq_id |
2715 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_CQ) |
2716 FIELD_PREP(IRDMA_CQPSQ_CQ_CQRESIZE, info->cq_resize) |
2717 FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, info->pbl_chunk_size) |
2718 FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, info->check_overflow) |
2719 FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, info->virtual_map) |
2720 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) |
2721 FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) |
2722 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT,
2723 cq->cq_uk.avoid_mem_cflct) |
2724 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2725 dma_wmb(); /* make sure WQE is written before valid bit is set */
2726
2727 set_64bit_val(wqe, 24, hdr);
2728
2729 print_hex_dump_debug("WQE: CQ_MODIFY WQE", DUMP_PREFIX_OFFSET, 16, 8,
2730 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2731 if (post_sq)
2732 irdma_sc_cqp_post_sq(cqp);
2733
2734 return 0;
2735 }
2736
2737 /**
2738 * irdma_check_cqp_progress - check cqp processing progress
2739 * @timeout: timeout info struct
2740 * @dev: sc device struct
2741 */
irdma_check_cqp_progress(struct irdma_cqp_timeout * timeout,struct irdma_sc_dev * dev)2742 void irdma_check_cqp_progress(struct irdma_cqp_timeout *timeout, struct irdma_sc_dev *dev)
2743 {
2744 if (timeout->compl_cqp_cmds != dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]) {
2745 timeout->compl_cqp_cmds = dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];
2746 timeout->count = 0;
2747 } else {
2748 if (dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] !=
2749 timeout->compl_cqp_cmds)
2750 timeout->count++;
2751 }
2752 }
2753
2754 /**
2755 * irdma_get_cqp_reg_info - get head and tail for cqp using registers
2756 * @cqp: struct for cqp hw
2757 * @val: cqp tail register value
2758 * @tail: wqtail register value
2759 * @error: cqp processing err
2760 */
irdma_get_cqp_reg_info(struct irdma_sc_cqp * cqp,u32 * val,u32 * tail,u32 * error)2761 static inline void irdma_get_cqp_reg_info(struct irdma_sc_cqp *cqp, u32 *val,
2762 u32 *tail, u32 *error)
2763 {
2764 *val = readl(cqp->dev->hw_regs[IRDMA_CQPTAIL]);
2765 *tail = FIELD_GET(IRDMA_CQPTAIL_WQTAIL, *val);
2766 *error = FIELD_GET(IRDMA_CQPTAIL_CQP_OP_ERR, *val);
2767 }
2768
2769 /**
2770 * irdma_cqp_poll_registers - poll cqp registers
2771 * @cqp: struct for cqp hw
2772 * @tail: wqtail register value
2773 * @count: how many times to try for completion
2774 */
irdma_cqp_poll_registers(struct irdma_sc_cqp * cqp,u32 tail,u32 count)2775 static enum irdma_status_code irdma_cqp_poll_registers(struct irdma_sc_cqp *cqp,
2776 u32 tail, u32 count)
2777 {
2778 u32 i = 0;
2779 u32 newtail, error, val;
2780
2781 while (i++ < count) {
2782 irdma_get_cqp_reg_info(cqp, &val, &newtail, &error);
2783 if (error) {
2784 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]);
2785 ibdev_dbg(to_ibdev(cqp->dev),
2786 "CQP: CQPERRCODES error_code[x%08X]\n",
2787 error);
2788 return IRDMA_ERR_CQP_COMPL_ERROR;
2789 }
2790 if (newtail != tail) {
2791 /* SUCCESS */
2792 IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
2793 cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++;
2794 return 0;
2795 }
2796 udelay(cqp->dev->hw_attrs.max_sleep_count);
2797 }
2798
2799 return IRDMA_ERR_TIMEOUT;
2800 }
2801
2802 /**
2803 * irdma_sc_decode_fpm_commit - decode a 64 bit value into count and base
2804 * @dev: sc device struct
2805 * @buf: pointer to commit buffer
2806 * @buf_idx: buffer index
2807 * @obj_info: object info pointer
2808 * @rsrc_idx: indexs of memory resource
2809 */
irdma_sc_decode_fpm_commit(struct irdma_sc_dev * dev,__le64 * buf,u32 buf_idx,struct irdma_hmc_obj_info * obj_info,u32 rsrc_idx)2810 static u64 irdma_sc_decode_fpm_commit(struct irdma_sc_dev *dev, __le64 *buf,
2811 u32 buf_idx, struct irdma_hmc_obj_info *obj_info,
2812 u32 rsrc_idx)
2813 {
2814 u64 temp;
2815
2816 get_64bit_val(buf, buf_idx, &temp);
2817
2818 switch (rsrc_idx) {
2819 case IRDMA_HMC_IW_QP:
2820 obj_info[rsrc_idx].cnt = (u32)FIELD_GET(IRDMA_COMMIT_FPM_QPCNT, temp);
2821 break;
2822 case IRDMA_HMC_IW_CQ:
2823 obj_info[rsrc_idx].cnt = (u32)FLD_RS_64(dev, temp, IRDMA_COMMIT_FPM_CQCNT);
2824 break;
2825 case IRDMA_HMC_IW_APBVT_ENTRY:
2826 obj_info[rsrc_idx].cnt = 1;
2827 break;
2828 default:
2829 obj_info[rsrc_idx].cnt = (u32)temp;
2830 break;
2831 }
2832
2833 obj_info[rsrc_idx].base = (temp >> IRDMA_COMMIT_FPM_BASE_S) * 512;
2834
2835 return temp;
2836 }
2837
2838 /**
2839 * irdma_sc_parse_fpm_commit_buf - parse fpm commit buffer
2840 * @dev: pointer to dev struct
2841 * @buf: ptr to fpm commit buffer
2842 * @info: ptr to irdma_hmc_obj_info struct
2843 * @sd: number of SDs for HMC objects
2844 *
2845 * parses fpm commit info and copy base value
2846 * of hmc objects in hmc_info
2847 */
2848 static void
irdma_sc_parse_fpm_commit_buf(struct irdma_sc_dev * dev,__le64 * buf,struct irdma_hmc_obj_info * info,u32 * sd)2849 irdma_sc_parse_fpm_commit_buf(struct irdma_sc_dev *dev, __le64 *buf,
2850 struct irdma_hmc_obj_info *info, u32 *sd)
2851 {
2852 u64 size;
2853 u32 i;
2854 u64 max_base = 0;
2855 u32 last_hmc_obj = 0;
2856
2857 irdma_sc_decode_fpm_commit(dev, buf, 0, info,
2858 IRDMA_HMC_IW_QP);
2859 irdma_sc_decode_fpm_commit(dev, buf, 8, info,
2860 IRDMA_HMC_IW_CQ);
2861 /* skiping RSRVD */
2862 irdma_sc_decode_fpm_commit(dev, buf, 24, info,
2863 IRDMA_HMC_IW_HTE);
2864 irdma_sc_decode_fpm_commit(dev, buf, 32, info,
2865 IRDMA_HMC_IW_ARP);
2866 irdma_sc_decode_fpm_commit(dev, buf, 40, info,
2867 IRDMA_HMC_IW_APBVT_ENTRY);
2868 irdma_sc_decode_fpm_commit(dev, buf, 48, info,
2869 IRDMA_HMC_IW_MR);
2870 irdma_sc_decode_fpm_commit(dev, buf, 56, info,
2871 IRDMA_HMC_IW_XF);
2872 irdma_sc_decode_fpm_commit(dev, buf, 64, info,
2873 IRDMA_HMC_IW_XFFL);
2874 irdma_sc_decode_fpm_commit(dev, buf, 72, info,
2875 IRDMA_HMC_IW_Q1);
2876 irdma_sc_decode_fpm_commit(dev, buf, 80, info,
2877 IRDMA_HMC_IW_Q1FL);
2878 irdma_sc_decode_fpm_commit(dev, buf, 88, info,
2879 IRDMA_HMC_IW_TIMER);
2880 irdma_sc_decode_fpm_commit(dev, buf, 112, info,
2881 IRDMA_HMC_IW_PBLE);
2882 /* skipping RSVD. */
2883 if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {
2884 irdma_sc_decode_fpm_commit(dev, buf, 96, info,
2885 IRDMA_HMC_IW_FSIMC);
2886 irdma_sc_decode_fpm_commit(dev, buf, 104, info,
2887 IRDMA_HMC_IW_FSIAV);
2888 irdma_sc_decode_fpm_commit(dev, buf, 128, info,
2889 IRDMA_HMC_IW_RRF);
2890 irdma_sc_decode_fpm_commit(dev, buf, 136, info,
2891 IRDMA_HMC_IW_RRFFL);
2892 irdma_sc_decode_fpm_commit(dev, buf, 144, info,
2893 IRDMA_HMC_IW_HDR);
2894 irdma_sc_decode_fpm_commit(dev, buf, 152, info,
2895 IRDMA_HMC_IW_MD);
2896 irdma_sc_decode_fpm_commit(dev, buf, 160, info,
2897 IRDMA_HMC_IW_OOISC);
2898 irdma_sc_decode_fpm_commit(dev, buf, 168, info,
2899 IRDMA_HMC_IW_OOISCFFL);
2900 }
2901
2902 /* searching for the last object in HMC to find the size of the HMC area. */
2903 for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) {
2904 if (info[i].base > max_base) {
2905 max_base = info[i].base;
2906 last_hmc_obj = i;
2907 }
2908 }
2909
2910 size = info[last_hmc_obj].cnt * info[last_hmc_obj].size +
2911 info[last_hmc_obj].base;
2912
2913 if (size & 0x1FFFFF)
2914 *sd = (u32)((size >> 21) + 1); /* add 1 for remainder */
2915 else
2916 *sd = (u32)(size >> 21);
2917
2918 }
2919
2920 /**
2921 * irdma_sc_decode_fpm_query() - Decode a 64 bit value into max count and size
2922 * @buf: ptr to fpm query buffer
2923 * @buf_idx: index into buf
2924 * @obj_info: ptr to irdma_hmc_obj_info struct
2925 * @rsrc_idx: resource index into info
2926 *
2927 * Decode a 64 bit value from fpm query buffer into max count and size
2928 */
irdma_sc_decode_fpm_query(__le64 * buf,u32 buf_idx,struct irdma_hmc_obj_info * obj_info,u32 rsrc_idx)2929 static u64 irdma_sc_decode_fpm_query(__le64 *buf, u32 buf_idx,
2930 struct irdma_hmc_obj_info *obj_info,
2931 u32 rsrc_idx)
2932 {
2933 u64 temp;
2934 u32 size;
2935
2936 get_64bit_val(buf, buf_idx, &temp);
2937 obj_info[rsrc_idx].max_cnt = (u32)temp;
2938 size = (u32)(temp >> 32);
2939 obj_info[rsrc_idx].size = BIT_ULL(size);
2940
2941 return temp;
2942 }
2943
2944 /**
2945 * irdma_sc_parse_fpm_query_buf() - parses fpm query buffer
2946 * @dev: ptr to shared code device
2947 * @buf: ptr to fpm query buffer
2948 * @hmc_info: ptr to irdma_hmc_obj_info struct
2949 * @hmc_fpm_misc: ptr to fpm data
2950 *
2951 * parses fpm query buffer and copy max_cnt and
2952 * size value of hmc objects in hmc_info
2953 */
2954 static enum irdma_status_code
irdma_sc_parse_fpm_query_buf(struct irdma_sc_dev * dev,__le64 * buf,struct irdma_hmc_info * hmc_info,struct irdma_hmc_fpm_misc * hmc_fpm_misc)2955 irdma_sc_parse_fpm_query_buf(struct irdma_sc_dev *dev, __le64 *buf,
2956 struct irdma_hmc_info *hmc_info,
2957 struct irdma_hmc_fpm_misc *hmc_fpm_misc)
2958 {
2959 struct irdma_hmc_obj_info *obj_info;
2960 u64 temp;
2961 u32 size;
2962 u16 max_pe_sds;
2963
2964 obj_info = hmc_info->hmc_obj;
2965
2966 get_64bit_val(buf, 0, &temp);
2967 hmc_info->first_sd_index = (u16)FIELD_GET(IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX, temp);
2968 max_pe_sds = (u16)FIELD_GET(IRDMA_QUERY_FPM_MAX_PE_SDS, temp);
2969
2970 hmc_fpm_misc->max_sds = max_pe_sds;
2971 hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index;
2972 get_64bit_val(buf, 8, &temp);
2973 obj_info[IRDMA_HMC_IW_QP].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_QPS, temp);
2974 size = (u32)(temp >> 32);
2975 obj_info[IRDMA_HMC_IW_QP].size = BIT_ULL(size);
2976
2977 get_64bit_val(buf, 16, &temp);
2978 obj_info[IRDMA_HMC_IW_CQ].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_CQS, temp);
2979 size = (u32)(temp >> 32);
2980 obj_info[IRDMA_HMC_IW_CQ].size = BIT_ULL(size);
2981
2982 irdma_sc_decode_fpm_query(buf, 32, obj_info, IRDMA_HMC_IW_HTE);
2983 irdma_sc_decode_fpm_query(buf, 40, obj_info, IRDMA_HMC_IW_ARP);
2984
2985 obj_info[IRDMA_HMC_IW_APBVT_ENTRY].size = 8192;
2986 obj_info[IRDMA_HMC_IW_APBVT_ENTRY].max_cnt = 1;
2987
2988 irdma_sc_decode_fpm_query(buf, 48, obj_info, IRDMA_HMC_IW_MR);
2989 irdma_sc_decode_fpm_query(buf, 56, obj_info, IRDMA_HMC_IW_XF);
2990
2991 get_64bit_val(buf, 64, &temp);
2992 obj_info[IRDMA_HMC_IW_XFFL].max_cnt = (u32)temp;
2993 obj_info[IRDMA_HMC_IW_XFFL].size = 4;
2994 hmc_fpm_misc->xf_block_size = FIELD_GET(IRDMA_QUERY_FPM_XFBLOCKSIZE, temp);
2995 if (!hmc_fpm_misc->xf_block_size)
2996 return IRDMA_ERR_INVALID_SIZE;
2997
2998 irdma_sc_decode_fpm_query(buf, 72, obj_info, IRDMA_HMC_IW_Q1);
2999 get_64bit_val(buf, 80, &temp);
3000 obj_info[IRDMA_HMC_IW_Q1FL].max_cnt = (u32)temp;
3001 obj_info[IRDMA_HMC_IW_Q1FL].size = 4;
3002
3003 hmc_fpm_misc->q1_block_size = FIELD_GET(IRDMA_QUERY_FPM_Q1BLOCKSIZE, temp);
3004 if (!hmc_fpm_misc->q1_block_size)
3005 return IRDMA_ERR_INVALID_SIZE;
3006
3007 irdma_sc_decode_fpm_query(buf, 88, obj_info, IRDMA_HMC_IW_TIMER);
3008
3009 get_64bit_val(buf, 112, &temp);
3010 obj_info[IRDMA_HMC_IW_PBLE].max_cnt = (u32)temp;
3011 obj_info[IRDMA_HMC_IW_PBLE].size = 8;
3012
3013 get_64bit_val(buf, 120, &temp);
3014 hmc_fpm_misc->max_ceqs = FIELD_GET(IRDMA_QUERY_FPM_MAX_CEQS, temp);
3015 hmc_fpm_misc->ht_multiplier = FIELD_GET(IRDMA_QUERY_FPM_HTMULTIPLIER, temp);
3016 hmc_fpm_misc->timer_bucket = FIELD_GET(IRDMA_QUERY_FPM_TIMERBUCKET, temp);
3017 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
3018 return 0;
3019 irdma_sc_decode_fpm_query(buf, 96, obj_info, IRDMA_HMC_IW_FSIMC);
3020 irdma_sc_decode_fpm_query(buf, 104, obj_info, IRDMA_HMC_IW_FSIAV);
3021 irdma_sc_decode_fpm_query(buf, 128, obj_info, IRDMA_HMC_IW_RRF);
3022
3023 get_64bit_val(buf, 136, &temp);
3024 obj_info[IRDMA_HMC_IW_RRFFL].max_cnt = (u32)temp;
3025 obj_info[IRDMA_HMC_IW_RRFFL].size = 4;
3026 hmc_fpm_misc->rrf_block_size = FIELD_GET(IRDMA_QUERY_FPM_RRFBLOCKSIZE, temp);
3027 if (!hmc_fpm_misc->rrf_block_size &&
3028 obj_info[IRDMA_HMC_IW_RRFFL].max_cnt)
3029 return IRDMA_ERR_INVALID_SIZE;
3030
3031 irdma_sc_decode_fpm_query(buf, 144, obj_info, IRDMA_HMC_IW_HDR);
3032 irdma_sc_decode_fpm_query(buf, 152, obj_info, IRDMA_HMC_IW_MD);
3033 irdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC);
3034
3035 get_64bit_val(buf, 168, &temp);
3036 obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp;
3037 obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4;
3038 hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp);
3039 if (!hmc_fpm_misc->ooiscf_block_size &&
3040 obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt)
3041 return IRDMA_ERR_INVALID_SIZE;
3042
3043 return 0;
3044 }
3045
3046 /**
3047 * irdma_sc_find_reg_cq - find cq ctx index
3048 * @ceq: ceq sc structure
3049 * @cq: cq sc structure
3050 */
irdma_sc_find_reg_cq(struct irdma_sc_ceq * ceq,struct irdma_sc_cq * cq)3051 static u32 irdma_sc_find_reg_cq(struct irdma_sc_ceq *ceq,
3052 struct irdma_sc_cq *cq)
3053 {
3054 u32 i;
3055
3056 for (i = 0; i < ceq->reg_cq_size; i++) {
3057 if (cq == ceq->reg_cq[i])
3058 return i;
3059 }
3060
3061 return IRDMA_INVALID_CQ_IDX;
3062 }
3063
3064 /**
3065 * irdma_sc_add_cq_ctx - add cq ctx tracking for ceq
3066 * @ceq: ceq sc structure
3067 * @cq: cq sc structure
3068 */
irdma_sc_add_cq_ctx(struct irdma_sc_ceq * ceq,struct irdma_sc_cq * cq)3069 enum irdma_status_code irdma_sc_add_cq_ctx(struct irdma_sc_ceq *ceq,
3070 struct irdma_sc_cq *cq)
3071 {
3072 unsigned long flags;
3073
3074 spin_lock_irqsave(&ceq->req_cq_lock, flags);
3075
3076 if (ceq->reg_cq_size == ceq->elem_cnt) {
3077 spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
3078 return IRDMA_ERR_REG_CQ_FULL;
3079 }
3080
3081 ceq->reg_cq[ceq->reg_cq_size++] = cq;
3082
3083 spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
3084
3085 return 0;
3086 }
3087
3088 /**
3089 * irdma_sc_remove_cq_ctx - remove cq ctx tracking for ceq
3090 * @ceq: ceq sc structure
3091 * @cq: cq sc structure
3092 */
irdma_sc_remove_cq_ctx(struct irdma_sc_ceq * ceq,struct irdma_sc_cq * cq)3093 void irdma_sc_remove_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq)
3094 {
3095 unsigned long flags;
3096 u32 cq_ctx_idx;
3097
3098 spin_lock_irqsave(&ceq->req_cq_lock, flags);
3099 cq_ctx_idx = irdma_sc_find_reg_cq(ceq, cq);
3100 if (cq_ctx_idx == IRDMA_INVALID_CQ_IDX)
3101 goto exit;
3102
3103 ceq->reg_cq_size--;
3104 if (cq_ctx_idx != ceq->reg_cq_size)
3105 ceq->reg_cq[cq_ctx_idx] = ceq->reg_cq[ceq->reg_cq_size];
3106 ceq->reg_cq[ceq->reg_cq_size] = NULL;
3107
3108 exit:
3109 spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
3110 }
3111
3112 /**
3113 * irdma_sc_cqp_init - Initialize buffers for a control Queue Pair
3114 * @cqp: IWARP control queue pair pointer
3115 * @info: IWARP control queue pair init info pointer
3116 *
3117 * Initializes the object and context buffers for a control Queue Pair.
3118 */
irdma_sc_cqp_init(struct irdma_sc_cqp * cqp,struct irdma_cqp_init_info * info)3119 enum irdma_status_code irdma_sc_cqp_init(struct irdma_sc_cqp *cqp,
3120 struct irdma_cqp_init_info *info)
3121 {
3122 u8 hw_sq_size;
3123
3124 if (info->sq_size > IRDMA_CQP_SW_SQSIZE_2048 ||
3125 info->sq_size < IRDMA_CQP_SW_SQSIZE_4 ||
3126 ((info->sq_size & (info->sq_size - 1))))
3127 return IRDMA_ERR_INVALID_SIZE;
3128
3129 hw_sq_size = irdma_get_encoded_wqe_size(info->sq_size,
3130 IRDMA_QUEUE_TYPE_CQP);
3131 cqp->size = sizeof(*cqp);
3132 cqp->sq_size = info->sq_size;
3133 cqp->hw_sq_size = hw_sq_size;
3134 cqp->sq_base = info->sq;
3135 cqp->host_ctx = info->host_ctx;
3136 cqp->sq_pa = info->sq_pa;
3137 cqp->host_ctx_pa = info->host_ctx_pa;
3138 cqp->dev = info->dev;
3139 cqp->struct_ver = info->struct_ver;
3140 cqp->hw_maj_ver = info->hw_maj_ver;
3141 cqp->hw_min_ver = info->hw_min_ver;
3142 cqp->scratch_array = info->scratch_array;
3143 cqp->polarity = 0;
3144 cqp->en_datacenter_tcp = info->en_datacenter_tcp;
3145 cqp->ena_vf_count = info->ena_vf_count;
3146 cqp->hmc_profile = info->hmc_profile;
3147 cqp->ceqs_per_vf = info->ceqs_per_vf;
3148 cqp->disable_packed = info->disable_packed;
3149 cqp->rocev2_rto_policy = info->rocev2_rto_policy;
3150 cqp->protocol_used = info->protocol_used;
3151 memcpy(&cqp->dcqcn_params, &info->dcqcn_params, sizeof(cqp->dcqcn_params));
3152 info->dev->cqp = cqp;
3153
3154 IRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size);
3155 cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] = 0;
3156 cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS] = 0;
3157 /* for the cqp commands backlog. */
3158 INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head);
3159
3160 writel(0, cqp->dev->hw_regs[IRDMA_CQPTAIL]);
3161 writel(0, cqp->dev->hw_regs[IRDMA_CQPDB]);
3162 writel(0, cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);
3163
3164 ibdev_dbg(to_ibdev(cqp->dev),
3165 "WQE: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%pK] cqp[%p] polarity[x%04x]\n",
3166 cqp->sq_size, cqp->hw_sq_size, cqp->sq_base,
3167 (u64 *)(uintptr_t)cqp->sq_pa, cqp, cqp->polarity);
3168 return 0;
3169 }
3170
3171 /**
3172 * irdma_sc_cqp_create - create cqp during bringup
3173 * @cqp: struct for cqp hw
3174 * @maj_err: If error, major err number
3175 * @min_err: If error, minor err number
3176 */
irdma_sc_cqp_create(struct irdma_sc_cqp * cqp,u16 * maj_err,u16 * min_err)3177 enum irdma_status_code irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err,
3178 u16 *min_err)
3179 {
3180 u64 temp;
3181 u8 hw_rev;
3182 u32 cnt = 0, p1, p2, val = 0, err_code;
3183 enum irdma_status_code ret_code;
3184
3185 hw_rev = cqp->dev->hw_attrs.uk_attrs.hw_rev;
3186 cqp->sdbuf.size = ALIGN(IRDMA_UPDATE_SD_BUFF_SIZE * cqp->sq_size,
3187 IRDMA_SD_BUF_ALIGNMENT);
3188 cqp->sdbuf.va = dma_alloc_coherent(cqp->dev->hw->device,
3189 cqp->sdbuf.size, &cqp->sdbuf.pa,
3190 GFP_KERNEL);
3191 if (!cqp->sdbuf.va)
3192 return IRDMA_ERR_NO_MEMORY;
3193
3194 spin_lock_init(&cqp->dev->cqp_lock);
3195
3196 temp = FIELD_PREP(IRDMA_CQPHC_SQSIZE, cqp->hw_sq_size) |
3197 FIELD_PREP(IRDMA_CQPHC_SVER, cqp->struct_ver) |
3198 FIELD_PREP(IRDMA_CQPHC_DISABLE_PFPDUS, cqp->disable_packed) |
3199 FIELD_PREP(IRDMA_CQPHC_CEQPERVF, cqp->ceqs_per_vf);
3200 if (hw_rev >= IRDMA_GEN_2) {
3201 temp |= FIELD_PREP(IRDMA_CQPHC_ROCEV2_RTO_POLICY,
3202 cqp->rocev2_rto_policy) |
3203 FIELD_PREP(IRDMA_CQPHC_PROTOCOL_USED,
3204 cqp->protocol_used);
3205 }
3206
3207 set_64bit_val(cqp->host_ctx, 0, temp);
3208 set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa);
3209
3210 temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) |
3211 FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile);
3212 set_64bit_val(cqp->host_ctx, 16, temp);
3213 set_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp);
3214 temp = FIELD_PREP(IRDMA_CQPHC_HW_MAJVER, cqp->hw_maj_ver) |
3215 FIELD_PREP(IRDMA_CQPHC_HW_MINVER, cqp->hw_min_ver);
3216 if (hw_rev >= IRDMA_GEN_2) {
3217 temp |= FIELD_PREP(IRDMA_CQPHC_MIN_RATE, cqp->dcqcn_params.min_rate) |
3218 FIELD_PREP(IRDMA_CQPHC_MIN_DEC_FACTOR, cqp->dcqcn_params.min_dec_factor);
3219 }
3220 set_64bit_val(cqp->host_ctx, 32, temp);
3221 set_64bit_val(cqp->host_ctx, 40, 0);
3222 temp = 0;
3223 if (hw_rev >= IRDMA_GEN_2) {
3224 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_T, cqp->dcqcn_params.dcqcn_t) |
3225 FIELD_PREP(IRDMA_CQPHC_RAI_FACTOR, cqp->dcqcn_params.rai_factor) |
3226 FIELD_PREP(IRDMA_CQPHC_HAI_FACTOR, cqp->dcqcn_params.hai_factor);
3227 }
3228 set_64bit_val(cqp->host_ctx, 48, temp);
3229 temp = 0;
3230 if (hw_rev >= IRDMA_GEN_2) {
3231 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_B, cqp->dcqcn_params.dcqcn_b) |
3232 FIELD_PREP(IRDMA_CQPHC_DCQCN_F, cqp->dcqcn_params.dcqcn_f) |
3233 FIELD_PREP(IRDMA_CQPHC_CC_CFG_VALID, cqp->dcqcn_params.cc_cfg_valid) |
3234 FIELD_PREP(IRDMA_CQPHC_RREDUCE_MPERIOD, cqp->dcqcn_params.rreduce_mperiod);
3235 }
3236 set_64bit_val(cqp->host_ctx, 56, temp);
3237 print_hex_dump_debug("WQE: CQP_HOST_CTX WQE", DUMP_PREFIX_OFFSET, 16,
3238 8, cqp->host_ctx, IRDMA_CQP_CTX_SIZE * 8, false);
3239 p1 = cqp->host_ctx_pa >> 32;
3240 p2 = (u32)cqp->host_ctx_pa;
3241
3242 writel(p1, cqp->dev->hw_regs[IRDMA_CCQPHIGH]);
3243 writel(p2, cqp->dev->hw_regs[IRDMA_CCQPLOW]);
3244
3245 do {
3246 if (cnt++ > cqp->dev->hw_attrs.max_done_count) {
3247 ret_code = IRDMA_ERR_TIMEOUT;
3248 goto err;
3249 }
3250 udelay(cqp->dev->hw_attrs.max_sleep_count);
3251 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);
3252 } while (!val);
3253
3254 if (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_ERR)) {
3255 ret_code = IRDMA_ERR_DEVICE_NOT_SUPPORTED;
3256 goto err;
3257 }
3258
3259 cqp->process_cqp_sds = irdma_update_sds_noccq;
3260 return 0;
3261
3262 err:
3263 dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size,
3264 cqp->sdbuf.va, cqp->sdbuf.pa);
3265 cqp->sdbuf.va = NULL;
3266 err_code = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]);
3267 *min_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MINOR_CODE, err_code);
3268 *maj_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MAJOR_CODE, err_code);
3269 return ret_code;
3270 }
3271
3272 /**
3273 * irdma_sc_cqp_post_sq - post of cqp's sq
3274 * @cqp: struct for cqp hw
3275 */
irdma_sc_cqp_post_sq(struct irdma_sc_cqp * cqp)3276 void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp)
3277 {
3278 writel(IRDMA_RING_CURRENT_HEAD(cqp->sq_ring), cqp->dev->cqp_db);
3279
3280 ibdev_dbg(to_ibdev(cqp->dev),
3281 "WQE: CQP SQ head 0x%x tail 0x%x size 0x%x\n",
3282 cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size);
3283 }
3284
3285 /**
3286 * irdma_sc_cqp_get_next_send_wqe_idx - get next wqe on cqp sq
3287 * and pass back index
3288 * @cqp: CQP HW structure
3289 * @scratch: private data for CQP WQE
3290 * @wqe_idx: WQE index of CQP SQ
3291 */
irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp * cqp,u64 scratch,u32 * wqe_idx)3292 __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch,
3293 u32 *wqe_idx)
3294 {
3295 __le64 *wqe = NULL;
3296 enum irdma_status_code ret_code;
3297
3298 if (IRDMA_RING_FULL_ERR(cqp->sq_ring)) {
3299 ibdev_dbg(to_ibdev(cqp->dev),
3300 "WQE: CQP SQ is full, head 0x%x tail 0x%x size 0x%x\n",
3301 cqp->sq_ring.head, cqp->sq_ring.tail,
3302 cqp->sq_ring.size);
3303 return NULL;
3304 }
3305 IRDMA_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code);
3306 if (ret_code)
3307 return NULL;
3308
3309 cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS]++;
3310 if (!*wqe_idx)
3311 cqp->polarity = !cqp->polarity;
3312 wqe = cqp->sq_base[*wqe_idx].elem;
3313 cqp->scratch_array[*wqe_idx] = scratch;
3314 IRDMA_CQP_INIT_WQE(wqe);
3315
3316 return wqe;
3317 }
3318
3319 /**
3320 * irdma_sc_cqp_destroy - destroy cqp during close
3321 * @cqp: struct for cqp hw
3322 */
irdma_sc_cqp_destroy(struct irdma_sc_cqp * cqp)3323 enum irdma_status_code irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp)
3324 {
3325 u32 cnt = 0, val;
3326 enum irdma_status_code ret_code = 0;
3327
3328 writel(0, cqp->dev->hw_regs[IRDMA_CCQPHIGH]);
3329 writel(0, cqp->dev->hw_regs[IRDMA_CCQPLOW]);
3330 do {
3331 if (cnt++ > cqp->dev->hw_attrs.max_done_count) {
3332 ret_code = IRDMA_ERR_TIMEOUT;
3333 break;
3334 }
3335 udelay(cqp->dev->hw_attrs.max_sleep_count);
3336 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);
3337 } while (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_DONE));
3338
3339 dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size,
3340 cqp->sdbuf.va, cqp->sdbuf.pa);
3341 cqp->sdbuf.va = NULL;
3342 return ret_code;
3343 }
3344
3345 /**
3346 * irdma_sc_ccq_arm - enable intr for control cq
3347 * @ccq: ccq sc struct
3348 */
irdma_sc_ccq_arm(struct irdma_sc_cq * ccq)3349 void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq)
3350 {
3351 u64 temp_val;
3352 u16 sw_cq_sel;
3353 u8 arm_next_se;
3354 u8 arm_seq_num;
3355
3356 get_64bit_val(ccq->cq_uk.shadow_area, 32, &temp_val);
3357 sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
3358 arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
3359 arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);
3360 arm_seq_num++;
3361 temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |
3362 FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |
3363 FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
3364 FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, 1);
3365 set_64bit_val(ccq->cq_uk.shadow_area, 32, temp_val);
3366
3367 dma_wmb(); /* make sure shadow area is updated before arming */
3368
3369 writel(ccq->cq_uk.cq_id, ccq->dev->cq_arm_db);
3370 }
3371
3372 /**
3373 * irdma_sc_ccq_get_cqe_info - get ccq's cq entry
3374 * @ccq: ccq sc struct
3375 * @info: completion q entry to return
3376 */
irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq * ccq,struct irdma_ccq_cqe_info * info)3377 enum irdma_status_code irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
3378 struct irdma_ccq_cqe_info *info)
3379 {
3380 u64 qp_ctx, temp, temp1;
3381 __le64 *cqe;
3382 struct irdma_sc_cqp *cqp;
3383 u32 wqe_idx;
3384 u32 error;
3385 u8 polarity;
3386 enum irdma_status_code ret_code = 0;
3387
3388 if (ccq->cq_uk.avoid_mem_cflct)
3389 cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(&ccq->cq_uk);
3390 else
3391 cqe = IRDMA_GET_CURRENT_CQ_ELEM(&ccq->cq_uk);
3392
3393 get_64bit_val(cqe, 24, &temp);
3394 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, temp);
3395 if (polarity != ccq->cq_uk.polarity)
3396 return IRDMA_ERR_Q_EMPTY;
3397
3398 get_64bit_val(cqe, 8, &qp_ctx);
3399 cqp = (struct irdma_sc_cqp *)(unsigned long)qp_ctx;
3400 info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, temp);
3401 info->maj_err_code = IRDMA_CQPSQ_MAJ_NO_ERROR;
3402 info->min_err_code = (u16)FIELD_GET(IRDMA_CQ_MINERR, temp);
3403 if (info->error) {
3404 info->maj_err_code = (u16)FIELD_GET(IRDMA_CQ_MAJERR, temp);
3405 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]);
3406 ibdev_dbg(to_ibdev(cqp->dev),
3407 "CQP: CQPERRCODES error_code[x%08X]\n", error);
3408 }
3409
3410 wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, temp);
3411 info->scratch = cqp->scratch_array[wqe_idx];
3412
3413 get_64bit_val(cqe, 16, &temp1);
3414 info->op_ret_val = (u32)FIELD_GET(IRDMA_CCQ_OPRETVAL, temp1);
3415 get_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1);
3416 info->op_code = (u8)FIELD_GET(IRDMA_CQPSQ_OPCODE, temp1);
3417 info->cqp = cqp;
3418
3419 /* move the head for cq */
3420 IRDMA_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code);
3421 if (!IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring))
3422 ccq->cq_uk.polarity ^= 1;
3423
3424 /* update cq tail in cq shadow memory also */
3425 IRDMA_RING_MOVE_TAIL(ccq->cq_uk.cq_ring);
3426 set_64bit_val(ccq->cq_uk.shadow_area, 0,
3427 IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring));
3428
3429 dma_wmb(); /* make sure shadow area is updated before moving tail */
3430
3431 IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
3432 ccq->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++;
3433
3434 return ret_code;
3435 }
3436
3437 /**
3438 * irdma_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ
3439 * @cqp: struct for cqp hw
3440 * @op_code: cqp opcode for completion
3441 * @compl_info: completion q entry to return
3442 */
irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp * cqp,u8 op_code,struct irdma_ccq_cqe_info * compl_info)3443 enum irdma_status_code irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 op_code,
3444 struct irdma_ccq_cqe_info *compl_info)
3445 {
3446 struct irdma_ccq_cqe_info info = {};
3447 struct irdma_sc_cq *ccq;
3448 enum irdma_status_code ret_code = 0;
3449 u32 cnt = 0;
3450
3451 ccq = cqp->dev->ccq;
3452 while (1) {
3453 if (cnt++ > 100 * cqp->dev->hw_attrs.max_done_count)
3454 return IRDMA_ERR_TIMEOUT;
3455
3456 if (irdma_sc_ccq_get_cqe_info(ccq, &info)) {
3457 udelay(cqp->dev->hw_attrs.max_sleep_count);
3458 continue;
3459 }
3460 if (info.error && info.op_code != IRDMA_CQP_OP_QUERY_STAG) {
3461 ret_code = IRDMA_ERR_CQP_COMPL_ERROR;
3462 break;
3463 }
3464 /* make sure op code matches*/
3465 if (op_code == info.op_code)
3466 break;
3467 ibdev_dbg(to_ibdev(cqp->dev),
3468 "WQE: opcode mismatch for my op code 0x%x, returned opcode %x\n",
3469 op_code, info.op_code);
3470 }
3471
3472 if (compl_info)
3473 memcpy(compl_info, &info, sizeof(*compl_info));
3474
3475 return ret_code;
3476 }
3477
3478 /**
3479 * irdma_sc_manage_hmc_pm_func_table - manage of function table
3480 * @cqp: struct for cqp hw
3481 * @scratch: u64 saved to be used during cqp completion
3482 * @info: info for the manage function table operation
3483 * @post_sq: flag for cqp db to ring
3484 */
3485 static enum irdma_status_code
irdma_sc_manage_hmc_pm_func_table(struct irdma_sc_cqp * cqp,struct irdma_hmc_fcn_info * info,u64 scratch,bool post_sq)3486 irdma_sc_manage_hmc_pm_func_table(struct irdma_sc_cqp *cqp,
3487 struct irdma_hmc_fcn_info *info,
3488 u64 scratch, bool post_sq)
3489 {
3490 __le64 *wqe;
3491 u64 hdr;
3492
3493 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3494 if (!wqe)
3495 return IRDMA_ERR_RING_FULL;
3496
3497 set_64bit_val(wqe, 0, 0);
3498 set_64bit_val(wqe, 8, 0);
3499 set_64bit_val(wqe, 16, 0);
3500 set_64bit_val(wqe, 32, 0);
3501 set_64bit_val(wqe, 40, 0);
3502 set_64bit_val(wqe, 48, 0);
3503 set_64bit_val(wqe, 56, 0);
3504
3505 hdr = FIELD_PREP(IRDMA_CQPSQ_MHMC_VFIDX, info->vf_id) |
3506 FIELD_PREP(IRDMA_CQPSQ_OPCODE,
3507 IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE) |
3508 FIELD_PREP(IRDMA_CQPSQ_MHMC_FREEPMFN, info->free_fcn) |
3509 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3510 dma_wmb(); /* make sure WQE is written before valid bit is set */
3511
3512 set_64bit_val(wqe, 24, hdr);
3513
3514 print_hex_dump_debug("WQE: MANAGE_HMC_PM_FUNC_TABLE WQE",
3515 DUMP_PREFIX_OFFSET, 16, 8, wqe,
3516 IRDMA_CQP_WQE_SIZE * 8, false);
3517 if (post_sq)
3518 irdma_sc_cqp_post_sq(cqp);
3519
3520 return 0;
3521 }
3522
3523 /**
3524 * irdma_sc_commit_fpm_val_done - wait for cqp eqe completion
3525 * for fpm commit
3526 * @cqp: struct for cqp hw
3527 */
3528 static enum irdma_status_code
irdma_sc_commit_fpm_val_done(struct irdma_sc_cqp * cqp)3529 irdma_sc_commit_fpm_val_done(struct irdma_sc_cqp *cqp)
3530 {
3531 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_COMMIT_FPM_VAL,
3532 NULL);
3533 }
3534
3535 /**
3536 * irdma_sc_commit_fpm_val - cqp wqe for commit fpm values
3537 * @cqp: struct for cqp hw
3538 * @scratch: u64 saved to be used during cqp completion
3539 * @hmc_fn_id: hmc function id
3540 * @commit_fpm_mem: Memory for fpm values
3541 * @post_sq: flag for cqp db to ring
3542 * @wait_type: poll ccq or cqp registers for cqp completion
3543 */
3544 static enum irdma_status_code
irdma_sc_commit_fpm_val(struct irdma_sc_cqp * cqp,u64 scratch,u8 hmc_fn_id,struct irdma_dma_mem * commit_fpm_mem,bool post_sq,u8 wait_type)3545 irdma_sc_commit_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, u8 hmc_fn_id,
3546 struct irdma_dma_mem *commit_fpm_mem, bool post_sq,
3547 u8 wait_type)
3548 {
3549 __le64 *wqe;
3550 u64 hdr;
3551 u32 tail, val, error;
3552 enum irdma_status_code ret_code = 0;
3553
3554 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3555 if (!wqe)
3556 return IRDMA_ERR_RING_FULL;
3557
3558 set_64bit_val(wqe, 16, hmc_fn_id);
3559 set_64bit_val(wqe, 32, commit_fpm_mem->pa);
3560
3561 hdr = FIELD_PREP(IRDMA_CQPSQ_BUFSIZE, IRDMA_COMMIT_FPM_BUF_SIZE) |
3562 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_COMMIT_FPM_VAL) |
3563 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3564
3565 dma_wmb(); /* make sure WQE is written before valid bit is set */
3566
3567 set_64bit_val(wqe, 24, hdr);
3568
3569 print_hex_dump_debug("WQE: COMMIT_FPM_VAL WQE", DUMP_PREFIX_OFFSET,
3570 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3571 irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
3572
3573 if (post_sq) {
3574 irdma_sc_cqp_post_sq(cqp);
3575 if (wait_type == IRDMA_CQP_WAIT_POLL_REGS)
3576 ret_code = irdma_cqp_poll_registers(cqp, tail,
3577 cqp->dev->hw_attrs.max_done_count);
3578 else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ)
3579 ret_code = irdma_sc_commit_fpm_val_done(cqp);
3580 }
3581
3582 return ret_code;
3583 }
3584
3585 /**
3586 * irdma_sc_query_fpm_val_done - poll for cqp wqe completion for
3587 * query fpm
3588 * @cqp: struct for cqp hw
3589 */
3590 static enum irdma_status_code
irdma_sc_query_fpm_val_done(struct irdma_sc_cqp * cqp)3591 irdma_sc_query_fpm_val_done(struct irdma_sc_cqp *cqp)
3592 {
3593 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_QUERY_FPM_VAL,
3594 NULL);
3595 }
3596
3597 /**
3598 * irdma_sc_query_fpm_val - cqp wqe query fpm values
3599 * @cqp: struct for cqp hw
3600 * @scratch: u64 saved to be used during cqp completion
3601 * @hmc_fn_id: hmc function id
3602 * @query_fpm_mem: memory for return fpm values
3603 * @post_sq: flag for cqp db to ring
3604 * @wait_type: poll ccq or cqp registers for cqp completion
3605 */
3606 static enum irdma_status_code
irdma_sc_query_fpm_val(struct irdma_sc_cqp * cqp,u64 scratch,u8 hmc_fn_id,struct irdma_dma_mem * query_fpm_mem,bool post_sq,u8 wait_type)3607 irdma_sc_query_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, u8 hmc_fn_id,
3608 struct irdma_dma_mem *query_fpm_mem, bool post_sq,
3609 u8 wait_type)
3610 {
3611 __le64 *wqe;
3612 u64 hdr;
3613 u32 tail, val, error;
3614 enum irdma_status_code ret_code = 0;
3615
3616 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3617 if (!wqe)
3618 return IRDMA_ERR_RING_FULL;
3619
3620 set_64bit_val(wqe, 16, hmc_fn_id);
3621 set_64bit_val(wqe, 32, query_fpm_mem->pa);
3622
3623 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_QUERY_FPM_VAL) |
3624 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3625 dma_wmb(); /* make sure WQE is written before valid bit is set */
3626
3627 set_64bit_val(wqe, 24, hdr);
3628
3629 print_hex_dump_debug("WQE: QUERY_FPM WQE", DUMP_PREFIX_OFFSET, 16, 8,
3630 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3631 irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
3632
3633 if (post_sq) {
3634 irdma_sc_cqp_post_sq(cqp);
3635 if (wait_type == IRDMA_CQP_WAIT_POLL_REGS)
3636 ret_code = irdma_cqp_poll_registers(cqp, tail,
3637 cqp->dev->hw_attrs.max_done_count);
3638 else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ)
3639 ret_code = irdma_sc_query_fpm_val_done(cqp);
3640 }
3641
3642 return ret_code;
3643 }
3644
3645 /**
3646 * irdma_sc_ceq_init - initialize ceq
3647 * @ceq: ceq sc structure
3648 * @info: ceq initialization info
3649 */
irdma_sc_ceq_init(struct irdma_sc_ceq * ceq,struct irdma_ceq_init_info * info)3650 enum irdma_status_code irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,
3651 struct irdma_ceq_init_info *info)
3652 {
3653 u32 pble_obj_cnt;
3654
3655 if (info->elem_cnt < info->dev->hw_attrs.min_hw_ceq_size ||
3656 info->elem_cnt > info->dev->hw_attrs.max_hw_ceq_size)
3657 return IRDMA_ERR_INVALID_SIZE;
3658
3659 if (info->ceq_id > (info->dev->hmc_fpm_misc.max_ceqs - 1))
3660 return IRDMA_ERR_INVALID_CEQ_ID;
3661 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
3662
3663 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
3664 return IRDMA_ERR_INVALID_PBLE_INDEX;
3665
3666 ceq->size = sizeof(*ceq);
3667 ceq->ceqe_base = (struct irdma_ceqe *)info->ceqe_base;
3668 ceq->ceq_id = info->ceq_id;
3669 ceq->dev = info->dev;
3670 ceq->elem_cnt = info->elem_cnt;
3671 ceq->ceq_elem_pa = info->ceqe_pa;
3672 ceq->virtual_map = info->virtual_map;
3673 ceq->itr_no_expire = info->itr_no_expire;
3674 ceq->reg_cq = info->reg_cq;
3675 ceq->reg_cq_size = 0;
3676 spin_lock_init(&ceq->req_cq_lock);
3677 ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0);
3678 ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0);
3679 ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL);
3680 ceq->tph_en = info->tph_en;
3681 ceq->tph_val = info->tph_val;
3682 ceq->vsi = info->vsi;
3683 ceq->polarity = 1;
3684 IRDMA_RING_INIT(ceq->ceq_ring, ceq->elem_cnt);
3685 ceq->dev->ceq[info->ceq_id] = ceq;
3686
3687 return 0;
3688 }
3689
3690 /**
3691 * irdma_sc_ceq_create - create ceq wqe
3692 * @ceq: ceq sc structure
3693 * @scratch: u64 saved to be used during cqp completion
3694 * @post_sq: flag for cqp db to ring
3695 */
3696
irdma_sc_ceq_create(struct irdma_sc_ceq * ceq,u64 scratch,bool post_sq)3697 static enum irdma_status_code irdma_sc_ceq_create(struct irdma_sc_ceq *ceq, u64 scratch,
3698 bool post_sq)
3699 {
3700 struct irdma_sc_cqp *cqp;
3701 __le64 *wqe;
3702 u64 hdr;
3703
3704 cqp = ceq->dev->cqp;
3705 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3706 if (!wqe)
3707 return IRDMA_ERR_RING_FULL;
3708 set_64bit_val(wqe, 16, ceq->elem_cnt);
3709 set_64bit_val(wqe, 32,
3710 (ceq->virtual_map ? 0 : ceq->ceq_elem_pa));
3711 set_64bit_val(wqe, 48,
3712 (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0));
3713 set_64bit_val(wqe, 56,
3714 FIELD_PREP(IRDMA_CQPSQ_TPHVAL, ceq->tph_val) |
3715 FIELD_PREP(IRDMA_CQPSQ_VSIIDX, ceq->vsi->vsi_idx));
3716 hdr = FIELD_PREP(IRDMA_CQPSQ_CEQ_CEQID, ceq->ceq_id) |
3717 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CEQ) |
3718 FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) |
3719 FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) |
3720 FIELD_PREP(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE, ceq->itr_no_expire) |
3721 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) |
3722 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3723 dma_wmb(); /* make sure WQE is written before valid bit is set */
3724
3725 set_64bit_val(wqe, 24, hdr);
3726
3727 print_hex_dump_debug("WQE: CEQ_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,
3728 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3729 if (post_sq)
3730 irdma_sc_cqp_post_sq(cqp);
3731
3732 return 0;
3733 }
3734
3735 /**
3736 * irdma_sc_cceq_create_done - poll for control ceq wqe to complete
3737 * @ceq: ceq sc structure
3738 */
3739 static enum irdma_status_code
irdma_sc_cceq_create_done(struct irdma_sc_ceq * ceq)3740 irdma_sc_cceq_create_done(struct irdma_sc_ceq *ceq)
3741 {
3742 struct irdma_sc_cqp *cqp;
3743
3744 cqp = ceq->dev->cqp;
3745 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CEQ,
3746 NULL);
3747 }
3748
3749 /**
3750 * irdma_sc_cceq_destroy_done - poll for destroy cceq to complete
3751 * @ceq: ceq sc structure
3752 */
irdma_sc_cceq_destroy_done(struct irdma_sc_ceq * ceq)3753 enum irdma_status_code irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq)
3754 {
3755 struct irdma_sc_cqp *cqp;
3756
3757 if (ceq->reg_cq)
3758 irdma_sc_remove_cq_ctx(ceq, ceq->dev->ccq);
3759
3760 cqp = ceq->dev->cqp;
3761 cqp->process_cqp_sds = irdma_update_sds_noccq;
3762
3763 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_DESTROY_CEQ,
3764 NULL);
3765 }
3766
3767 /**
3768 * irdma_sc_cceq_create - create cceq
3769 * @ceq: ceq sc structure
3770 * @scratch: u64 saved to be used during cqp completion
3771 */
irdma_sc_cceq_create(struct irdma_sc_ceq * ceq,u64 scratch)3772 enum irdma_status_code irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch)
3773 {
3774 enum irdma_status_code ret_code;
3775 struct irdma_sc_dev *dev = ceq->dev;
3776
3777 dev->ccq->vsi = ceq->vsi;
3778 if (ceq->reg_cq) {
3779 ret_code = irdma_sc_add_cq_ctx(ceq, ceq->dev->ccq);
3780 if (ret_code)
3781 return ret_code;
3782 }
3783
3784 ret_code = irdma_sc_ceq_create(ceq, scratch, true);
3785 if (!ret_code)
3786 return irdma_sc_cceq_create_done(ceq);
3787
3788 return ret_code;
3789 }
3790
3791 /**
3792 * irdma_sc_ceq_destroy - destroy ceq
3793 * @ceq: ceq sc structure
3794 * @scratch: u64 saved to be used during cqp completion
3795 * @post_sq: flag for cqp db to ring
3796 */
irdma_sc_ceq_destroy(struct irdma_sc_ceq * ceq,u64 scratch,bool post_sq)3797 enum irdma_status_code irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch,
3798 bool post_sq)
3799 {
3800 struct irdma_sc_cqp *cqp;
3801 __le64 *wqe;
3802 u64 hdr;
3803
3804 cqp = ceq->dev->cqp;
3805 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3806 if (!wqe)
3807 return IRDMA_ERR_RING_FULL;
3808
3809 set_64bit_val(wqe, 16, ceq->elem_cnt);
3810 set_64bit_val(wqe, 48, ceq->first_pm_pbl_idx);
3811 hdr = ceq->ceq_id |
3812 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CEQ) |
3813 FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) |
3814 FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) |
3815 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) |
3816 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3817 dma_wmb(); /* make sure WQE is written before valid bit is set */
3818
3819 set_64bit_val(wqe, 24, hdr);
3820
3821 print_hex_dump_debug("WQE: CEQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16,
3822 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3823 if (post_sq)
3824 irdma_sc_cqp_post_sq(cqp);
3825
3826 return 0;
3827 }
3828
3829 /**
3830 * irdma_sc_process_ceq - process ceq
3831 * @dev: sc device struct
3832 * @ceq: ceq sc structure
3833 *
3834 * It is expected caller serializes this function with cleanup_ceqes()
3835 * because these functions manipulate the same ceq
3836 */
irdma_sc_process_ceq(struct irdma_sc_dev * dev,struct irdma_sc_ceq * ceq)3837 void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq)
3838 {
3839 u64 temp;
3840 __le64 *ceqe;
3841 struct irdma_sc_cq *cq = NULL;
3842 struct irdma_sc_cq *temp_cq;
3843 u8 polarity;
3844 u32 cq_idx;
3845 unsigned long flags;
3846
3847 do {
3848 cq_idx = 0;
3849 ceqe = IRDMA_GET_CURRENT_CEQ_ELEM(ceq);
3850 get_64bit_val(ceqe, 0, &temp);
3851 polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp);
3852 if (polarity != ceq->polarity)
3853 return NULL;
3854
3855 temp_cq = (struct irdma_sc_cq *)(unsigned long)(temp << 1);
3856 if (!temp_cq) {
3857 cq_idx = IRDMA_INVALID_CQ_IDX;
3858 IRDMA_RING_MOVE_TAIL(ceq->ceq_ring);
3859
3860 if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring))
3861 ceq->polarity ^= 1;
3862 continue;
3863 }
3864
3865 cq = temp_cq;
3866 if (ceq->reg_cq) {
3867 spin_lock_irqsave(&ceq->req_cq_lock, flags);
3868 cq_idx = irdma_sc_find_reg_cq(ceq, cq);
3869 spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
3870 }
3871
3872 IRDMA_RING_MOVE_TAIL(ceq->ceq_ring);
3873 if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring))
3874 ceq->polarity ^= 1;
3875 } while (cq_idx == IRDMA_INVALID_CQ_IDX);
3876
3877 if (cq)
3878 irdma_sc_cq_ack(cq);
3879 return cq;
3880 }
3881
3882 /**
3883 * irdma_sc_cleanup_ceqes - clear the valid ceqes ctx matching the cq
3884 * @cq: cq for which the ceqes need to be cleaned up
3885 * @ceq: ceq ptr
3886 *
3887 * The function is called after the cq is destroyed to cleanup
3888 * its pending ceqe entries. It is expected caller serializes this
3889 * function with process_ceq() in interrupt context.
3890 */
irdma_sc_cleanup_ceqes(struct irdma_sc_cq * cq,struct irdma_sc_ceq * ceq)3891 void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq)
3892 {
3893 struct irdma_sc_cq *next_cq;
3894 u8 ceq_polarity = ceq->polarity;
3895 __le64 *ceqe;
3896 u8 polarity;
3897 u64 temp;
3898 int next;
3899 u32 i;
3900
3901 next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, 0);
3902
3903 for (i = 1; i <= IRDMA_RING_SIZE(*ceq); i++) {
3904 ceqe = IRDMA_GET_CEQ_ELEM_AT_POS(ceq, next);
3905
3906 get_64bit_val(ceqe, 0, &temp);
3907 polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp);
3908 if (polarity != ceq_polarity)
3909 return;
3910
3911 next_cq = (struct irdma_sc_cq *)(unsigned long)(temp << 1);
3912 if (cq == next_cq)
3913 set_64bit_val(ceqe, 0, temp & IRDMA_CEQE_VALID);
3914
3915 next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, i);
3916 if (!next)
3917 ceq_polarity ^= 1;
3918 }
3919 }
3920
3921 /**
3922 * irdma_sc_aeq_init - initialize aeq
3923 * @aeq: aeq structure ptr
3924 * @info: aeq initialization info
3925 */
irdma_sc_aeq_init(struct irdma_sc_aeq * aeq,struct irdma_aeq_init_info * info)3926 enum irdma_status_code irdma_sc_aeq_init(struct irdma_sc_aeq *aeq,
3927 struct irdma_aeq_init_info *info)
3928 {
3929 u32 pble_obj_cnt;
3930
3931 if (info->elem_cnt < info->dev->hw_attrs.min_hw_aeq_size ||
3932 info->elem_cnt > info->dev->hw_attrs.max_hw_aeq_size)
3933 return IRDMA_ERR_INVALID_SIZE;
3934
3935 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
3936
3937 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
3938 return IRDMA_ERR_INVALID_PBLE_INDEX;
3939
3940 aeq->size = sizeof(*aeq);
3941 aeq->polarity = 1;
3942 aeq->aeqe_base = (struct irdma_sc_aeqe *)info->aeqe_base;
3943 aeq->dev = info->dev;
3944 aeq->elem_cnt = info->elem_cnt;
3945 aeq->aeq_elem_pa = info->aeq_elem_pa;
3946 IRDMA_RING_INIT(aeq->aeq_ring, aeq->elem_cnt);
3947 aeq->virtual_map = info->virtual_map;
3948 aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL);
3949 aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0);
3950 aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0);
3951 aeq->msix_idx = info->msix_idx;
3952 info->dev->aeq = aeq;
3953
3954 return 0;
3955 }
3956
3957 /**
3958 * irdma_sc_aeq_create - create aeq
3959 * @aeq: aeq structure ptr
3960 * @scratch: u64 saved to be used during cqp completion
3961 * @post_sq: flag for cqp db to ring
3962 */
irdma_sc_aeq_create(struct irdma_sc_aeq * aeq,u64 scratch,bool post_sq)3963 static enum irdma_status_code irdma_sc_aeq_create(struct irdma_sc_aeq *aeq,
3964 u64 scratch, bool post_sq)
3965 {
3966 __le64 *wqe;
3967 struct irdma_sc_cqp *cqp;
3968 u64 hdr;
3969
3970 cqp = aeq->dev->cqp;
3971 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3972 if (!wqe)
3973 return IRDMA_ERR_RING_FULL;
3974 set_64bit_val(wqe, 16, aeq->elem_cnt);
3975 set_64bit_val(wqe, 32,
3976 (aeq->virtual_map ? 0 : aeq->aeq_elem_pa));
3977 set_64bit_val(wqe, 48,
3978 (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0));
3979
3980 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_AEQ) |
3981 FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) |
3982 FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) |
3983 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3984 dma_wmb(); /* make sure WQE is written before valid bit is set */
3985
3986 set_64bit_val(wqe, 24, hdr);
3987
3988 print_hex_dump_debug("WQE: AEQ_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,
3989 wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3990 if (post_sq)
3991 irdma_sc_cqp_post_sq(cqp);
3992
3993 return 0;
3994 }
3995
3996 /**
3997 * irdma_sc_aeq_destroy - destroy aeq during close
3998 * @aeq: aeq structure ptr
3999 * @scratch: u64 saved to be used during cqp completion
4000 * @post_sq: flag for cqp db to ring
4001 */
irdma_sc_aeq_destroy(struct irdma_sc_aeq * aeq,u64 scratch,bool post_sq)4002 static enum irdma_status_code irdma_sc_aeq_destroy(struct irdma_sc_aeq *aeq,
4003 u64 scratch, bool post_sq)
4004 {
4005 __le64 *wqe;
4006 struct irdma_sc_cqp *cqp;
4007 struct irdma_sc_dev *dev;
4008 u64 hdr;
4009
4010 dev = aeq->dev;
4011 writel(0, dev->hw_regs[IRDMA_PFINT_AEQCTL]);
4012
4013 cqp = dev->cqp;
4014 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
4015 if (!wqe)
4016 return IRDMA_ERR_RING_FULL;
4017 set_64bit_val(wqe, 16, aeq->elem_cnt);
4018 set_64bit_val(wqe, 48, aeq->first_pm_pbl_idx);
4019 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_AEQ) |
4020 FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) |
4021 FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) |
4022 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
4023 dma_wmb(); /* make sure WQE is written before valid bit is set */
4024
4025 set_64bit_val(wqe, 24, hdr);
4026
4027 print_hex_dump_debug("WQE: AEQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16,
4028 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
4029 if (post_sq)
4030 irdma_sc_cqp_post_sq(cqp);
4031 return 0;
4032 }
4033
4034 /**
4035 * irdma_sc_get_next_aeqe - get next aeq entry
4036 * @aeq: aeq structure ptr
4037 * @info: aeqe info to be returned
4038 */
irdma_sc_get_next_aeqe(struct irdma_sc_aeq * aeq,struct irdma_aeqe_info * info)4039 enum irdma_status_code irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,
4040 struct irdma_aeqe_info *info)
4041 {
4042 u64 temp, compl_ctx;
4043 __le64 *aeqe;
4044 u16 wqe_idx;
4045 u8 ae_src;
4046 u8 polarity;
4047
4048 aeqe = IRDMA_GET_CURRENT_AEQ_ELEM(aeq);
4049 get_64bit_val(aeqe, 0, &compl_ctx);
4050 get_64bit_val(aeqe, 8, &temp);
4051 polarity = (u8)FIELD_GET(IRDMA_AEQE_VALID, temp);
4052
4053 if (aeq->polarity != polarity)
4054 return IRDMA_ERR_Q_EMPTY;
4055
4056 print_hex_dump_debug("WQE: AEQ_ENTRY WQE", DUMP_PREFIX_OFFSET, 16, 8,
4057 aeqe, 16, false);
4058
4059 ae_src = (u8)FIELD_GET(IRDMA_AEQE_AESRC, temp);
4060 wqe_idx = (u16)FIELD_GET(IRDMA_AEQE_WQDESCIDX, temp);
4061 info->qp_cq_id = (u32)FIELD_GET(IRDMA_AEQE_QPCQID_LOW, temp) |
4062 ((u32)FIELD_GET(IRDMA_AEQE_QPCQID_HI, temp) << 18);
4063 info->ae_id = (u16)FIELD_GET(IRDMA_AEQE_AECODE, temp);
4064 info->tcp_state = (u8)FIELD_GET(IRDMA_AEQE_TCPSTATE, temp);
4065 info->iwarp_state = (u8)FIELD_GET(IRDMA_AEQE_IWSTATE, temp);
4066 info->q2_data_written = (u8)FIELD_GET(IRDMA_AEQE_Q2DATA, temp);
4067 info->aeqe_overflow = (bool)FIELD_GET(IRDMA_AEQE_OVERFLOW, temp);
4068
4069 info->ae_src = ae_src;
4070 switch (info->ae_id) {
4071 case IRDMA_AE_PRIV_OPERATION_DENIED:
4072 case IRDMA_AE_AMP_INVALIDATE_TYPE1_MW:
4073 case IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW:
4074 case IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG:
4075 case IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH:
4076 case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
4077 case IRDMA_AE_UDA_XMIT_BAD_PD:
4078 case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:
4079 case IRDMA_AE_BAD_CLOSE:
4080 case IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO:
4081 case IRDMA_AE_STAG_ZERO_INVALID:
4082 case IRDMA_AE_IB_RREQ_AND_Q1_FULL:
4083 case IRDMA_AE_IB_INVALID_REQUEST:
4084 case IRDMA_AE_WQE_UNEXPECTED_OPCODE:
4085 case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:
4086 case IRDMA_AE_IB_REMOTE_OP_ERROR:
4087 case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:
4088 case IRDMA_AE_DDP_UBE_INVALID_MO:
4089 case IRDMA_AE_DDP_UBE_INVALID_QN:
4090 case IRDMA_AE_DDP_NO_L_BIT:
4091 case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
4092 case IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
4093 case IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST:
4094 case IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
4095 case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
4096 case IRDMA_AE_INVALID_ARP_ENTRY:
4097 case IRDMA_AE_INVALID_TCP_OPTION_RCVD:
4098 case IRDMA_AE_STALE_ARP_ENTRY:
4099 case IRDMA_AE_INVALID_AH_ENTRY:
4100 case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
4101 case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
4102 case IRDMA_AE_LLP_TOO_MANY_RETRIES:
4103 case IRDMA_AE_LLP_DOUBT_REACHABILITY:
4104 case IRDMA_AE_LLP_CONNECTION_ESTABLISHED:
4105 case IRDMA_AE_RESET_SENT:
4106 case IRDMA_AE_TERMINATE_SENT:
4107 case IRDMA_AE_RESET_NOT_SENT:
4108 case IRDMA_AE_LCE_QP_CATASTROPHIC:
4109 case IRDMA_AE_QP_SUSPEND_COMPLETE:
4110 case IRDMA_AE_UDA_L4LEN_INVALID:
4111 info->qp = true;
4112 info->compl_ctx = compl_ctx;
4113 break;
4114 case IRDMA_AE_LCE_CQ_CATASTROPHIC:
4115 info->cq = true;
4116 info->compl_ctx = compl_ctx << 1;
4117 ae_src = IRDMA_AE_SOURCE_RSVD;
4118 break;
4119 case IRDMA_AE_ROCE_EMPTY_MCG:
4120 case IRDMA_AE_ROCE_BAD_MC_IP_ADDR:
4121 case IRDMA_AE_ROCE_BAD_MC_QPID:
4122 case IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH:
4123 fallthrough;
4124 case IRDMA_AE_LLP_CONNECTION_RESET:
4125 case IRDMA_AE_LLP_SYN_RECEIVED:
4126 case IRDMA_AE_LLP_FIN_RECEIVED:
4127 case IRDMA_AE_LLP_CLOSE_COMPLETE:
4128 case IRDMA_AE_LLP_TERMINATE_RECEIVED:
4129 case IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE:
4130 ae_src = IRDMA_AE_SOURCE_RSVD;
4131 info->qp = true;
4132 info->compl_ctx = compl_ctx;
4133 break;
4134 default:
4135 break;
4136 }
4137
4138 switch (ae_src) {
4139 case IRDMA_AE_SOURCE_RQ:
4140 case IRDMA_AE_SOURCE_RQ_0011:
4141 info->qp = true;
4142 info->rq = true;
4143 info->wqe_idx = wqe_idx;
4144 info->compl_ctx = compl_ctx;
4145 break;
4146 case IRDMA_AE_SOURCE_CQ:
4147 case IRDMA_AE_SOURCE_CQ_0110:
4148 case IRDMA_AE_SOURCE_CQ_1010:
4149 case IRDMA_AE_SOURCE_CQ_1110:
4150 info->cq = true;
4151 info->compl_ctx = compl_ctx << 1;
4152 break;
4153 case IRDMA_AE_SOURCE_SQ:
4154 case IRDMA_AE_SOURCE_SQ_0111:
4155 info->qp = true;
4156 info->sq = true;
4157 info->wqe_idx = wqe_idx;
4158 info->compl_ctx = compl_ctx;
4159 break;
4160 case IRDMA_AE_SOURCE_IN_RR_WR:
4161 case IRDMA_AE_SOURCE_IN_RR_WR_1011:
4162 info->qp = true;
4163 info->compl_ctx = compl_ctx;
4164 info->in_rdrsp_wr = true;
4165 break;
4166 case IRDMA_AE_SOURCE_OUT_RR:
4167 case IRDMA_AE_SOURCE_OUT_RR_1111:
4168 info->qp = true;
4169 info->compl_ctx = compl_ctx;
4170 info->out_rdrsp = true;
4171 break;
4172 case IRDMA_AE_SOURCE_RSVD:
4173 default:
4174 break;
4175 }
4176
4177 IRDMA_RING_MOVE_TAIL(aeq->aeq_ring);
4178 if (!IRDMA_RING_CURRENT_TAIL(aeq->aeq_ring))
4179 aeq->polarity ^= 1;
4180
4181 return 0;
4182 }
4183
4184 /**
4185 * irdma_sc_repost_aeq_entries - repost completed aeq entries
4186 * @dev: sc device struct
4187 * @count: allocate count
4188 */
irdma_sc_repost_aeq_entries(struct irdma_sc_dev * dev,u32 count)4189 void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count)
4190 {
4191 writel(count, dev->hw_regs[IRDMA_AEQALLOC]);
4192 }
4193
4194 /**
4195 * irdma_sc_ccq_init - initialize control cq
4196 * @cq: sc's cq ctruct
4197 * @info: info for control cq initialization
4198 */
irdma_sc_ccq_init(struct irdma_sc_cq * cq,struct irdma_ccq_init_info * info)4199 enum irdma_status_code irdma_sc_ccq_init(struct irdma_sc_cq *cq,
4200 struct irdma_ccq_init_info *info)
4201 {
4202 u32 pble_obj_cnt;
4203
4204 if (info->num_elem < info->dev->hw_attrs.uk_attrs.min_hw_cq_size ||
4205 info->num_elem > info->dev->hw_attrs.uk_attrs.max_hw_cq_size)
4206 return IRDMA_ERR_INVALID_SIZE;
4207
4208 if (info->ceq_id > (info->dev->hmc_fpm_misc.max_ceqs - 1))
4209 return IRDMA_ERR_INVALID_CEQ_ID;
4210
4211 pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
4212
4213 if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
4214 return IRDMA_ERR_INVALID_PBLE_INDEX;
4215
4216 cq->cq_pa = info->cq_pa;
4217 cq->cq_uk.cq_base = info->cq_base;
4218 cq->shadow_area_pa = info->shadow_area_pa;
4219 cq->cq_uk.shadow_area = info->shadow_area;
4220 cq->shadow_read_threshold = info->shadow_read_threshold;
4221 cq->dev = info->dev;
4222 cq->ceq_id = info->ceq_id;
4223 cq->cq_uk.cq_size = info->num_elem;
4224 cq->cq_type = IRDMA_CQ_TYPE_CQP;
4225 cq->ceqe_mask = info->ceqe_mask;
4226 IRDMA_RING_INIT(cq->cq_uk.cq_ring, info->num_elem);
4227 cq->cq_uk.cq_id = 0; /* control cq is id 0 always */
4228 cq->ceq_id_valid = info->ceq_id_valid;
4229 cq->tph_en = info->tph_en;
4230 cq->tph_val = info->tph_val;
4231 cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct;
4232 cq->pbl_list = info->pbl_list;
4233 cq->virtual_map = info->virtual_map;
4234 cq->pbl_chunk_size = info->pbl_chunk_size;
4235 cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
4236 cq->cq_uk.polarity = true;
4237 cq->vsi = info->vsi;
4238 cq->cq_uk.cq_ack_db = cq->dev->cq_ack_db;
4239
4240 /* Only applicable to CQs other than CCQ so initialize to zero */
4241 cq->cq_uk.cqe_alloc_db = NULL;
4242
4243 info->dev->ccq = cq;
4244 return 0;
4245 }
4246
4247 /**
4248 * irdma_sc_ccq_create_done - poll cqp for ccq create
4249 * @ccq: ccq sc struct
4250 */
irdma_sc_ccq_create_done(struct irdma_sc_cq * ccq)4251 static inline enum irdma_status_code irdma_sc_ccq_create_done(struct irdma_sc_cq *ccq)
4252 {
4253 struct irdma_sc_cqp *cqp;
4254
4255 cqp = ccq->dev->cqp;
4256
4257 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CQ, NULL);
4258 }
4259
4260 /**
4261 * irdma_sc_ccq_create - create control cq
4262 * @ccq: ccq sc struct
4263 * @scratch: u64 saved to be used during cqp completion
4264 * @check_overflow: overlow flag for ccq
4265 * @post_sq: flag for cqp db to ring
4266 */
irdma_sc_ccq_create(struct irdma_sc_cq * ccq,u64 scratch,bool check_overflow,bool post_sq)4267 enum irdma_status_code irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch,
4268 bool check_overflow, bool post_sq)
4269 {
4270 enum irdma_status_code ret_code;
4271
4272 ret_code = irdma_sc_cq_create(ccq, scratch, check_overflow, post_sq);
4273 if (ret_code)
4274 return ret_code;
4275
4276 if (post_sq) {
4277 ret_code = irdma_sc_ccq_create_done(ccq);
4278 if (ret_code)
4279 return ret_code;
4280 }
4281 ccq->dev->cqp->process_cqp_sds = irdma_cqp_sds_cmd;
4282
4283 return 0;
4284 }
4285
4286 /**
4287 * irdma_sc_ccq_destroy - destroy ccq during close
4288 * @ccq: ccq sc struct
4289 * @scratch: u64 saved to be used during cqp completion
4290 * @post_sq: flag for cqp db to ring
4291 */
irdma_sc_ccq_destroy(struct irdma_sc_cq * ccq,u64 scratch,bool post_sq)4292 enum irdma_status_code irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch,
4293 bool post_sq)
4294 {
4295 struct irdma_sc_cqp *cqp;
4296 __le64 *wqe;
4297 u64 hdr;
4298 enum irdma_status_code ret_code = 0;
4299 u32 tail, val, error;
4300
4301 cqp = ccq->dev->cqp;
4302 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
4303 if (!wqe)
4304 return IRDMA_ERR_RING_FULL;
4305
4306 set_64bit_val(wqe, 0, ccq->cq_uk.cq_size);
4307 set_64bit_val(wqe, 8, (uintptr_t)ccq >> 1);
4308 set_64bit_val(wqe, 40, ccq->shadow_area_pa);
4309
4310 hdr = ccq->cq_uk.cq_id |
4311 FLD_LS_64(ccq->dev, (ccq->ceq_id_valid ? ccq->ceq_id : 0),
4312 IRDMA_CQPSQ_CQ_CEQID) |
4313 FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) |
4314 FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, ccq->ceqe_mask) |
4315 FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, ccq->ceq_id_valid) |
4316 FIELD_PREP(IRDMA_CQPSQ_TPHEN, ccq->tph_en) |
4317 FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, ccq->cq_uk.avoid_mem_cflct) |
4318 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
4319 dma_wmb(); /* make sure WQE is written before valid bit is set */
4320
4321 set_64bit_val(wqe, 24, hdr);
4322
4323 print_hex_dump_debug("WQE: CCQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16,
4324 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
4325 irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
4326
4327 if (post_sq) {
4328 irdma_sc_cqp_post_sq(cqp);
4329 ret_code = irdma_cqp_poll_registers(cqp, tail,
4330 cqp->dev->hw_attrs.max_done_count);
4331 }
4332
4333 cqp->process_cqp_sds = irdma_update_sds_noccq;
4334
4335 return ret_code;
4336 }
4337
4338 /**
4339 * irdma_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info
4340 * @dev : ptr to irdma_dev struct
4341 * @hmc_fn_id: hmc function id
4342 */
irdma_sc_init_iw_hmc(struct irdma_sc_dev * dev,u8 hmc_fn_id)4343 enum irdma_status_code irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev,
4344 u8 hmc_fn_id)
4345 {
4346 struct irdma_hmc_info *hmc_info;
4347 struct irdma_hmc_fpm_misc *hmc_fpm_misc;
4348 struct irdma_dma_mem query_fpm_mem;
4349 enum irdma_status_code ret_code = 0;
4350 u8 wait_type;
4351
4352 hmc_info = dev->hmc_info;
4353 hmc_fpm_misc = &dev->hmc_fpm_misc;
4354 query_fpm_mem.pa = dev->fpm_query_buf_pa;
4355 query_fpm_mem.va = dev->fpm_query_buf;
4356 hmc_info->hmc_fn_id = hmc_fn_id;
4357 wait_type = (u8)IRDMA_CQP_WAIT_POLL_REGS;
4358
4359 ret_code = irdma_sc_query_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id,
4360 &query_fpm_mem, true, wait_type);
4361 if (ret_code)
4362 return ret_code;
4363
4364 /* parse the fpm_query_buf and fill hmc obj info */
4365 ret_code = irdma_sc_parse_fpm_query_buf(dev, query_fpm_mem.va, hmc_info,
4366 hmc_fpm_misc);
4367
4368 print_hex_dump_debug("HMC: QUERY FPM BUFFER", DUMP_PREFIX_OFFSET, 16,
4369 8, query_fpm_mem.va, IRDMA_QUERY_FPM_BUF_SIZE,
4370 false);
4371 return ret_code;
4372 }
4373
4374 /**
4375 * irdma_sc_cfg_iw_fpm() - commits hmc obj cnt values using cqp
4376 * command and populates fpm base address in hmc_info
4377 * @dev : ptr to irdma_dev struct
4378 * @hmc_fn_id: hmc function id
4379 */
irdma_sc_cfg_iw_fpm(struct irdma_sc_dev * dev,u8 hmc_fn_id)4380 static enum irdma_status_code irdma_sc_cfg_iw_fpm(struct irdma_sc_dev *dev,
4381 u8 hmc_fn_id)
4382 {
4383 struct irdma_hmc_info *hmc_info;
4384 struct irdma_hmc_obj_info *obj_info;
4385 __le64 *buf;
4386 struct irdma_dma_mem commit_fpm_mem;
4387 enum irdma_status_code ret_code = 0;
4388 u8 wait_type;
4389
4390 hmc_info = dev->hmc_info;
4391 obj_info = hmc_info->hmc_obj;
4392 buf = dev->fpm_commit_buf;
4393
4394 set_64bit_val(buf, 0, (u64)obj_info[IRDMA_HMC_IW_QP].cnt);
4395 set_64bit_val(buf, 8, (u64)obj_info[IRDMA_HMC_IW_CQ].cnt);
4396 set_64bit_val(buf, 16, (u64)0); /* RSRVD */
4397 set_64bit_val(buf, 24, (u64)obj_info[IRDMA_HMC_IW_HTE].cnt);
4398 set_64bit_val(buf, 32, (u64)obj_info[IRDMA_HMC_IW_ARP].cnt);
4399 set_64bit_val(buf, 40, (u64)0); /* RSVD */
4400 set_64bit_val(buf, 48, (u64)obj_info[IRDMA_HMC_IW_MR].cnt);
4401 set_64bit_val(buf, 56, (u64)obj_info[IRDMA_HMC_IW_XF].cnt);
4402 set_64bit_val(buf, 64, (u64)obj_info[IRDMA_HMC_IW_XFFL].cnt);
4403 set_64bit_val(buf, 72, (u64)obj_info[IRDMA_HMC_IW_Q1].cnt);
4404 set_64bit_val(buf, 80, (u64)obj_info[IRDMA_HMC_IW_Q1FL].cnt);
4405 set_64bit_val(buf, 88,
4406 (u64)obj_info[IRDMA_HMC_IW_TIMER].cnt);
4407 set_64bit_val(buf, 96,
4408 (u64)obj_info[IRDMA_HMC_IW_FSIMC].cnt);
4409 set_64bit_val(buf, 104,
4410 (u64)obj_info[IRDMA_HMC_IW_FSIAV].cnt);
4411 set_64bit_val(buf, 112,
4412 (u64)obj_info[IRDMA_HMC_IW_PBLE].cnt);
4413 set_64bit_val(buf, 120, (u64)0); /* RSVD */
4414 set_64bit_val(buf, 128, (u64)obj_info[IRDMA_HMC_IW_RRF].cnt);
4415 set_64bit_val(buf, 136,
4416 (u64)obj_info[IRDMA_HMC_IW_RRFFL].cnt);
4417 set_64bit_val(buf, 144, (u64)obj_info[IRDMA_HMC_IW_HDR].cnt);
4418 set_64bit_val(buf, 152, (u64)obj_info[IRDMA_HMC_IW_MD].cnt);
4419 set_64bit_val(buf, 160,
4420 (u64)obj_info[IRDMA_HMC_IW_OOISC].cnt);
4421 set_64bit_val(buf, 168,
4422 (u64)obj_info[IRDMA_HMC_IW_OOISCFFL].cnt);
4423
4424 commit_fpm_mem.pa = dev->fpm_commit_buf_pa;
4425 commit_fpm_mem.va = dev->fpm_commit_buf;
4426
4427 wait_type = (u8)IRDMA_CQP_WAIT_POLL_REGS;
4428 print_hex_dump_debug("HMC: COMMIT FPM BUFFER", DUMP_PREFIX_OFFSET, 16,
4429 8, commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE,
4430 false);
4431 ret_code = irdma_sc_commit_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id,
4432 &commit_fpm_mem, true, wait_type);
4433 if (!ret_code)
4434 irdma_sc_parse_fpm_commit_buf(dev, dev->fpm_commit_buf,
4435 hmc_info->hmc_obj,
4436 &hmc_info->sd_table.sd_cnt);
4437 print_hex_dump_debug("HMC: COMMIT FPM BUFFER", DUMP_PREFIX_OFFSET, 16,
4438 8, commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE,
4439 false);
4440
4441 return ret_code;
4442 }
4443
4444 /**
4445 * cqp_sds_wqe_fill - fill cqp wqe doe sd
4446 * @cqp: struct for cqp hw
4447 * @info: sd info for wqe
4448 * @scratch: u64 saved to be used during cqp completion
4449 */
4450 static enum irdma_status_code
cqp_sds_wqe_fill(struct irdma_sc_cqp * cqp,struct irdma_update_sds_info * info,u64 scratch)4451 cqp_sds_wqe_fill(struct irdma_sc_cqp *cqp, struct irdma_update_sds_info *info,
4452 u64 scratch)
4453 {
4454 u64 data;
4455 u64 hdr;
4456 __le64 *wqe;
4457 int mem_entries, wqe_entries;
4458 struct irdma_dma_mem *sdbuf = &cqp->sdbuf;
4459 u64 offset = 0;
4460 u32 wqe_idx;
4461
4462 wqe = irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
4463 if (!wqe)
4464 return IRDMA_ERR_RING_FULL;
4465
4466 wqe_entries = (info->cnt > 3) ? 3 : info->cnt;
4467 mem_entries = info->cnt - wqe_entries;
4468
4469 if (mem_entries) {
4470 offset = wqe_idx * IRDMA_UPDATE_SD_BUFF_SIZE;
4471 memcpy(((char *)sdbuf->va + offset), &info->entry[3], mem_entries << 4);
4472
4473 data = (u64)sdbuf->pa + offset;
4474 } else {
4475 data = 0;
4476 }
4477 data |= FIELD_PREP(IRDMA_CQPSQ_UPESD_HMCFNID, info->hmc_fn_id);
4478 set_64bit_val(wqe, 16, data);
4479
4480 switch (wqe_entries) {
4481 case 3:
4482 set_64bit_val(wqe, 48,
4483 (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[2].cmd) |
4484 FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1)));
4485
4486 set_64bit_val(wqe, 56, info->entry[2].data);
4487 fallthrough;
4488 case 2:
4489 set_64bit_val(wqe, 32,
4490 (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[1].cmd) |
4491 FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1)));
4492
4493 set_64bit_val(wqe, 40, info->entry[1].data);
4494 fallthrough;
4495 case 1:
4496 set_64bit_val(wqe, 0,
4497 FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[0].cmd));
4498
4499 set_64bit_val(wqe, 8, info->entry[0].data);
4500 break;
4501 default:
4502 break;
4503 }
4504
4505 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPDATE_PE_SDS) |
4506 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) |
4507 FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_COUNT, mem_entries);
4508 dma_wmb(); /* make sure WQE is written before valid bit is set */
4509
4510 set_64bit_val(wqe, 24, hdr);
4511
4512 if (mem_entries)
4513 print_hex_dump_debug("WQE: UPDATE_PE_SDS WQE Buffer",
4514 DUMP_PREFIX_OFFSET, 16, 8,
4515 (char *)sdbuf->va + offset,
4516 mem_entries << 4, false);
4517
4518 print_hex_dump_debug("WQE: UPDATE_PE_SDS WQE", DUMP_PREFIX_OFFSET, 16,
4519 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
4520
4521 return 0;
4522 }
4523
4524 /**
4525 * irdma_update_pe_sds - cqp wqe for sd
4526 * @dev: ptr to irdma_dev struct
4527 * @info: sd info for sd's
4528 * @scratch: u64 saved to be used during cqp completion
4529 */
4530 static enum irdma_status_code
irdma_update_pe_sds(struct irdma_sc_dev * dev,struct irdma_update_sds_info * info,u64 scratch)4531 irdma_update_pe_sds(struct irdma_sc_dev *dev,
4532 struct irdma_update_sds_info *info, u64 scratch)
4533 {
4534 struct irdma_sc_cqp *cqp = dev->cqp;
4535 enum irdma_status_code ret_code;
4536
4537 ret_code = cqp_sds_wqe_fill(cqp, info, scratch);
4538 if (!ret_code)
4539 irdma_sc_cqp_post_sq(cqp);
4540
4541 return ret_code;
4542 }
4543
4544 /**
4545 * irdma_update_sds_noccq - update sd before ccq created
4546 * @dev: sc device struct
4547 * @info: sd info for sd's
4548 */
4549 enum irdma_status_code
irdma_update_sds_noccq(struct irdma_sc_dev * dev,struct irdma_update_sds_info * info)4550 irdma_update_sds_noccq(struct irdma_sc_dev *dev,
4551 struct irdma_update_sds_info *info)
4552 {
4553 u32 error, val, tail;
4554 struct irdma_sc_cqp *cqp = dev->cqp;
4555 enum irdma_status_code ret_code;
4556
4557 ret_code = cqp_sds_wqe_fill(cqp, info, 0);
4558 if (ret_code)
4559 return ret_code;
4560
4561 irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
4562
4563 irdma_sc_cqp_post_sq(cqp);
4564 return irdma_cqp_poll_registers(cqp, tail,
4565 cqp->dev->hw_attrs.max_done_count);
4566 }
4567
4568 /**
4569 * irdma_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages
4570 * @cqp: struct for cqp hw
4571 * @scratch: u64 saved to be used during cqp completion
4572 * @hmc_fn_id: hmc function id
4573 * @post_sq: flag for cqp db to ring
4574 * @poll_registers: flag to poll register for cqp completion
4575 */
4576 enum irdma_status_code
irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp * cqp,u64 scratch,u8 hmc_fn_id,bool post_sq,bool poll_registers)4577 irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
4578 u8 hmc_fn_id, bool post_sq,
4579 bool poll_registers)
4580 {
4581 u64 hdr;
4582 __le64 *wqe;
4583 u32 tail, val, error;
4584
4585 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
4586 if (!wqe)
4587 return IRDMA_ERR_RING_FULL;
4588
4589 set_64bit_val(wqe, 16,
4590 FIELD_PREP(IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID, hmc_fn_id));
4591
4592 hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE,
4593 IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED) |
4594 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
4595 dma_wmb(); /* make sure WQE is written before valid bit is set */
4596
4597 set_64bit_val(wqe, 24, hdr);
4598
4599 print_hex_dump_debug("WQE: SHMC_PAGES_ALLOCATED WQE",
4600 DUMP_PREFIX_OFFSET, 16, 8, wqe,
4601 IRDMA_CQP_WQE_SIZE * 8, false);
4602 irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
4603
4604 if (post_sq) {
4605 irdma_sc_cqp_post_sq(cqp);
4606 if (poll_registers)
4607 /* check for cqp sq tail update */
4608 return irdma_cqp_poll_registers(cqp, tail,
4609 cqp->dev->hw_attrs.max_done_count);
4610 else
4611 return irdma_sc_poll_for_cqp_op_done(cqp,
4612 IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED,
4613 NULL);
4614 }
4615
4616 return 0;
4617 }
4618
4619 /**
4620 * irdma_cqp_ring_full - check if cqp ring is full
4621 * @cqp: struct for cqp hw
4622 */
irdma_cqp_ring_full(struct irdma_sc_cqp * cqp)4623 static bool irdma_cqp_ring_full(struct irdma_sc_cqp *cqp)
4624 {
4625 return IRDMA_RING_FULL_ERR(cqp->sq_ring);
4626 }
4627
4628 /**
4629 * irdma_est_sd - returns approximate number of SDs for HMC
4630 * @dev: sc device struct
4631 * @hmc_info: hmc structure, size and count for HMC objects
4632 */
irdma_est_sd(struct irdma_sc_dev * dev,struct irdma_hmc_info * hmc_info)4633 static u32 irdma_est_sd(struct irdma_sc_dev *dev,
4634 struct irdma_hmc_info *hmc_info)
4635 {
4636 int i;
4637 u64 size = 0;
4638 u64 sd;
4639
4640 for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++)
4641 if (i != IRDMA_HMC_IW_PBLE)
4642 size += round_up(hmc_info->hmc_obj[i].cnt *
4643 hmc_info->hmc_obj[i].size, 512);
4644 size += round_up(hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt *
4645 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].size, 512);
4646 if (size & 0x1FFFFF)
4647 sd = (size >> 21) + 1; /* add 1 for remainder */
4648 else
4649 sd = size >> 21;
4650 if (sd > 0xFFFFFFFF) {
4651 ibdev_dbg(to_ibdev(dev), "HMC: sd overflow[%lld]\n", sd);
4652 sd = 0xFFFFFFFF - 1;
4653 }
4654
4655 return (u32)sd;
4656 }
4657
4658 /**
4659 * irdma_sc_query_rdma_features_done - poll cqp for query features done
4660 * @cqp: struct for cqp hw
4661 */
4662 static enum irdma_status_code
irdma_sc_query_rdma_features_done(struct irdma_sc_cqp * cqp)4663 irdma_sc_query_rdma_features_done(struct irdma_sc_cqp *cqp)
4664 {
4665 return irdma_sc_poll_for_cqp_op_done(cqp,
4666 IRDMA_CQP_OP_QUERY_RDMA_FEATURES,
4667 NULL);
4668 }
4669
4670 /**
4671 * irdma_sc_query_rdma_features - query RDMA features and FW ver
4672 * @cqp: struct for cqp hw
4673 * @buf: buffer to hold query info
4674 * @scratch: u64 saved to be used during cqp completion
4675 */
4676 static enum irdma_status_code
irdma_sc_query_rdma_features(struct irdma_sc_cqp * cqp,struct irdma_dma_mem * buf,u64 scratch)4677 irdma_sc_query_rdma_features(struct irdma_sc_cqp *cqp,
4678 struct irdma_dma_mem *buf, u64 scratch)
4679 {
4680 __le64 *wqe;
4681 u64 temp;
4682
4683 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
4684 if (!wqe)
4685 return IRDMA_ERR_RING_FULL;
4686
4687 temp = buf->pa;
4688 set_64bit_val(wqe, 32, temp);
4689
4690 temp = FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID,
4691 cqp->polarity) |
4692 FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN, buf->size) |
4693 FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_QUERY_RDMA_FEATURES);
4694 dma_wmb(); /* make sure WQE is written before valid bit is set */
4695
4696 set_64bit_val(wqe, 24, temp);
4697
4698 print_hex_dump_debug("WQE: QUERY RDMA FEATURES", DUMP_PREFIX_OFFSET,
4699 16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
4700 irdma_sc_cqp_post_sq(cqp);
4701
4702 return 0;
4703 }
4704
4705 /**
4706 * irdma_get_rdma_features - get RDMA features
4707 * @dev: sc device struct
4708 */
irdma_get_rdma_features(struct irdma_sc_dev * dev)4709 enum irdma_status_code irdma_get_rdma_features(struct irdma_sc_dev *dev)
4710 {
4711 enum irdma_status_code ret_code;
4712 struct irdma_dma_mem feat_buf;
4713 u64 temp;
4714 u16 byte_idx, feat_type, feat_cnt, feat_idx;
4715
4716 feat_buf.size = ALIGN(IRDMA_FEATURE_BUF_SIZE,
4717 IRDMA_FEATURE_BUF_ALIGNMENT);
4718 feat_buf.va = dma_alloc_coherent(dev->hw->device, feat_buf.size,
4719 &feat_buf.pa, GFP_KERNEL);
4720 if (!feat_buf.va)
4721 return IRDMA_ERR_NO_MEMORY;
4722
4723 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0);
4724 if (!ret_code)
4725 ret_code = irdma_sc_query_rdma_features_done(dev->cqp);
4726 if (ret_code)
4727 goto exit;
4728
4729 get_64bit_val(feat_buf.va, 0, &temp);
4730 feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp);
4731 if (feat_cnt < 2) {
4732 ret_code = IRDMA_ERR_INVALID_FEAT_CNT;
4733 goto exit;
4734 } else if (feat_cnt > IRDMA_MAX_FEATURES) {
4735 ibdev_dbg(to_ibdev(dev),
4736 "DEV: feature buf size insufficient, retrying with larger buffer\n");
4737 dma_free_coherent(dev->hw->device, feat_buf.size, feat_buf.va,
4738 feat_buf.pa);
4739 feat_buf.va = NULL;
4740 feat_buf.size = ALIGN(8 * feat_cnt,
4741 IRDMA_FEATURE_BUF_ALIGNMENT);
4742 feat_buf.va = dma_alloc_coherent(dev->hw->device,
4743 feat_buf.size, &feat_buf.pa,
4744 GFP_KERNEL);
4745 if (!feat_buf.va)
4746 return IRDMA_ERR_NO_MEMORY;
4747
4748 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0);
4749 if (!ret_code)
4750 ret_code = irdma_sc_query_rdma_features_done(dev->cqp);
4751 if (ret_code)
4752 goto exit;
4753
4754 get_64bit_val(feat_buf.va, 0, &temp);
4755 feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp);
4756 if (feat_cnt < 2) {
4757 ret_code = IRDMA_ERR_INVALID_FEAT_CNT;
4758 goto exit;
4759 }
4760 }
4761
4762 print_hex_dump_debug("WQE: QUERY RDMA FEATURES", DUMP_PREFIX_OFFSET,
4763 16, 8, feat_buf.va, feat_cnt * 8, false);
4764
4765 for (byte_idx = 0, feat_idx = 0; feat_idx < min(feat_cnt, (u16)IRDMA_MAX_FEATURES);
4766 feat_idx++, byte_idx += 8) {
4767 get_64bit_val(feat_buf.va, byte_idx, &temp);
4768 feat_type = FIELD_GET(IRDMA_FEATURE_TYPE, temp);
4769 if (feat_type >= IRDMA_MAX_FEATURES) {
4770 ibdev_dbg(to_ibdev(dev),
4771 "DEV: found unrecognized feature type %d\n",
4772 feat_type);
4773 continue;
4774 }
4775 dev->feature_info[feat_type] = temp;
4776 }
4777 exit:
4778 dma_free_coherent(dev->hw->device, feat_buf.size, feat_buf.va,
4779 feat_buf.pa);
4780 feat_buf.va = NULL;
4781 return ret_code;
4782 }
4783
irdma_q1_cnt(struct irdma_sc_dev * dev,struct irdma_hmc_info * hmc_info,u32 qpwanted)4784 static u32 irdma_q1_cnt(struct irdma_sc_dev *dev,
4785 struct irdma_hmc_info *hmc_info, u32 qpwanted)
4786 {
4787 u32 q1_cnt;
4788
4789 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {
4790 q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted);
4791 } else {
4792 if (dev->cqp->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
4793 q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted + 512);
4794 else
4795 q1_cnt = dev->hw_attrs.max_hw_ird * 2 * qpwanted;
4796 }
4797
4798 return q1_cnt;
4799 }
4800
cfg_fpm_value_gen_1(struct irdma_sc_dev * dev,struct irdma_hmc_info * hmc_info,u32 qpwanted)4801 static void cfg_fpm_value_gen_1(struct irdma_sc_dev *dev,
4802 struct irdma_hmc_info *hmc_info, u32 qpwanted)
4803 {
4804 hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt = roundup_pow_of_two(qpwanted * dev->hw_attrs.max_hw_wqes);
4805 }
4806
cfg_fpm_value_gen_2(struct irdma_sc_dev * dev,struct irdma_hmc_info * hmc_info,u32 qpwanted)4807 static void cfg_fpm_value_gen_2(struct irdma_sc_dev *dev,
4808 struct irdma_hmc_info *hmc_info, u32 qpwanted)
4809 {
4810 struct irdma_hmc_fpm_misc *hmc_fpm_misc = &dev->hmc_fpm_misc;
4811
4812 hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt =
4813 4 * hmc_fpm_misc->xf_block_size * qpwanted;
4814
4815 hmc_info->hmc_obj[IRDMA_HMC_IW_HDR].cnt = qpwanted;
4816
4817 if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].max_cnt)
4818 hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt = 32 * qpwanted;
4819 if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].max_cnt)
4820 hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].cnt =
4821 hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt /
4822 hmc_fpm_misc->rrf_block_size;
4823 if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].max_cnt)
4824 hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt = 32 * qpwanted;
4825 if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].max_cnt)
4826 hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].cnt =
4827 hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt /
4828 hmc_fpm_misc->ooiscf_block_size;
4829 }
4830
4831 /**
4832 * irdma_cfg_fpm_val - configure HMC objects
4833 * @dev: sc device struct
4834 * @qp_count: desired qp count
4835 */
irdma_cfg_fpm_val(struct irdma_sc_dev * dev,u32 qp_count)4836 enum irdma_status_code irdma_cfg_fpm_val(struct irdma_sc_dev *dev, u32 qp_count)
4837 {
4838 struct irdma_virt_mem virt_mem;
4839 u32 i, mem_size;
4840 u32 qpwanted, mrwanted, pblewanted;
4841 u32 powerof2, hte;
4842 u32 sd_needed;
4843 u32 sd_diff;
4844 u32 loop_count = 0;
4845 struct irdma_hmc_info *hmc_info;
4846 struct irdma_hmc_fpm_misc *hmc_fpm_misc;
4847 enum irdma_status_code ret_code = 0;
4848
4849 hmc_info = dev->hmc_info;
4850 hmc_fpm_misc = &dev->hmc_fpm_misc;
4851
4852 ret_code = irdma_sc_init_iw_hmc(dev, dev->hmc_fn_id);
4853 if (ret_code) {
4854 ibdev_dbg(to_ibdev(dev),
4855 "HMC: irdma_sc_init_iw_hmc returned error_code = %d\n",
4856 ret_code);
4857 return ret_code;
4858 }
4859
4860 for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++)
4861 hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt;
4862 sd_needed = irdma_est_sd(dev, hmc_info);
4863 ibdev_dbg(to_ibdev(dev),
4864 "HMC: FW max resources sd_needed[%08d] first_sd_index[%04d]\n",
4865 sd_needed, hmc_info->first_sd_index);
4866 ibdev_dbg(to_ibdev(dev), "HMC: sd count %d where max sd is %d\n",
4867 hmc_info->sd_table.sd_cnt, hmc_fpm_misc->max_sds);
4868
4869 qpwanted = min(qp_count, hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt);
4870
4871 powerof2 = 1;
4872 while (powerof2 <= qpwanted)
4873 powerof2 *= 2;
4874 powerof2 /= 2;
4875 qpwanted = powerof2;
4876
4877 mrwanted = hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt;
4878 pblewanted = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt;
4879
4880 ibdev_dbg(to_ibdev(dev),
4881 "HMC: req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d, mc=%d, av=%d\n",
4882 qp_count, hmc_fpm_misc->max_sds,
4883 hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt,
4884 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt,
4885 hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt,
4886 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt,
4887 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt,
4888 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt);
4889 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt =
4890 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt;
4891 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt =
4892 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt;
4893 hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt =
4894 hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].max_cnt;
4895
4896 hmc_info->hmc_obj[IRDMA_HMC_IW_APBVT_ENTRY].cnt = 1;
4897
4898 while (irdma_q1_cnt(dev, hmc_info, qpwanted) > hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].max_cnt)
4899 qpwanted /= 2;
4900
4901 do {
4902 ++loop_count;
4903 hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt = qpwanted;
4904 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt =
4905 min(2 * qpwanted, hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt);
4906 hmc_info->hmc_obj[IRDMA_HMC_IW_RESERVED].cnt = 0; /* Reserved */
4907 hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt = mrwanted;
4908
4909 hte = round_up(qpwanted + hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt, 512);
4910 powerof2 = 1;
4911 while (powerof2 < hte)
4912 powerof2 *= 2;
4913 hmc_info->hmc_obj[IRDMA_HMC_IW_HTE].cnt =
4914 powerof2 * hmc_fpm_misc->ht_multiplier;
4915 if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
4916 cfg_fpm_value_gen_1(dev, hmc_info, qpwanted);
4917 else
4918 cfg_fpm_value_gen_2(dev, hmc_info, qpwanted);
4919
4920 hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt = irdma_q1_cnt(dev, hmc_info, qpwanted);
4921 hmc_info->hmc_obj[IRDMA_HMC_IW_XFFL].cnt =
4922 hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size;
4923 hmc_info->hmc_obj[IRDMA_HMC_IW_Q1FL].cnt =
4924 hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size;
4925 hmc_info->hmc_obj[IRDMA_HMC_IW_TIMER].cnt =
4926 (round_up(qpwanted, 512) / 512 + 1) * hmc_fpm_misc->timer_bucket;
4927
4928 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted;
4929 sd_needed = irdma_est_sd(dev, hmc_info);
4930 ibdev_dbg(to_ibdev(dev),
4931 "HMC: sd_needed = %d, hmc_fpm_misc->max_sds=%d, mrwanted=%d, pblewanted=%d qpwanted=%d\n",
4932 sd_needed, hmc_fpm_misc->max_sds, mrwanted,
4933 pblewanted, qpwanted);
4934
4935 /* Do not reduce resources further. All objects fit with max SDs */
4936 if (sd_needed <= hmc_fpm_misc->max_sds)
4937 break;
4938
4939 sd_diff = sd_needed - hmc_fpm_misc->max_sds;
4940 if (sd_diff > 128) {
4941 if (qpwanted > 128 && sd_diff > 144)
4942 qpwanted /= 2;
4943 mrwanted /= 2;
4944 pblewanted /= 2;
4945 continue;
4946 }
4947 if (dev->cqp->hmc_profile != IRDMA_HMC_PROFILE_FAVOR_VF &&
4948 pblewanted > (512 * FPM_MULTIPLIER * sd_diff)) {
4949 pblewanted -= 256 * FPM_MULTIPLIER * sd_diff;
4950 continue;
4951 } else if (pblewanted > (100 * FPM_MULTIPLIER)) {
4952 pblewanted -= 10 * FPM_MULTIPLIER;
4953 } else if (pblewanted > FPM_MULTIPLIER) {
4954 pblewanted -= FPM_MULTIPLIER;
4955 } else if (qpwanted <= 128) {
4956 if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt > 256)
4957 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt /= 2;
4958 if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256)
4959 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2;
4960 }
4961 if (mrwanted > FPM_MULTIPLIER)
4962 mrwanted -= FPM_MULTIPLIER;
4963 if (!(loop_count % 10) && qpwanted > 128) {
4964 qpwanted /= 2;
4965 if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256)
4966 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2;
4967 }
4968 } while (loop_count < 2000);
4969
4970 if (sd_needed > hmc_fpm_misc->max_sds) {
4971 ibdev_dbg(to_ibdev(dev),
4972 "HMC: cfg_fpm failed loop_cnt=%d, sd_needed=%d, max sd count %d\n",
4973 loop_count, sd_needed, hmc_info->sd_table.sd_cnt);
4974 return IRDMA_ERR_CFG;
4975 }
4976
4977 if (loop_count > 1 && sd_needed < hmc_fpm_misc->max_sds) {
4978 pblewanted += (hmc_fpm_misc->max_sds - sd_needed) * 256 *
4979 FPM_MULTIPLIER;
4980 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted;
4981 sd_needed = irdma_est_sd(dev, hmc_info);
4982 }
4983
4984 ibdev_dbg(to_ibdev(dev),
4985 "HMC: loop_cnt=%d, sd_needed=%d, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d, mc=%d, ah=%d, max sd count %d, first sd index %d\n",
4986 loop_count, sd_needed,
4987 hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt,
4988 hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt,
4989 hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt,
4990 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt,
4991 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt,
4992 hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt,
4993 hmc_info->sd_table.sd_cnt, hmc_info->first_sd_index);
4994
4995 ret_code = irdma_sc_cfg_iw_fpm(dev, dev->hmc_fn_id);
4996 if (ret_code) {
4997 ibdev_dbg(to_ibdev(dev),
4998 "HMC: cfg_iw_fpm returned error_code[x%08X]\n",
4999 readl(dev->hw_regs[IRDMA_CQPERRCODES]));
5000 return ret_code;
5001 }
5002
5003 mem_size = sizeof(struct irdma_hmc_sd_entry) *
5004 (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1);
5005 virt_mem.size = mem_size;
5006 virt_mem.va = kzalloc(virt_mem.size, GFP_KERNEL);
5007 if (!virt_mem.va) {
5008 ibdev_dbg(to_ibdev(dev),
5009 "HMC: failed to allocate memory for sd_entry buffer\n");
5010 return IRDMA_ERR_NO_MEMORY;
5011 }
5012 hmc_info->sd_table.sd_entry = virt_mem.va;
5013
5014 return ret_code;
5015 }
5016
5017 /**
5018 * irdma_exec_cqp_cmd - execute cqp cmd when wqe are available
5019 * @dev: rdma device
5020 * @pcmdinfo: cqp command info
5021 */
irdma_exec_cqp_cmd(struct irdma_sc_dev * dev,struct cqp_cmds_info * pcmdinfo)5022 static enum irdma_status_code irdma_exec_cqp_cmd(struct irdma_sc_dev *dev,
5023 struct cqp_cmds_info *pcmdinfo)
5024 {
5025 enum irdma_status_code status;
5026 struct irdma_dma_mem val_mem;
5027 bool alloc = false;
5028
5029 dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++;
5030 switch (pcmdinfo->cqp_cmd) {
5031 case IRDMA_OP_CEQ_DESTROY:
5032 status = irdma_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq,
5033 pcmdinfo->in.u.ceq_destroy.scratch,
5034 pcmdinfo->post_sq);
5035 break;
5036 case IRDMA_OP_AEQ_DESTROY:
5037 status = irdma_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq,
5038 pcmdinfo->in.u.aeq_destroy.scratch,
5039 pcmdinfo->post_sq);
5040
5041 break;
5042 case IRDMA_OP_CEQ_CREATE:
5043 status = irdma_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq,
5044 pcmdinfo->in.u.ceq_create.scratch,
5045 pcmdinfo->post_sq);
5046 break;
5047 case IRDMA_OP_AEQ_CREATE:
5048 status = irdma_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq,
5049 pcmdinfo->in.u.aeq_create.scratch,
5050 pcmdinfo->post_sq);
5051 break;
5052 case IRDMA_OP_QP_UPLOAD_CONTEXT:
5053 status = irdma_sc_qp_upload_context(pcmdinfo->in.u.qp_upload_context.dev,
5054 &pcmdinfo->in.u.qp_upload_context.info,
5055 pcmdinfo->in.u.qp_upload_context.scratch,
5056 pcmdinfo->post_sq);
5057 break;
5058 case IRDMA_OP_CQ_CREATE:
5059 status = irdma_sc_cq_create(pcmdinfo->in.u.cq_create.cq,
5060 pcmdinfo->in.u.cq_create.scratch,
5061 pcmdinfo->in.u.cq_create.check_overflow,
5062 pcmdinfo->post_sq);
5063 break;
5064 case IRDMA_OP_CQ_MODIFY:
5065 status = irdma_sc_cq_modify(pcmdinfo->in.u.cq_modify.cq,
5066 &pcmdinfo->in.u.cq_modify.info,
5067 pcmdinfo->in.u.cq_modify.scratch,
5068 pcmdinfo->post_sq);
5069 break;
5070 case IRDMA_OP_CQ_DESTROY:
5071 status = irdma_sc_cq_destroy(pcmdinfo->in.u.cq_destroy.cq,
5072 pcmdinfo->in.u.cq_destroy.scratch,
5073 pcmdinfo->post_sq);
5074 break;
5075 case IRDMA_OP_QP_FLUSH_WQES:
5076 status = irdma_sc_qp_flush_wqes(pcmdinfo->in.u.qp_flush_wqes.qp,
5077 &pcmdinfo->in.u.qp_flush_wqes.info,
5078 pcmdinfo->in.u.qp_flush_wqes.scratch,
5079 pcmdinfo->post_sq);
5080 break;
5081 case IRDMA_OP_GEN_AE:
5082 status = irdma_sc_gen_ae(pcmdinfo->in.u.gen_ae.qp,
5083 &pcmdinfo->in.u.gen_ae.info,
5084 pcmdinfo->in.u.gen_ae.scratch,
5085 pcmdinfo->post_sq);
5086 break;
5087 case IRDMA_OP_MANAGE_PUSH_PAGE:
5088 status = irdma_sc_manage_push_page(pcmdinfo->in.u.manage_push_page.cqp,
5089 &pcmdinfo->in.u.manage_push_page.info,
5090 pcmdinfo->in.u.manage_push_page.scratch,
5091 pcmdinfo->post_sq);
5092 break;
5093 case IRDMA_OP_UPDATE_PE_SDS:
5094 status = irdma_update_pe_sds(pcmdinfo->in.u.update_pe_sds.dev,
5095 &pcmdinfo->in.u.update_pe_sds.info,
5096 pcmdinfo->in.u.update_pe_sds.scratch);
5097 break;
5098 case IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE:
5099 /* switch to calling through the call table */
5100 status =
5101 irdma_sc_manage_hmc_pm_func_table(pcmdinfo->in.u.manage_hmc_pm.dev->cqp,
5102 &pcmdinfo->in.u.manage_hmc_pm.info,
5103 pcmdinfo->in.u.manage_hmc_pm.scratch,
5104 true);
5105 break;
5106 case IRDMA_OP_SUSPEND:
5107 status = irdma_sc_suspend_qp(pcmdinfo->in.u.suspend_resume.cqp,
5108 pcmdinfo->in.u.suspend_resume.qp,
5109 pcmdinfo->in.u.suspend_resume.scratch);
5110 break;
5111 case IRDMA_OP_RESUME:
5112 status = irdma_sc_resume_qp(pcmdinfo->in.u.suspend_resume.cqp,
5113 pcmdinfo->in.u.suspend_resume.qp,
5114 pcmdinfo->in.u.suspend_resume.scratch);
5115 break;
5116 case IRDMA_OP_QUERY_FPM_VAL:
5117 val_mem.pa = pcmdinfo->in.u.query_fpm_val.fpm_val_pa;
5118 val_mem.va = pcmdinfo->in.u.query_fpm_val.fpm_val_va;
5119 status = irdma_sc_query_fpm_val(pcmdinfo->in.u.query_fpm_val.cqp,
5120 pcmdinfo->in.u.query_fpm_val.scratch,
5121 pcmdinfo->in.u.query_fpm_val.hmc_fn_id,
5122 &val_mem, true, IRDMA_CQP_WAIT_EVENT);
5123 break;
5124 case IRDMA_OP_COMMIT_FPM_VAL:
5125 val_mem.pa = pcmdinfo->in.u.commit_fpm_val.fpm_val_pa;
5126 val_mem.va = pcmdinfo->in.u.commit_fpm_val.fpm_val_va;
5127 status = irdma_sc_commit_fpm_val(pcmdinfo->in.u.commit_fpm_val.cqp,
5128 pcmdinfo->in.u.commit_fpm_val.scratch,
5129 pcmdinfo->in.u.commit_fpm_val.hmc_fn_id,
5130 &val_mem,
5131 true,
5132 IRDMA_CQP_WAIT_EVENT);
5133 break;
5134 case IRDMA_OP_STATS_ALLOCATE:
5135 alloc = true;
5136 fallthrough;
5137 case IRDMA_OP_STATS_FREE:
5138 status = irdma_sc_manage_stats_inst(pcmdinfo->in.u.stats_manage.cqp,
5139 &pcmdinfo->in.u.stats_manage.info,
5140 alloc,
5141 pcmdinfo->in.u.stats_manage.scratch);
5142 break;
5143 case IRDMA_OP_STATS_GATHER:
5144 status = irdma_sc_gather_stats(pcmdinfo->in.u.stats_gather.cqp,
5145 &pcmdinfo->in.u.stats_gather.info,
5146 pcmdinfo->in.u.stats_gather.scratch);
5147 break;
5148 case IRDMA_OP_WS_MODIFY_NODE:
5149 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,
5150 &pcmdinfo->in.u.ws_node.info,
5151 IRDMA_MODIFY_NODE,
5152 pcmdinfo->in.u.ws_node.scratch);
5153 break;
5154 case IRDMA_OP_WS_DELETE_NODE:
5155 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,
5156 &pcmdinfo->in.u.ws_node.info,
5157 IRDMA_DEL_NODE,
5158 pcmdinfo->in.u.ws_node.scratch);
5159 break;
5160 case IRDMA_OP_WS_ADD_NODE:
5161 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,
5162 &pcmdinfo->in.u.ws_node.info,
5163 IRDMA_ADD_NODE,
5164 pcmdinfo->in.u.ws_node.scratch);
5165 break;
5166 case IRDMA_OP_SET_UP_MAP:
5167 status = irdma_sc_set_up_map(pcmdinfo->in.u.up_map.cqp,
5168 &pcmdinfo->in.u.up_map.info,
5169 pcmdinfo->in.u.up_map.scratch);
5170 break;
5171 case IRDMA_OP_QUERY_RDMA_FEATURES:
5172 status = irdma_sc_query_rdma_features(pcmdinfo->in.u.query_rdma.cqp,
5173 &pcmdinfo->in.u.query_rdma.query_buff_mem,
5174 pcmdinfo->in.u.query_rdma.scratch);
5175 break;
5176 case IRDMA_OP_DELETE_ARP_CACHE_ENTRY:
5177 status = irdma_sc_del_arp_cache_entry(pcmdinfo->in.u.del_arp_cache_entry.cqp,
5178 pcmdinfo->in.u.del_arp_cache_entry.scratch,
5179 pcmdinfo->in.u.del_arp_cache_entry.arp_index,
5180 pcmdinfo->post_sq);
5181 break;
5182 case IRDMA_OP_MANAGE_APBVT_ENTRY:
5183 status = irdma_sc_manage_apbvt_entry(pcmdinfo->in.u.manage_apbvt_entry.cqp,
5184 &pcmdinfo->in.u.manage_apbvt_entry.info,
5185 pcmdinfo->in.u.manage_apbvt_entry.scratch,
5186 pcmdinfo->post_sq);
5187 break;
5188 case IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY:
5189 status = irdma_sc_manage_qhash_table_entry(pcmdinfo->in.u.manage_qhash_table_entry.cqp,
5190 &pcmdinfo->in.u.manage_qhash_table_entry.info,
5191 pcmdinfo->in.u.manage_qhash_table_entry.scratch,
5192 pcmdinfo->post_sq);
5193 break;
5194 case IRDMA_OP_QP_MODIFY:
5195 status = irdma_sc_qp_modify(pcmdinfo->in.u.qp_modify.qp,
5196 &pcmdinfo->in.u.qp_modify.info,
5197 pcmdinfo->in.u.qp_modify.scratch,
5198 pcmdinfo->post_sq);
5199 break;
5200 case IRDMA_OP_QP_CREATE:
5201 status = irdma_sc_qp_create(pcmdinfo->in.u.qp_create.qp,
5202 &pcmdinfo->in.u.qp_create.info,
5203 pcmdinfo->in.u.qp_create.scratch,
5204 pcmdinfo->post_sq);
5205 break;
5206 case IRDMA_OP_QP_DESTROY:
5207 status = irdma_sc_qp_destroy(pcmdinfo->in.u.qp_destroy.qp,
5208 pcmdinfo->in.u.qp_destroy.scratch,
5209 pcmdinfo->in.u.qp_destroy.remove_hash_idx,
5210 pcmdinfo->in.u.qp_destroy.ignore_mw_bnd,
5211 pcmdinfo->post_sq);
5212 break;
5213 case IRDMA_OP_ALLOC_STAG:
5214 status = irdma_sc_alloc_stag(pcmdinfo->in.u.alloc_stag.dev,
5215 &pcmdinfo->in.u.alloc_stag.info,
5216 pcmdinfo->in.u.alloc_stag.scratch,
5217 pcmdinfo->post_sq);
5218 break;
5219 case IRDMA_OP_MR_REG_NON_SHARED:
5220 status = irdma_sc_mr_reg_non_shared(pcmdinfo->in.u.mr_reg_non_shared.dev,
5221 &pcmdinfo->in.u.mr_reg_non_shared.info,
5222 pcmdinfo->in.u.mr_reg_non_shared.scratch,
5223 pcmdinfo->post_sq);
5224 break;
5225 case IRDMA_OP_DEALLOC_STAG:
5226 status = irdma_sc_dealloc_stag(pcmdinfo->in.u.dealloc_stag.dev,
5227 &pcmdinfo->in.u.dealloc_stag.info,
5228 pcmdinfo->in.u.dealloc_stag.scratch,
5229 pcmdinfo->post_sq);
5230 break;
5231 case IRDMA_OP_MW_ALLOC:
5232 status = irdma_sc_mw_alloc(pcmdinfo->in.u.mw_alloc.dev,
5233 &pcmdinfo->in.u.mw_alloc.info,
5234 pcmdinfo->in.u.mw_alloc.scratch,
5235 pcmdinfo->post_sq);
5236 break;
5237 case IRDMA_OP_ADD_ARP_CACHE_ENTRY:
5238 status = irdma_sc_add_arp_cache_entry(pcmdinfo->in.u.add_arp_cache_entry.cqp,
5239 &pcmdinfo->in.u.add_arp_cache_entry.info,
5240 pcmdinfo->in.u.add_arp_cache_entry.scratch,
5241 pcmdinfo->post_sq);
5242 break;
5243 case IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY:
5244 status = irdma_sc_alloc_local_mac_entry(pcmdinfo->in.u.alloc_local_mac_entry.cqp,
5245 pcmdinfo->in.u.alloc_local_mac_entry.scratch,
5246 pcmdinfo->post_sq);
5247 break;
5248 case IRDMA_OP_ADD_LOCAL_MAC_ENTRY:
5249 status = irdma_sc_add_local_mac_entry(pcmdinfo->in.u.add_local_mac_entry.cqp,
5250 &pcmdinfo->in.u.add_local_mac_entry.info,
5251 pcmdinfo->in.u.add_local_mac_entry.scratch,
5252 pcmdinfo->post_sq);
5253 break;
5254 case IRDMA_OP_DELETE_LOCAL_MAC_ENTRY:
5255 status = irdma_sc_del_local_mac_entry(pcmdinfo->in.u.del_local_mac_entry.cqp,
5256 pcmdinfo->in.u.del_local_mac_entry.scratch,
5257 pcmdinfo->in.u.del_local_mac_entry.entry_idx,
5258 pcmdinfo->in.u.del_local_mac_entry.ignore_ref_count,
5259 pcmdinfo->post_sq);
5260 break;
5261 case IRDMA_OP_AH_CREATE:
5262 status = irdma_sc_create_ah(pcmdinfo->in.u.ah_create.cqp,
5263 &pcmdinfo->in.u.ah_create.info,
5264 pcmdinfo->in.u.ah_create.scratch);
5265 break;
5266 case IRDMA_OP_AH_DESTROY:
5267 status = irdma_sc_destroy_ah(pcmdinfo->in.u.ah_destroy.cqp,
5268 &pcmdinfo->in.u.ah_destroy.info,
5269 pcmdinfo->in.u.ah_destroy.scratch);
5270 break;
5271 case IRDMA_OP_MC_CREATE:
5272 status = irdma_sc_create_mcast_grp(pcmdinfo->in.u.mc_create.cqp,
5273 &pcmdinfo->in.u.mc_create.info,
5274 pcmdinfo->in.u.mc_create.scratch);
5275 break;
5276 case IRDMA_OP_MC_DESTROY:
5277 status = irdma_sc_destroy_mcast_grp(pcmdinfo->in.u.mc_destroy.cqp,
5278 &pcmdinfo->in.u.mc_destroy.info,
5279 pcmdinfo->in.u.mc_destroy.scratch);
5280 break;
5281 case IRDMA_OP_MC_MODIFY:
5282 status = irdma_sc_modify_mcast_grp(pcmdinfo->in.u.mc_modify.cqp,
5283 &pcmdinfo->in.u.mc_modify.info,
5284 pcmdinfo->in.u.mc_modify.scratch);
5285 break;
5286 default:
5287 status = IRDMA_NOT_SUPPORTED;
5288 break;
5289 }
5290
5291 return status;
5292 }
5293
5294 /**
5295 * irdma_process_cqp_cmd - process all cqp commands
5296 * @dev: sc device struct
5297 * @pcmdinfo: cqp command info
5298 */
irdma_process_cqp_cmd(struct irdma_sc_dev * dev,struct cqp_cmds_info * pcmdinfo)5299 enum irdma_status_code irdma_process_cqp_cmd(struct irdma_sc_dev *dev,
5300 struct cqp_cmds_info *pcmdinfo)
5301 {
5302 enum irdma_status_code status = 0;
5303 unsigned long flags;
5304
5305 spin_lock_irqsave(&dev->cqp_lock, flags);
5306 if (list_empty(&dev->cqp_cmd_head) && !irdma_cqp_ring_full(dev->cqp))
5307 status = irdma_exec_cqp_cmd(dev, pcmdinfo);
5308 else
5309 list_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head);
5310 spin_unlock_irqrestore(&dev->cqp_lock, flags);
5311 return status;
5312 }
5313
5314 /**
5315 * irdma_process_bh - called from tasklet for cqp list
5316 * @dev: sc device struct
5317 */
irdma_process_bh(struct irdma_sc_dev * dev)5318 enum irdma_status_code irdma_process_bh(struct irdma_sc_dev *dev)
5319 {
5320 enum irdma_status_code status = 0;
5321 struct cqp_cmds_info *pcmdinfo;
5322 unsigned long flags;
5323
5324 spin_lock_irqsave(&dev->cqp_lock, flags);
5325 while (!list_empty(&dev->cqp_cmd_head) &&
5326 !irdma_cqp_ring_full(dev->cqp)) {
5327 pcmdinfo = (struct cqp_cmds_info *)irdma_remove_cqp_head(dev);
5328 status = irdma_exec_cqp_cmd(dev, pcmdinfo);
5329 if (status)
5330 break;
5331 }
5332 spin_unlock_irqrestore(&dev->cqp_lock, flags);
5333 return status;
5334 }
5335
5336 /**
5337 * irdma_cfg_aeq- Configure AEQ interrupt
5338 * @dev: pointer to the device structure
5339 * @idx: vector index
5340 * @enable: True to enable, False disables
5341 */
irdma_cfg_aeq(struct irdma_sc_dev * dev,u32 idx,bool enable)5342 void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable)
5343 {
5344 u32 reg_val;
5345
5346 reg_val = FIELD_PREP(IRDMA_PFINT_AEQCTL_CAUSE_ENA, enable) |
5347 FIELD_PREP(IRDMA_PFINT_AEQCTL_MSIX_INDX, idx) |
5348 FIELD_PREP(IRDMA_PFINT_AEQCTL_ITR_INDX, 3);
5349 writel(reg_val, dev->hw_regs[IRDMA_PFINT_AEQCTL]);
5350 }
5351
5352 /**
5353 * sc_vsi_update_stats - Update statistics
5354 * @vsi: sc_vsi instance to update
5355 */
sc_vsi_update_stats(struct irdma_sc_vsi * vsi)5356 void sc_vsi_update_stats(struct irdma_sc_vsi *vsi)
5357 {
5358 struct irdma_gather_stats *gather_stats;
5359 struct irdma_gather_stats *last_gather_stats;
5360
5361 gather_stats = vsi->pestat->gather_info.gather_stats_va;
5362 last_gather_stats = vsi->pestat->gather_info.last_gather_stats_va;
5363 irdma_update_stats(&vsi->pestat->hw_stats, gather_stats,
5364 last_gather_stats);
5365 }
5366
5367 /**
5368 * irdma_wait_pe_ready - Check if firmware is ready
5369 * @dev: provides access to registers
5370 */
irdma_wait_pe_ready(struct irdma_sc_dev * dev)5371 static int irdma_wait_pe_ready(struct irdma_sc_dev *dev)
5372 {
5373 u32 statuscpu0;
5374 u32 statuscpu1;
5375 u32 statuscpu2;
5376 u32 retrycount = 0;
5377
5378 do {
5379 statuscpu0 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS0]);
5380 statuscpu1 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS1]);
5381 statuscpu2 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS2]);
5382 if (statuscpu0 == 0x80 && statuscpu1 == 0x80 &&
5383 statuscpu2 == 0x80)
5384 return 0;
5385 mdelay(1000);
5386 } while (retrycount++ < dev->hw_attrs.max_pe_ready_count);
5387 return -1;
5388 }
5389
irdma_sc_init_hw(struct irdma_sc_dev * dev)5390 static inline void irdma_sc_init_hw(struct irdma_sc_dev *dev)
5391 {
5392 switch (dev->hw_attrs.uk_attrs.hw_rev) {
5393 case IRDMA_GEN_1:
5394 i40iw_init_hw(dev);
5395 break;
5396 case IRDMA_GEN_2:
5397 icrdma_init_hw(dev);
5398 break;
5399 }
5400 }
5401
5402 /**
5403 * irdma_sc_dev_init - Initialize control part of device
5404 * @ver: version
5405 * @dev: Device pointer
5406 * @info: Device init info
5407 */
irdma_sc_dev_init(enum irdma_vers ver,struct irdma_sc_dev * dev,struct irdma_device_init_info * info)5408 enum irdma_status_code irdma_sc_dev_init(enum irdma_vers ver,
5409 struct irdma_sc_dev *dev,
5410 struct irdma_device_init_info *info)
5411 {
5412 u32 val;
5413 enum irdma_status_code ret_code = 0;
5414 u8 db_size;
5415
5416 INIT_LIST_HEAD(&dev->cqp_cmd_head); /* for CQP command backlog */
5417 mutex_init(&dev->ws_mutex);
5418 dev->hmc_fn_id = info->hmc_fn_id;
5419 dev->fpm_query_buf_pa = info->fpm_query_buf_pa;
5420 dev->fpm_query_buf = info->fpm_query_buf;
5421 dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa;
5422 dev->fpm_commit_buf = info->fpm_commit_buf;
5423 dev->hw = info->hw;
5424 dev->hw->hw_addr = info->bar0;
5425 /* Setup the hardware limits, hmc may limit further */
5426 dev->hw_attrs.min_hw_qp_id = IRDMA_MIN_IW_QP_ID;
5427 dev->hw_attrs.min_hw_aeq_size = IRDMA_MIN_AEQ_ENTRIES;
5428 dev->hw_attrs.max_hw_aeq_size = IRDMA_MAX_AEQ_ENTRIES;
5429 dev->hw_attrs.min_hw_ceq_size = IRDMA_MIN_CEQ_ENTRIES;
5430 dev->hw_attrs.max_hw_ceq_size = IRDMA_MAX_CEQ_ENTRIES;
5431 dev->hw_attrs.uk_attrs.min_hw_cq_size = IRDMA_MIN_CQ_SIZE;
5432 dev->hw_attrs.uk_attrs.max_hw_cq_size = IRDMA_MAX_CQ_SIZE;
5433 dev->hw_attrs.uk_attrs.max_hw_wq_frags = IRDMA_MAX_WQ_FRAGMENT_COUNT;
5434 dev->hw_attrs.uk_attrs.max_hw_read_sges = IRDMA_MAX_SGE_RD;
5435 dev->hw_attrs.max_hw_outbound_msg_size = IRDMA_MAX_OUTBOUND_MSG_SIZE;
5436 dev->hw_attrs.max_mr_size = IRDMA_MAX_MR_SIZE;
5437 dev->hw_attrs.max_hw_inbound_msg_size = IRDMA_MAX_INBOUND_MSG_SIZE;
5438 dev->hw_attrs.max_hw_device_pages = IRDMA_MAX_PUSH_PAGE_COUNT;
5439 dev->hw_attrs.uk_attrs.max_hw_inline = IRDMA_MAX_INLINE_DATA_SIZE;
5440 dev->hw_attrs.max_hw_wqes = IRDMA_MAX_WQ_ENTRIES;
5441 dev->hw_attrs.max_qp_wr = IRDMA_MAX_QP_WRS(IRDMA_MAX_QUANTA_PER_WR);
5442
5443 dev->hw_attrs.uk_attrs.max_hw_rq_quanta = IRDMA_QP_SW_MAX_RQ_QUANTA;
5444 dev->hw_attrs.uk_attrs.max_hw_wq_quanta = IRDMA_QP_SW_MAX_WQ_QUANTA;
5445 dev->hw_attrs.max_hw_pds = IRDMA_MAX_PDS;
5446 dev->hw_attrs.max_hw_ena_vf_count = IRDMA_MAX_PE_ENA_VF_COUNT;
5447
5448 dev->hw_attrs.max_pe_ready_count = 14;
5449 dev->hw_attrs.max_done_count = IRDMA_DONE_COUNT;
5450 dev->hw_attrs.max_sleep_count = IRDMA_SLEEP_COUNT;
5451 dev->hw_attrs.max_cqp_compl_wait_time_ms = CQP_COMPL_WAIT_TIME_MS;
5452
5453 dev->hw_attrs.uk_attrs.hw_rev = ver;
5454 irdma_sc_init_hw(dev);
5455
5456 if (irdma_wait_pe_ready(dev))
5457 return IRDMA_ERR_TIMEOUT;
5458
5459 val = readl(dev->hw_regs[IRDMA_GLPCI_LBARCTRL]);
5460 db_size = (u8)FIELD_GET(IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE, val);
5461 if (db_size != IRDMA_PE_DB_SIZE_4M && db_size != IRDMA_PE_DB_SIZE_8M) {
5462 ibdev_dbg(to_ibdev(dev),
5463 "DEV: RDMA PE doorbell is not enabled in CSR val 0x%x db_size=%d\n",
5464 val, db_size);
5465 return IRDMA_ERR_PE_DOORBELL_NOT_ENA;
5466 }
5467 dev->db_addr = dev->hw->hw_addr + (uintptr_t)dev->hw_regs[IRDMA_DB_ADDR_OFFSET];
5468
5469 return ret_code;
5470 }
5471
5472 /**
5473 * irdma_update_stats - Update statistics
5474 * @hw_stats: hw_stats instance to update
5475 * @gather_stats: updated stat counters
5476 * @last_gather_stats: last stat counters
5477 */
irdma_update_stats(struct irdma_dev_hw_stats * hw_stats,struct irdma_gather_stats * gather_stats,struct irdma_gather_stats * last_gather_stats)5478 void irdma_update_stats(struct irdma_dev_hw_stats *hw_stats,
5479 struct irdma_gather_stats *gather_stats,
5480 struct irdma_gather_stats *last_gather_stats)
5481 {
5482 u64 *stats_val = hw_stats->stats_val_32;
5483
5484 stats_val[IRDMA_HW_STAT_INDEX_RXVLANERR] +=
5485 IRDMA_STATS_DELTA(gather_stats->rxvlanerr,
5486 last_gather_stats->rxvlanerr,
5487 IRDMA_MAX_STATS_32);
5488 stats_val[IRDMA_HW_STAT_INDEX_IP4RXDISCARD] +=
5489 IRDMA_STATS_DELTA(gather_stats->ip4rxdiscard,
5490 last_gather_stats->ip4rxdiscard,
5491 IRDMA_MAX_STATS_32);
5492 stats_val[IRDMA_HW_STAT_INDEX_IP4RXTRUNC] +=
5493 IRDMA_STATS_DELTA(gather_stats->ip4rxtrunc,
5494 last_gather_stats->ip4rxtrunc,
5495 IRDMA_MAX_STATS_32);
5496 stats_val[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE] +=
5497 IRDMA_STATS_DELTA(gather_stats->ip4txnoroute,
5498 last_gather_stats->ip4txnoroute,
5499 IRDMA_MAX_STATS_32);
5500 stats_val[IRDMA_HW_STAT_INDEX_IP6RXDISCARD] +=
5501 IRDMA_STATS_DELTA(gather_stats->ip6rxdiscard,
5502 last_gather_stats->ip6rxdiscard,
5503 IRDMA_MAX_STATS_32);
5504 stats_val[IRDMA_HW_STAT_INDEX_IP6RXTRUNC] +=
5505 IRDMA_STATS_DELTA(gather_stats->ip6rxtrunc,
5506 last_gather_stats->ip6rxtrunc,
5507 IRDMA_MAX_STATS_32);
5508 stats_val[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE] +=
5509 IRDMA_STATS_DELTA(gather_stats->ip6txnoroute,
5510 last_gather_stats->ip6txnoroute,
5511 IRDMA_MAX_STATS_32);
5512 stats_val[IRDMA_HW_STAT_INDEX_TCPRTXSEG] +=
5513 IRDMA_STATS_DELTA(gather_stats->tcprtxseg,
5514 last_gather_stats->tcprtxseg,
5515 IRDMA_MAX_STATS_32);
5516 stats_val[IRDMA_HW_STAT_INDEX_TCPRXOPTERR] +=
5517 IRDMA_STATS_DELTA(gather_stats->tcprxopterr,
5518 last_gather_stats->tcprxopterr,
5519 IRDMA_MAX_STATS_32);
5520 stats_val[IRDMA_HW_STAT_INDEX_TCPRXPROTOERR] +=
5521 IRDMA_STATS_DELTA(gather_stats->tcprxprotoerr,
5522 last_gather_stats->tcprxprotoerr,
5523 IRDMA_MAX_STATS_32);
5524 stats_val[IRDMA_HW_STAT_INDEX_RXRPCNPHANDLED] +=
5525 IRDMA_STATS_DELTA(gather_stats->rxrpcnphandled,
5526 last_gather_stats->rxrpcnphandled,
5527 IRDMA_MAX_STATS_32);
5528 stats_val[IRDMA_HW_STAT_INDEX_RXRPCNPIGNORED] +=
5529 IRDMA_STATS_DELTA(gather_stats->rxrpcnpignored,
5530 last_gather_stats->rxrpcnpignored,
5531 IRDMA_MAX_STATS_32);
5532 stats_val[IRDMA_HW_STAT_INDEX_TXNPCNPSENT] +=
5533 IRDMA_STATS_DELTA(gather_stats->txnpcnpsent,
5534 last_gather_stats->txnpcnpsent,
5535 IRDMA_MAX_STATS_32);
5536 stats_val = hw_stats->stats_val_64;
5537 stats_val[IRDMA_HW_STAT_INDEX_IP4RXOCTS] +=
5538 IRDMA_STATS_DELTA(gather_stats->ip4rxocts,
5539 last_gather_stats->ip4rxocts,
5540 IRDMA_MAX_STATS_48);
5541 stats_val[IRDMA_HW_STAT_INDEX_IP4RXPKTS] +=
5542 IRDMA_STATS_DELTA(gather_stats->ip4rxpkts,
5543 last_gather_stats->ip4rxpkts,
5544 IRDMA_MAX_STATS_48);
5545 stats_val[IRDMA_HW_STAT_INDEX_IP4RXFRAGS] +=
5546 IRDMA_STATS_DELTA(gather_stats->ip4txfrag,
5547 last_gather_stats->ip4txfrag,
5548 IRDMA_MAX_STATS_48);
5549 stats_val[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS] +=
5550 IRDMA_STATS_DELTA(gather_stats->ip4rxmcpkts,
5551 last_gather_stats->ip4rxmcpkts,
5552 IRDMA_MAX_STATS_48);
5553 stats_val[IRDMA_HW_STAT_INDEX_IP4TXOCTS] +=
5554 IRDMA_STATS_DELTA(gather_stats->ip4txocts,
5555 last_gather_stats->ip4txocts,
5556 IRDMA_MAX_STATS_48);
5557 stats_val[IRDMA_HW_STAT_INDEX_IP4TXPKTS] +=
5558 IRDMA_STATS_DELTA(gather_stats->ip4txpkts,
5559 last_gather_stats->ip4txpkts,
5560 IRDMA_MAX_STATS_48);
5561 stats_val[IRDMA_HW_STAT_INDEX_IP4TXFRAGS] +=
5562 IRDMA_STATS_DELTA(gather_stats->ip4txfrag,
5563 last_gather_stats->ip4txfrag,
5564 IRDMA_MAX_STATS_48);
5565 stats_val[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS] +=
5566 IRDMA_STATS_DELTA(gather_stats->ip4txmcpkts,
5567 last_gather_stats->ip4txmcpkts,
5568 IRDMA_MAX_STATS_48);
5569 stats_val[IRDMA_HW_STAT_INDEX_IP6RXOCTS] +=
5570 IRDMA_STATS_DELTA(gather_stats->ip6rxocts,
5571 last_gather_stats->ip6rxocts,
5572 IRDMA_MAX_STATS_48);
5573 stats_val[IRDMA_HW_STAT_INDEX_IP6RXPKTS] +=
5574 IRDMA_STATS_DELTA(gather_stats->ip6rxpkts,
5575 last_gather_stats->ip6rxpkts,
5576 IRDMA_MAX_STATS_48);
5577 stats_val[IRDMA_HW_STAT_INDEX_IP6RXFRAGS] +=
5578 IRDMA_STATS_DELTA(gather_stats->ip6txfrags,
5579 last_gather_stats->ip6txfrags,
5580 IRDMA_MAX_STATS_48);
5581 stats_val[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS] +=
5582 IRDMA_STATS_DELTA(gather_stats->ip6rxmcpkts,
5583 last_gather_stats->ip6rxmcpkts,
5584 IRDMA_MAX_STATS_48);
5585 stats_val[IRDMA_HW_STAT_INDEX_IP6TXOCTS] +=
5586 IRDMA_STATS_DELTA(gather_stats->ip6txocts,
5587 last_gather_stats->ip6txocts,
5588 IRDMA_MAX_STATS_48);
5589 stats_val[IRDMA_HW_STAT_INDEX_IP6TXPKTS] +=
5590 IRDMA_STATS_DELTA(gather_stats->ip6txpkts,
5591 last_gather_stats->ip6txpkts,
5592 IRDMA_MAX_STATS_48);
5593 stats_val[IRDMA_HW_STAT_INDEX_IP6TXFRAGS] +=
5594 IRDMA_STATS_DELTA(gather_stats->ip6txfrags,
5595 last_gather_stats->ip6txfrags,
5596 IRDMA_MAX_STATS_48);
5597 stats_val[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS] +=
5598 IRDMA_STATS_DELTA(gather_stats->ip6txmcpkts,
5599 last_gather_stats->ip6txmcpkts,
5600 IRDMA_MAX_STATS_48);
5601 stats_val[IRDMA_HW_STAT_INDEX_TCPRXSEGS] +=
5602 IRDMA_STATS_DELTA(gather_stats->tcprxsegs,
5603 last_gather_stats->tcprxsegs,
5604 IRDMA_MAX_STATS_48);
5605 stats_val[IRDMA_HW_STAT_INDEX_TCPTXSEG] +=
5606 IRDMA_STATS_DELTA(gather_stats->tcptxsegs,
5607 last_gather_stats->tcptxsegs,
5608 IRDMA_MAX_STATS_48);
5609 stats_val[IRDMA_HW_STAT_INDEX_RDMARXRDS] +=
5610 IRDMA_STATS_DELTA(gather_stats->rdmarxrds,
5611 last_gather_stats->rdmarxrds,
5612 IRDMA_MAX_STATS_48);
5613 stats_val[IRDMA_HW_STAT_INDEX_RDMARXSNDS] +=
5614 IRDMA_STATS_DELTA(gather_stats->rdmarxsnds,
5615 last_gather_stats->rdmarxsnds,
5616 IRDMA_MAX_STATS_48);
5617 stats_val[IRDMA_HW_STAT_INDEX_RDMARXWRS] +=
5618 IRDMA_STATS_DELTA(gather_stats->rdmarxwrs,
5619 last_gather_stats->rdmarxwrs,
5620 IRDMA_MAX_STATS_48);
5621 stats_val[IRDMA_HW_STAT_INDEX_RDMATXRDS] +=
5622 IRDMA_STATS_DELTA(gather_stats->rdmatxrds,
5623 last_gather_stats->rdmatxrds,
5624 IRDMA_MAX_STATS_48);
5625 stats_val[IRDMA_HW_STAT_INDEX_RDMATXSNDS] +=
5626 IRDMA_STATS_DELTA(gather_stats->rdmatxsnds,
5627 last_gather_stats->rdmatxsnds,
5628 IRDMA_MAX_STATS_48);
5629 stats_val[IRDMA_HW_STAT_INDEX_RDMATXWRS] +=
5630 IRDMA_STATS_DELTA(gather_stats->rdmatxwrs,
5631 last_gather_stats->rdmatxwrs,
5632 IRDMA_MAX_STATS_48);
5633 stats_val[IRDMA_HW_STAT_INDEX_RDMAVBND] +=
5634 IRDMA_STATS_DELTA(gather_stats->rdmavbn,
5635 last_gather_stats->rdmavbn,
5636 IRDMA_MAX_STATS_48);
5637 stats_val[IRDMA_HW_STAT_INDEX_RDMAVINV] +=
5638 IRDMA_STATS_DELTA(gather_stats->rdmavinv,
5639 last_gather_stats->rdmavinv,
5640 IRDMA_MAX_STATS_48);
5641 stats_val[IRDMA_HW_STAT_INDEX_UDPRXPKTS] +=
5642 IRDMA_STATS_DELTA(gather_stats->udprxpkts,
5643 last_gather_stats->udprxpkts,
5644 IRDMA_MAX_STATS_48);
5645 stats_val[IRDMA_HW_STAT_INDEX_UDPTXPKTS] +=
5646 IRDMA_STATS_DELTA(gather_stats->udptxpkts,
5647 last_gather_stats->udptxpkts,
5648 IRDMA_MAX_STATS_48);
5649 stats_val[IRDMA_HW_STAT_INDEX_RXNPECNMARKEDPKTS] +=
5650 IRDMA_STATS_DELTA(gather_stats->rxnpecnmrkpkts,
5651 last_gather_stats->rxnpecnmrkpkts,
5652 IRDMA_MAX_STATS_48);
5653 memcpy(last_gather_stats, gather_stats, sizeof(*last_gather_stats));
5654 }
5655
