1 /*
2 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
3 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #ifndef RXE_VERBS_H
35 #define RXE_VERBS_H
36
37 #include <linux/interrupt.h>
38 #include <linux/workqueue.h>
39 #include <rdma/rdma_user_rxe.h>
40 #include "rxe_pool.h"
41 #include "rxe_task.h"
42 #include "rxe_hw_counters.h"
43
pkey_match(u16 key1,u16 key2)44 static inline int pkey_match(u16 key1, u16 key2)
45 {
46 return (((key1 & 0x7fff) != 0) &&
47 ((key1 & 0x7fff) == (key2 & 0x7fff)) &&
48 ((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0;
49 }
50
51 /* Return >0 if psn_a > psn_b
52 * 0 if psn_a == psn_b
53 * <0 if psn_a < psn_b
54 */
psn_compare(u32 psn_a,u32 psn_b)55 static inline int psn_compare(u32 psn_a, u32 psn_b)
56 {
57 s32 diff;
58
59 diff = (psn_a - psn_b) << 8;
60 return diff;
61 }
62
63 struct rxe_ucontext {
64 struct rxe_pool_entry pelem;
65 struct ib_ucontext ibuc;
66 };
67
68 struct rxe_pd {
69 struct rxe_pool_entry pelem;
70 struct ib_pd ibpd;
71 };
72
73 struct rxe_ah {
74 struct rxe_pool_entry pelem;
75 struct ib_ah ibah;
76 struct rxe_pd *pd;
77 struct rxe_av av;
78 };
79
80 struct rxe_cqe {
81 union {
82 struct ib_wc ibwc;
83 struct ib_uverbs_wc uibwc;
84 };
85 };
86
87 struct rxe_cq {
88 struct rxe_pool_entry pelem;
89 struct ib_cq ibcq;
90 struct rxe_queue *queue;
91 spinlock_t cq_lock;
92 u8 notify;
93 bool is_dying;
94 int is_user;
95 struct tasklet_struct comp_task;
96 };
97
98 enum wqe_state {
99 wqe_state_posted,
100 wqe_state_processing,
101 wqe_state_pending,
102 wqe_state_done,
103 wqe_state_error,
104 };
105
106 struct rxe_sq {
107 int max_wr;
108 int max_sge;
109 int max_inline;
110 spinlock_t sq_lock; /* guard queue */
111 struct rxe_queue *queue;
112 };
113
114 struct rxe_rq {
115 int max_wr;
116 int max_sge;
117 spinlock_t producer_lock; /* guard queue producer */
118 spinlock_t consumer_lock; /* guard queue consumer */
119 struct rxe_queue *queue;
120 };
121
122 struct rxe_srq {
123 struct rxe_pool_entry pelem;
124 struct ib_srq ibsrq;
125 struct rxe_pd *pd;
126 struct rxe_rq rq;
127 u32 srq_num;
128
129 int limit;
130 int error;
131 };
132
133 enum rxe_qp_state {
134 QP_STATE_RESET,
135 QP_STATE_INIT,
136 QP_STATE_READY,
137 QP_STATE_DRAIN, /* req only */
138 QP_STATE_DRAINED, /* req only */
139 QP_STATE_ERROR
140 };
141
142 struct rxe_req_info {
143 enum rxe_qp_state state;
144 int wqe_index;
145 u32 psn;
146 int opcode;
147 atomic_t rd_atomic;
148 int wait_fence;
149 int need_rd_atomic;
150 int wait_psn;
151 int need_retry;
152 int noack_pkts;
153 struct rxe_task task;
154 };
155
156 struct rxe_comp_info {
157 u32 psn;
158 int opcode;
159 int timeout;
160 int timeout_retry;
161 u32 retry_cnt;
162 u32 rnr_retry;
163 struct rxe_task task;
164 };
165
166 enum rdatm_res_state {
167 rdatm_res_state_next,
168 rdatm_res_state_new,
169 rdatm_res_state_replay,
170 };
171
172 struct resp_res {
173 int type;
174 u32 first_psn;
175 u32 last_psn;
176 u32 cur_psn;
177 enum rdatm_res_state state;
178
179 union {
180 struct {
181 struct sk_buff *skb;
182 } atomic;
183 struct {
184 struct rxe_mem *mr;
185 u64 va_org;
186 u32 rkey;
187 u32 length;
188 u64 va;
189 u32 resid;
190 } read;
191 };
192 };
193
194 struct rxe_resp_info {
195 enum rxe_qp_state state;
196 u32 msn;
197 u32 psn;
198 int opcode;
199 int drop_msg;
200 int goto_error;
201 int sent_psn_nak;
202 enum ib_wc_status status;
203 u8 aeth_syndrome;
204
205 /* Receive only */
206 struct rxe_recv_wqe *wqe;
207
208 /* RDMA read / atomic only */
209 u64 va;
210 struct rxe_mem *mr;
211 u32 resid;
212 u32 rkey;
213 u64 atomic_orig;
214
215 /* SRQ only */
216 struct {
217 struct rxe_recv_wqe wqe;
218 struct ib_sge sge[RXE_MAX_SGE];
219 } srq_wqe;
220
221 /* Responder resources. It's a circular list where the oldest
222 * resource is dropped first.
223 */
224 struct resp_res *resources;
225 unsigned int res_head;
226 unsigned int res_tail;
227 struct resp_res *res;
228 struct rxe_task task;
229 };
230
231 struct rxe_qp {
232 struct rxe_pool_entry pelem;
233 struct ib_qp ibqp;
234 struct ib_qp_attr attr;
235 unsigned int valid;
236 unsigned int mtu;
237 int is_user;
238
239 struct rxe_pd *pd;
240 struct rxe_srq *srq;
241 struct rxe_cq *scq;
242 struct rxe_cq *rcq;
243
244 enum ib_sig_type sq_sig_type;
245
246 struct rxe_sq sq;
247 struct rxe_rq rq;
248
249 struct socket *sk;
250 u32 dst_cookie;
251
252 struct rxe_av pri_av;
253 struct rxe_av alt_av;
254
255 /* list of mcast groups qp has joined (for cleanup) */
256 struct list_head grp_list;
257 spinlock_t grp_lock; /* guard grp_list */
258
259 struct sk_buff_head req_pkts;
260 struct sk_buff_head resp_pkts;
261 struct sk_buff_head send_pkts;
262
263 struct rxe_req_info req;
264 struct rxe_comp_info comp;
265 struct rxe_resp_info resp;
266
267 atomic_t ssn;
268 atomic_t skb_out;
269 int need_req_skb;
270
271 /* Timer for retranmitting packet when ACKs have been lost. RC
272 * only. The requester sets it when it is not already
273 * started. The responder resets it whenever an ack is
274 * received.
275 */
276 struct timer_list retrans_timer;
277 u64 qp_timeout_jiffies;
278
279 /* Timer for handling RNR NAKS. */
280 struct timer_list rnr_nak_timer;
281
282 spinlock_t state_lock; /* guard requester and completer */
283
284 struct execute_work cleanup_work;
285 };
286
287 enum rxe_mem_state {
288 RXE_MEM_STATE_ZOMBIE,
289 RXE_MEM_STATE_INVALID,
290 RXE_MEM_STATE_FREE,
291 RXE_MEM_STATE_VALID,
292 };
293
294 enum rxe_mem_type {
295 RXE_MEM_TYPE_NONE,
296 RXE_MEM_TYPE_DMA,
297 RXE_MEM_TYPE_MR,
298 RXE_MEM_TYPE_FMR,
299 RXE_MEM_TYPE_MW,
300 };
301
302 #define RXE_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rxe_phys_buf))
303
304 struct rxe_phys_buf {
305 u64 addr;
306 u64 size;
307 };
308
309 struct rxe_map {
310 struct rxe_phys_buf buf[RXE_BUF_PER_MAP];
311 };
312
313 struct rxe_mem {
314 struct rxe_pool_entry pelem;
315 union {
316 struct ib_mr ibmr;
317 struct ib_mw ibmw;
318 };
319
320 struct rxe_pd *pd;
321 struct ib_umem *umem;
322
323 u32 lkey;
324 u32 rkey;
325
326 enum rxe_mem_state state;
327 enum rxe_mem_type type;
328 u64 va;
329 u64 iova;
330 size_t length;
331 u32 offset;
332 int access;
333
334 int page_shift;
335 int page_mask;
336 int map_shift;
337 int map_mask;
338
339 u32 num_buf;
340 u32 nbuf;
341
342 u32 max_buf;
343 u32 num_map;
344
345 struct rxe_map **map;
346 };
347
348 struct rxe_mc_grp {
349 struct rxe_pool_entry pelem;
350 spinlock_t mcg_lock; /* guard group */
351 struct rxe_dev *rxe;
352 struct list_head qp_list;
353 union ib_gid mgid;
354 int num_qp;
355 u32 qkey;
356 u16 pkey;
357 };
358
359 struct rxe_mc_elem {
360 struct rxe_pool_entry pelem;
361 struct list_head qp_list;
362 struct list_head grp_list;
363 struct rxe_qp *qp;
364 struct rxe_mc_grp *grp;
365 };
366
367 struct rxe_port {
368 struct ib_port_attr attr;
369 u16 *pkey_tbl;
370 __be64 port_guid;
371 __be64 subnet_prefix;
372 spinlock_t port_lock; /* guard port */
373 unsigned int mtu_cap;
374 /* special QPs */
375 u32 qp_smi_index;
376 u32 qp_gsi_index;
377 };
378
379 struct rxe_dev {
380 struct ib_device ib_dev;
381 struct ib_device_attr attr;
382 int max_ucontext;
383 int max_inline_data;
384 struct kref ref_cnt;
385 struct mutex usdev_lock;
386
387 struct net_device *ndev;
388
389 int xmit_errors;
390
391 struct rxe_pool uc_pool;
392 struct rxe_pool pd_pool;
393 struct rxe_pool ah_pool;
394 struct rxe_pool srq_pool;
395 struct rxe_pool qp_pool;
396 struct rxe_pool cq_pool;
397 struct rxe_pool mr_pool;
398 struct rxe_pool mw_pool;
399 struct rxe_pool mc_grp_pool;
400 struct rxe_pool mc_elem_pool;
401
402 spinlock_t pending_lock; /* guard pending_mmaps */
403 struct list_head pending_mmaps;
404
405 spinlock_t mmap_offset_lock; /* guard mmap_offset */
406 int mmap_offset;
407
408 u64 stats_counters[RXE_NUM_OF_COUNTERS];
409
410 struct rxe_port port;
411 struct list_head list;
412 struct crypto_shash *tfm;
413 };
414
rxe_counter_inc(struct rxe_dev * rxe,enum rxe_counters cnt)415 static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters cnt)
416 {
417 rxe->stats_counters[cnt]++;
418 }
419
to_rdev(struct ib_device * dev)420 static inline struct rxe_dev *to_rdev(struct ib_device *dev)
421 {
422 return dev ? container_of(dev, struct rxe_dev, ib_dev) : NULL;
423 }
424
to_ruc(struct ib_ucontext * uc)425 static inline struct rxe_ucontext *to_ruc(struct ib_ucontext *uc)
426 {
427 return uc ? container_of(uc, struct rxe_ucontext, ibuc) : NULL;
428 }
429
to_rpd(struct ib_pd * pd)430 static inline struct rxe_pd *to_rpd(struct ib_pd *pd)
431 {
432 return pd ? container_of(pd, struct rxe_pd, ibpd) : NULL;
433 }
434
to_rah(struct ib_ah * ah)435 static inline struct rxe_ah *to_rah(struct ib_ah *ah)
436 {
437 return ah ? container_of(ah, struct rxe_ah, ibah) : NULL;
438 }
439
to_rsrq(struct ib_srq * srq)440 static inline struct rxe_srq *to_rsrq(struct ib_srq *srq)
441 {
442 return srq ? container_of(srq, struct rxe_srq, ibsrq) : NULL;
443 }
444
to_rqp(struct ib_qp * qp)445 static inline struct rxe_qp *to_rqp(struct ib_qp *qp)
446 {
447 return qp ? container_of(qp, struct rxe_qp, ibqp) : NULL;
448 }
449
to_rcq(struct ib_cq * cq)450 static inline struct rxe_cq *to_rcq(struct ib_cq *cq)
451 {
452 return cq ? container_of(cq, struct rxe_cq, ibcq) : NULL;
453 }
454
to_rmr(struct ib_mr * mr)455 static inline struct rxe_mem *to_rmr(struct ib_mr *mr)
456 {
457 return mr ? container_of(mr, struct rxe_mem, ibmr) : NULL;
458 }
459
to_rmw(struct ib_mw * mw)460 static inline struct rxe_mem *to_rmw(struct ib_mw *mw)
461 {
462 return mw ? container_of(mw, struct rxe_mem, ibmw) : NULL;
463 }
464
465 int rxe_register_device(struct rxe_dev *rxe);
466 int rxe_unregister_device(struct rxe_dev *rxe);
467
468 void rxe_mc_cleanup(struct rxe_pool_entry *arg);
469
470 #endif /* RXE_VERBS_H */
471
