1 #ifndef DEF_RDMA_VT_H
2 #define DEF_RDMA_VT_H
3
4 /*
5 * Copyright(c) 2016 - 2019 Intel Corporation.
6 *
7 * This file is provided under a dual BSD/GPLv2 license. When using or
8 * redistributing this file, you may do so under either license.
9 *
10 * GPL LICENSE SUMMARY
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of version 2 of the GNU General Public License as
14 * published by the Free Software Foundation.
15 *
16 * This program is distributed in the hope that it will be useful, but
17 * WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 * General Public License for more details.
20 *
21 * BSD LICENSE
22 *
23 * Redistribution and use in source and binary forms, with or without
24 * modification, are permitted provided that the following conditions
25 * are met:
26 *
27 * - Redistributions of source code must retain the above copyright
28 * notice, this list of conditions and the following disclaimer.
29 * - Redistributions in binary form must reproduce the above copyright
30 * notice, this list of conditions and the following disclaimer in
31 * the documentation and/or other materials provided with the
32 * distribution.
33 * - Neither the name of Intel Corporation nor the names of its
34 * contributors may be used to endorse or promote products derived
35 * from this software without specific prior written permission.
36 *
37 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
38 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
39 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
40 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
41 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
42 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
43 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
44 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
45 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
46 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
47 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
48 *
49 */
50
51 /*
52 * Structure that low level drivers will populate in order to register with the
53 * rdmavt layer.
54 */
55
56 #include <linux/spinlock.h>
57 #include <linux/list.h>
58 #include <linux/hash.h>
59 #include <rdma/ib_verbs.h>
60 #include <rdma/ib_mad.h>
61 #include <rdma/rdmavt_mr.h>
62
63 #define RVT_MAX_PKEY_VALUES 16
64
65 #define RVT_MAX_TRAP_LEN 100 /* Limit pending trap list */
66 #define RVT_MAX_TRAP_LISTS 5 /*((IB_NOTICE_TYPE_INFO & 0x0F) + 1)*/
67 #define RVT_TRAP_TIMEOUT 4096 /* 4.096 usec */
68
69 struct trap_list {
70 u32 list_len;
71 struct list_head list;
72 };
73
74 struct rvt_qp;
75 struct rvt_qpn_table;
76 struct rvt_ibport {
77 struct rvt_qp __rcu *qp[2];
78 struct ib_mad_agent *send_agent; /* agent for SMI (traps) */
79 struct rb_root mcast_tree;
80 spinlock_t lock; /* protect changes in this struct */
81
82 /* non-zero when timer is set */
83 unsigned long mkey_lease_timeout;
84 unsigned long trap_timeout;
85 __be64 gid_prefix; /* in network order */
86 __be64 mkey;
87 u64 tid;
88 u32 port_cap_flags;
89 u16 port_cap3_flags;
90 u32 pma_sample_start;
91 u32 pma_sample_interval;
92 __be16 pma_counter_select[5];
93 u16 pma_tag;
94 u16 mkey_lease_period;
95 u32 sm_lid;
96 u8 sm_sl;
97 u8 mkeyprot;
98 u8 subnet_timeout;
99 u8 vl_high_limit;
100
101 /*
102 * Driver is expected to keep these up to date. These
103 * counters are informational only and not required to be
104 * completely accurate.
105 */
106 u64 n_rc_resends;
107 u64 n_seq_naks;
108 u64 n_rdma_seq;
109 u64 n_rnr_naks;
110 u64 n_other_naks;
111 u64 n_loop_pkts;
112 u64 n_pkt_drops;
113 u64 n_vl15_dropped;
114 u64 n_rc_timeouts;
115 u64 n_dmawait;
116 u64 n_unaligned;
117 u64 n_rc_dupreq;
118 u64 n_rc_seqnak;
119 u64 n_rc_crwaits;
120 u16 pkey_violations;
121 u16 qkey_violations;
122 u16 mkey_violations;
123
124 /* Hot-path per CPU counters to avoid cacheline trading to update */
125 u64 z_rc_acks;
126 u64 z_rc_qacks;
127 u64 z_rc_delayed_comp;
128 u64 __percpu *rc_acks;
129 u64 __percpu *rc_qacks;
130 u64 __percpu *rc_delayed_comp;
131
132 void *priv; /* driver private data */
133
134 /*
135 * The pkey table is allocated and maintained by the driver. Drivers
136 * need to have access to this before registering with rdmav. However
137 * rdmavt will need access to it so drivers need to proviee this during
138 * the attach port API call.
139 */
140 u16 *pkey_table;
141
142 struct rvt_ah *sm_ah;
143
144 /*
145 * Keep a list of traps that have not been repressed. They will be
146 * resent based on trap_timer.
147 */
148 struct trap_list trap_lists[RVT_MAX_TRAP_LISTS];
149 struct timer_list trap_timer;
150 };
151
152 #define RVT_CQN_MAX 16 /* maximum length of cq name */
153
154 #define RVT_SGE_COPY_MEMCPY 0
155 #define RVT_SGE_COPY_CACHELESS 1
156 #define RVT_SGE_COPY_ADAPTIVE 2
157
158 /*
159 * Things that are driver specific, module parameters in hfi1 and qib
160 */
161 struct rvt_driver_params {
162 struct ib_device_attr props;
163
164 /*
165 * Anything driver specific that is not covered by props
166 * For instance special module parameters. Goes here.
167 */
168 unsigned int lkey_table_size;
169 unsigned int qp_table_size;
170 unsigned int sge_copy_mode;
171 unsigned int wss_threshold;
172 unsigned int wss_clean_period;
173 int qpn_start;
174 int qpn_inc;
175 int qpn_res_start;
176 int qpn_res_end;
177 int nports;
178 int npkeys;
179 int node;
180 int psn_mask;
181 int psn_shift;
182 int psn_modify_mask;
183 u32 core_cap_flags;
184 u32 max_mad_size;
185 u8 qos_shift;
186 u8 max_rdma_atomic;
187 u8 extra_rdma_atomic;
188 u8 reserved_operations;
189 };
190
191 /* User context */
192 struct rvt_ucontext {
193 struct ib_ucontext ibucontext;
194 };
195
196 /* Protection domain */
197 struct rvt_pd {
198 struct ib_pd ibpd;
199 bool user;
200 };
201
202 /* Address handle */
203 struct rvt_ah {
204 struct ib_ah ibah;
205 struct rdma_ah_attr attr;
206 u8 vl;
207 u8 log_pmtu;
208 };
209
210 /*
211 * This structure is used by rvt_mmap() to validate an offset
212 * when an mmap() request is made. The vm_area_struct then uses
213 * this as its vm_private_data.
214 */
215 struct rvt_mmap_info {
216 struct list_head pending_mmaps;
217 struct ib_ucontext *context;
218 void *obj;
219 __u64 offset;
220 struct kref ref;
221 u32 size;
222 };
223
224 /* memory working set size */
225 struct rvt_wss {
226 unsigned long *entries;
227 atomic_t total_count;
228 atomic_t clean_counter;
229 atomic_t clean_entry;
230
231 int threshold;
232 int num_entries;
233 long pages_mask;
234 unsigned int clean_period;
235 };
236
237 struct rvt_dev_info;
238 struct rvt_swqe;
239 struct rvt_driver_provided {
240 /*
241 * Which functions are required depends on which verbs rdmavt is
242 * providing and which verbs the driver is overriding. See
243 * check_support() for details.
244 */
245
246 /* hot path calldowns in a single cacheline */
247
248 /*
249 * Give the driver a notice that there is send work to do. It is up to
250 * the driver to generally push the packets out, this just queues the
251 * work with the driver. There are two variants here. The no_lock
252 * version requires the s_lock not to be held. The other assumes the
253 * s_lock is held.
254 */
255 bool (*schedule_send)(struct rvt_qp *qp);
256 bool (*schedule_send_no_lock)(struct rvt_qp *qp);
257
258 /*
259 * Driver specific work request setup and checking.
260 * This function is allowed to perform any setup, checks, or
261 * adjustments required to the SWQE in order to be usable by
262 * underlying protocols. This includes private data structure
263 * allocations.
264 */
265 int (*setup_wqe)(struct rvt_qp *qp, struct rvt_swqe *wqe,
266 bool *call_send);
267
268 /*
269 * Sometimes rdmavt needs to kick the driver's send progress. That is
270 * done by this call back.
271 */
272 void (*do_send)(struct rvt_qp *qp);
273
274 /*
275 * Returns a pointer to the undelying hardware's PCI device. This is
276 * used to display information as to what hardware is being referenced
277 * in an output message
278 */
279 struct pci_dev * (*get_pci_dev)(struct rvt_dev_info *rdi);
280
281 /*
282 * Allocate a private queue pair data structure for driver specific
283 * information which is opaque to rdmavt. Errors are returned via
284 * ERR_PTR(err). The driver is free to return NULL or a valid
285 * pointer.
286 */
287 void * (*qp_priv_alloc)(struct rvt_dev_info *rdi, struct rvt_qp *qp);
288
289 /*
290 * Init a struture allocated with qp_priv_alloc(). This should be
291 * called after all qp fields have been initialized in rdmavt.
292 */
293 int (*qp_priv_init)(struct rvt_dev_info *rdi, struct rvt_qp *qp,
294 struct ib_qp_init_attr *init_attr);
295
296 /*
297 * Free the driver's private qp structure.
298 */
299 void (*qp_priv_free)(struct rvt_dev_info *rdi, struct rvt_qp *qp);
300
301 /*
302 * Inform the driver the particular qp in quesiton has been reset so
303 * that it can clean up anything it needs to.
304 */
305 void (*notify_qp_reset)(struct rvt_qp *qp);
306
307 /*
308 * Get a path mtu from the driver based on qp attributes.
309 */
310 int (*get_pmtu_from_attr)(struct rvt_dev_info *rdi, struct rvt_qp *qp,
311 struct ib_qp_attr *attr);
312
313 /*
314 * Notify driver that it needs to flush any outstanding IO requests that
315 * are waiting on a qp.
316 */
317 void (*flush_qp_waiters)(struct rvt_qp *qp);
318
319 /*
320 * Notify driver to stop its queue of sending packets. Nothing else
321 * should be posted to the queue pair after this has been called.
322 */
323 void (*stop_send_queue)(struct rvt_qp *qp);
324
325 /*
326 * Have the drivr drain any in progress operations
327 */
328 void (*quiesce_qp)(struct rvt_qp *qp);
329
330 /*
331 * Inform the driver a qp has went to error state.
332 */
333 void (*notify_error_qp)(struct rvt_qp *qp);
334
335 /*
336 * Get an MTU for a qp.
337 */
338 u32 (*mtu_from_qp)(struct rvt_dev_info *rdi, struct rvt_qp *qp,
339 u32 pmtu);
340 /*
341 * Convert an mtu to a path mtu
342 */
343 int (*mtu_to_path_mtu)(u32 mtu);
344
345 /*
346 * Get the guid of a port in big endian byte order
347 */
348 int (*get_guid_be)(struct rvt_dev_info *rdi, struct rvt_ibport *rvp,
349 int guid_index, __be64 *guid);
350
351 /*
352 * Query driver for the state of the port.
353 */
354 int (*query_port_state)(struct rvt_dev_info *rdi, u8 port_num,
355 struct ib_port_attr *props);
356
357 /*
358 * Tell driver to shutdown a port
359 */
360 int (*shut_down_port)(struct rvt_dev_info *rdi, u8 port_num);
361
362 /* Tell driver to send a trap for changed port capabilities */
363 void (*cap_mask_chg)(struct rvt_dev_info *rdi, u8 port_num);
364
365 /*
366 * The following functions can be safely ignored completely. Any use of
367 * these is checked for NULL before blindly calling. Rdmavt should also
368 * be functional if drivers omit these.
369 */
370
371 /* Called to inform the driver that all qps should now be freed. */
372 unsigned (*free_all_qps)(struct rvt_dev_info *rdi);
373
374 /* Driver specific AH validation */
375 int (*check_ah)(struct ib_device *, struct rdma_ah_attr *);
376
377 /* Inform the driver a new AH has been created */
378 void (*notify_new_ah)(struct ib_device *, struct rdma_ah_attr *,
379 struct rvt_ah *);
380
381 /* Let the driver pick the next queue pair number*/
382 int (*alloc_qpn)(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt,
383 enum ib_qp_type type, u8 port_num);
384
385 /* Determine if its safe or allowed to modify the qp */
386 int (*check_modify_qp)(struct rvt_qp *qp, struct ib_qp_attr *attr,
387 int attr_mask, struct ib_udata *udata);
388
389 /* Driver specific QP modification/notification-of */
390 void (*modify_qp)(struct rvt_qp *qp, struct ib_qp_attr *attr,
391 int attr_mask, struct ib_udata *udata);
392
393 /* Notify driver a mad agent has been created */
394 void (*notify_create_mad_agent)(struct rvt_dev_info *rdi, int port_idx);
395
396 /* Notify driver a mad agent has been removed */
397 void (*notify_free_mad_agent)(struct rvt_dev_info *rdi, int port_idx);
398
399 /* Notify driver to restart rc */
400 void (*notify_restart_rc)(struct rvt_qp *qp, u32 psn, int wait);
401
402 /* Get and return CPU to pin CQ processing thread */
403 int (*comp_vect_cpu_lookup)(struct rvt_dev_info *rdi, int comp_vect);
404 };
405
406 struct rvt_dev_info {
407 struct ib_device ibdev; /* Keep this first. Nothing above here */
408
409 /*
410 * Prior to calling for registration the driver will be responsible for
411 * allocating space for this structure.
412 *
413 * The driver will also be responsible for filling in certain members of
414 * dparms.props. The driver needs to fill in dparms exactly as it would
415 * want values reported to a ULP. This will be returned to the caller
416 * in rdmavt's device. The driver should also therefore refrain from
417 * modifying this directly after registration with rdmavt.
418 */
419
420 /* Driver specific properties */
421 struct rvt_driver_params dparms;
422
423 /* post send table */
424 const struct rvt_operation_params *post_parms;
425
426 /* opcode translation table */
427 const enum ib_wc_opcode *wc_opcode;
428
429 /* Driver specific helper functions */
430 struct rvt_driver_provided driver_f;
431
432 struct rvt_mregion __rcu *dma_mr;
433 struct rvt_lkey_table lkey_table;
434
435 /* Internal use */
436 int n_pds_allocated;
437 spinlock_t n_pds_lock; /* Protect pd allocated count */
438
439 int n_ahs_allocated;
440 spinlock_t n_ahs_lock; /* Protect ah allocated count */
441
442 u32 n_srqs_allocated;
443 spinlock_t n_srqs_lock; /* Protect srqs allocated count */
444
445 int flags;
446 struct rvt_ibport **ports;
447
448 /* QP */
449 struct rvt_qp_ibdev *qp_dev;
450 u32 n_qps_allocated; /* number of QPs allocated for device */
451 u32 n_rc_qps; /* number of RC QPs allocated for device */
452 u32 busy_jiffies; /* timeout scaling based on RC QP count */
453 spinlock_t n_qps_lock; /* protect qps, rc qps and busy jiffy counts */
454
455 /* memory maps */
456 struct list_head pending_mmaps;
457 spinlock_t mmap_offset_lock; /* protect mmap_offset */
458 u32 mmap_offset;
459 spinlock_t pending_lock; /* protect pending mmap list */
460
461 /* CQ */
462 u32 n_cqs_allocated; /* number of CQs allocated for device */
463 spinlock_t n_cqs_lock; /* protect count of in use cqs */
464
465 /* Multicast */
466 u32 n_mcast_grps_allocated; /* number of mcast groups allocated */
467 spinlock_t n_mcast_grps_lock;
468
469 /* Memory Working Set Size */
470 struct rvt_wss *wss;
471 };
472
473 /**
474 * rvt_set_ibdev_name - Craft an IB device name from client info
475 * @rdi: pointer to the client rvt_dev_info structure
476 * @name: client specific name
477 * @unit: client specific unit number.
478 */
rvt_set_ibdev_name(struct rvt_dev_info * rdi,const char * fmt,const char * name,const int unit)479 static inline void rvt_set_ibdev_name(struct rvt_dev_info *rdi,
480 const char *fmt, const char *name,
481 const int unit)
482 {
483 /*
484 * FIXME: rvt and its users want to touch the ibdev before
485 * registration and have things like the name work. We don't have the
486 * infrastructure in the core to support this directly today, hack it
487 * to work by setting the name manually here.
488 */
489 dev_set_name(&rdi->ibdev.dev, fmt, name, unit);
490 strlcpy(rdi->ibdev.name, dev_name(&rdi->ibdev.dev), IB_DEVICE_NAME_MAX);
491 }
492
493 /**
494 * rvt_get_ibdev_name - return the IB name
495 * @rdi: rdmavt device
496 *
497 * Return the registered name of the device.
498 */
rvt_get_ibdev_name(const struct rvt_dev_info * rdi)499 static inline const char *rvt_get_ibdev_name(const struct rvt_dev_info *rdi)
500 {
501 return dev_name(&rdi->ibdev.dev);
502 }
503
ibpd_to_rvtpd(struct ib_pd * ibpd)504 static inline struct rvt_pd *ibpd_to_rvtpd(struct ib_pd *ibpd)
505 {
506 return container_of(ibpd, struct rvt_pd, ibpd);
507 }
508
ibah_to_rvtah(struct ib_ah * ibah)509 static inline struct rvt_ah *ibah_to_rvtah(struct ib_ah *ibah)
510 {
511 return container_of(ibah, struct rvt_ah, ibah);
512 }
513
ib_to_rvt(struct ib_device * ibdev)514 static inline struct rvt_dev_info *ib_to_rvt(struct ib_device *ibdev)
515 {
516 return container_of(ibdev, struct rvt_dev_info, ibdev);
517 }
518
rvt_get_npkeys(struct rvt_dev_info * rdi)519 static inline unsigned rvt_get_npkeys(struct rvt_dev_info *rdi)
520 {
521 /*
522 * All ports have same number of pkeys.
523 */
524 return rdi->dparms.npkeys;
525 }
526
527 /*
528 * Return the max atomic suitable for determining
529 * the size of the ack ring buffer in a QP.
530 */
rvt_max_atomic(struct rvt_dev_info * rdi)531 static inline unsigned int rvt_max_atomic(struct rvt_dev_info *rdi)
532 {
533 return rdi->dparms.max_rdma_atomic +
534 rdi->dparms.extra_rdma_atomic + 1;
535 }
536
rvt_size_atomic(struct rvt_dev_info * rdi)537 static inline unsigned int rvt_size_atomic(struct rvt_dev_info *rdi)
538 {
539 return rdi->dparms.max_rdma_atomic +
540 rdi->dparms.extra_rdma_atomic;
541 }
542
543 /*
544 * Return the indexed PKEY from the port PKEY table.
545 */
rvt_get_pkey(struct rvt_dev_info * rdi,int port_index,unsigned index)546 static inline u16 rvt_get_pkey(struct rvt_dev_info *rdi,
547 int port_index,
548 unsigned index)
549 {
550 if (index >= rvt_get_npkeys(rdi))
551 return 0;
552 else
553 return rdi->ports[port_index]->pkey_table[index];
554 }
555
556 struct rvt_dev_info *rvt_alloc_device(size_t size, int nports);
557 void rvt_dealloc_device(struct rvt_dev_info *rdi);
558 int rvt_register_device(struct rvt_dev_info *rvd);
559 void rvt_unregister_device(struct rvt_dev_info *rvd);
560 int rvt_check_ah(struct ib_device *ibdev, struct rdma_ah_attr *ah_attr);
561 int rvt_init_port(struct rvt_dev_info *rdi, struct rvt_ibport *port,
562 int port_index, u16 *pkey_table);
563 int rvt_fast_reg_mr(struct rvt_qp *qp, struct ib_mr *ibmr, u32 key,
564 int access);
565 int rvt_invalidate_rkey(struct rvt_qp *qp, u32 rkey);
566 int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge,
567 u32 len, u64 vaddr, u32 rkey, int acc);
568 int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
569 struct rvt_sge *isge, struct rvt_sge *last_sge,
570 struct ib_sge *sge, int acc);
571 struct rvt_mcast *rvt_mcast_find(struct rvt_ibport *ibp, union ib_gid *mgid,
572 u16 lid);
573
574 #endif /* DEF_RDMA_VT_H */
575
