1 /*
2 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
3 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #include "rxe.h"
35 #include "rxe_loc.h"
36
37 /* info about object pools
38 * note that mr and mw share a single index space
39 * so that one can map an lkey to the correct type of object
40 */
41 struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
42 [RXE_TYPE_UC] = {
43 .name = "rxe-uc",
44 .size = sizeof(struct rxe_ucontext),
45 },
46 [RXE_TYPE_PD] = {
47 .name = "rxe-pd",
48 .size = sizeof(struct rxe_pd),
49 },
50 [RXE_TYPE_AH] = {
51 .name = "rxe-ah",
52 .size = sizeof(struct rxe_ah),
53 .flags = RXE_POOL_ATOMIC,
54 },
55 [RXE_TYPE_SRQ] = {
56 .name = "rxe-srq",
57 .size = sizeof(struct rxe_srq),
58 .flags = RXE_POOL_INDEX,
59 .min_index = RXE_MIN_SRQ_INDEX,
60 .max_index = RXE_MAX_SRQ_INDEX,
61 },
62 [RXE_TYPE_QP] = {
63 .name = "rxe-qp",
64 .size = sizeof(struct rxe_qp),
65 .cleanup = rxe_qp_cleanup,
66 .flags = RXE_POOL_INDEX,
67 .min_index = RXE_MIN_QP_INDEX,
68 .max_index = RXE_MAX_QP_INDEX,
69 },
70 [RXE_TYPE_CQ] = {
71 .name = "rxe-cq",
72 .size = sizeof(struct rxe_cq),
73 .cleanup = rxe_cq_cleanup,
74 },
75 [RXE_TYPE_MR] = {
76 .name = "rxe-mr",
77 .size = sizeof(struct rxe_mem),
78 .cleanup = rxe_mem_cleanup,
79 .flags = RXE_POOL_INDEX,
80 .max_index = RXE_MAX_MR_INDEX,
81 .min_index = RXE_MIN_MR_INDEX,
82 },
83 [RXE_TYPE_MW] = {
84 .name = "rxe-mw",
85 .size = sizeof(struct rxe_mem),
86 .flags = RXE_POOL_INDEX,
87 .max_index = RXE_MAX_MW_INDEX,
88 .min_index = RXE_MIN_MW_INDEX,
89 },
90 [RXE_TYPE_MC_GRP] = {
91 .name = "rxe-mc_grp",
92 .size = sizeof(struct rxe_mc_grp),
93 .cleanup = rxe_mc_cleanup,
94 .flags = RXE_POOL_KEY,
95 .key_offset = offsetof(struct rxe_mc_grp, mgid),
96 .key_size = sizeof(union ib_gid),
97 },
98 [RXE_TYPE_MC_ELEM] = {
99 .name = "rxe-mc_elem",
100 .size = sizeof(struct rxe_mc_elem),
101 .flags = RXE_POOL_ATOMIC,
102 },
103 };
104
pool_name(struct rxe_pool * pool)105 static inline const char *pool_name(struct rxe_pool *pool)
106 {
107 return rxe_type_info[pool->type].name;
108 }
109
pool_cache(struct rxe_pool * pool)110 static inline struct kmem_cache *pool_cache(struct rxe_pool *pool)
111 {
112 return rxe_type_info[pool->type].cache;
113 }
114
rxe_cache_init(void)115 int rxe_cache_init(void)
116 {
117 int err;
118 int i;
119 size_t size;
120 struct rxe_type_info *type;
121
122 for (i = 0; i < RXE_NUM_TYPES; i++) {
123 type = &rxe_type_info[i];
124 size = ALIGN(type->size, RXE_POOL_ALIGN);
125 type->cache = kmem_cache_create(type->name, size,
126 RXE_POOL_ALIGN,
127 RXE_POOL_CACHE_FLAGS, NULL);
128 if (!type->cache) {
129 pr_err("Unable to init kmem cache for %s\n",
130 type->name);
131 err = -ENOMEM;
132 goto err1;
133 }
134 }
135
136 return 0;
137
138 err1:
139 while (--i >= 0) {
140 kmem_cache_destroy(type->cache);
141 type->cache = NULL;
142 }
143
144 return err;
145 }
146
rxe_cache_exit(void)147 void rxe_cache_exit(void)
148 {
149 int i;
150 struct rxe_type_info *type;
151
152 for (i = 0; i < RXE_NUM_TYPES; i++) {
153 type = &rxe_type_info[i];
154 kmem_cache_destroy(type->cache);
155 type->cache = NULL;
156 }
157 }
158
rxe_pool_init_index(struct rxe_pool * pool,u32 max,u32 min)159 static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)
160 {
161 int err = 0;
162 size_t size;
163
164 if ((max - min + 1) < pool->max_elem) {
165 pr_warn("not enough indices for max_elem\n");
166 err = -EINVAL;
167 goto out;
168 }
169
170 pool->max_index = max;
171 pool->min_index = min;
172
173 size = BITS_TO_LONGS(max - min + 1) * sizeof(long);
174 pool->table = kmalloc(size, GFP_KERNEL);
175 if (!pool->table) {
176 err = -ENOMEM;
177 goto out;
178 }
179
180 pool->table_size = size;
181 bitmap_zero(pool->table, max - min + 1);
182
183 out:
184 return err;
185 }
186
rxe_pool_init(struct rxe_dev * rxe,struct rxe_pool * pool,enum rxe_elem_type type,unsigned int max_elem)187 int rxe_pool_init(
188 struct rxe_dev *rxe,
189 struct rxe_pool *pool,
190 enum rxe_elem_type type,
191 unsigned int max_elem)
192 {
193 int err = 0;
194 size_t size = rxe_type_info[type].size;
195
196 memset(pool, 0, sizeof(*pool));
197
198 pool->rxe = rxe;
199 pool->type = type;
200 pool->max_elem = max_elem;
201 pool->elem_size = ALIGN(size, RXE_POOL_ALIGN);
202 pool->flags = rxe_type_info[type].flags;
203 pool->tree = RB_ROOT;
204 pool->cleanup = rxe_type_info[type].cleanup;
205
206 atomic_set(&pool->num_elem, 0);
207
208 kref_init(&pool->ref_cnt);
209
210 spin_lock_init(&pool->pool_lock);
211
212 if (rxe_type_info[type].flags & RXE_POOL_INDEX) {
213 err = rxe_pool_init_index(pool,
214 rxe_type_info[type].max_index,
215 rxe_type_info[type].min_index);
216 if (err)
217 goto out;
218 }
219
220 if (rxe_type_info[type].flags & RXE_POOL_KEY) {
221 pool->key_offset = rxe_type_info[type].key_offset;
222 pool->key_size = rxe_type_info[type].key_size;
223 }
224
225 pool->state = rxe_pool_valid;
226
227 out:
228 return err;
229 }
230
rxe_pool_release(struct kref * kref)231 static void rxe_pool_release(struct kref *kref)
232 {
233 struct rxe_pool *pool = container_of(kref, struct rxe_pool, ref_cnt);
234
235 pool->state = rxe_pool_invalid;
236 kfree(pool->table);
237 }
238
rxe_pool_put(struct rxe_pool * pool)239 static void rxe_pool_put(struct rxe_pool *pool)
240 {
241 kref_put(&pool->ref_cnt, rxe_pool_release);
242 }
243
rxe_pool_cleanup(struct rxe_pool * pool)244 int rxe_pool_cleanup(struct rxe_pool *pool)
245 {
246 unsigned long flags;
247
248 spin_lock_irqsave(&pool->pool_lock, flags);
249 pool->state = rxe_pool_invalid;
250 if (atomic_read(&pool->num_elem) > 0)
251 pr_warn("%s pool destroyed with unfree'd elem\n",
252 pool_name(pool));
253 spin_unlock_irqrestore(&pool->pool_lock, flags);
254
255 rxe_pool_put(pool);
256
257 return 0;
258 }
259
alloc_index(struct rxe_pool * pool)260 static u32 alloc_index(struct rxe_pool *pool)
261 {
262 u32 index;
263 u32 range = pool->max_index - pool->min_index + 1;
264
265 index = find_next_zero_bit(pool->table, range, pool->last);
266 if (index >= range)
267 index = find_first_zero_bit(pool->table, range);
268
269 WARN_ON_ONCE(index >= range);
270 set_bit(index, pool->table);
271 pool->last = index;
272 return index + pool->min_index;
273 }
274
insert_index(struct rxe_pool * pool,struct rxe_pool_entry * new)275 static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
276 {
277 struct rb_node **link = &pool->tree.rb_node;
278 struct rb_node *parent = NULL;
279 struct rxe_pool_entry *elem;
280
281 while (*link) {
282 parent = *link;
283 elem = rb_entry(parent, struct rxe_pool_entry, node);
284
285 if (elem->index == new->index) {
286 pr_warn("element already exists!\n");
287 goto out;
288 }
289
290 if (elem->index > new->index)
291 link = &(*link)->rb_left;
292 else
293 link = &(*link)->rb_right;
294 }
295
296 rb_link_node(&new->node, parent, link);
297 rb_insert_color(&new->node, &pool->tree);
298 out:
299 return;
300 }
301
insert_key(struct rxe_pool * pool,struct rxe_pool_entry * new)302 static void insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)
303 {
304 struct rb_node **link = &pool->tree.rb_node;
305 struct rb_node *parent = NULL;
306 struct rxe_pool_entry *elem;
307 int cmp;
308
309 while (*link) {
310 parent = *link;
311 elem = rb_entry(parent, struct rxe_pool_entry, node);
312
313 cmp = memcmp((u8 *)elem + pool->key_offset,
314 (u8 *)new + pool->key_offset, pool->key_size);
315
316 if (cmp == 0) {
317 pr_warn("key already exists!\n");
318 goto out;
319 }
320
321 if (cmp > 0)
322 link = &(*link)->rb_left;
323 else
324 link = &(*link)->rb_right;
325 }
326
327 rb_link_node(&new->node, parent, link);
328 rb_insert_color(&new->node, &pool->tree);
329 out:
330 return;
331 }
332
rxe_add_key(void * arg,void * key)333 void rxe_add_key(void *arg, void *key)
334 {
335 struct rxe_pool_entry *elem = arg;
336 struct rxe_pool *pool = elem->pool;
337 unsigned long flags;
338
339 spin_lock_irqsave(&pool->pool_lock, flags);
340 memcpy((u8 *)elem + pool->key_offset, key, pool->key_size);
341 insert_key(pool, elem);
342 spin_unlock_irqrestore(&pool->pool_lock, flags);
343 }
344
rxe_drop_key(void * arg)345 void rxe_drop_key(void *arg)
346 {
347 struct rxe_pool_entry *elem = arg;
348 struct rxe_pool *pool = elem->pool;
349 unsigned long flags;
350
351 spin_lock_irqsave(&pool->pool_lock, flags);
352 rb_erase(&elem->node, &pool->tree);
353 spin_unlock_irqrestore(&pool->pool_lock, flags);
354 }
355
rxe_add_index(void * arg)356 void rxe_add_index(void *arg)
357 {
358 struct rxe_pool_entry *elem = arg;
359 struct rxe_pool *pool = elem->pool;
360 unsigned long flags;
361
362 spin_lock_irqsave(&pool->pool_lock, flags);
363 elem->index = alloc_index(pool);
364 insert_index(pool, elem);
365 spin_unlock_irqrestore(&pool->pool_lock, flags);
366 }
367
rxe_drop_index(void * arg)368 void rxe_drop_index(void *arg)
369 {
370 struct rxe_pool_entry *elem = arg;
371 struct rxe_pool *pool = elem->pool;
372 unsigned long flags;
373
374 spin_lock_irqsave(&pool->pool_lock, flags);
375 clear_bit(elem->index - pool->min_index, pool->table);
376 rb_erase(&elem->node, &pool->tree);
377 spin_unlock_irqrestore(&pool->pool_lock, flags);
378 }
379
rxe_alloc(struct rxe_pool * pool)380 void *rxe_alloc(struct rxe_pool *pool)
381 {
382 struct rxe_pool_entry *elem;
383 unsigned long flags;
384
385 might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
386
387 spin_lock_irqsave(&pool->pool_lock, flags);
388 if (pool->state != rxe_pool_valid) {
389 spin_unlock_irqrestore(&pool->pool_lock, flags);
390 return NULL;
391 }
392 kref_get(&pool->ref_cnt);
393 spin_unlock_irqrestore(&pool->pool_lock, flags);
394
395 kref_get(&pool->rxe->ref_cnt);
396
397 if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
398 goto out_put_pool;
399
400 elem = kmem_cache_zalloc(pool_cache(pool),
401 (pool->flags & RXE_POOL_ATOMIC) ?
402 GFP_ATOMIC : GFP_KERNEL);
403 if (!elem)
404 goto out_put_pool;
405
406 elem->pool = pool;
407 kref_init(&elem->ref_cnt);
408
409 return elem;
410
411 out_put_pool:
412 atomic_dec(&pool->num_elem);
413 rxe_dev_put(pool->rxe);
414 rxe_pool_put(pool);
415 return NULL;
416 }
417
rxe_elem_release(struct kref * kref)418 void rxe_elem_release(struct kref *kref)
419 {
420 struct rxe_pool_entry *elem =
421 container_of(kref, struct rxe_pool_entry, ref_cnt);
422 struct rxe_pool *pool = elem->pool;
423
424 if (pool->cleanup)
425 pool->cleanup(elem);
426
427 kmem_cache_free(pool_cache(pool), elem);
428 atomic_dec(&pool->num_elem);
429 rxe_dev_put(pool->rxe);
430 rxe_pool_put(pool);
431 }
432
rxe_pool_get_index(struct rxe_pool * pool,u32 index)433 void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
434 {
435 struct rb_node *node = NULL;
436 struct rxe_pool_entry *elem = NULL;
437 unsigned long flags;
438
439 spin_lock_irqsave(&pool->pool_lock, flags);
440
441 if (pool->state != rxe_pool_valid)
442 goto out;
443
444 node = pool->tree.rb_node;
445
446 while (node) {
447 elem = rb_entry(node, struct rxe_pool_entry, node);
448
449 if (elem->index > index)
450 node = node->rb_left;
451 else if (elem->index < index)
452 node = node->rb_right;
453 else
454 break;
455 }
456
457 if (node)
458 kref_get(&elem->ref_cnt);
459
460 out:
461 spin_unlock_irqrestore(&pool->pool_lock, flags);
462 return node ? elem : NULL;
463 }
464
rxe_pool_get_key(struct rxe_pool * pool,void * key)465 void *rxe_pool_get_key(struct rxe_pool *pool, void *key)
466 {
467 struct rb_node *node = NULL;
468 struct rxe_pool_entry *elem = NULL;
469 int cmp;
470 unsigned long flags;
471
472 spin_lock_irqsave(&pool->pool_lock, flags);
473
474 if (pool->state != rxe_pool_valid)
475 goto out;
476
477 node = pool->tree.rb_node;
478
479 while (node) {
480 elem = rb_entry(node, struct rxe_pool_entry, node);
481
482 cmp = memcmp((u8 *)elem + pool->key_offset,
483 key, pool->key_size);
484
485 if (cmp > 0)
486 node = node->rb_left;
487 else if (cmp < 0)
488 node = node->rb_right;
489 else
490 break;
491 }
492
493 if (node)
494 kref_get(&elem->ref_cnt);
495
496 out:
497 spin_unlock_irqrestore(&pool->pool_lock, flags);
498 return node ? elem : NULL;
499 }
500
