1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2019 Mellanox Technologies. */
3 
4 #include "dr_types.h"
5 
6 #define DR_ICM_MODIFY_HDR_ALIGN_BASE 64
7 #define DR_ICM_SYNC_THRESHOLD (64 * 1024 * 1024)
8 
9 struct mlx5dr_icm_pool;
10 
11 struct mlx5dr_icm_bucket {
12 	struct mlx5dr_icm_pool *pool;
13 
14 	/* Chunks that aren't visible to HW not directly and not in cache */
15 	struct list_head free_list;
16 	unsigned int free_list_count;
17 
18 	/* Used chunks, HW may be accessing this memory */
19 	struct list_head used_list;
20 	unsigned int used_list_count;
21 
22 	/* HW may be accessing this memory but at some future,
23 	 * undetermined time, it might cease to do so. Before deciding to call
24 	 * sync_ste, this list is moved to sync_list
25 	 */
26 	struct list_head hot_list;
27 	unsigned int hot_list_count;
28 
29 	/* Pending sync list, entries from the hot list are moved to this list.
30 	 * sync_ste is executed and then sync_list is concatenated to the free list
31 	 */
32 	struct list_head sync_list;
33 	unsigned int sync_list_count;
34 
35 	u32 total_chunks;
36 	u32 num_of_entries;
37 	u32 entry_size;
38 	/* protect the ICM bucket */
39 	struct mutex mutex;
40 };
41 
42 struct mlx5dr_icm_pool {
43 	struct mlx5dr_icm_bucket *buckets;
44 	enum mlx5dr_icm_type icm_type;
45 	enum mlx5dr_icm_chunk_size max_log_chunk_sz;
46 	enum mlx5dr_icm_chunk_size num_of_buckets;
47 	struct list_head icm_mr_list;
48 	/* protect the ICM MR list */
49 	struct mutex mr_mutex;
50 	struct mlx5dr_domain *dmn;
51 };
52 
53 struct mlx5dr_icm_dm {
54 	u32 obj_id;
55 	enum mlx5_sw_icm_type type;
56 	phys_addr_t addr;
57 	size_t length;
58 };
59 
60 struct mlx5dr_icm_mr {
61 	struct mlx5dr_icm_pool *pool;
62 	struct mlx5_core_mkey mkey;
63 	struct mlx5dr_icm_dm dm;
64 	size_t used_length;
65 	size_t length;
66 	u64 icm_start_addr;
67 	struct list_head mr_list;
68 };
69 
dr_icm_create_dm_mkey(struct mlx5_core_dev * mdev,u32 pd,u64 length,u64 start_addr,int mode,struct mlx5_core_mkey * mkey)70 static int dr_icm_create_dm_mkey(struct mlx5_core_dev *mdev,
71 				 u32 pd, u64 length, u64 start_addr, int mode,
72 				 struct mlx5_core_mkey *mkey)
73 {
74 	u32 inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
75 	u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {};
76 	void *mkc;
77 
78 	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
79 
80 	MLX5_SET(mkc, mkc, access_mode_1_0, mode);
81 	MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7);
82 	MLX5_SET(mkc, mkc, lw, 1);
83 	MLX5_SET(mkc, mkc, lr, 1);
84 	if (mode == MLX5_MKC_ACCESS_MODE_SW_ICM) {
85 		MLX5_SET(mkc, mkc, rw, 1);
86 		MLX5_SET(mkc, mkc, rr, 1);
87 	}
88 
89 	MLX5_SET64(mkc, mkc, len, length);
90 	MLX5_SET(mkc, mkc, pd, pd);
91 	MLX5_SET(mkc, mkc, qpn, 0xffffff);
92 	MLX5_SET64(mkc, mkc, start_addr, start_addr);
93 
94 	return mlx5_core_create_mkey(mdev, mkey, in, inlen);
95 }
96 
97 static struct mlx5dr_icm_mr *
dr_icm_pool_mr_create(struct mlx5dr_icm_pool * pool,enum mlx5_sw_icm_type type,size_t align_base)98 dr_icm_pool_mr_create(struct mlx5dr_icm_pool *pool,
99 		      enum mlx5_sw_icm_type type,
100 		      size_t align_base)
101 {
102 	struct mlx5_core_dev *mdev = pool->dmn->mdev;
103 	struct mlx5dr_icm_mr *icm_mr;
104 	size_t align_diff;
105 	int err;
106 
107 	icm_mr = kvzalloc(sizeof(*icm_mr), GFP_KERNEL);
108 	if (!icm_mr)
109 		return NULL;
110 
111 	icm_mr->pool = pool;
112 	INIT_LIST_HEAD(&icm_mr->mr_list);
113 
114 	icm_mr->dm.type = type;
115 
116 	/* 2^log_biggest_table * entry-size * double-for-alignment */
117 	icm_mr->dm.length = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
118 							       pool->icm_type) * 2;
119 
120 	err = mlx5_dm_sw_icm_alloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 0,
121 				   &icm_mr->dm.addr, &icm_mr->dm.obj_id);
122 	if (err) {
123 		mlx5dr_err(pool->dmn, "Failed to allocate SW ICM memory, err (%d)\n", err);
124 		goto free_icm_mr;
125 	}
126 
127 	/* Register device memory */
128 	err = dr_icm_create_dm_mkey(mdev, pool->dmn->pdn,
129 				    icm_mr->dm.length,
130 				    icm_mr->dm.addr,
131 				    MLX5_MKC_ACCESS_MODE_SW_ICM,
132 				    &icm_mr->mkey);
133 	if (err) {
134 		mlx5dr_err(pool->dmn, "Failed to create SW ICM MKEY, err (%d)\n", err);
135 		goto free_dm;
136 	}
137 
138 	icm_mr->icm_start_addr = icm_mr->dm.addr;
139 
140 	/* align_base is always a power of 2 */
141 	align_diff = icm_mr->icm_start_addr & (align_base - 1);
142 	if (align_diff)
143 		icm_mr->used_length = align_base - align_diff;
144 
145 	list_add_tail(&icm_mr->mr_list, &pool->icm_mr_list);
146 
147 	return icm_mr;
148 
149 free_dm:
150 	mlx5_dm_sw_icm_dealloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 0,
151 			       icm_mr->dm.addr, icm_mr->dm.obj_id);
152 free_icm_mr:
153 	kvfree(icm_mr);
154 	return NULL;
155 }
156 
dr_icm_pool_mr_destroy(struct mlx5dr_icm_mr * icm_mr)157 static void dr_icm_pool_mr_destroy(struct mlx5dr_icm_mr *icm_mr)
158 {
159 	struct mlx5_core_dev *mdev = icm_mr->pool->dmn->mdev;
160 	struct mlx5dr_icm_dm *dm = &icm_mr->dm;
161 
162 	list_del(&icm_mr->mr_list);
163 	mlx5_core_destroy_mkey(mdev, &icm_mr->mkey);
164 	mlx5_dm_sw_icm_dealloc(mdev, dm->type, dm->length, 0,
165 			       dm->addr, dm->obj_id);
166 	kvfree(icm_mr);
167 }
168 
dr_icm_chunk_ste_init(struct mlx5dr_icm_chunk * chunk)169 static int dr_icm_chunk_ste_init(struct mlx5dr_icm_chunk *chunk)
170 {
171 	struct mlx5dr_icm_bucket *bucket = chunk->bucket;
172 
173 	chunk->ste_arr = kvzalloc(bucket->num_of_entries *
174 				  sizeof(chunk->ste_arr[0]), GFP_KERNEL);
175 	if (!chunk->ste_arr)
176 		return -ENOMEM;
177 
178 	chunk->hw_ste_arr = kvzalloc(bucket->num_of_entries *
179 				     DR_STE_SIZE_REDUCED, GFP_KERNEL);
180 	if (!chunk->hw_ste_arr)
181 		goto out_free_ste_arr;
182 
183 	chunk->miss_list = kvmalloc(bucket->num_of_entries *
184 				    sizeof(chunk->miss_list[0]), GFP_KERNEL);
185 	if (!chunk->miss_list)
186 		goto out_free_hw_ste_arr;
187 
188 	return 0;
189 
190 out_free_hw_ste_arr:
191 	kvfree(chunk->hw_ste_arr);
192 out_free_ste_arr:
193 	kvfree(chunk->ste_arr);
194 	return -ENOMEM;
195 }
196 
dr_icm_chunks_create(struct mlx5dr_icm_bucket * bucket)197 static int dr_icm_chunks_create(struct mlx5dr_icm_bucket *bucket)
198 {
199 	size_t mr_free_size, mr_req_size, mr_row_size;
200 	struct mlx5dr_icm_pool *pool = bucket->pool;
201 	struct mlx5dr_icm_mr *icm_mr = NULL;
202 	struct mlx5dr_icm_chunk *chunk;
203 	enum mlx5_sw_icm_type dm_type;
204 	size_t align_base;
205 	int i, err = 0;
206 
207 	mr_req_size = bucket->num_of_entries * bucket->entry_size;
208 	mr_row_size = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz,
209 							 pool->icm_type);
210 
211 	if (pool->icm_type == DR_ICM_TYPE_STE) {
212 		dm_type = MLX5_SW_ICM_TYPE_STEERING;
213 		/* Align base is the biggest chunk size / row size */
214 		align_base = mr_row_size;
215 	} else {
216 		dm_type = MLX5_SW_ICM_TYPE_HEADER_MODIFY;
217 		/* Align base is 64B */
218 		align_base = DR_ICM_MODIFY_HDR_ALIGN_BASE;
219 	}
220 
221 	mutex_lock(&pool->mr_mutex);
222 	if (!list_empty(&pool->icm_mr_list)) {
223 		icm_mr = list_last_entry(&pool->icm_mr_list,
224 					 struct mlx5dr_icm_mr, mr_list);
225 
226 		if (icm_mr)
227 			mr_free_size = icm_mr->dm.length - icm_mr->used_length;
228 	}
229 
230 	if (!icm_mr || mr_free_size < mr_row_size) {
231 		icm_mr = dr_icm_pool_mr_create(pool, dm_type, align_base);
232 		if (!icm_mr) {
233 			err = -ENOMEM;
234 			goto out_err;
235 		}
236 	}
237 
238 	/* Create memory aligned chunks */
239 	for (i = 0; i < mr_row_size / mr_req_size; i++) {
240 		chunk = kvzalloc(sizeof(*chunk), GFP_KERNEL);
241 		if (!chunk) {
242 			err = -ENOMEM;
243 			goto out_err;
244 		}
245 
246 		chunk->bucket = bucket;
247 		chunk->rkey = icm_mr->mkey.key;
248 		/* mr start addr is zero based */
249 		chunk->mr_addr = icm_mr->used_length;
250 		chunk->icm_addr = (uintptr_t)icm_mr->icm_start_addr + icm_mr->used_length;
251 		icm_mr->used_length += mr_req_size;
252 		chunk->num_of_entries = bucket->num_of_entries;
253 		chunk->byte_size = chunk->num_of_entries * bucket->entry_size;
254 
255 		if (pool->icm_type == DR_ICM_TYPE_STE) {
256 			err = dr_icm_chunk_ste_init(chunk);
257 			if (err)
258 				goto out_free_chunk;
259 		}
260 
261 		INIT_LIST_HEAD(&chunk->chunk_list);
262 		list_add(&chunk->chunk_list, &bucket->free_list);
263 		bucket->free_list_count++;
264 		bucket->total_chunks++;
265 	}
266 	mutex_unlock(&pool->mr_mutex);
267 	return 0;
268 
269 out_free_chunk:
270 	kvfree(chunk);
271 out_err:
272 	mutex_unlock(&pool->mr_mutex);
273 	return err;
274 }
275 
dr_icm_chunk_ste_cleanup(struct mlx5dr_icm_chunk * chunk)276 static void dr_icm_chunk_ste_cleanup(struct mlx5dr_icm_chunk *chunk)
277 {
278 	kvfree(chunk->miss_list);
279 	kvfree(chunk->hw_ste_arr);
280 	kvfree(chunk->ste_arr);
281 }
282 
dr_icm_chunk_destroy(struct mlx5dr_icm_chunk * chunk)283 static void dr_icm_chunk_destroy(struct mlx5dr_icm_chunk *chunk)
284 {
285 	struct mlx5dr_icm_bucket *bucket = chunk->bucket;
286 
287 	list_del(&chunk->chunk_list);
288 	bucket->total_chunks--;
289 
290 	if (bucket->pool->icm_type == DR_ICM_TYPE_STE)
291 		dr_icm_chunk_ste_cleanup(chunk);
292 
293 	kvfree(chunk);
294 }
295 
dr_icm_bucket_init(struct mlx5dr_icm_pool * pool,struct mlx5dr_icm_bucket * bucket,enum mlx5dr_icm_chunk_size chunk_size)296 static void dr_icm_bucket_init(struct mlx5dr_icm_pool *pool,
297 			       struct mlx5dr_icm_bucket *bucket,
298 			       enum mlx5dr_icm_chunk_size chunk_size)
299 {
300 	if (pool->icm_type == DR_ICM_TYPE_STE)
301 		bucket->entry_size = DR_STE_SIZE;
302 	else
303 		bucket->entry_size = DR_MODIFY_ACTION_SIZE;
304 
305 	bucket->num_of_entries = mlx5dr_icm_pool_chunk_size_to_entries(chunk_size);
306 	bucket->pool = pool;
307 	mutex_init(&bucket->mutex);
308 	INIT_LIST_HEAD(&bucket->free_list);
309 	INIT_LIST_HEAD(&bucket->used_list);
310 	INIT_LIST_HEAD(&bucket->hot_list);
311 	INIT_LIST_HEAD(&bucket->sync_list);
312 }
313 
dr_icm_bucket_cleanup(struct mlx5dr_icm_bucket * bucket)314 static void dr_icm_bucket_cleanup(struct mlx5dr_icm_bucket *bucket)
315 {
316 	struct mlx5dr_icm_chunk *chunk, *next;
317 
318 	mutex_destroy(&bucket->mutex);
319 	list_splice_tail_init(&bucket->sync_list, &bucket->free_list);
320 	list_splice_tail_init(&bucket->hot_list, &bucket->free_list);
321 
322 	list_for_each_entry_safe(chunk, next, &bucket->free_list, chunk_list)
323 		dr_icm_chunk_destroy(chunk);
324 
325 	WARN_ON(bucket->total_chunks != 0);
326 
327 	/* Cleanup of unreturned chunks */
328 	list_for_each_entry_safe(chunk, next, &bucket->used_list, chunk_list)
329 		dr_icm_chunk_destroy(chunk);
330 }
331 
dr_icm_hot_mem_size(struct mlx5dr_icm_pool * pool)332 static u64 dr_icm_hot_mem_size(struct mlx5dr_icm_pool *pool)
333 {
334 	u64 hot_size = 0;
335 	int chunk_order;
336 
337 	for (chunk_order = 0; chunk_order < pool->num_of_buckets; chunk_order++)
338 		hot_size += pool->buckets[chunk_order].hot_list_count *
339 			    mlx5dr_icm_pool_chunk_size_to_byte(chunk_order, pool->icm_type);
340 
341 	return hot_size;
342 }
343 
dr_icm_reuse_hot_entries(struct mlx5dr_icm_pool * pool,struct mlx5dr_icm_bucket * bucket)344 static bool dr_icm_reuse_hot_entries(struct mlx5dr_icm_pool *pool,
345 				     struct mlx5dr_icm_bucket *bucket)
346 {
347 	u64 bytes_for_sync;
348 
349 	bytes_for_sync = dr_icm_hot_mem_size(pool);
350 	if (bytes_for_sync < DR_ICM_SYNC_THRESHOLD || !bucket->hot_list_count)
351 		return false;
352 
353 	return true;
354 }
355 
dr_icm_chill_bucket_start(struct mlx5dr_icm_bucket * bucket)356 static void dr_icm_chill_bucket_start(struct mlx5dr_icm_bucket *bucket)
357 {
358 	list_splice_tail_init(&bucket->hot_list, &bucket->sync_list);
359 	bucket->sync_list_count += bucket->hot_list_count;
360 	bucket->hot_list_count = 0;
361 }
362 
dr_icm_chill_bucket_end(struct mlx5dr_icm_bucket * bucket)363 static void dr_icm_chill_bucket_end(struct mlx5dr_icm_bucket *bucket)
364 {
365 	list_splice_tail_init(&bucket->sync_list, &bucket->free_list);
366 	bucket->free_list_count += bucket->sync_list_count;
367 	bucket->sync_list_count = 0;
368 }
369 
dr_icm_chill_bucket_abort(struct mlx5dr_icm_bucket * bucket)370 static void dr_icm_chill_bucket_abort(struct mlx5dr_icm_bucket *bucket)
371 {
372 	list_splice_tail_init(&bucket->sync_list, &bucket->hot_list);
373 	bucket->hot_list_count += bucket->sync_list_count;
374 	bucket->sync_list_count = 0;
375 }
376 
dr_icm_chill_buckets_start(struct mlx5dr_icm_pool * pool,struct mlx5dr_icm_bucket * cb,bool buckets[DR_CHUNK_SIZE_MAX])377 static void dr_icm_chill_buckets_start(struct mlx5dr_icm_pool *pool,
378 				       struct mlx5dr_icm_bucket *cb,
379 				       bool buckets[DR_CHUNK_SIZE_MAX])
380 {
381 	struct mlx5dr_icm_bucket *bucket;
382 	int i;
383 
384 	for (i = 0; i < pool->num_of_buckets; i++) {
385 		bucket = &pool->buckets[i];
386 		if (bucket == cb) {
387 			dr_icm_chill_bucket_start(bucket);
388 			continue;
389 		}
390 
391 		/* Freeing the mutex is done at the end of that process, after
392 		 * sync_ste was executed at dr_icm_chill_buckets_end func.
393 		 */
394 		if (mutex_trylock(&bucket->mutex)) {
395 			dr_icm_chill_bucket_start(bucket);
396 			buckets[i] = true;
397 		}
398 	}
399 }
400 
dr_icm_chill_buckets_end(struct mlx5dr_icm_pool * pool,struct mlx5dr_icm_bucket * cb,bool buckets[DR_CHUNK_SIZE_MAX])401 static void dr_icm_chill_buckets_end(struct mlx5dr_icm_pool *pool,
402 				     struct mlx5dr_icm_bucket *cb,
403 				     bool buckets[DR_CHUNK_SIZE_MAX])
404 {
405 	struct mlx5dr_icm_bucket *bucket;
406 	int i;
407 
408 	for (i = 0; i < pool->num_of_buckets; i++) {
409 		bucket = &pool->buckets[i];
410 		if (bucket == cb) {
411 			dr_icm_chill_bucket_end(bucket);
412 			continue;
413 		}
414 
415 		if (!buckets[i])
416 			continue;
417 
418 		dr_icm_chill_bucket_end(bucket);
419 		mutex_unlock(&bucket->mutex);
420 	}
421 }
422 
dr_icm_chill_buckets_abort(struct mlx5dr_icm_pool * pool,struct mlx5dr_icm_bucket * cb,bool buckets[DR_CHUNK_SIZE_MAX])423 static void dr_icm_chill_buckets_abort(struct mlx5dr_icm_pool *pool,
424 				       struct mlx5dr_icm_bucket *cb,
425 				       bool buckets[DR_CHUNK_SIZE_MAX])
426 {
427 	struct mlx5dr_icm_bucket *bucket;
428 	int i;
429 
430 	for (i = 0; i < pool->num_of_buckets; i++) {
431 		bucket = &pool->buckets[i];
432 		if (bucket == cb) {
433 			dr_icm_chill_bucket_abort(bucket);
434 			continue;
435 		}
436 
437 		if (!buckets[i])
438 			continue;
439 
440 		dr_icm_chill_bucket_abort(bucket);
441 		mutex_unlock(&bucket->mutex);
442 	}
443 }
444 
445 /* Allocate an ICM chunk, each chunk holds a piece of ICM memory and
446  * also memory used for HW STE management for optimizations.
447  */
448 struct mlx5dr_icm_chunk *
mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool * pool,enum mlx5dr_icm_chunk_size chunk_size)449 mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool,
450 		       enum mlx5dr_icm_chunk_size chunk_size)
451 {
452 	struct mlx5dr_icm_chunk *chunk = NULL; /* Fix compilation warning */
453 	bool buckets[DR_CHUNK_SIZE_MAX] = {};
454 	struct mlx5dr_icm_bucket *bucket;
455 	int err;
456 
457 	if (chunk_size > pool->max_log_chunk_sz)
458 		return NULL;
459 
460 	bucket = &pool->buckets[chunk_size];
461 
462 	mutex_lock(&bucket->mutex);
463 
464 	/* Take chunk from pool if available, otherwise allocate new chunks */
465 	if (list_empty(&bucket->free_list)) {
466 		if (dr_icm_reuse_hot_entries(pool, bucket)) {
467 			dr_icm_chill_buckets_start(pool, bucket, buckets);
468 			err = mlx5dr_cmd_sync_steering(pool->dmn->mdev);
469 			if (err) {
470 				dr_icm_chill_buckets_abort(pool, bucket, buckets);
471 				mlx5dr_dbg(pool->dmn, "Sync_steering failed\n");
472 				chunk = NULL;
473 				goto out;
474 			}
475 			dr_icm_chill_buckets_end(pool, bucket, buckets);
476 		} else {
477 			dr_icm_chunks_create(bucket);
478 		}
479 	}
480 
481 	if (!list_empty(&bucket->free_list)) {
482 		chunk = list_last_entry(&bucket->free_list,
483 					struct mlx5dr_icm_chunk,
484 					chunk_list);
485 		if (chunk) {
486 			list_del_init(&chunk->chunk_list);
487 			list_add_tail(&chunk->chunk_list, &bucket->used_list);
488 			bucket->free_list_count--;
489 			bucket->used_list_count++;
490 		}
491 	}
492 out:
493 	mutex_unlock(&bucket->mutex);
494 	return chunk;
495 }
496 
mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk * chunk)497 void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk)
498 {
499 	struct mlx5dr_icm_bucket *bucket = chunk->bucket;
500 
501 	if (bucket->pool->icm_type == DR_ICM_TYPE_STE) {
502 		memset(chunk->ste_arr, 0,
503 		       bucket->num_of_entries * sizeof(chunk->ste_arr[0]));
504 		memset(chunk->hw_ste_arr, 0,
505 		       bucket->num_of_entries * DR_STE_SIZE_REDUCED);
506 	}
507 
508 	mutex_lock(&bucket->mutex);
509 	list_del_init(&chunk->chunk_list);
510 	list_add_tail(&chunk->chunk_list, &bucket->hot_list);
511 	bucket->hot_list_count++;
512 	bucket->used_list_count--;
513 	mutex_unlock(&bucket->mutex);
514 }
515 
mlx5dr_icm_pool_create(struct mlx5dr_domain * dmn,enum mlx5dr_icm_type icm_type)516 struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn,
517 					       enum mlx5dr_icm_type icm_type)
518 {
519 	enum mlx5dr_icm_chunk_size max_log_chunk_sz;
520 	struct mlx5dr_icm_pool *pool;
521 	int i;
522 
523 	if (icm_type == DR_ICM_TYPE_STE)
524 		max_log_chunk_sz = dmn->info.max_log_sw_icm_sz;
525 	else
526 		max_log_chunk_sz = dmn->info.max_log_action_icm_sz;
527 
528 	pool = kvzalloc(sizeof(*pool), GFP_KERNEL);
529 	if (!pool)
530 		return NULL;
531 
532 	pool->buckets = kcalloc(max_log_chunk_sz + 1,
533 				sizeof(pool->buckets[0]),
534 				GFP_KERNEL);
535 	if (!pool->buckets)
536 		goto free_pool;
537 
538 	pool->dmn = dmn;
539 	pool->icm_type = icm_type;
540 	pool->max_log_chunk_sz = max_log_chunk_sz;
541 	pool->num_of_buckets = max_log_chunk_sz + 1;
542 	INIT_LIST_HEAD(&pool->icm_mr_list);
543 
544 	for (i = 0; i < pool->num_of_buckets; i++)
545 		dr_icm_bucket_init(pool, &pool->buckets[i], i);
546 
547 	mutex_init(&pool->mr_mutex);
548 
549 	return pool;
550 
551 free_pool:
552 	kvfree(pool);
553 	return NULL;
554 }
555 
mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool * pool)556 void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool)
557 {
558 	struct mlx5dr_icm_mr *icm_mr, *next;
559 	int i;
560 
561 	mutex_destroy(&pool->mr_mutex);
562 
563 	list_for_each_entry_safe(icm_mr, next, &pool->icm_mr_list, mr_list)
564 		dr_icm_pool_mr_destroy(icm_mr);
565 
566 	for (i = 0; i < pool->num_of_buckets; i++)
567 		dr_icm_bucket_cleanup(&pool->buckets[i]);
568 
569 	kfree(pool->buckets);
570 	kvfree(pool);
571 }
572