14 static void cb_fini(struct hl_device *hdev, struct hl_cb *cb)  in cb_fini()  argument
16 	hdev->asic_funcs->asic_dma_free_coherent(hdev, cb->size,  in cb_fini()
17 			(void *) (uintptr_t) cb->kernel_address,  in cb_fini()
18 			cb->bus_address);  in cb_fini()
19 	kfree(cb);  in cb_fini()
22 static void cb_do_release(struct hl_device *hdev, struct hl_cb *cb)  in cb_do_release()  argument
24 	if (cb->is_pool) {  in cb_do_release()
26 		list_add(&cb->pool_list, &hdev->cb_pool);  in cb_do_release()
29 		cb_fini(hdev, cb);  in cb_do_release()
36 	struct hl_cb *cb;  in cb_release()  local
38 	cb = container_of(ref, struct hl_cb, refcount);  in cb_release()
39 	hdev = cb->hdev;  in cb_release()
41 	hl_debugfs_remove_cb(cb);  in cb_release()
43 	cb_do_release(hdev, cb);  in cb_release()
49 	struct hl_cb *cb;  in hl_cb_alloc()  local
61 		cb = kzalloc(sizeof(*cb), GFP_ATOMIC);  in hl_cb_alloc()
63 		cb = kzalloc(sizeof(*cb), GFP_KERNEL);  in hl_cb_alloc()
65 	if (!cb)  in hl_cb_alloc()
70 						&cb->bus_address, GFP_ATOMIC);  in hl_cb_alloc()
73 						&cb->bus_address,  in hl_cb_alloc()
79 		kfree(cb);  in hl_cb_alloc()
83 	cb->kernel_address = (u64) (uintptr_t) p;  in hl_cb_alloc()
84 	cb->size = cb_size;  in hl_cb_alloc()
86 	return cb;  in hl_cb_alloc()
92 	struct hl_cb *cb;  in hl_cb_create()  local
125 			cb = list_first_entry(&hdev->cb_pool, typeof(*cb),  in hl_cb_create()
127 			list_del(&cb->pool_list);  in hl_cb_create()
137 		cb = hl_cb_alloc(hdev, cb_size, ctx_id);  in hl_cb_create()
138 		if (!cb) {  in hl_cb_create()
144 	cb->hdev = hdev;  in hl_cb_create()
145 	cb->ctx_id = ctx_id;  in hl_cb_create()
148 	rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);  in hl_cb_create()
156 	cb->id = rc;  in hl_cb_create()
158 	kref_init(&cb->refcount);  in hl_cb_create()
159 	spin_lock_init(&cb->lock);  in hl_cb_create()
165 	*handle = cb->id | HL_MMAP_CB_MASK;  in hl_cb_create()
168 	hl_debugfs_add_cb(cb);  in hl_cb_create()
173 	cb_do_release(hdev, cb);  in hl_cb_create()
182 	struct hl_cb *cb;  in hl_cb_destroy()  local
195 	cb = idr_find(&mgr->cb_handles, handle);  in hl_cb_destroy()
196 	if (cb) {  in hl_cb_destroy()
199 		kref_put(&cb->refcount, cb_release);  in hl_cb_destroy()
245 	struct hl_cb *cb = (struct hl_cb *) vma->vm_private_data;  in cb_vm_close()  local
248 	new_mmap_size = cb->mmap_size - (vma->vm_end - vma->vm_start);  in cb_vm_close()
251 		cb->mmap_size = new_mmap_size;  in cb_vm_close()
255 	spin_lock(&cb->lock);  in cb_vm_close()
256 	cb->mmap = false;  in cb_vm_close()
257 	spin_unlock(&cb->lock);  in cb_vm_close()
259 	hl_cb_put(cb);  in cb_vm_close()
270 	struct hl_cb *cb;  in hl_cb_mmap()  local
278 	cb = hl_cb_get(hdev, &hpriv->cb_mgr, handle);  in hl_cb_mmap()
279 	if (!cb) {  in hl_cb_mmap()
286 	if ((vma->vm_end - vma->vm_start) != ALIGN(cb->size, PAGE_SIZE)) {  in hl_cb_mmap()
289 			vma->vm_end - vma->vm_start, cb->size);  in hl_cb_mmap()
294 	spin_lock(&cb->lock);  in hl_cb_mmap()
296 	if (cb->mmap) {  in hl_cb_mmap()
303 	cb->mmap = true;  in hl_cb_mmap()
305 	spin_unlock(&cb->lock);  in hl_cb_mmap()
314 	vma->vm_private_data = cb;  in hl_cb_mmap()
317 	address = virt_to_phys((void *) (uintptr_t) cb->kernel_address);  in hl_cb_mmap()
319 	rc = hdev->asic_funcs->cb_mmap(hdev, vma, cb->kernel_address,  in hl_cb_mmap()
320 					address, cb->size);  in hl_cb_mmap()
323 		spin_lock(&cb->lock);  in hl_cb_mmap()
324 		cb->mmap = false;  in hl_cb_mmap()
328 	cb->mmap_size = cb->size;  in hl_cb_mmap()
333 	spin_unlock(&cb->lock);  in hl_cb_mmap()
335 	hl_cb_put(cb);  in hl_cb_mmap()
342 	struct hl_cb *cb;  in hl_cb_get()  local
345 	cb = idr_find(&mgr->cb_handles, handle);  in hl_cb_get()
347 	if (!cb) {  in hl_cb_get()
354 	kref_get(&cb->refcount);  in hl_cb_get()
358 	return cb;  in hl_cb_get()
362 void hl_cb_put(struct hl_cb *cb)  in hl_cb_put()  argument
364 	kref_put(&cb->refcount, cb_release);  in hl_cb_put()
375 	struct hl_cb *cb;  in hl_cb_mgr_fini()  local
381 	idr_for_each_entry(idp, cb, id) {  in hl_cb_mgr_fini()
382 		if (kref_put(&cb->refcount, cb_release) != 1)  in hl_cb_mgr_fini()
385 				id, cb->ctx_id);  in hl_cb_mgr_fini()
394 	struct hl_cb *cb;  in hl_cb_kernel_create()  local
406 	cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) cb_handle);  in hl_cb_kernel_create()
408 	WARN(!cb, "Kernel CB handle invalid 0x%x\n", (u32) cb_handle);  in hl_cb_kernel_create()
409 	if (!cb)  in hl_cb_kernel_create()
412 	return cb;  in hl_cb_kernel_create()
422 	struct hl_cb *cb;  in hl_cb_pool_init()  local
429 		cb = hl_cb_alloc(hdev, hdev->asic_prop.cb_pool_cb_size,  in hl_cb_pool_init()
431 		if (cb) {  in hl_cb_pool_init()
432 			cb->is_pool = true;  in hl_cb_pool_init()
433 			list_add(&cb->pool_list, &hdev->cb_pool);  in hl_cb_pool_init()
445 	struct hl_cb *cb, *tmp;  in hl_cb_pool_fini()  local
447 	list_for_each_entry_safe(cb, tmp, &hdev->cb_pool, pool_list) {  in hl_cb_pool_fini()
448 		list_del(&cb->pool_list);  in hl_cb_pool_fini()
449 		cb_fini(hdev, cb);  in hl_cb_pool_fini()