7  * DOC: VC4 GEM BO management support
9  * The VC4 GPU architecture (both scanout and rendering) has direct
44 static void vc4_bo_stats_print(struct drm_printer *p, struct vc4_dev *vc4)  in vc4_bo_stats_print()  argument
48 	for (i = 0; i < vc4->num_labels; i++) {  in vc4_bo_stats_print()
49 		if (!vc4->bo_labels[i].num_allocated)  in vc4_bo_stats_print()
53 			   vc4->bo_labels[i].name,  in vc4_bo_stats_print()
54 			   vc4->bo_labels[i].size_allocated / 1024,  in vc4_bo_stats_print()
55 			   vc4->bo_labels[i].num_allocated);  in vc4_bo_stats_print()
58 	mutex_lock(&vc4->purgeable.lock);  in vc4_bo_stats_print()
59 	if (vc4->purgeable.num)  in vc4_bo_stats_print()
61 			   vc4->purgeable.size / 1024, vc4->purgeable.num);  in vc4_bo_stats_print()
63 	if (vc4->purgeable.purged_num)  in vc4_bo_stats_print()
65 			   vc4->purgeable.purged_size / 1024,  in vc4_bo_stats_print()
66 			   vc4->purgeable.purged_num);  in vc4_bo_stats_print()
67 	mutex_unlock(&vc4->purgeable.lock);  in vc4_bo_stats_print()
74 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_stats_debugfs()  local
77 	vc4_bo_stats_print(&p, vc4);  in vc4_bo_stats_debugfs()
90 static int vc4_get_user_label(struct vc4_dev *vc4, const char *name)  in vc4_get_user_label()  argument
95 	for (i = 0; i < vc4->num_labels; i++) {  in vc4_get_user_label()
96 		if (!vc4->bo_labels[i].name) {  in vc4_get_user_label()
98 		} else if (strcmp(vc4->bo_labels[i].name, name) == 0) {  in vc4_get_user_label()
105 		WARN_ON(vc4->bo_labels[free_slot].num_allocated != 0);  in vc4_get_user_label()
106 		vc4->bo_labels[free_slot].name = name;  in vc4_get_user_label()
109 		u32 new_label_count = vc4->num_labels + 1;  in vc4_get_user_label()
111 			krealloc(vc4->bo_labels,  in vc4_get_user_label()
120 		free_slot = vc4->num_labels;  in vc4_get_user_label()
121 		vc4->bo_labels = new_labels;  in vc4_get_user_label()
122 		vc4->num_labels = new_label_count;  in vc4_get_user_label()
124 		vc4->bo_labels[free_slot].name = name;  in vc4_get_user_label()
125 		vc4->bo_labels[free_slot].num_allocated = 0;  in vc4_get_user_label()
126 		vc4->bo_labels[free_slot].size_allocated = 0;  in vc4_get_user_label()
135 	struct vc4_dev *vc4 = to_vc4_dev(gem_obj->dev);  in vc4_bo_set_label()  local
137 	lockdep_assert_held(&vc4->bo_lock);  in vc4_bo_set_label()
140 		vc4->bo_labels[label].num_allocated++;  in vc4_bo_set_label()
141 		vc4->bo_labels[label].size_allocated += gem_obj->size;  in vc4_bo_set_label()
144 	vc4->bo_labels[bo->label].num_allocated--;  in vc4_bo_set_label()
145 	vc4->bo_labels[bo->label].size_allocated -= gem_obj->size;  in vc4_bo_set_label()
147 	if (vc4->bo_labels[bo->label].num_allocated == 0 &&  in vc4_bo_set_label()
154 		kfree(vc4->bo_labels[bo->label].name);  in vc4_bo_set_label()
155 		vc4->bo_labels[bo->label].name = NULL;  in vc4_bo_set_label()
169 	struct vc4_dev *vc4 = to_vc4_dev(obj->dev);  in vc4_bo_destroy()  local
171 	lockdep_assert_held(&vc4->bo_lock);  in vc4_bo_destroy()
187 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_remove_from_cache()  local
189 	lockdep_assert_held(&vc4->bo_lock);  in vc4_bo_remove_from_cache()
197 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_get_cache_list_for_size()  local
200 	if (vc4->bo_cache.size_list_size <= page_index) {  in vc4_get_cache_list_for_size()
201 		uint32_t new_size = max(vc4->bo_cache.size_list_size * 2,  in vc4_get_cache_list_for_size()
214 		for (i = 0; i < vc4->bo_cache.size_list_size; i++) {  in vc4_get_cache_list_for_size()
216 				&vc4->bo_cache.size_list[i];  in vc4_get_cache_list_for_size()
224 		for (i = vc4->bo_cache.size_list_size; i < new_size; i++)  in vc4_get_cache_list_for_size()
227 		kfree(vc4->bo_cache.size_list);  in vc4_get_cache_list_for_size()
228 		vc4->bo_cache.size_list = new_list;  in vc4_get_cache_list_for_size()
229 		vc4->bo_cache.size_list_size = new_size;  in vc4_get_cache_list_for_size()
232 	return &vc4->bo_cache.size_list[page_index];  in vc4_get_cache_list_for_size()
237 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_cache_purge()  local
239 	mutex_lock(&vc4->bo_lock);  in vc4_bo_cache_purge()
240 	while (!list_empty(&vc4->bo_cache.time_list)) {  in vc4_bo_cache_purge()
241 		struct vc4_bo *bo = list_last_entry(&vc4->bo_cache.time_list,  in vc4_bo_cache_purge()
246 	mutex_unlock(&vc4->bo_lock);  in vc4_bo_cache_purge()
251 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_add_to_purgeable_pool()  local
253 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_bo_add_to_purgeable_pool()
256 	mutex_lock(&vc4->purgeable.lock);  in vc4_bo_add_to_purgeable_pool()
257 	list_add_tail(&bo->size_head, &vc4->purgeable.list);  in vc4_bo_add_to_purgeable_pool()
258 	vc4->purgeable.num++;  in vc4_bo_add_to_purgeable_pool()
259 	vc4->purgeable.size += bo->base.base.size;  in vc4_bo_add_to_purgeable_pool()
260 	mutex_unlock(&vc4->purgeable.lock);  in vc4_bo_add_to_purgeable_pool()
265 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_remove_from_purgeable_pool_locked()  local
267 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_bo_remove_from_purgeable_pool_locked()
283 	vc4->purgeable.num--;  in vc4_bo_remove_from_purgeable_pool_locked()
284 	vc4->purgeable.size -= bo->base.base.size;  in vc4_bo_remove_from_purgeable_pool_locked()
289 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_remove_from_purgeable_pool()  local
291 	mutex_lock(&vc4->purgeable.lock);  in vc4_bo_remove_from_purgeable_pool()
293 	mutex_unlock(&vc4->purgeable.lock);  in vc4_bo_remove_from_purgeable_pool()
313 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_userspace_cache_purge()  local
315 	mutex_lock(&vc4->purgeable.lock);  in vc4_bo_userspace_cache_purge()
316 	while (!list_empty(&vc4->purgeable.list)) {  in vc4_bo_userspace_cache_purge()
317 		struct vc4_bo *bo = list_first_entry(&vc4->purgeable.list,  in vc4_bo_userspace_cache_purge()
329 		mutex_unlock(&vc4->purgeable.lock);  in vc4_bo_userspace_cache_purge()
348 		mutex_lock(&vc4->purgeable.lock);  in vc4_bo_userspace_cache_purge()
351 			vc4->purgeable.purged_size += purged_size;  in vc4_bo_userspace_cache_purge()
352 			vc4->purgeable.purged_num++;  in vc4_bo_userspace_cache_purge()
355 	mutex_unlock(&vc4->purgeable.lock);  in vc4_bo_userspace_cache_purge()
362 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_get_from_cache()  local
366 	mutex_lock(&vc4->bo_lock);  in vc4_bo_get_from_cache()
367 	if (page_index >= vc4->bo_cache.size_list_size)  in vc4_bo_get_from_cache()
370 	if (list_empty(&vc4->bo_cache.size_list[page_index]))  in vc4_bo_get_from_cache()
373 	bo = list_first_entry(&vc4->bo_cache.size_list[page_index],  in vc4_bo_get_from_cache()
381 	mutex_unlock(&vc4->bo_lock);  in vc4_bo_get_from_cache()
395 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_create_object()  local
399 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_create_object()
413 	mutex_lock(&vc4->bo_lock);  in vc4_create_object()
415 	vc4->bo_labels[VC4_BO_TYPE_KERNEL].num_allocated++;  in vc4_create_object()
416 	vc4->bo_labels[VC4_BO_TYPE_KERNEL].size_allocated += size;  in vc4_create_object()
417 	mutex_unlock(&vc4->bo_lock);  in vc4_create_object()
428 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_create()  local
432 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_bo_create()
473 		struct drm_printer p = drm_info_printer(vc4->base.dev);  in vc4_bo_create()
475 		vc4_bo_stats_print(&p, vc4);  in vc4_bo_create()
486 	mutex_lock(&vc4->bo_lock);  in vc4_bo_create()
488 	mutex_unlock(&vc4->bo_lock);  in vc4_bo_create()
497 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_dumb_create()  local
501 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_bo_dumb_create()
522 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_cache_free_old()  local
525 	lockdep_assert_held(&vc4->bo_lock);  in vc4_bo_cache_free_old()
527 	while (!list_empty(&vc4->bo_cache.time_list)) {  in vc4_bo_cache_free_old()
528 		struct vc4_bo *bo = list_last_entry(&vc4->bo_cache.time_list,  in vc4_bo_cache_free_old()
531 			mod_timer(&vc4->bo_cache.time_timer,  in vc4_bo_cache_free_old()
548 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_free_object()  local
558 	mutex_lock(&vc4->bo_lock);  in vc4_free_object()
601 	list_add(&bo->unref_head, &vc4->bo_cache.time_list);  in vc4_free_object()
608 	mutex_unlock(&vc4->bo_lock);  in vc4_free_object()
613 	struct vc4_dev *vc4 =  in vc4_bo_cache_time_work()  local
615 	struct drm_device *dev = &vc4->base;  in vc4_bo_cache_time_work()
617 	mutex_lock(&vc4->bo_lock);  in vc4_bo_cache_time_work()
619 	mutex_unlock(&vc4->bo_lock);  in vc4_bo_cache_time_work()
624 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_inc_usecnt()  local
627 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_bo_inc_usecnt()
664 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_dec_usecnt()  local
666 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_bo_dec_usecnt()
684 	struct vc4_dev *vc4 = from_timer(vc4, t, bo_cache.time_timer);  in vc4_bo_cache_time_timer()  local
686 	schedule_work(&vc4->bo_cache.time_work);  in vc4_bo_cache_time_timer()
768 static int vc4_grab_bin_bo(struct vc4_dev *vc4, struct vc4_file *vc4file)  in vc4_grab_bin_bo()  argument
770 	if (!vc4->v3d)  in vc4_grab_bin_bo()
776 	return vc4_v3d_bin_bo_get(vc4, &vc4file->bin_bo_used);  in vc4_grab_bin_bo()
784 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_create_bo_ioctl()  local
788 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_create_bo_ioctl()
791 	ret = vc4_grab_bin_bo(vc4, vc4file);  in vc4_create_bo_ioctl()
814 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_mmap_bo_ioctl()  local
818 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_mmap_bo_ioctl()
840 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_create_shader_bo_ioctl()  local
844 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_create_shader_bo_ioctl()
863 	ret = vc4_grab_bin_bo(vc4, vc4file);  in vc4_create_shader_bo_ioctl()
917 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_set_tiling_ioctl()  local
923 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_set_tiling_ioctl()
964 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_get_tiling_ioctl()  local
969 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_get_tiling_ioctl()
995 	struct vc4_dev *vc4 = to_vc4_dev(drm);  in vc4_bo_debugfs_init()  local
998 	if (!vc4->v3d)  in vc4_bo_debugfs_init()
1012 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_cache_init()  local
1016 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_bo_cache_init()
1023 	vc4->bo_labels = kcalloc(VC4_BO_TYPE_COUNT, sizeof(*vc4->bo_labels),  in vc4_bo_cache_init()
1025 	if (!vc4->bo_labels)  in vc4_bo_cache_init()
1027 	vc4->num_labels = VC4_BO_TYPE_COUNT;  in vc4_bo_cache_init()
1031 		vc4->bo_labels[i].name = bo_type_names[i];  in vc4_bo_cache_init()
1033 	ret = drmm_mutex_init(dev, &vc4->bo_lock);  in vc4_bo_cache_init()
1035 		kfree(vc4->bo_labels);  in vc4_bo_cache_init()
1039 	INIT_LIST_HEAD(&vc4->bo_cache.time_list);  in vc4_bo_cache_init()
1041 	INIT_WORK(&vc4->bo_cache.time_work, vc4_bo_cache_time_work);  in vc4_bo_cache_init()
1042 	timer_setup(&vc4->bo_cache.time_timer, vc4_bo_cache_time_timer, 0);  in vc4_bo_cache_init()
1049 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_cache_destroy()  local
1052 	del_timer(&vc4->bo_cache.time_timer);  in vc4_bo_cache_destroy()
1053 	cancel_work_sync(&vc4->bo_cache.time_work);  in vc4_bo_cache_destroy()
1057 	for (i = 0; i < vc4->num_labels; i++) {  in vc4_bo_cache_destroy()
1058 		if (vc4->bo_labels[i].num_allocated) {  in vc4_bo_cache_destroy()
1061 				  vc4->bo_labels[i].num_allocated,  in vc4_bo_cache_destroy()
1062 				  vc4->bo_labels[i].name);  in vc4_bo_cache_destroy()
1066 			kfree(vc4->bo_labels[i].name);  in vc4_bo_cache_destroy()
1068 	kfree(vc4->bo_labels);  in vc4_bo_cache_destroy()
1074 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_label_bo_ioctl()  local
1080 	if (WARN_ON_ONCE(vc4->is_vc5))  in vc4_label_bo_ioctl()
1097 	mutex_lock(&vc4->bo_lock);  in vc4_label_bo_ioctl()
1098 	label = vc4_get_user_label(vc4, name);  in vc4_label_bo_ioctl()
1103 	mutex_unlock(&vc4->bo_lock);  in vc4_label_bo_ioctl()