7  * DOC: VC4 GEM BO management support
9  * The VC4 GPU architecture (both scanout and rendering) has direct
40 static void vc4_bo_stats_print(struct drm_printer *p, struct vc4_dev *vc4)  in vc4_bo_stats_print()  argument
44 	for (i = 0; i < vc4->num_labels; i++) {  in vc4_bo_stats_print()
45 		if (!vc4->bo_labels[i].num_allocated)  in vc4_bo_stats_print()
49 			   vc4->bo_labels[i].name,  in vc4_bo_stats_print()
50 			   vc4->bo_labels[i].size_allocated / 1024,  in vc4_bo_stats_print()
51 			   vc4->bo_labels[i].num_allocated);  in vc4_bo_stats_print()
54 	mutex_lock(&vc4->purgeable.lock);  in vc4_bo_stats_print()
55 	if (vc4->purgeable.num)  in vc4_bo_stats_print()
57 			   vc4->purgeable.size / 1024, vc4->purgeable.num);  in vc4_bo_stats_print()
59 	if (vc4->purgeable.purged_num)  in vc4_bo_stats_print()
61 			   vc4->purgeable.purged_size / 1024,  in vc4_bo_stats_print()
62 			   vc4->purgeable.purged_num);  in vc4_bo_stats_print()
63 	mutex_unlock(&vc4->purgeable.lock);  in vc4_bo_stats_print()
70 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_stats_debugfs()  local
73 	vc4_bo_stats_print(&p, vc4);  in vc4_bo_stats_debugfs()
86 static int vc4_get_user_label(struct vc4_dev *vc4, const char *name)  in vc4_get_user_label()  argument
91 	for (i = 0; i < vc4->num_labels; i++) {  in vc4_get_user_label()
92 		if (!vc4->bo_labels[i].name) {  in vc4_get_user_label()
94 		} else if (strcmp(vc4->bo_labels[i].name, name) == 0) {  in vc4_get_user_label()
101 		WARN_ON(vc4->bo_labels[free_slot].num_allocated != 0);  in vc4_get_user_label()
102 		vc4->bo_labels[free_slot].name = name;  in vc4_get_user_label()
105 		u32 new_label_count = vc4->num_labels + 1;  in vc4_get_user_label()
107 			krealloc(vc4->bo_labels,  in vc4_get_user_label()
116 		free_slot = vc4->num_labels;  in vc4_get_user_label()
117 		vc4->bo_labels = new_labels;  in vc4_get_user_label()
118 		vc4->num_labels = new_label_count;  in vc4_get_user_label()
120 		vc4->bo_labels[free_slot].name = name;  in vc4_get_user_label()
121 		vc4->bo_labels[free_slot].num_allocated = 0;  in vc4_get_user_label()
122 		vc4->bo_labels[free_slot].size_allocated = 0;  in vc4_get_user_label()
131 	struct vc4_dev *vc4 = to_vc4_dev(gem_obj->dev);  in vc4_bo_set_label()  local
133 	lockdep_assert_held(&vc4->bo_lock);  in vc4_bo_set_label()
136 		vc4->bo_labels[label].num_allocated++;  in vc4_bo_set_label()
137 		vc4->bo_labels[label].size_allocated += gem_obj->size;  in vc4_bo_set_label()
140 	vc4->bo_labels[bo->label].num_allocated--;  in vc4_bo_set_label()
141 	vc4->bo_labels[bo->label].size_allocated -= gem_obj->size;  in vc4_bo_set_label()
143 	if (vc4->bo_labels[bo->label].num_allocated == 0 &&  in vc4_bo_set_label()
150 		kfree(vc4->bo_labels[bo->label].name);  in vc4_bo_set_label()
151 		vc4->bo_labels[bo->label].name = NULL;  in vc4_bo_set_label()
165 	struct vc4_dev *vc4 = to_vc4_dev(obj->dev);  in vc4_bo_destroy()  local
167 	lockdep_assert_held(&vc4->bo_lock);  in vc4_bo_destroy()
183 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_remove_from_cache()  local
185 	lockdep_assert_held(&vc4->bo_lock);  in vc4_bo_remove_from_cache()
193 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_get_cache_list_for_size()  local
196 	if (vc4->bo_cache.size_list_size <= page_index) {  in vc4_get_cache_list_for_size()
197 		uint32_t new_size = max(vc4->bo_cache.size_list_size * 2,  in vc4_get_cache_list_for_size()
210 		for (i = 0; i < vc4->bo_cache.size_list_size; i++) {  in vc4_get_cache_list_for_size()
212 				&vc4->bo_cache.size_list[i];  in vc4_get_cache_list_for_size()
220 		for (i = vc4->bo_cache.size_list_size; i < new_size; i++)  in vc4_get_cache_list_for_size()
223 		kfree(vc4->bo_cache.size_list);  in vc4_get_cache_list_for_size()
224 		vc4->bo_cache.size_list = new_list;  in vc4_get_cache_list_for_size()
225 		vc4->bo_cache.size_list_size = new_size;  in vc4_get_cache_list_for_size()
228 	return &vc4->bo_cache.size_list[page_index];  in vc4_get_cache_list_for_size()
233 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_cache_purge()  local
235 	mutex_lock(&vc4->bo_lock);  in vc4_bo_cache_purge()
236 	while (!list_empty(&vc4->bo_cache.time_list)) {  in vc4_bo_cache_purge()
237 		struct vc4_bo *bo = list_last_entry(&vc4->bo_cache.time_list,  in vc4_bo_cache_purge()
242 	mutex_unlock(&vc4->bo_lock);  in vc4_bo_cache_purge()
247 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_add_to_purgeable_pool()  local
249 	mutex_lock(&vc4->purgeable.lock);  in vc4_bo_add_to_purgeable_pool()
250 	list_add_tail(&bo->size_head, &vc4->purgeable.list);  in vc4_bo_add_to_purgeable_pool()
251 	vc4->purgeable.num++;  in vc4_bo_add_to_purgeable_pool()
252 	vc4->purgeable.size += bo->base.base.size;  in vc4_bo_add_to_purgeable_pool()
253 	mutex_unlock(&vc4->purgeable.lock);  in vc4_bo_add_to_purgeable_pool()
258 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_remove_from_purgeable_pool_locked()  local
273 	vc4->purgeable.num--;  in vc4_bo_remove_from_purgeable_pool_locked()
274 	vc4->purgeable.size -= bo->base.base.size;  in vc4_bo_remove_from_purgeable_pool_locked()
279 	struct vc4_dev *vc4 = to_vc4_dev(bo->base.base.dev);  in vc4_bo_remove_from_purgeable_pool()  local
281 	mutex_lock(&vc4->purgeable.lock);  in vc4_bo_remove_from_purgeable_pool()
283 	mutex_unlock(&vc4->purgeable.lock);  in vc4_bo_remove_from_purgeable_pool()
303 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_userspace_cache_purge()  local
305 	mutex_lock(&vc4->purgeable.lock);  in vc4_bo_userspace_cache_purge()
306 	while (!list_empty(&vc4->purgeable.list)) {  in vc4_bo_userspace_cache_purge()
307 		struct vc4_bo *bo = list_first_entry(&vc4->purgeable.list,  in vc4_bo_userspace_cache_purge()
319 		mutex_unlock(&vc4->purgeable.lock);  in vc4_bo_userspace_cache_purge()
338 		mutex_lock(&vc4->purgeable.lock);  in vc4_bo_userspace_cache_purge()
341 			vc4->purgeable.purged_size += purged_size;  in vc4_bo_userspace_cache_purge()
342 			vc4->purgeable.purged_num++;  in vc4_bo_userspace_cache_purge()
345 	mutex_unlock(&vc4->purgeable.lock);  in vc4_bo_userspace_cache_purge()
352 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_get_from_cache()  local
358 	mutex_lock(&vc4->bo_lock);  in vc4_bo_get_from_cache()
359 	if (page_index >= vc4->bo_cache.size_list_size)  in vc4_bo_get_from_cache()
362 	if (list_empty(&vc4->bo_cache.size_list[page_index]))  in vc4_bo_get_from_cache()
365 	bo = list_first_entry(&vc4->bo_cache.size_list[page_index],  in vc4_bo_get_from_cache()
373 	mutex_unlock(&vc4->bo_lock);  in vc4_bo_get_from_cache()
387 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_create_object()  local
397 	mutex_lock(&vc4->bo_lock);  in vc4_create_object()
399 	vc4->bo_labels[VC4_BO_TYPE_KERNEL].num_allocated++;  in vc4_create_object()
400 	vc4->bo_labels[VC4_BO_TYPE_KERNEL].size_allocated += size;  in vc4_create_object()
401 	mutex_unlock(&vc4->bo_lock);  in vc4_create_object()
410 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_create()  local
452 		struct drm_printer p = drm_info_printer(vc4->base.dev);  in vc4_bo_create()
454 		vc4_bo_stats_print(&p, vc4);  in vc4_bo_create()
465 	mutex_lock(&vc4->bo_lock);  in vc4_bo_create()
467 	mutex_unlock(&vc4->bo_lock);  in vc4_bo_create()
500 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_cache_free_old()  local
503 	lockdep_assert_held(&vc4->bo_lock);  in vc4_bo_cache_free_old()
505 	while (!list_empty(&vc4->bo_cache.time_list)) {  in vc4_bo_cache_free_old()
506 		struct vc4_bo *bo = list_last_entry(&vc4->bo_cache.time_list,  in vc4_bo_cache_free_old()
509 			mod_timer(&vc4->bo_cache.time_timer,  in vc4_bo_cache_free_old()
526 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_free_object()  local
536 	mutex_lock(&vc4->bo_lock);  in vc4_free_object()
579 	list_add(&bo->unref_head, &vc4->bo_cache.time_list);  in vc4_free_object()
586 	mutex_unlock(&vc4->bo_lock);  in vc4_free_object()
591 	struct vc4_dev *vc4 =  in vc4_bo_cache_time_work()  local
593 	struct drm_device *dev = &vc4->base;  in vc4_bo_cache_time_work()
595 	mutex_lock(&vc4->bo_lock);  in vc4_bo_cache_time_work()
597 	mutex_unlock(&vc4->bo_lock);  in vc4_bo_cache_time_work()
653 	struct vc4_dev *vc4 = from_timer(vc4, t, bo_cache.time_timer);  in vc4_bo_cache_time_timer()  local
655 	schedule_work(&vc4->bo_cache.time_work);  in vc4_bo_cache_time_timer()
796 static int vc4_grab_bin_bo(struct vc4_dev *vc4, struct vc4_file *vc4file)  in vc4_grab_bin_bo()  argument
800 	if (!vc4->v3d)  in vc4_grab_bin_bo()
806 	ret = vc4_v3d_bin_bo_get(vc4, &vc4file->bin_bo_used);  in vc4_grab_bin_bo()
818 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_create_bo_ioctl()  local
822 	ret = vc4_grab_bin_bo(vc4, vc4file);  in vc4_create_bo_ioctl()
867 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_create_shader_bo_ioctl()  local
887 	ret = vc4_grab_bin_bo(vc4, vc4file);  in vc4_create_shader_bo_ioctl()
1011 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_cache_init()  local
1018 	vc4->bo_labels = kcalloc(VC4_BO_TYPE_COUNT, sizeof(*vc4->bo_labels),  in vc4_bo_cache_init()
1020 	if (!vc4->bo_labels)  in vc4_bo_cache_init()
1022 	vc4->num_labels = VC4_BO_TYPE_COUNT;  in vc4_bo_cache_init()
1026 		vc4->bo_labels[i].name = bo_type_names[i];  in vc4_bo_cache_init()
1028 	mutex_init(&vc4->bo_lock);  in vc4_bo_cache_init()
1032 	INIT_LIST_HEAD(&vc4->bo_cache.time_list);  in vc4_bo_cache_init()
1034 	INIT_WORK(&vc4->bo_cache.time_work, vc4_bo_cache_time_work);  in vc4_bo_cache_init()
1035 	timer_setup(&vc4->bo_cache.time_timer, vc4_bo_cache_time_timer, 0);  in vc4_bo_cache_init()
1042 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_bo_cache_destroy()  local
1045 	del_timer(&vc4->bo_cache.time_timer);  in vc4_bo_cache_destroy()
1046 	cancel_work_sync(&vc4->bo_cache.time_work);  in vc4_bo_cache_destroy()
1050 	for (i = 0; i < vc4->num_labels; i++) {  in vc4_bo_cache_destroy()
1051 		if (vc4->bo_labels[i].num_allocated) {  in vc4_bo_cache_destroy()
1054 				  vc4->bo_labels[i].num_allocated,  in vc4_bo_cache_destroy()
1055 				  vc4->bo_labels[i].name);  in vc4_bo_cache_destroy()
1059 			kfree(vc4->bo_labels[i].name);  in vc4_bo_cache_destroy()
1061 	kfree(vc4->bo_labels);  in vc4_bo_cache_destroy()
1067 	struct vc4_dev *vc4 = to_vc4_dev(dev);  in vc4_label_bo_ioctl()  local
1087 	mutex_lock(&vc4->bo_lock);  in vc4_label_bo_ioctl()
1088 	label = vc4_get_user_label(vc4, name);  in vc4_label_bo_ioctl()
1093 	mutex_unlock(&vc4->bo_lock);  in vc4_label_bo_ioctl()