116 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_obj_destroy()  local
118 	spin_lock(&fman->lock);  in vmw_fence_obj_destroy()
120 	--fman->num_fence_objects;  in vmw_fence_obj_destroy()
121 	spin_unlock(&fman->lock);  in vmw_fence_obj_destroy()
140 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_enable_signaling()  local
141 	struct vmw_private *dev_priv = fman->dev_priv;  in vmw_fence_enable_signaling()
167 static void __vmw_fences_update(struct vmw_fence_manager *fman);
174 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_wait()  local
175 	struct vmw_private *dev_priv = fman->dev_priv;  in vmw_fence_wait()
197 		__vmw_fences_update(fman);  in vmw_fence_wait()
258 	struct vmw_fence_manager *fman =  in vmw_fence_work_func()  local
266 		mutex_lock(&fman->goal_irq_mutex);  in vmw_fence_work_func()
268 		spin_lock(&fman->lock);  in vmw_fence_work_func()
269 		list_splice_init(&fman->cleanup_list, &list);  in vmw_fence_work_func()
270 		seqno_valid = fman->seqno_valid;  in vmw_fence_work_func()
271 		spin_unlock(&fman->lock);  in vmw_fence_work_func()
273 		if (!seqno_valid && fman->goal_irq_on) {  in vmw_fence_work_func()
274 			fman->goal_irq_on = false;  in vmw_fence_work_func()
275 			vmw_goal_waiter_remove(fman->dev_priv);  in vmw_fence_work_func()
277 		mutex_unlock(&fman->goal_irq_mutex);  in vmw_fence_work_func()
298 	struct vmw_fence_manager *fman = kzalloc(sizeof(*fman), GFP_KERNEL);  in vmw_fence_manager_init()  local
300 	if (unlikely(!fman))  in vmw_fence_manager_init()
303 	fman->dev_priv = dev_priv;  in vmw_fence_manager_init()
304 	spin_lock_init(&fman->lock);  in vmw_fence_manager_init()
305 	INIT_LIST_HEAD(&fman->fence_list);  in vmw_fence_manager_init()
306 	INIT_LIST_HEAD(&fman->cleanup_list);  in vmw_fence_manager_init()
307 	INIT_WORK(&fman->work, &vmw_fence_work_func);  in vmw_fence_manager_init()
308 	fman->fifo_down = true;  in vmw_fence_manager_init()
309 	fman->user_fence_size = ttm_round_pot(sizeof(struct vmw_user_fence));  in vmw_fence_manager_init()
310 	fman->fence_size = ttm_round_pot(sizeof(struct vmw_fence_obj));  in vmw_fence_manager_init()
311 	fman->event_fence_action_size =  in vmw_fence_manager_init()
313 	mutex_init(&fman->goal_irq_mutex);  in vmw_fence_manager_init()
314 	fman->ctx = dma_fence_context_alloc(1);  in vmw_fence_manager_init()
316 	return fman;  in vmw_fence_manager_init()
319 void vmw_fence_manager_takedown(struct vmw_fence_manager *fman)  in vmw_fence_manager_takedown()  argument
323 	(void) cancel_work_sync(&fman->work);  in vmw_fence_manager_takedown()
325 	spin_lock(&fman->lock);  in vmw_fence_manager_takedown()
326 	lists_empty = list_empty(&fman->fence_list) &&  in vmw_fence_manager_takedown()
327 		list_empty(&fman->cleanup_list);  in vmw_fence_manager_takedown()
328 	spin_unlock(&fman->lock);  in vmw_fence_manager_takedown()
331 	kfree(fman);  in vmw_fence_manager_takedown()
334 static int vmw_fence_obj_init(struct vmw_fence_manager *fman,  in vmw_fence_obj_init()  argument
340 	dma_fence_init(&fence->base, &vmw_fence_ops, &fman->lock,  in vmw_fence_obj_init()
341 		       fman->ctx, seqno);  in vmw_fence_obj_init()
345 	spin_lock(&fman->lock);  in vmw_fence_obj_init()
346 	if (unlikely(fman->fifo_down)) {  in vmw_fence_obj_init()
350 	list_add_tail(&fence->head, &fman->fence_list);  in vmw_fence_obj_init()
351 	++fman->num_fence_objects;  in vmw_fence_obj_init()
354 	spin_unlock(&fman->lock);  in vmw_fence_obj_init()
359 static void vmw_fences_perform_actions(struct vmw_fence_manager *fman,  in vmw_fences_perform_actions()  argument
366 		fman->pending_actions[action->type]--;  in vmw_fences_perform_actions()
375 		list_add_tail(&action->head, &fman->cleanup_list);  in vmw_fences_perform_actions()
395 static bool vmw_fence_goal_new_locked(struct vmw_fence_manager *fman,  in vmw_fence_goal_new_locked()  argument
402 	if (likely(!fman->seqno_valid))  in vmw_fence_goal_new_locked()
405 	fifo_mem = fman->dev_priv->mmio_virt;  in vmw_fence_goal_new_locked()
410 	fman->seqno_valid = false;  in vmw_fence_goal_new_locked()
411 	list_for_each_entry(fence, &fman->fence_list, head) {  in vmw_fence_goal_new_locked()
413 			fman->seqno_valid = true;  in vmw_fence_goal_new_locked()
441 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_goal_check_locked()  local
448 	fifo_mem = fman->dev_priv->mmio_virt;  in vmw_fence_goal_check_locked()
450 	if (likely(fman->seqno_valid &&  in vmw_fence_goal_check_locked()
455 	fman->seqno_valid = true;  in vmw_fence_goal_check_locked()
460 static void __vmw_fences_update(struct vmw_fence_manager *fman)  in __vmw_fences_update()  argument
466 	u32 *fifo_mem = fman->dev_priv->mmio_virt;  in __vmw_fences_update()
470 	list_for_each_entry_safe(fence, next_fence, &fman->fence_list, head) {  in __vmw_fences_update()
477 			vmw_fences_perform_actions(fman, &action_list);  in __vmw_fences_update()
488 	needs_rerun = vmw_fence_goal_new_locked(fman, seqno);  in __vmw_fences_update()
497 	if (!list_empty(&fman->cleanup_list))  in __vmw_fences_update()
498 		(void) schedule_work(&fman->work);  in __vmw_fences_update()
501 void vmw_fences_update(struct vmw_fence_manager *fman)  in vmw_fences_update()  argument
503 	spin_lock(&fman->lock);  in vmw_fences_update()
504 	__vmw_fences_update(fman);  in vmw_fences_update()
505 	spin_unlock(&fman->lock);  in vmw_fences_update()
510 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_obj_signaled()  local
515 	vmw_fences_update(fman);  in vmw_fence_obj_signaled()
545 int vmw_fence_create(struct vmw_fence_manager *fman,  in vmw_fence_create()  argument
556 	ret = vmw_fence_obj_init(fman, fence, seqno,  in vmw_fence_create()
574 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_user_fence_destroy()  local
580 	ttm_mem_global_free(vmw_mem_glob(fman->dev_priv),  in vmw_user_fence_destroy()
581 			    fman->user_fence_size);  in vmw_user_fence_destroy()
596 			  struct vmw_fence_manager *fman,  in vmw_user_fence_create()  argument
604 	struct ttm_mem_global *mem_glob = vmw_mem_glob(fman->dev_priv);  in vmw_user_fence_create()
616 	ret = ttm_mem_global_alloc(mem_glob, fman->user_fence_size,  in vmw_user_fence_create()
627 	ret = vmw_fence_obj_init(fman, &ufence->fence, seqno,  in vmw_user_fence_create()
660 	ttm_mem_global_free(mem_glob, fman->user_fence_size);  in vmw_user_fence_create()
674 int vmw_wait_dma_fence(struct vmw_fence_manager *fman,  in vmw_wait_dma_fence()  argument
714 void vmw_fence_fifo_down(struct vmw_fence_manager *fman)  in vmw_fence_fifo_down()  argument
724 	spin_lock(&fman->lock);  in vmw_fence_fifo_down()
725 	fman->fifo_down = true;  in vmw_fence_fifo_down()
726 	while (!list_empty(&fman->fence_list)) {  in vmw_fence_fifo_down()
728 			list_entry(fman->fence_list.prev, struct vmw_fence_obj,  in vmw_fence_fifo_down()
731 		spin_unlock(&fman->lock);  in vmw_fence_fifo_down()
742 			vmw_fences_perform_actions(fman, &action_list);  in vmw_fence_fifo_down()
747 		spin_lock(&fman->lock);  in vmw_fence_fifo_down()
749 	spin_unlock(&fman->lock);  in vmw_fence_fifo_down()
752 void vmw_fence_fifo_up(struct vmw_fence_manager *fman)  in vmw_fence_fifo_up()  argument
754 	spin_lock(&fman->lock);  in vmw_fence_fifo_up()
755 	fman->fifo_down = false;  in vmw_fence_fifo_up()
756 	spin_unlock(&fman->lock);  in vmw_fence_fifo_up()
857 	struct vmw_fence_manager *fman;  in vmw_fence_obj_signaled_ioctl()  local
866 	fman = fman_from_fence(fence);  in vmw_fence_obj_signaled_ioctl()
871 	spin_lock(&fman->lock);  in vmw_fence_obj_signaled_ioctl()
873 	spin_unlock(&fman->lock);  in vmw_fence_obj_signaled_ioctl()
962 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_fence_obj_add_action()  local
965 	mutex_lock(&fman->goal_irq_mutex);  in vmw_fence_obj_add_action()
966 	spin_lock(&fman->lock);  in vmw_fence_obj_add_action()
968 	fman->pending_actions[action->type]++;  in vmw_fence_obj_add_action()
974 		vmw_fences_perform_actions(fman, &action_list);  in vmw_fence_obj_add_action()
985 	spin_unlock(&fman->lock);  in vmw_fence_obj_add_action()
988 		if (!fman->goal_irq_on) {  in vmw_fence_obj_add_action()
989 			fman->goal_irq_on = true;  in vmw_fence_obj_add_action()
990 			vmw_goal_waiter_add(fman->dev_priv);  in vmw_fence_obj_add_action()
992 		vmw_fences_update(fman);  in vmw_fence_obj_add_action()
994 	mutex_unlock(&fman->goal_irq_mutex);  in vmw_fence_obj_add_action()
1021 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_event_fence_action_queue()  local
1034 	eaction->dev = fman->dev_priv->dev;  in vmw_event_fence_action_queue()
1055 	struct vmw_fence_manager *fman = fman_from_fence(fence);  in vmw_event_fence_action_create()  local
1056 	struct drm_device *dev = fman->dev_priv->dev;  in vmw_event_fence_action_create()