36 struct pasid_state {  struct
58 	struct pasid_state **states;  argument
71 	struct pasid_state *state;
154 static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state,  in __get_pasid_state_ptr()
157 	struct pasid_state **root, **ptr;  in __get_pasid_state_ptr()
180 		root   = (struct pasid_state **)*ptr;  in __get_pasid_state_ptr()
188 			   struct pasid_state *pasid_state,  in set_pasid_state()  argument
191 	struct pasid_state **ptr;  in set_pasid_state()
206 	*ptr = pasid_state;  in set_pasid_state()
218 	struct pasid_state **ptr;  in clear_pasid_state()
233 static struct pasid_state *get_pasid_state(struct device_state *dev_state,  in get_pasid_state()
236 	struct pasid_state **ptr, *ret = NULL;  in get_pasid_state()
255 static void free_pasid_state(struct pasid_state *pasid_state)  in free_pasid_state()  argument
257 	kfree(pasid_state);  in free_pasid_state()
260 static void put_pasid_state(struct pasid_state *pasid_state)  in put_pasid_state()  argument
262 	if (atomic_dec_and_test(&pasid_state->count))  in put_pasid_state()
263 		wake_up(&pasid_state->wq);  in put_pasid_state()
266 static void put_pasid_state_wait(struct pasid_state *pasid_state)  in put_pasid_state_wait()  argument
268 	atomic_dec(&pasid_state->count);  in put_pasid_state_wait()
269 	wait_event(pasid_state->wq, !atomic_read(&pasid_state->count));  in put_pasid_state_wait()
270 	free_pasid_state(pasid_state);  in put_pasid_state_wait()
273 static void unbind_pasid(struct pasid_state *pasid_state)  in unbind_pasid()  argument
277 	domain = pasid_state->device_state->domain;  in unbind_pasid()
283 	pasid_state->invalid = true;  in unbind_pasid()
289 	amd_iommu_domain_clear_gcr3(domain, pasid_state->pasid);  in unbind_pasid()
295 static void free_pasid_states_level1(struct pasid_state **tbl)  in free_pasid_states_level1()
307 static void free_pasid_states_level2(struct pasid_state **tbl)  in free_pasid_states_level2()
309 	struct pasid_state **ptr;  in free_pasid_states_level2()
316 		ptr = (struct pasid_state **)tbl[i];  in free_pasid_states_level2()
323 	struct pasid_state *pasid_state;  in free_pasid_states()  local
327 		pasid_state = get_pasid_state(dev_state, i);  in free_pasid_states()
328 		if (pasid_state == NULL)  in free_pasid_states()
331 		put_pasid_state(pasid_state);  in free_pasid_states()
337 		mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);  in free_pasid_states()
339 		put_pasid_state_wait(pasid_state); /* Reference taken in  in free_pasid_states()
356 static struct pasid_state *mn_to_state(struct mmu_notifier *mn)  in mn_to_state()
358 	return container_of(mn, struct pasid_state, mn);  in mn_to_state()
365 	struct pasid_state *pasid_state;  in mn_invalidate_range()  local
368 	pasid_state = mn_to_state(mn);  in mn_invalidate_range()
369 	dev_state   = pasid_state->device_state;  in mn_invalidate_range()
372 		amd_iommu_flush_page(dev_state->domain, pasid_state->pasid,  in mn_invalidate_range()
375 		amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid);  in mn_invalidate_range()
380 	struct pasid_state *pasid_state;  in mn_release()  local
386 	pasid_state    = mn_to_state(mn);  in mn_release()
387 	dev_state      = pasid_state->device_state;  in mn_release()
388 	run_inv_ctx_cb = !pasid_state->invalid;  in mn_release()
391 		dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid);  in mn_release()
393 	unbind_pasid(pasid_state);  in mn_release()
401 static void set_pri_tag_status(struct pasid_state *pasid_state,  in set_pri_tag_status()  argument
406 	spin_lock_irqsave(&pasid_state->lock, flags);  in set_pri_tag_status()
407 	pasid_state->pri[tag].status = status;  in set_pri_tag_status()
408 	spin_unlock_irqrestore(&pasid_state->lock, flags);  in set_pri_tag_status()
412 			   struct pasid_state *pasid_state,  in finish_pri_tag()  argument
417 	spin_lock_irqsave(&pasid_state->lock, flags);  in finish_pri_tag()
418 	if (atomic_dec_and_test(&pasid_state->pri[tag].inflight) &&  in finish_pri_tag()
419 	    pasid_state->pri[tag].finish) {  in finish_pri_tag()
420 		amd_iommu_complete_ppr(dev_state->pdev, pasid_state->pasid,  in finish_pri_tag()
421 				       pasid_state->pri[tag].status, tag);  in finish_pri_tag()
422 		pasid_state->pri[tag].finish = false;  in finish_pri_tag()
423 		pasid_state->pri[tag].status = PPR_SUCCESS;  in finish_pri_tag()
425 	spin_unlock_irqrestore(&pasid_state->lock, flags);  in finish_pri_tag()
518 	struct pasid_state *pasid_state;  in ppr_notifier()  local
552 	pasid_state = get_pasid_state(dev_state, iommu_fault->pasid);  in ppr_notifier()
553 	if (pasid_state == NULL || pasid_state->invalid) {  in ppr_notifier()
560 	spin_lock_irqsave(&pasid_state->lock, flags);  in ppr_notifier()
561 	atomic_inc(&pasid_state->pri[tag].inflight);  in ppr_notifier()
563 		pasid_state->pri[tag].finish = true;  in ppr_notifier()
564 	spin_unlock_irqrestore(&pasid_state->lock, flags);  in ppr_notifier()
569 		finish_pri_tag(dev_state, pasid_state, tag);  in ppr_notifier()
575 	fault->state     = pasid_state;  in ppr_notifier()
588 	if (ret != NOTIFY_OK && pasid_state)  in ppr_notifier()
589 		put_pasid_state(pasid_state);  in ppr_notifier()
604 	struct pasid_state *pasid_state;  in amd_iommu_bind_pasid()  local
626 	pasid_state = kzalloc(sizeof(*pasid_state), GFP_KERNEL);  in amd_iommu_bind_pasid()
627 	if (pasid_state == NULL)  in amd_iommu_bind_pasid()
631 	atomic_set(&pasid_state->count, 1);  in amd_iommu_bind_pasid()
632 	init_waitqueue_head(&pasid_state->wq);  in amd_iommu_bind_pasid()
633 	spin_lock_init(&pasid_state->lock);  in amd_iommu_bind_pasid()
636 	pasid_state->mm           = mm;  in amd_iommu_bind_pasid()
637 	pasid_state->device_state = dev_state;  in amd_iommu_bind_pasid()
638 	pasid_state->pasid        = pasid;  in amd_iommu_bind_pasid()
639 	pasid_state->invalid      = true; /* Mark as valid only if we are  in amd_iommu_bind_pasid()
641 	pasid_state->mn.ops       = &iommu_mn;  in amd_iommu_bind_pasid()
643 	if (pasid_state->mm == NULL)  in amd_iommu_bind_pasid()
646 	mmu_notifier_register(&pasid_state->mn, mm);  in amd_iommu_bind_pasid()
648 	ret = set_pasid_state(dev_state, pasid_state, pasid);  in amd_iommu_bind_pasid()
653 					__pa(pasid_state->mm->pgd));  in amd_iommu_bind_pasid()
658 	pasid_state->invalid = false;  in amd_iommu_bind_pasid()
673 	mmu_notifier_unregister(&pasid_state->mn, mm);  in amd_iommu_bind_pasid()
677 	free_pasid_state(pasid_state);  in amd_iommu_bind_pasid()
688 	struct pasid_state *pasid_state;  in amd_iommu_unbind_pasid()  local
705 	pasid_state = get_pasid_state(dev_state, pasid);  in amd_iommu_unbind_pasid()
706 	if (pasid_state == NULL)  in amd_iommu_unbind_pasid()
712 	put_pasid_state(pasid_state);  in amd_iommu_unbind_pasid()
715 	clear_pasid_state(dev_state, pasid_state->pasid);  in amd_iommu_unbind_pasid()
721 	mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);  in amd_iommu_unbind_pasid()
723 	put_pasid_state_wait(pasid_state); /* Reference taken in  in amd_iommu_unbind_pasid()