46 struct pasid_state {  struct
68 	struct pasid_state **states;  argument
81 	struct pasid_state *state;
164 static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state,  in __get_pasid_state_ptr()
167 	struct pasid_state **root, **ptr;  in __get_pasid_state_ptr()
190 		root   = (struct pasid_state **)*ptr;  in __get_pasid_state_ptr()
198 			   struct pasid_state *pasid_state,  in set_pasid_state()  argument
201 	struct pasid_state **ptr;  in set_pasid_state()
216 	*ptr = pasid_state;  in set_pasid_state()
228 	struct pasid_state **ptr;  in clear_pasid_state()
243 static struct pasid_state *get_pasid_state(struct device_state *dev_state,  in get_pasid_state()
246 	struct pasid_state **ptr, *ret = NULL;  in get_pasid_state()
265 static void free_pasid_state(struct pasid_state *pasid_state)  in free_pasid_state()  argument
267 	kfree(pasid_state);  in free_pasid_state()
270 static void put_pasid_state(struct pasid_state *pasid_state)  in put_pasid_state()  argument
272 	if (atomic_dec_and_test(&pasid_state->count))  in put_pasid_state()
273 		wake_up(&pasid_state->wq);  in put_pasid_state()
276 static void put_pasid_state_wait(struct pasid_state *pasid_state)  in put_pasid_state_wait()  argument
278 	atomic_dec(&pasid_state->count);  in put_pasid_state_wait()
279 	wait_event(pasid_state->wq, !atomic_read(&pasid_state->count));  in put_pasid_state_wait()
280 	free_pasid_state(pasid_state);  in put_pasid_state_wait()
283 static void unbind_pasid(struct pasid_state *pasid_state)  in unbind_pasid()  argument
287 	domain = pasid_state->device_state->domain;  in unbind_pasid()
293 	pasid_state->invalid = true;  in unbind_pasid()
299 	amd_iommu_domain_clear_gcr3(domain, pasid_state->pasid);  in unbind_pasid()
305 static void free_pasid_states_level1(struct pasid_state **tbl)  in free_pasid_states_level1()
317 static void free_pasid_states_level2(struct pasid_state **tbl)  in free_pasid_states_level2()
319 	struct pasid_state **ptr;  in free_pasid_states_level2()
326 		ptr = (struct pasid_state **)tbl[i];  in free_pasid_states_level2()
333 	struct pasid_state *pasid_state;  in free_pasid_states()  local
337 		pasid_state = get_pasid_state(dev_state, i);  in free_pasid_states()
338 		if (pasid_state == NULL)  in free_pasid_states()
341 		put_pasid_state(pasid_state);  in free_pasid_states()
347 		mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);  in free_pasid_states()
349 		put_pasid_state_wait(pasid_state); /* Reference taken in  in free_pasid_states()
366 static struct pasid_state *mn_to_state(struct mmu_notifier *mn)  in mn_to_state()
368 	return container_of(mn, struct pasid_state, mn);  in mn_to_state()
374 	struct pasid_state *pasid_state;  in __mn_flush_page()  local
377 	pasid_state = mn_to_state(mn);  in __mn_flush_page()
378 	dev_state   = pasid_state->device_state;  in __mn_flush_page()
380 	amd_iommu_flush_page(dev_state->domain, pasid_state->pasid, address);  in __mn_flush_page()
398 	struct pasid_state *pasid_state;  in mn_invalidate_range()  local
401 	pasid_state = mn_to_state(mn);  in mn_invalidate_range()
402 	dev_state   = pasid_state->device_state;  in mn_invalidate_range()
405 		amd_iommu_flush_page(dev_state->domain, pasid_state->pasid,  in mn_invalidate_range()
408 		amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid);  in mn_invalidate_range()
413 	struct pasid_state *pasid_state;  in mn_release()  local
419 	pasid_state    = mn_to_state(mn);  in mn_release()
420 	dev_state      = pasid_state->device_state;  in mn_release()
421 	run_inv_ctx_cb = !pasid_state->invalid;  in mn_release()
424 		dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid);  in mn_release()
426 	unbind_pasid(pasid_state);  in mn_release()
436 static void set_pri_tag_status(struct pasid_state *pasid_state,  in set_pri_tag_status()  argument
441 	spin_lock_irqsave(&pasid_state->lock, flags);  in set_pri_tag_status()
442 	pasid_state->pri[tag].status = status;  in set_pri_tag_status()
443 	spin_unlock_irqrestore(&pasid_state->lock, flags);  in set_pri_tag_status()
447 			   struct pasid_state *pasid_state,  in finish_pri_tag()  argument
452 	spin_lock_irqsave(&pasid_state->lock, flags);  in finish_pri_tag()
453 	if (atomic_dec_and_test(&pasid_state->pri[tag].inflight) &&  in finish_pri_tag()
454 	    pasid_state->pri[tag].finish) {  in finish_pri_tag()
455 		amd_iommu_complete_ppr(dev_state->pdev, pasid_state->pasid,  in finish_pri_tag()
456 				       pasid_state->pri[tag].status, tag);  in finish_pri_tag()
457 		pasid_state->pri[tag].finish = false;  in finish_pri_tag()
458 		pasid_state->pri[tag].status = PPR_SUCCESS;  in finish_pri_tag()
460 	spin_unlock_irqrestore(&pasid_state->lock, flags);  in finish_pri_tag()
553 	struct pasid_state *pasid_state;  in ppr_notifier()  local
587 	pasid_state = get_pasid_state(dev_state, iommu_fault->pasid);  in ppr_notifier()
588 	if (pasid_state == NULL || pasid_state->invalid) {  in ppr_notifier()
595 	spin_lock_irqsave(&pasid_state->lock, flags);  in ppr_notifier()
596 	atomic_inc(&pasid_state->pri[tag].inflight);  in ppr_notifier()
598 		pasid_state->pri[tag].finish = true;  in ppr_notifier()
599 	spin_unlock_irqrestore(&pasid_state->lock, flags);  in ppr_notifier()
604 		finish_pri_tag(dev_state, pasid_state, tag);  in ppr_notifier()
610 	fault->state     = pasid_state;  in ppr_notifier()
623 	if (ret != NOTIFY_OK && pasid_state)  in ppr_notifier()
624 		put_pasid_state(pasid_state);  in ppr_notifier()
639 	struct pasid_state *pasid_state;  in amd_iommu_bind_pasid()  local
661 	pasid_state = kzalloc(sizeof(*pasid_state), GFP_KERNEL);  in amd_iommu_bind_pasid()
662 	if (pasid_state == NULL)  in amd_iommu_bind_pasid()
666 	atomic_set(&pasid_state->count, 1);  in amd_iommu_bind_pasid()
667 	init_waitqueue_head(&pasid_state->wq);  in amd_iommu_bind_pasid()
668 	spin_lock_init(&pasid_state->lock);  in amd_iommu_bind_pasid()
671 	pasid_state->mm           = mm;  in amd_iommu_bind_pasid()
672 	pasid_state->device_state = dev_state;  in amd_iommu_bind_pasid()
673 	pasid_state->pasid        = pasid;  in amd_iommu_bind_pasid()
674 	pasid_state->invalid      = true; /* Mark as valid only if we are  in amd_iommu_bind_pasid()
676 	pasid_state->mn.ops       = &iommu_mn;  in amd_iommu_bind_pasid()
678 	if (pasid_state->mm == NULL)  in amd_iommu_bind_pasid()
681 	mmu_notifier_register(&pasid_state->mn, mm);  in amd_iommu_bind_pasid()
683 	ret = set_pasid_state(dev_state, pasid_state, pasid);  in amd_iommu_bind_pasid()
688 					__pa(pasid_state->mm->pgd));  in amd_iommu_bind_pasid()
693 	pasid_state->invalid = false;  in amd_iommu_bind_pasid()
708 	mmu_notifier_unregister(&pasid_state->mn, mm);  in amd_iommu_bind_pasid()
712 	free_pasid_state(pasid_state);  in amd_iommu_bind_pasid()
723 	struct pasid_state *pasid_state;  in amd_iommu_unbind_pasid()  local
740 	pasid_state = get_pasid_state(dev_state, pasid);  in amd_iommu_unbind_pasid()
741 	if (pasid_state == NULL)  in amd_iommu_unbind_pasid()
747 	put_pasid_state(pasid_state);  in amd_iommu_unbind_pasid()
750 	clear_pasid_state(dev_state, pasid_state->pasid);  in amd_iommu_unbind_pasid()
756 	mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm);  in amd_iommu_unbind_pasid()
758 	put_pasid_state_wait(pasid_state); /* Reference taken in  in amd_iommu_unbind_pasid()