Lines Matching refs:pasid_state
35 struct pasid_state { struct
57 struct pasid_state **states; argument
70 struct pasid_state *state;
151 static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state, in __get_pasid_state_ptr()
154 struct pasid_state **root, **ptr; in __get_pasid_state_ptr()
177 root = (struct pasid_state **)*ptr; in __get_pasid_state_ptr()
185 struct pasid_state *pasid_state, in set_pasid_state() argument
188 struct pasid_state **ptr; in set_pasid_state()
203 *ptr = pasid_state; in set_pasid_state()
215 struct pasid_state **ptr; in clear_pasid_state()
230 static struct pasid_state *get_pasid_state(struct device_state *dev_state, in get_pasid_state()
233 struct pasid_state **ptr, *ret = NULL; in get_pasid_state()
252 static void free_pasid_state(struct pasid_state *pasid_state) in free_pasid_state() argument
254 kfree(pasid_state); in free_pasid_state()
257 static void put_pasid_state(struct pasid_state *pasid_state) in put_pasid_state() argument
259 if (refcount_dec_and_test(&pasid_state->count)) in put_pasid_state()
260 wake_up(&pasid_state->wq); in put_pasid_state()
263 static void put_pasid_state_wait(struct pasid_state *pasid_state) in put_pasid_state_wait() argument
265 refcount_dec(&pasid_state->count); in put_pasid_state_wait()
266 wait_event(pasid_state->wq, !refcount_read(&pasid_state->count)); in put_pasid_state_wait()
267 free_pasid_state(pasid_state); in put_pasid_state_wait()
270 static void unbind_pasid(struct pasid_state *pasid_state) in unbind_pasid() argument
274 domain = pasid_state->device_state->domain; in unbind_pasid()
280 pasid_state->invalid = true; in unbind_pasid()
286 amd_iommu_domain_clear_gcr3(domain, pasid_state->pasid); in unbind_pasid()
292 static void free_pasid_states_level1(struct pasid_state **tbl) in free_pasid_states_level1()
304 static void free_pasid_states_level2(struct pasid_state **tbl) in free_pasid_states_level2()
306 struct pasid_state **ptr; in free_pasid_states_level2()
313 ptr = (struct pasid_state **)tbl[i]; in free_pasid_states_level2()
320 struct pasid_state *pasid_state; in free_pasid_states() local
324 pasid_state = get_pasid_state(dev_state, i); in free_pasid_states()
325 if (pasid_state == NULL) in free_pasid_states()
328 put_pasid_state(pasid_state); in free_pasid_states()
334 mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm); in free_pasid_states()
336 put_pasid_state_wait(pasid_state); /* Reference taken in in free_pasid_states()
353 static struct pasid_state *mn_to_state(struct mmu_notifier *mn) in mn_to_state()
355 return container_of(mn, struct pasid_state, mn); in mn_to_state()
362 struct pasid_state *pasid_state; in mn_invalidate_range() local
365 pasid_state = mn_to_state(mn); in mn_invalidate_range()
366 dev_state = pasid_state->device_state; in mn_invalidate_range()
369 amd_iommu_flush_page(dev_state->domain, pasid_state->pasid, in mn_invalidate_range()
372 amd_iommu_flush_tlb(dev_state->domain, pasid_state->pasid); in mn_invalidate_range()
377 struct pasid_state *pasid_state; in mn_release() local
383 pasid_state = mn_to_state(mn); in mn_release()
384 dev_state = pasid_state->device_state; in mn_release()
385 run_inv_ctx_cb = !pasid_state->invalid; in mn_release()
388 dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid); in mn_release()
390 unbind_pasid(pasid_state); in mn_release()
398 static void set_pri_tag_status(struct pasid_state *pasid_state, in set_pri_tag_status() argument
403 spin_lock_irqsave(&pasid_state->lock, flags); in set_pri_tag_status()
404 pasid_state->pri[tag].status = status; in set_pri_tag_status()
405 spin_unlock_irqrestore(&pasid_state->lock, flags); in set_pri_tag_status()
409 struct pasid_state *pasid_state, in finish_pri_tag() argument
414 spin_lock_irqsave(&pasid_state->lock, flags); in finish_pri_tag()
415 if (atomic_dec_and_test(&pasid_state->pri[tag].inflight) && in finish_pri_tag()
416 pasid_state->pri[tag].finish) { in finish_pri_tag()
417 amd_iommu_complete_ppr(dev_state->pdev, pasid_state->pasid, in finish_pri_tag()
418 pasid_state->pri[tag].status, tag); in finish_pri_tag()
419 pasid_state->pri[tag].finish = false; in finish_pri_tag()
420 pasid_state->pri[tag].status = PPR_SUCCESS; in finish_pri_tag()
422 spin_unlock_irqrestore(&pasid_state->lock, flags); in finish_pri_tag()
515 struct pasid_state *pasid_state; in ppr_notifier() local
548 pasid_state = get_pasid_state(dev_state, iommu_fault->pasid); in ppr_notifier()
549 if (pasid_state == NULL || pasid_state->invalid) { in ppr_notifier()
556 spin_lock_irqsave(&pasid_state->lock, flags); in ppr_notifier()
557 atomic_inc(&pasid_state->pri[tag].inflight); in ppr_notifier()
559 pasid_state->pri[tag].finish = true; in ppr_notifier()
560 spin_unlock_irqrestore(&pasid_state->lock, flags); in ppr_notifier()
565 finish_pri_tag(dev_state, pasid_state, tag); in ppr_notifier()
571 fault->state = pasid_state; in ppr_notifier()
584 if (ret != NOTIFY_OK && pasid_state) in ppr_notifier()
585 put_pasid_state(pasid_state); in ppr_notifier()
600 struct pasid_state *pasid_state; in amd_iommu_bind_pasid() local
622 pasid_state = kzalloc(sizeof(*pasid_state), GFP_KERNEL); in amd_iommu_bind_pasid()
623 if (pasid_state == NULL) in amd_iommu_bind_pasid()
627 refcount_set(&pasid_state->count, 1); in amd_iommu_bind_pasid()
628 init_waitqueue_head(&pasid_state->wq); in amd_iommu_bind_pasid()
629 spin_lock_init(&pasid_state->lock); in amd_iommu_bind_pasid()
632 pasid_state->mm = mm; in amd_iommu_bind_pasid()
633 pasid_state->device_state = dev_state; in amd_iommu_bind_pasid()
634 pasid_state->pasid = pasid; in amd_iommu_bind_pasid()
635 pasid_state->invalid = true; /* Mark as valid only if we are in amd_iommu_bind_pasid()
637 pasid_state->mn.ops = &iommu_mn; in amd_iommu_bind_pasid()
639 if (pasid_state->mm == NULL) in amd_iommu_bind_pasid()
642 mmu_notifier_register(&pasid_state->mn, mm); in amd_iommu_bind_pasid()
644 ret = set_pasid_state(dev_state, pasid_state, pasid); in amd_iommu_bind_pasid()
649 __pa(pasid_state->mm->pgd)); in amd_iommu_bind_pasid()
654 pasid_state->invalid = false; in amd_iommu_bind_pasid()
669 mmu_notifier_unregister(&pasid_state->mn, mm); in amd_iommu_bind_pasid()
673 free_pasid_state(pasid_state); in amd_iommu_bind_pasid()
684 struct pasid_state *pasid_state; in amd_iommu_unbind_pasid() local
701 pasid_state = get_pasid_state(dev_state, pasid); in amd_iommu_unbind_pasid()
702 if (pasid_state == NULL) in amd_iommu_unbind_pasid()
708 put_pasid_state(pasid_state); in amd_iommu_unbind_pasid()
711 clear_pasid_state(dev_state, pasid_state->pasid); in amd_iommu_unbind_pasid()
717 mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm); in amd_iommu_unbind_pasid()
719 put_pasid_state_wait(pasid_state); /* Reference taken in in amd_iommu_unbind_pasid()