547 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in _ppgtt_get_root_entry()
574 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in _ppgtt_set_root_entry()
596 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_get_guest_entry()
608 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_set_guest_entry()
619 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_get_host_entry()
629 	struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_set_host_entry()
646 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in ppgtt_spt_get_entry()
675 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in ppgtt_spt_set_entry()
738 	radix_tree_delete(&spt->vgpu->gtt.spt_tree, spt->shadow_page.mfn);  in ppgtt_free_spt()
757 	radix_tree_for_each_slot(slot, &vgpu->gtt.spt_tree, &iter, 0) {  in ppgtt_free_all_spt()
801 	return radix_tree_lookup(&vgpu->gtt.spt_tree, mfn);  in intel_vgpu_find_spt_by_mfn()
843 	ret = radix_tree_insert(&vgpu->gtt.spt_tree, spt->shadow_page.mfn, spt);  in ppgtt_alloc_spt()
897 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
903 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
931 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_invalidate_spt_by_shadow_entry()
941 			vgpu->gtt.scratch_pt[cur_pt_type].page_mfn)  in ppgtt_invalidate_spt_by_shadow_entry()
957 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_invalidate_pte()
965 	if (!pfn || pfn == vgpu->gtt.scratch_pt[type].page_mfn)  in ppgtt_invalidate_pte()
1045 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_populate_spt_by_guest_entry()
1103 	struct intel_gvt_gtt_pte_ops *ops = s->vgpu->gvt->gtt.pte_ops;  in ppgtt_generate_shadow_entry()
1122 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in is_2MB_gtt_possible()
1139 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in split_2MB_gtt_entry()
1186 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in split_64KB_gtt_entry()
1217 	struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_populate_shadow_entry()
1271 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in ppgtt_populate_spt()
1293 				ops->set_pfn(&se, gvt->gtt.scratch_mfn);  in ppgtt_populate_spt()
1314 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_entry_removal()
1327 	    vgpu->gtt.scratch_pt[spt->shadow_page.type].page_mfn)  in ppgtt_handle_guest_entry_removal()
1395 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in sync_oos_page()
1447 	list_move_tail(&oos_page->list, &gvt->gtt.oos_page_free_list_head);  in detach_oos_page()
1467 	list_move_tail(&oos_page->list, &gvt->gtt.oos_page_use_list_head);  in attach_oos_page()
1493 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in ppgtt_allocate_oos_page()  local
1499 	if (list_empty(&gtt->oos_page_free_list_head)) {  in ppgtt_allocate_oos_page()
1500 		oos_page = container_of(gtt->oos_page_use_list_head.next,  in ppgtt_allocate_oos_page()
1509 		oos_page = container_of(gtt->oos_page_free_list_head.next,  in ppgtt_allocate_oos_page()
1524 	list_add_tail(&oos_page->vm_list, &spt->vgpu->gtt.oos_page_list_head);  in ppgtt_set_guest_page_oos()
1547 	list_for_each_safe(pos, n, &vgpu->gtt.oos_page_list_head) {  in intel_vgpu_sync_oos_pages()
1566 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_write_page_table()
1598 					vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table()
1605 				     vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table()
1609 				     vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table()
1638 			&spt->vgpu->gtt.post_shadow_list_head);  in ppgtt_set_post_shadow()
1659 	list_for_each_safe(pos, n, &vgpu->gtt.post_shadow_list_head) {  in intel_vgpu_flush_post_shadow()
1683 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_write_page_table_bytes()
1717 			ops->set_pfn(&se, vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table_bytes()
1747 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in invalidate_ppgtt_mm()  local
1748 	struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;  in invalidate_ppgtt_mm()
1777 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in shadow_ppgtt_mm()  local
1778 	struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;  in shadow_ppgtt_mm()
1880 	list_add_tail(&mm->ppgtt_mm.list, &vgpu->gtt.ppgtt_mm_list_head);  in intel_vgpu_create_ppgtt_mm()
1881 	list_add_tail(&mm->ppgtt_mm.lru_list, &gvt->gtt.ppgtt_mm_lru_list_head);  in intel_vgpu_create_ppgtt_mm()
1969 			       &mm->vgpu->gvt->gtt.ppgtt_mm_lru_list_head);  in intel_vgpu_pin_mm()
1981 	list_for_each_safe(pos, n, &gvt->gtt.ppgtt_mm_lru_list_head) {  in reclaim_one_ppgtt_mm()
2001 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_get_next_level_entry()
2030 	struct intel_gvt_gtt_pte_ops *pte_ops = gvt->gtt.pte_ops;  in intel_vgpu_gma_to_gpa()
2031 	struct intel_gvt_gtt_gma_ops *gma_ops = gvt->gtt.gma_ops;  in intel_vgpu_gma_to_gpa()
2103 	struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;  in emulate_ggtt_mmio_read()
2146 	struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in ggtt_invalidate_pte()
2150 	if (pfn != vgpu->gvt->gtt.scratch_mfn)  in ggtt_invalidate_pte()
2160 	struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;  in emulate_ggtt_mmio_write()
2161 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in emulate_ggtt_mmio_write()
2222 			ops->set_pfn(&m, gvt->gtt.scratch_mfn);  in emulate_ggtt_mmio_write()
2246 			ops->set_pfn(&m, gvt->gtt.scratch_mfn);  in emulate_ggtt_mmio_write()
2258 			ops->set_pfn(&m, gvt->gtt.scratch_mfn);  in emulate_ggtt_mmio_write()
2264 		ops->set_pfn(&m, gvt->gtt.scratch_mfn);  in emulate_ggtt_mmio_write()
2304 	struct intel_vgpu_gtt *gtt = &vgpu->gtt;  in alloc_scratch_pages()  local
2305 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in alloc_scratch_pages()
2329 	gtt->scratch_pt[type].page_mfn =  in alloc_scratch_pages()
2331 	gtt->scratch_pt[type].page = virt_to_page(scratch_pt);  in alloc_scratch_pages()
2333 			vgpu->id, type, gtt->scratch_pt[type].page_mfn);  in alloc_scratch_pages()
2348 		ops->set_pfn(&se, gtt->scratch_pt[type - 1].page_mfn);  in alloc_scratch_pages()
2371 		if (vgpu->gtt.scratch_pt[i].page != NULL) {  in release_scratch_page_tree()
2372 			daddr = (dma_addr_t)(vgpu->gtt.scratch_pt[i].page_mfn <<  in release_scratch_page_tree()
2375 			__free_page(vgpu->gtt.scratch_pt[i].page);  in release_scratch_page_tree()
2376 			vgpu->gtt.scratch_pt[i].page = NULL;  in release_scratch_page_tree()
2377 			vgpu->gtt.scratch_pt[i].page_mfn = 0;  in release_scratch_page_tree()
2413 	struct intel_vgpu_gtt *gtt = &vgpu->gtt;  in intel_vgpu_init_gtt()  local
2415 	INIT_RADIX_TREE(&gtt->spt_tree, GFP_KERNEL);  in intel_vgpu_init_gtt()
2417 	INIT_LIST_HEAD(&gtt->ppgtt_mm_list_head);  in intel_vgpu_init_gtt()
2418 	INIT_LIST_HEAD(&gtt->oos_page_list_head);  in intel_vgpu_init_gtt()
2419 	INIT_LIST_HEAD(&gtt->post_shadow_list_head);  in intel_vgpu_init_gtt()
2421 	gtt->ggtt_mm = intel_vgpu_create_ggtt_mm(vgpu);  in intel_vgpu_init_gtt()
2422 	if (IS_ERR(gtt->ggtt_mm)) {  in intel_vgpu_init_gtt()
2424 		return PTR_ERR(gtt->ggtt_mm);  in intel_vgpu_init_gtt()
2437 	list_for_each_safe(pos, n, &vgpu->gtt.ppgtt_mm_list_head) {  in intel_vgpu_destroy_all_ppgtt_mm()
2442 	if (GEM_WARN_ON(!list_empty(&vgpu->gtt.ppgtt_mm_list_head)))  in intel_vgpu_destroy_all_ppgtt_mm()
2445 	if (GEM_WARN_ON(!radix_tree_empty(&vgpu->gtt.spt_tree))) {  in intel_vgpu_destroy_all_ppgtt_mm()
2453 	intel_vgpu_destroy_mm(vgpu->gtt.ggtt_mm);  in intel_vgpu_destroy_ggtt_mm()
2454 	vgpu->gtt.ggtt_mm = NULL;  in intel_vgpu_destroy_ggtt_mm()
2476 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in clean_spt_oos()  local
2480 	WARN(!list_empty(&gtt->oos_page_use_list_head),  in clean_spt_oos()
2483 	list_for_each_safe(pos, n, &gtt->oos_page_free_list_head) {  in clean_spt_oos()
2492 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in setup_spt_oos()  local
2497 	INIT_LIST_HEAD(&gtt->oos_page_free_list_head);  in setup_spt_oos()
2498 	INIT_LIST_HEAD(&gtt->oos_page_use_list_head);  in setup_spt_oos()
2510 		list_add_tail(&oos_page->list, &gtt->oos_page_free_list_head);  in setup_spt_oos()
2538 	list_for_each(pos, &vgpu->gtt.ppgtt_mm_list_head) {  in intel_vgpu_find_ppgtt_mm()
2627 	gvt->gtt.pte_ops = &gen8_gtt_pte_ops;  in intel_gvt_init_gtt()
2628 	gvt->gtt.gma_ops = &gen8_gtt_gma_ops;  in intel_gvt_init_gtt()
2644 	gvt->gtt.scratch_page = virt_to_page(page);  in intel_gvt_init_gtt()
2645 	gvt->gtt.scratch_mfn = (unsigned long)(daddr >> I915_GTT_PAGE_SHIFT);  in intel_gvt_init_gtt()
2652 			__free_page(gvt->gtt.scratch_page);  in intel_gvt_init_gtt()
2656 	INIT_LIST_HEAD(&gvt->gtt.ppgtt_mm_lru_list_head);  in intel_gvt_init_gtt()
2671 	dma_addr_t daddr = (dma_addr_t)(gvt->gtt.scratch_mfn <<  in intel_gvt_clean_gtt()
2676 	__free_page(gvt->gtt.scratch_page);  in intel_gvt_clean_gtt()
2694 	list_for_each_safe(pos, n, &vgpu->gtt.ppgtt_mm_list_head) {  in intel_vgpu_invalidate_ppgtt()
2717 	struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in intel_vgpu_reset_ggtt()
2723 	pte_ops->set_pfn(&entry, gvt->gtt.scratch_mfn);  in intel_vgpu_reset_ggtt()
2730 			ggtt_get_host_entry(vgpu->gtt.ggtt_mm, &old_entry, index);  in intel_vgpu_reset_ggtt()
2733 		ggtt_set_host_entry(vgpu->gtt.ggtt_mm, &entry, index++);  in intel_vgpu_reset_ggtt()
2740 			ggtt_get_host_entry(vgpu->gtt.ggtt_mm, &old_entry, index);  in intel_vgpu_reset_ggtt()
2743 		ggtt_set_host_entry(vgpu->gtt.ggtt_mm, &entry, index++);  in intel_vgpu_reset_ggtt()