26 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in physaddr()  local
28 	return (((dma_addr_t)msm_obj->vram_node->start) << PAGE_SHIFT) +  in physaddr()
34 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in use_pages()  local
35 	return !msm_obj->vram_node;  in use_pages()
52 static void sync_for_device(struct msm_gem_object *msm_obj)  in sync_for_device()  argument
54 	struct device *dev = msm_obj->base.dev->dev;  in sync_for_device()
56 	dma_map_sgtable(dev, msm_obj->sgt, DMA_BIDIRECTIONAL, 0);  in sync_for_device()
59 static void sync_for_cpu(struct msm_gem_object *msm_obj)  in sync_for_cpu()  argument
61 	struct device *dev = msm_obj->base.dev->dev;  in sync_for_cpu()
63 	dma_unmap_sgtable(dev, msm_obj->sgt, DMA_BIDIRECTIONAL, 0);  in sync_for_cpu()
69 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in get_pages_vram()  local
80 	ret = drm_mm_insert_node(&priv->vram.mm, msm_obj->vram_node, npages);  in get_pages_vram()
98 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in get_pages()  local
102 	if (!msm_obj->pages) {  in get_pages()
118 		msm_obj->pages = p;  in get_pages()
120 		msm_obj->sgt = drm_prime_pages_to_sg(obj->dev, p, npages);  in get_pages()
121 		if (IS_ERR(msm_obj->sgt)) {  in get_pages()
122 			void *ptr = ERR_CAST(msm_obj->sgt);  in get_pages()
125 			msm_obj->sgt = NULL;  in get_pages()
132 		if (msm_obj->flags & MSM_BO_WC)  in get_pages()
133 			sync_for_device(msm_obj);  in get_pages()
138 	return msm_obj->pages;  in get_pages()
143 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in put_pages_vram()  local
147 	drm_mm_remove_node(msm_obj->vram_node);  in put_pages_vram()
150 	kvfree(msm_obj->pages);  in put_pages_vram()
155 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in put_pages()  local
157 	if (msm_obj->pages) {  in put_pages()
158 		if (msm_obj->sgt) {  in put_pages()
163 			if (msm_obj->flags & MSM_BO_WC)  in put_pages()
164 				sync_for_cpu(msm_obj);  in put_pages()
166 			sg_free_table(msm_obj->sgt);  in put_pages()
167 			kfree(msm_obj->sgt);  in put_pages()
168 			msm_obj->sgt = NULL;  in put_pages()
172 			drm_gem_put_pages(obj, msm_obj->pages, true, false);  in put_pages()
176 		msm_obj->pages = NULL;  in put_pages()
183 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_pin_pages_locked()  local
188 	if (GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) {  in msm_gem_pin_pages_locked()
219 static pgprot_t msm_gem_pgprot(struct msm_gem_object *msm_obj, pgprot_t prot)  in msm_gem_pgprot()  argument
221 	if (msm_obj->flags & MSM_BO_WC)  in msm_gem_pgprot()
230 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_fault()  local
247 	if (GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) {  in msm_gem_fault()
307 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in add_vma()  local
318 	list_add_tail(&vma->list, &msm_obj->vmas);  in add_vma()
326 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in lookup_vma()  local
331 	list_for_each_entry(vma, &msm_obj->vmas, list) {  in lookup_vma()
357 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in put_iova_spaces()  local
362 	list_for_each_entry(vma, &msm_obj->vmas, list) {  in put_iova_spaces()
375 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in put_iova_vmas()  local
380 	list_for_each_entry_safe(vma, tmp, &msm_obj->vmas, list) {  in put_iova_vmas()
418 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_pin_vma_locked()  local
422 	if (!(msm_obj->flags & MSM_BO_GPU_READONLY))  in msm_gem_pin_vma_locked()
425 	if (msm_obj->flags & MSM_BO_MAP_PRIV)  in msm_gem_pin_vma_locked()
428 	if (msm_obj->flags & MSM_BO_CACHED_COHERENT)  in msm_gem_pin_vma_locked()
433 	if (GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED))  in msm_gem_pin_vma_locked()
440 	ret = msm_gem_map_vma(vma->aspace, vma, prot, msm_obj->sgt, obj->size);  in msm_gem_pin_vma_locked()
449 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_unpin_locked()  local
453 	msm_obj->pin_count--;  in msm_gem_unpin_locked()
454 	GEM_WARN_ON(msm_obj->pin_count < 0);  in msm_gem_unpin_locked()
630 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in get_vaddr()  local
638 	if (GEM_WARN_ON(msm_obj->madv > madv)) {  in get_vaddr()
640 			msm_obj->madv, madv);  in get_vaddr()
650 	msm_obj->vmap_count++;  in get_vaddr()
652 	if (!msm_obj->vaddr) {  in get_vaddr()
658 		msm_obj->vaddr = vmap(pages, obj->size >> PAGE_SHIFT,  in get_vaddr()
659 				VM_MAP, msm_gem_pgprot(msm_obj, PAGE_KERNEL));  in get_vaddr()
660 		if (msm_obj->vaddr == NULL) {  in get_vaddr()
668 	return msm_obj->vaddr;  in get_vaddr()
671 	msm_obj->vmap_count--;  in get_vaddr()
704 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_put_vaddr_locked()  local
707 	GEM_WARN_ON(msm_obj->vmap_count < 1);  in msm_gem_put_vaddr_locked()
709 	msm_obj->vmap_count--;  in msm_gem_put_vaddr_locked()
724 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_madvise()  local
728 	if (msm_obj->madv != __MSM_MADV_PURGED)  in msm_gem_madvise()
729 		msm_obj->madv = madv;  in msm_gem_madvise()
731 	madv = msm_obj->madv;  in msm_gem_madvise()
746 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_purge()  local
749 	GEM_WARN_ON(!is_purgeable(msm_obj));  in msm_gem_purge()
762 	msm_obj->madv = __MSM_MADV_PURGED;  in msm_gem_purge()
783 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_evict()  local
786 	GEM_WARN_ON(is_unevictable(msm_obj));  in msm_gem_evict()
798 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_vunmap()  local
802 	if (!msm_obj->vaddr || GEM_WARN_ON(!is_vunmapable(msm_obj)))  in msm_gem_vunmap()
805 	vunmap(msm_obj->vaddr);  in msm_gem_vunmap()
806 	msm_obj->vaddr = NULL;  in msm_gem_vunmap()
812 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in update_lru()  local
814 	msm_gem_assert_locked(&msm_obj->base);  in update_lru()
816 	if (!msm_obj->pages) {  in update_lru()
817 		GEM_WARN_ON(msm_obj->pin_count);  in update_lru()
818 		GEM_WARN_ON(msm_obj->vmap_count);  in update_lru()
821 	} else if (msm_obj->pin_count || msm_obj->vmap_count) {  in update_lru()
823 	} else if (msm_obj->madv == MSM_MADV_WILLNEED) {  in update_lru()
826 		GEM_WARN_ON(msm_obj->madv != MSM_MADV_DONTNEED);  in update_lru()
871 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_describe()  local
887 	if (msm_obj->pages) {  in msm_gem_describe()
892 	switch (msm_obj->madv) {  in msm_gem_describe()
910 			msm_obj->flags, msm_gem_active(obj) ? 'A' : 'I',  in msm_gem_describe()
912 			off, msm_obj->vaddr);  in msm_gem_describe()
914 	seq_printf(m, " %08zu %9s %-32s\n", obj->size, madv, msm_obj->name);  in msm_gem_describe()
916 	if (!list_empty(&msm_obj->vmas)) {  in msm_gem_describe()
920 		list_for_each_entry(vma, &msm_obj->vmas, list) {  in msm_gem_describe()
954 	struct msm_gem_object *msm_obj;  in msm_gem_describe_objects()  local
957 	list_for_each_entry(msm_obj, list, node) {  in msm_gem_describe_objects()
958 		struct drm_gem_object *obj = &msm_obj->base;  in msm_gem_describe_objects()
979 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_free_object()  local
984 	list_del(&msm_obj->node);  in msm_gem_free_object()
990 		GEM_WARN_ON(msm_obj->vaddr);  in msm_gem_free_object()
995 		kvfree(msm_obj->pages);  in msm_gem_free_object()
999 		drm_prime_gem_destroy(obj, msm_obj->sgt);  in msm_gem_free_object()
1008 	kfree(msm_obj);  in msm_gem_free_object()
1013 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_object_mmap()  local
1016 	vma->vm_page_prot = msm_gem_pgprot(msm_obj, vm_get_page_prot(vma->vm_flags));  in msm_gem_object_mmap()
1067 	struct msm_gem_object *msm_obj;  in msm_gem_new_impl()  local
1083 	msm_obj = kzalloc(sizeof(*msm_obj), GFP_KERNEL);  in msm_gem_new_impl()
1084 	if (!msm_obj)  in msm_gem_new_impl()
1087 	msm_obj->flags = flags;  in msm_gem_new_impl()
1088 	msm_obj->madv = MSM_MADV_WILLNEED;  in msm_gem_new_impl()
1090 	INIT_LIST_HEAD(&msm_obj->node);  in msm_gem_new_impl()
1091 	INIT_LIST_HEAD(&msm_obj->vmas);  in msm_gem_new_impl()
1093 	*obj = &msm_obj->base;  in msm_gem_new_impl()
1102 	struct msm_gem_object *msm_obj;  in msm_gem_new()  local
1127 	msm_obj = to_msm_bo(obj);  in msm_gem_new()
1171 	list_add_tail(&msm_obj->node, &priv->objects);  in msm_gem_new()
1185 	struct msm_gem_object *msm_obj;  in msm_gem_import()  local
1206 	msm_obj = to_msm_bo(obj);  in msm_gem_import()
1208 	msm_obj->sgt = sgt;  in msm_gem_import()
1209 	msm_obj->pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);  in msm_gem_import()
1210 	if (!msm_obj->pages) {  in msm_gem_import()
1216 	ret = drm_prime_sg_to_page_array(sgt, msm_obj->pages, npages);  in msm_gem_import()
1227 	list_add_tail(&msm_obj->node, &priv->objects);  in msm_gem_import()
1285 	struct msm_gem_object *msm_obj = to_msm_bo(bo);  in msm_gem_object_set_name()  local
1292 	vsnprintf(msm_obj->name, sizeof(msm_obj->name), fmt, ap);  in msm_gem_object_set_name()