21 static void update_inactive(struct msm_gem_object *msm_obj);
25 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in physaddr()  local
27 	return (((dma_addr_t)msm_obj->vram_node->start) << PAGE_SHIFT) +  in physaddr()
33 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in use_pages()  local
34 	return !msm_obj->vram_node;  in use_pages()
51 static void sync_for_device(struct msm_gem_object *msm_obj)  in sync_for_device()  argument
53 	struct device *dev = msm_obj->base.dev->dev;  in sync_for_device()
55 	dma_map_sgtable(dev, msm_obj->sgt, DMA_BIDIRECTIONAL, 0);  in sync_for_device()
58 static void sync_for_cpu(struct msm_gem_object *msm_obj)  in sync_for_cpu()  argument
60 	struct device *dev = msm_obj->base.dev->dev;  in sync_for_cpu()
62 	dma_unmap_sgtable(dev, msm_obj->sgt, DMA_BIDIRECTIONAL, 0);  in sync_for_cpu()
68 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in get_pages_vram()  local
79 	ret = drm_mm_insert_node(&priv->vram.mm, msm_obj->vram_node, npages);  in get_pages_vram()
97 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in get_pages()  local
101 	if (!msm_obj->pages) {  in get_pages()
117 		msm_obj->pages = p;  in get_pages()
119 		msm_obj->sgt = drm_prime_pages_to_sg(obj->dev, p, npages);  in get_pages()
120 		if (IS_ERR(msm_obj->sgt)) {  in get_pages()
121 			void *ptr = ERR_CAST(msm_obj->sgt);  in get_pages()
124 			msm_obj->sgt = NULL;  in get_pages()
131 		if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED))  in get_pages()
132 			sync_for_device(msm_obj);  in get_pages()
134 		update_inactive(msm_obj);  in get_pages()
137 	return msm_obj->pages;  in get_pages()
142 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in put_pages_vram()  local
146 	drm_mm_remove_node(msm_obj->vram_node);  in put_pages_vram()
149 	kvfree(msm_obj->pages);  in put_pages_vram()
154 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in put_pages()  local
156 	if (msm_obj->pages) {  in put_pages()
157 		if (msm_obj->sgt) {  in put_pages()
162 			if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED))  in put_pages()
163 				sync_for_cpu(msm_obj);  in put_pages()
165 			sg_free_table(msm_obj->sgt);  in put_pages()
166 			kfree(msm_obj->sgt);  in put_pages()
167 			msm_obj->sgt = NULL;  in put_pages()
171 			drm_gem_put_pages(obj, msm_obj->pages, true, false);  in put_pages()
175 		msm_obj->pages = NULL;  in put_pages()
181 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_get_pages()  local
186 	if (GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) {  in msm_gem_get_pages()
194 		msm_obj->pin_count++;  in msm_gem_get_pages()
195 		update_inactive(msm_obj);  in msm_gem_get_pages()
204 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_put_pages()  local
207 	msm_obj->pin_count--;  in msm_gem_put_pages()
208 	GEM_WARN_ON(msm_obj->pin_count < 0);  in msm_gem_put_pages()
209 	update_inactive(msm_obj);  in msm_gem_put_pages()
213 static pgprot_t msm_gem_pgprot(struct msm_gem_object *msm_obj, pgprot_t prot)  in msm_gem_pgprot()  argument
215 	if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED))  in msm_gem_pgprot()
224 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_fault()  local
241 	if (GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) {  in msm_gem_fault()
300 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in add_vma()  local
311 	list_add_tail(&vma->list, &msm_obj->vmas);  in add_vma()
319 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in lookup_vma()  local
324 	list_for_each_entry(vma, &msm_obj->vmas, list) {  in lookup_vma()
350 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in put_iova_spaces()  local
355 	list_for_each_entry(vma, &msm_obj->vmas, list) {  in put_iova_spaces()
368 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in put_iova_vmas()  local
373 	list_for_each_entry_safe(vma, tmp, &msm_obj->vmas, list) {  in put_iova_vmas()
409 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_pin_iova()  local
414 	if (!(msm_obj->flags & MSM_BO_GPU_READONLY))  in msm_gem_pin_iova()
417 	if (msm_obj->flags & MSM_BO_MAP_PRIV)  in msm_gem_pin_iova()
420 	if (msm_obj->flags & MSM_BO_CACHED_COHERENT)  in msm_gem_pin_iova()
425 	if (GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED))  in msm_gem_pin_iova()
437 			msm_obj->sgt, obj->size >> PAGE_SHIFT);  in msm_gem_pin_iova()
440 		msm_obj->pin_count++;  in msm_gem_pin_iova()
534 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_unpin_iova_locked()  local
544 		msm_obj->pin_count--;  in msm_gem_unpin_iova_locked()
545 		GEM_WARN_ON(msm_obj->pin_count < 0);  in msm_gem_unpin_iova_locked()
547 		update_inactive(msm_obj);  in msm_gem_unpin_iova_locked()
596 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in get_vaddr()  local
604 	if (GEM_WARN_ON(msm_obj->madv > madv)) {  in get_vaddr()
606 			msm_obj->madv, madv);  in get_vaddr()
616 	msm_obj->vmap_count++;  in get_vaddr()
618 	if (!msm_obj->vaddr) {  in get_vaddr()
624 		msm_obj->vaddr = vmap(pages, obj->size >> PAGE_SHIFT,  in get_vaddr()
625 				VM_MAP, msm_gem_pgprot(msm_obj, PAGE_KERNEL));  in get_vaddr()
626 		if (msm_obj->vaddr == NULL) {  in get_vaddr()
631 		update_inactive(msm_obj);  in get_vaddr()
634 	return msm_obj->vaddr;  in get_vaddr()
637 	msm_obj->vmap_count--;  in get_vaddr()
670 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_put_vaddr_locked()  local
673 	GEM_WARN_ON(msm_obj->vmap_count < 1);  in msm_gem_put_vaddr_locked()
675 	msm_obj->vmap_count--;  in msm_gem_put_vaddr_locked()
690 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_madvise()  local
694 	if (msm_obj->madv != __MSM_MADV_PURGED)  in msm_gem_madvise()
695 		msm_obj->madv = madv;  in msm_gem_madvise()
697 	madv = msm_obj->madv;  in msm_gem_madvise()
702 	if (msm_obj->active_count == 0)  in msm_gem_madvise()
703 		update_inactive(msm_obj);  in msm_gem_madvise()
713 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_purge()  local
716 	GEM_WARN_ON(!is_purgeable(msm_obj));  in msm_gem_purge()
729 	msm_obj->madv = __MSM_MADV_PURGED;  in msm_gem_purge()
730 	update_inactive(msm_obj);  in msm_gem_purge()
751 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_evict()  local
754 	GEM_WARN_ON(is_unevictable(msm_obj));  in msm_gem_evict()
755 	GEM_WARN_ON(!msm_obj->evictable);  in msm_gem_evict()
756 	GEM_WARN_ON(msm_obj->active_count);  in msm_gem_evict()
765 	update_inactive(msm_obj);  in msm_gem_evict()
770 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_vunmap()  local
774 	if (!msm_obj->vaddr || GEM_WARN_ON(!is_vunmapable(msm_obj)))  in msm_gem_vunmap()
777 	vunmap(msm_obj->vaddr);  in msm_gem_vunmap()
778 	msm_obj->vaddr = NULL;  in msm_gem_vunmap()
783 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_active_get()  local
788 	GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED);  in msm_gem_active_get()
789 	GEM_WARN_ON(msm_obj->dontneed);  in msm_gem_active_get()
791 	if (msm_obj->active_count++ == 0) {  in msm_gem_active_get()
793 		if (msm_obj->evictable)  in msm_gem_active_get()
794 			mark_unevictable(msm_obj);  in msm_gem_active_get()
795 		list_move_tail(&msm_obj->mm_list, &gpu->active_list);  in msm_gem_active_get()
802 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_active_put()  local
807 	if (--msm_obj->active_count == 0) {  in msm_gem_active_put()
808 		update_inactive(msm_obj);  in msm_gem_active_put()
812 static void update_inactive(struct msm_gem_object *msm_obj)  in update_inactive()  argument
814 	struct msm_drm_private *priv = msm_obj->base.dev->dev_private;  in update_inactive()
816 	GEM_WARN_ON(!msm_gem_is_locked(&msm_obj->base));  in update_inactive()
818 	if (msm_obj->active_count != 0)  in update_inactive()
823 	if (msm_obj->dontneed)  in update_inactive()
824 		mark_unpurgeable(msm_obj);  in update_inactive()
825 	if (msm_obj->evictable)  in update_inactive()
826 		mark_unevictable(msm_obj);  in update_inactive()
828 	list_del(&msm_obj->mm_list);  in update_inactive()
829 	if ((msm_obj->madv == MSM_MADV_WILLNEED) && msm_obj->sgt) {  in update_inactive()
830 		list_add_tail(&msm_obj->mm_list, &priv->inactive_willneed);  in update_inactive()
831 		mark_evictable(msm_obj);  in update_inactive()
832 	} else if (msm_obj->madv == MSM_MADV_DONTNEED) {  in update_inactive()
833 		list_add_tail(&msm_obj->mm_list, &priv->inactive_dontneed);  in update_inactive()
834 		mark_purgeable(msm_obj);  in update_inactive()
836 		GEM_WARN_ON((msm_obj->madv != __MSM_MADV_PURGED) && msm_obj->sgt);  in update_inactive()
837 		list_add_tail(&msm_obj->mm_list, &priv->inactive_unpinned);  in update_inactive()
881 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_describe()  local
894 	if (is_active(msm_obj)) {  in msm_gem_describe()
899 	if (msm_obj->pages) {  in msm_gem_describe()
904 	switch (msm_obj->madv) {  in msm_gem_describe()
922 			msm_obj->flags, is_active(msm_obj) ? 'A' : 'I',  in msm_gem_describe()
924 			off, msm_obj->vaddr);  in msm_gem_describe()
926 	seq_printf(m, " %08zu %9s %-32s\n", obj->size, madv, msm_obj->name);  in msm_gem_describe()
928 	if (!list_empty(&msm_obj->vmas)) {  in msm_gem_describe()
932 		list_for_each_entry(vma, &msm_obj->vmas, list) {  in msm_gem_describe()
980 	struct msm_gem_object *msm_obj;  in msm_gem_describe_objects()  local
983 	list_for_each_entry(msm_obj, list, node) {  in msm_gem_describe_objects()
984 		struct drm_gem_object *obj = &msm_obj->base;  in msm_gem_describe_objects()
1005 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_free_object()  local
1010 	list_del(&msm_obj->node);  in msm_gem_free_object()
1014 	if (msm_obj->dontneed)  in msm_gem_free_object()
1015 		mark_unpurgeable(msm_obj);  in msm_gem_free_object()
1016 	list_del(&msm_obj->mm_list);  in msm_gem_free_object()
1022 	GEM_WARN_ON(is_active(msm_obj));  in msm_gem_free_object()
1027 		GEM_WARN_ON(msm_obj->vaddr);  in msm_gem_free_object()
1032 		kvfree(msm_obj->pages);  in msm_gem_free_object()
1041 		drm_prime_gem_destroy(obj, msm_obj->sgt);  in msm_gem_free_object()
1051 	kfree(msm_obj);  in msm_gem_free_object()
1056 	struct msm_gem_object *msm_obj = to_msm_bo(obj);  in msm_gem_object_mmap()  local
1060 	vma->vm_page_prot = msm_gem_pgprot(msm_obj, vm_get_page_prot(vma->vm_flags));  in msm_gem_object_mmap()
1111 	struct msm_gem_object *msm_obj;  in msm_gem_new_impl()  local
1128 	msm_obj = kzalloc(sizeof(*msm_obj), GFP_KERNEL);  in msm_gem_new_impl()
1129 	if (!msm_obj)  in msm_gem_new_impl()
1132 	msm_obj->flags = flags;  in msm_gem_new_impl()
1133 	msm_obj->madv = MSM_MADV_WILLNEED;  in msm_gem_new_impl()
1135 	INIT_LIST_HEAD(&msm_obj->vmas);  in msm_gem_new_impl()
1137 	*obj = &msm_obj->base;  in msm_gem_new_impl()
1146 	struct msm_gem_object *msm_obj;  in msm_gem_new()  local
1171 	msm_obj = to_msm_bo(obj);  in msm_gem_new()
1195 		INIT_LIST_HEAD(&msm_obj->mm_list);  in msm_gem_new()
1220 	list_add_tail(&msm_obj->mm_list, &priv->inactive_unpinned);  in msm_gem_new()
1224 	list_add_tail(&msm_obj->node, &priv->objects);  in msm_gem_new()
1238 	struct msm_gem_object *msm_obj;  in msm_gem_import()  local
1259 	msm_obj = to_msm_bo(obj);  in msm_gem_import()
1261 	msm_obj->sgt = sgt;  in msm_gem_import()
1262 	msm_obj->pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);  in msm_gem_import()
1263 	if (!msm_obj->pages) {  in msm_gem_import()
1269 	ret = drm_prime_sg_to_page_array(sgt, msm_obj->pages, npages);  in msm_gem_import()
1278 	list_add_tail(&msm_obj->mm_list, &priv->inactive_unpinned);  in msm_gem_import()
1282 	list_add_tail(&msm_obj->node, &priv->objects);  in msm_gem_import()
1340 	struct msm_gem_object *msm_obj = to_msm_bo(bo);  in msm_gem_object_set_name()  local
1347 	vsnprintf(msm_obj->name, sizeof(msm_obj->name), fmt, ap);  in msm_gem_object_set_name()