16 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)  in __i915_gem_object_flush_for_display()  argument
22 	i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);  in __i915_gem_object_flush_for_display()
23 	if (obj->cache_dirty)  in __i915_gem_object_flush_for_display()
24 		i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);  in __i915_gem_object_flush_for_display()
25 	obj->write_domain = 0;  in __i915_gem_object_flush_for_display()
28 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)  in i915_gem_object_flush_if_display()  argument
30 	if (!READ_ONCE(obj->pin_global))  in i915_gem_object_flush_if_display()
33 	i915_gem_object_lock(obj);  in i915_gem_object_flush_if_display()
34 	__i915_gem_object_flush_for_display(obj);  in i915_gem_object_flush_if_display()
35 	i915_gem_object_unlock(obj);  in i915_gem_object_flush_if_display()
47 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)  in i915_gem_object_set_to_wc_domain()  argument
51 	assert_object_held(obj);  in i915_gem_object_set_to_wc_domain()
53 	ret = i915_gem_object_wait(obj,  in i915_gem_object_set_to_wc_domain()
60 	if (obj->write_domain == I915_GEM_DOMAIN_WC)  in i915_gem_object_set_to_wc_domain()
71 	ret = i915_gem_object_pin_pages(obj);  in i915_gem_object_set_to_wc_domain()
75 	i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);  in i915_gem_object_set_to_wc_domain()
81 	if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0)  in i915_gem_object_set_to_wc_domain()
87 	GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0);  in i915_gem_object_set_to_wc_domain()
88 	obj->read_domains |= I915_GEM_DOMAIN_WC;  in i915_gem_object_set_to_wc_domain()
90 		obj->read_domains = I915_GEM_DOMAIN_WC;  in i915_gem_object_set_to_wc_domain()
91 		obj->write_domain = I915_GEM_DOMAIN_WC;  in i915_gem_object_set_to_wc_domain()
92 		obj->mm.dirty = true;  in i915_gem_object_set_to_wc_domain()
95 	i915_gem_object_unpin_pages(obj);  in i915_gem_object_set_to_wc_domain()
108 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)  in i915_gem_object_set_to_gtt_domain()  argument
112 	assert_object_held(obj);  in i915_gem_object_set_to_gtt_domain()
114 	ret = i915_gem_object_wait(obj,  in i915_gem_object_set_to_gtt_domain()
121 	if (obj->write_domain == I915_GEM_DOMAIN_GTT)  in i915_gem_object_set_to_gtt_domain()
132 	ret = i915_gem_object_pin_pages(obj);  in i915_gem_object_set_to_gtt_domain()
136 	i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);  in i915_gem_object_set_to_gtt_domain()
142 	if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0)  in i915_gem_object_set_to_gtt_domain()
148 	GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);  in i915_gem_object_set_to_gtt_domain()
149 	obj->read_domains |= I915_GEM_DOMAIN_GTT;  in i915_gem_object_set_to_gtt_domain()
151 		obj->read_domains = I915_GEM_DOMAIN_GTT;  in i915_gem_object_set_to_gtt_domain()
152 		obj->write_domain = I915_GEM_DOMAIN_GTT;  in i915_gem_object_set_to_gtt_domain()
153 		obj->mm.dirty = true;  in i915_gem_object_set_to_gtt_domain()
156 	i915_gem_object_unpin_pages(obj);  in i915_gem_object_set_to_gtt_domain()
175 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,  in i915_gem_object_set_cache_level()  argument
181 	assert_object_held(obj);  in i915_gem_object_set_cache_level()
183 	if (obj->cache_level == cache_level)  in i915_gem_object_set_cache_level()
192 	list_for_each_entry(vma, &obj->vma.list, obj_link) {  in i915_gem_object_set_cache_level()
223 	if (atomic_read(&obj->bind_count)) {  in i915_gem_object_set_cache_level()
224 		struct drm_i915_private *i915 = to_i915(obj->base.dev);  in i915_gem_object_set_cache_level()
230 		ret = i915_gem_object_wait(obj,  in i915_gem_object_set_cache_level()
256 			if (obj->userfault_count)  in i915_gem_object_set_cache_level()
257 				__i915_gem_object_release_mmap(obj);  in i915_gem_object_set_cache_level()
267 			for_each_ggtt_vma(vma, obj) {  in i915_gem_object_set_cache_level()
287 		list_for_each_entry(vma, &obj->vma.list, obj_link) {  in i915_gem_object_set_cache_level()
297 	list_for_each_entry(vma, &obj->vma.list, obj_link)  in i915_gem_object_set_cache_level()
299 	i915_gem_object_set_cache_coherency(obj, cache_level);  in i915_gem_object_set_cache_level()
300 	obj->cache_dirty = true; /* Always invalidate stale cachelines */  in i915_gem_object_set_cache_level()
309 	struct drm_i915_gem_object *obj;  in i915_gem_get_caching_ioctl()  local
313 	obj = i915_gem_object_lookup_rcu(file, args->handle);  in i915_gem_get_caching_ioctl()
314 	if (!obj) {  in i915_gem_get_caching_ioctl()
319 	switch (obj->cache_level) {  in i915_gem_get_caching_ioctl()
343 	struct drm_i915_gem_object *obj;  in i915_gem_set_caching_ioctl()  local
370 	obj = i915_gem_object_lookup(file, args->handle);  in i915_gem_set_caching_ioctl()
371 	if (!obj)  in i915_gem_set_caching_ioctl()
378 	if (i915_gem_object_is_proxy(obj)) {  in i915_gem_set_caching_ioctl()
383 	if (obj->cache_level == level)  in i915_gem_set_caching_ioctl()
386 	ret = i915_gem_object_wait(obj,  in i915_gem_set_caching_ioctl()
396 	ret = i915_gem_object_lock_interruptible(obj);  in i915_gem_set_caching_ioctl()
398 		ret = i915_gem_object_set_cache_level(obj, level);  in i915_gem_set_caching_ioctl()
399 		i915_gem_object_unlock(obj);  in i915_gem_set_caching_ioctl()
404 	i915_gem_object_put(obj);  in i915_gem_set_caching_ioctl()
415 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,  in i915_gem_object_pin_to_display_plane()  argument
423 	assert_object_held(obj);  in i915_gem_object_pin_to_display_plane()
428 	obj->pin_global++;  in i915_gem_object_pin_to_display_plane()
439 	ret = i915_gem_object_set_cache_level(obj,  in i915_gem_object_pin_to_display_plane()
440 					      HAS_WT(to_i915(obj->base.dev)) ?  in i915_gem_object_pin_to_display_plane()
457 		vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment,  in i915_gem_object_pin_to_display_plane()
462 		vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags);  in i915_gem_object_pin_to_display_plane()
468 	__i915_gem_object_flush_for_display(obj);  in i915_gem_object_pin_to_display_plane()
473 	obj->read_domains |= I915_GEM_DOMAIN_GTT;  in i915_gem_object_pin_to_display_plane()
478 	obj->pin_global--;  in i915_gem_object_pin_to_display_plane()
482 static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)  in i915_gem_object_bump_inactive_ggtt()  argument
484 	struct drm_i915_private *i915 = to_i915(obj->base.dev);  in i915_gem_object_bump_inactive_ggtt()
487 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));  in i915_gem_object_bump_inactive_ggtt()
490 	for_each_ggtt_vma(vma, obj) {  in i915_gem_object_bump_inactive_ggtt()
498 	if (i915_gem_object_is_shrinkable(obj)) {  in i915_gem_object_bump_inactive_ggtt()
503 		if (obj->mm.madv == I915_MADV_WILLNEED)  in i915_gem_object_bump_inactive_ggtt()
504 			list_move_tail(&obj->mm.link, &i915->mm.shrink_list);  in i915_gem_object_bump_inactive_ggtt()
513 	struct drm_i915_gem_object *obj = vma->obj;  in i915_gem_object_unpin_from_display_plane()  local
515 	assert_object_held(obj);  in i915_gem_object_unpin_from_display_plane()
517 	if (WARN_ON(obj->pin_global == 0))  in i915_gem_object_unpin_from_display_plane()
520 	if (--obj->pin_global == 0)  in i915_gem_object_unpin_from_display_plane()
524 	i915_gem_object_bump_inactive_ggtt(obj);  in i915_gem_object_unpin_from_display_plane()
538 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)  in i915_gem_object_set_to_cpu_domain()  argument
542 	assert_object_held(obj);  in i915_gem_object_set_to_cpu_domain()
544 	ret = i915_gem_object_wait(obj,  in i915_gem_object_set_to_cpu_domain()
551 	i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);  in i915_gem_object_set_to_cpu_domain()
554 	if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {  in i915_gem_object_set_to_cpu_domain()
555 		i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);  in i915_gem_object_set_to_cpu_domain()
556 		obj->read_domains |= I915_GEM_DOMAIN_CPU;  in i915_gem_object_set_to_cpu_domain()
562 	GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU);  in i915_gem_object_set_to_cpu_domain()
568 		__start_cpu_write(obj);  in i915_gem_object_set_to_cpu_domain()
585 	struct drm_i915_gem_object *obj;  in i915_gem_set_domain_ioctl()  local
604 	obj = i915_gem_object_lookup(file, args->handle);  in i915_gem_set_domain_ioctl()
605 	if (!obj)  in i915_gem_set_domain_ioctl()
618 	if (READ_ONCE(obj->write_domain) == read_domains) {  in i915_gem_set_domain_ioctl()
628 	err = i915_gem_object_wait(obj,  in i915_gem_set_domain_ioctl()
642 	if (i915_gem_object_is_proxy(obj)) {  in i915_gem_set_domain_ioctl()
656 	err = i915_gem_object_pin_pages(obj);  in i915_gem_set_domain_ioctl()
660 	err = i915_gem_object_lock_interruptible(obj);  in i915_gem_set_domain_ioctl()
665 		err = i915_gem_object_set_to_wc_domain(obj, write_domain);  in i915_gem_set_domain_ioctl()
667 		err = i915_gem_object_set_to_gtt_domain(obj, write_domain);  in i915_gem_set_domain_ioctl()
669 		err = i915_gem_object_set_to_cpu_domain(obj, write_domain);  in i915_gem_set_domain_ioctl()
672 	i915_gem_object_bump_inactive_ggtt(obj);  in i915_gem_set_domain_ioctl()
674 	i915_gem_object_unlock(obj);  in i915_gem_set_domain_ioctl()
677 		intel_frontbuffer_invalidate(obj->frontbuffer, ORIGIN_CPU);  in i915_gem_set_domain_ioctl()
680 	i915_gem_object_unpin_pages(obj);  in i915_gem_set_domain_ioctl()
682 	i915_gem_object_put(obj);  in i915_gem_set_domain_ioctl()
691 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,  in i915_gem_object_prepare_read()  argument
697 	if (!i915_gem_object_has_struct_page(obj))  in i915_gem_object_prepare_read()
700 	ret = i915_gem_object_lock_interruptible(obj);  in i915_gem_object_prepare_read()
704 	ret = i915_gem_object_wait(obj,  in i915_gem_object_prepare_read()
710 	ret = i915_gem_object_pin_pages(obj);  in i915_gem_object_prepare_read()
714 	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||  in i915_gem_object_prepare_read()
716 		ret = i915_gem_object_set_to_cpu_domain(obj, false);  in i915_gem_object_prepare_read()
723 	i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);  in i915_gem_object_prepare_read()
730 	if (!obj->cache_dirty &&  in i915_gem_object_prepare_read()
731 	    !(obj->read_domains & I915_GEM_DOMAIN_CPU))  in i915_gem_object_prepare_read()
739 	i915_gem_object_unpin_pages(obj);  in i915_gem_object_prepare_read()
741 	i915_gem_object_unlock(obj);  in i915_gem_object_prepare_read()
745 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,  in i915_gem_object_prepare_write()  argument
751 	if (!i915_gem_object_has_struct_page(obj))  in i915_gem_object_prepare_write()
754 	ret = i915_gem_object_lock_interruptible(obj);  in i915_gem_object_prepare_write()
758 	ret = i915_gem_object_wait(obj,  in i915_gem_object_prepare_write()
765 	ret = i915_gem_object_pin_pages(obj);  in i915_gem_object_prepare_write()
769 	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||  in i915_gem_object_prepare_write()
771 		ret = i915_gem_object_set_to_cpu_domain(obj, true);  in i915_gem_object_prepare_write()
778 	i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);  in i915_gem_object_prepare_write()
785 	if (!obj->cache_dirty) {  in i915_gem_object_prepare_write()
792 		if (!(obj->read_domains & I915_GEM_DOMAIN_CPU))  in i915_gem_object_prepare_write()
797 	intel_frontbuffer_invalidate(obj->frontbuffer, ORIGIN_CPU);  in i915_gem_object_prepare_write()
798 	obj->mm.dirty = true;  in i915_gem_object_prepare_write()
803 	i915_gem_object_unpin_pages(obj);  in i915_gem_object_prepare_write()
805 	i915_gem_object_unlock(obj);  in i915_gem_object_prepare_write()