20 static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)  in gpu_write_needs_clflush()  argument
22 	struct drm_i915_private *i915 = to_i915(obj->base.dev);  in gpu_write_needs_clflush()
27 	return !(obj->cache_level == I915_CACHE_NONE ||  in gpu_write_needs_clflush()
28 		 obj->cache_level == I915_CACHE_WT);  in gpu_write_needs_clflush()
31 bool i915_gem_cpu_write_needs_clflush(struct drm_i915_gem_object *obj)  in i915_gem_cpu_write_needs_clflush()  argument
33 	struct drm_i915_private *i915 = to_i915(obj->base.dev);  in i915_gem_cpu_write_needs_clflush()
35 	if (obj->cache_dirty)  in i915_gem_cpu_write_needs_clflush()
41 	if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))  in i915_gem_cpu_write_needs_clflush()
45 	return i915_gem_object_is_framebuffer(obj);  in i915_gem_cpu_write_needs_clflush()
49 flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)  in flush_write_domain()  argument
53 	assert_object_held(obj);  in flush_write_domain()
55 	if (!(obj->write_domain & flush_domains))  in flush_write_domain()
58 	switch (obj->write_domain) {  in flush_write_domain()
60 		spin_lock(&obj->vma.lock);  in flush_write_domain()
61 		for_each_ggtt_vma(vma, obj) {  in flush_write_domain()
65 		spin_unlock(&obj->vma.lock);  in flush_write_domain()
67 		i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);  in flush_write_domain()
75 		i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);  in flush_write_domain()
79 		if (gpu_write_needs_clflush(obj))  in flush_write_domain()
80 			obj->cache_dirty = true;  in flush_write_domain()
84 	obj->write_domain = 0;  in flush_write_domain()
87 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)  in __i915_gem_object_flush_for_display()  argument
93 	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);  in __i915_gem_object_flush_for_display()
94 	if (obj->cache_dirty)  in __i915_gem_object_flush_for_display()
95 		i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);  in __i915_gem_object_flush_for_display()
96 	obj->write_domain = 0;  in __i915_gem_object_flush_for_display()
99 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)  in i915_gem_object_flush_if_display()  argument
101 	if (!i915_gem_object_is_framebuffer(obj))  in i915_gem_object_flush_if_display()
104 	i915_gem_object_lock(obj, NULL);  in i915_gem_object_flush_if_display()
105 	__i915_gem_object_flush_for_display(obj);  in i915_gem_object_flush_if_display()
106 	i915_gem_object_unlock(obj);  in i915_gem_object_flush_if_display()
109 void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj)  in i915_gem_object_flush_if_display_locked()  argument
111 	if (i915_gem_object_is_framebuffer(obj))  in i915_gem_object_flush_if_display_locked()
112 		__i915_gem_object_flush_for_display(obj);  in i915_gem_object_flush_if_display_locked()
124 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)  in i915_gem_object_set_to_wc_domain()  argument
128 	assert_object_held(obj);  in i915_gem_object_set_to_wc_domain()
130 	ret = i915_gem_object_wait(obj,  in i915_gem_object_set_to_wc_domain()
137 	if (obj->write_domain == I915_GEM_DOMAIN_WC)  in i915_gem_object_set_to_wc_domain()
148 	ret = i915_gem_object_pin_pages(obj);  in i915_gem_object_set_to_wc_domain()
152 	flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);  in i915_gem_object_set_to_wc_domain()
158 	if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0)  in i915_gem_object_set_to_wc_domain()
164 	GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0);  in i915_gem_object_set_to_wc_domain()
165 	obj->read_domains |= I915_GEM_DOMAIN_WC;  in i915_gem_object_set_to_wc_domain()
167 		obj->read_domains = I915_GEM_DOMAIN_WC;  in i915_gem_object_set_to_wc_domain()
168 		obj->write_domain = I915_GEM_DOMAIN_WC;  in i915_gem_object_set_to_wc_domain()
169 		obj->mm.dirty = true;  in i915_gem_object_set_to_wc_domain()
172 	i915_gem_object_unpin_pages(obj);  in i915_gem_object_set_to_wc_domain()
185 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)  in i915_gem_object_set_to_gtt_domain()  argument
189 	assert_object_held(obj);  in i915_gem_object_set_to_gtt_domain()
191 	ret = i915_gem_object_wait(obj,  in i915_gem_object_set_to_gtt_domain()
198 	if (obj->write_domain == I915_GEM_DOMAIN_GTT)  in i915_gem_object_set_to_gtt_domain()
209 	ret = i915_gem_object_pin_pages(obj);  in i915_gem_object_set_to_gtt_domain()
213 	flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);  in i915_gem_object_set_to_gtt_domain()
219 	if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0)  in i915_gem_object_set_to_gtt_domain()
225 	GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);  in i915_gem_object_set_to_gtt_domain()
226 	obj->read_domains |= I915_GEM_DOMAIN_GTT;  in i915_gem_object_set_to_gtt_domain()
230 		obj->read_domains = I915_GEM_DOMAIN_GTT;  in i915_gem_object_set_to_gtt_domain()
231 		obj->write_domain = I915_GEM_DOMAIN_GTT;  in i915_gem_object_set_to_gtt_domain()
232 		obj->mm.dirty = true;  in i915_gem_object_set_to_gtt_domain()
234 		spin_lock(&obj->vma.lock);  in i915_gem_object_set_to_gtt_domain()
235 		for_each_ggtt_vma(vma, obj)  in i915_gem_object_set_to_gtt_domain()
238 		spin_unlock(&obj->vma.lock);  in i915_gem_object_set_to_gtt_domain()
241 	i915_gem_object_unpin_pages(obj);  in i915_gem_object_set_to_gtt_domain()
260 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,  in i915_gem_object_set_cache_level()  argument
265 	if (obj->cache_level == cache_level)  in i915_gem_object_set_cache_level()
268 	ret = i915_gem_object_wait(obj,  in i915_gem_object_set_cache_level()
276 	if (obj->cache_level != cache_level) {  in i915_gem_object_set_cache_level()
277 		i915_gem_object_set_cache_coherency(obj, cache_level);  in i915_gem_object_set_cache_level()
278 		obj->cache_dirty = true;  in i915_gem_object_set_cache_level()
282 	return i915_gem_object_unbind(obj,  in i915_gem_object_set_cache_level()
291 	struct drm_i915_gem_object *obj;  in i915_gem_get_caching_ioctl()  local
298 	obj = i915_gem_object_lookup_rcu(file, args->handle);  in i915_gem_get_caching_ioctl()
299 	if (!obj) {  in i915_gem_get_caching_ioctl()
304 	switch (obj->cache_level) {  in i915_gem_get_caching_ioctl()
328 	struct drm_i915_gem_object *obj;  in i915_gem_set_caching_ioctl()  local
358 	obj = i915_gem_object_lookup(file, args->handle);  in i915_gem_set_caching_ioctl()
359 	if (!obj)  in i915_gem_set_caching_ioctl()
366 	if (i915_gem_object_is_proxy(obj)) {  in i915_gem_set_caching_ioctl()
371 		if (!i915_gem_object_is_userptr(obj) ||  in i915_gem_set_caching_ioctl()
378 	ret = i915_gem_object_lock_interruptible(obj, NULL);  in i915_gem_set_caching_ioctl()
382 	ret = i915_gem_object_set_cache_level(obj, level);  in i915_gem_set_caching_ioctl()
383 	i915_gem_object_unlock(obj);  in i915_gem_set_caching_ioctl()
386 	i915_gem_object_put(obj);  in i915_gem_set_caching_ioctl()
397 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,  in i915_gem_object_pin_to_display_plane()  argument
403 	struct drm_i915_private *i915 = to_i915(obj->base.dev);  in i915_gem_object_pin_to_display_plane()
408 	if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj))  in i915_gem_object_pin_to_display_plane()
421 	ret = i915_gem_object_set_cache_level(obj,  in i915_gem_object_pin_to_display_plane()
438 		vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0, alignment,  in i915_gem_object_pin_to_display_plane()
442 		vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0,  in i915_gem_object_pin_to_display_plane()
450 	i915_gem_object_flush_if_display_locked(obj);  in i915_gem_object_pin_to_display_plane()
464 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)  in i915_gem_object_set_to_cpu_domain()  argument
468 	assert_object_held(obj);  in i915_gem_object_set_to_cpu_domain()
470 	ret = i915_gem_object_wait(obj,  in i915_gem_object_set_to_cpu_domain()
477 	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);  in i915_gem_object_set_to_cpu_domain()
480 	if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {  in i915_gem_object_set_to_cpu_domain()
481 		i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);  in i915_gem_object_set_to_cpu_domain()
482 		obj->read_domains |= I915_GEM_DOMAIN_CPU;  in i915_gem_object_set_to_cpu_domain()
488 	GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU);  in i915_gem_object_set_to_cpu_domain()
494 		__start_cpu_write(obj);  in i915_gem_object_set_to_cpu_domain()
511 	struct drm_i915_gem_object *obj;  in i915_gem_set_domain_ioctl()  local
533 	obj = i915_gem_object_lookup(file, args->handle);  in i915_gem_set_domain_ioctl()
534 	if (!obj)  in i915_gem_set_domain_ioctl()
542 	err = i915_gem_object_wait(obj,  in i915_gem_set_domain_ioctl()
550 	if (i915_gem_object_is_userptr(obj)) {  in i915_gem_set_domain_ioctl()
555 		err = i915_gem_object_userptr_validate(obj);  in i915_gem_set_domain_ioctl()
557 			err = i915_gem_object_wait(obj,  in i915_gem_set_domain_ioctl()
571 	if (i915_gem_object_is_proxy(obj)) {  in i915_gem_set_domain_ioctl()
576 	err = i915_gem_object_lock_interruptible(obj, NULL);  in i915_gem_set_domain_ioctl()
589 	err = i915_gem_object_pin_pages(obj);  in i915_gem_set_domain_ioctl()
603 	if (READ_ONCE(obj->write_domain) == read_domains)  in i915_gem_set_domain_ioctl()
607 		err = i915_gem_object_set_to_wc_domain(obj, write_domain);  in i915_gem_set_domain_ioctl()
609 		err = i915_gem_object_set_to_gtt_domain(obj, write_domain);  in i915_gem_set_domain_ioctl()
611 		err = i915_gem_object_set_to_cpu_domain(obj, write_domain);  in i915_gem_set_domain_ioctl()
614 	i915_gem_object_unpin_pages(obj);  in i915_gem_set_domain_ioctl()
617 	i915_gem_object_unlock(obj);  in i915_gem_set_domain_ioctl()
620 		i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);  in i915_gem_set_domain_ioctl()
623 	i915_gem_object_put(obj);  in i915_gem_set_domain_ioctl()
632 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,  in i915_gem_object_prepare_read()  argument
638 	if (!i915_gem_object_has_struct_page(obj))  in i915_gem_object_prepare_read()
641 	assert_object_held(obj);  in i915_gem_object_prepare_read()
643 	ret = i915_gem_object_wait(obj,  in i915_gem_object_prepare_read()
649 	ret = i915_gem_object_pin_pages(obj);  in i915_gem_object_prepare_read()
653 	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||  in i915_gem_object_prepare_read()
655 		ret = i915_gem_object_set_to_cpu_domain(obj, false);  in i915_gem_object_prepare_read()
662 	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);  in i915_gem_object_prepare_read()
669 	if (!obj->cache_dirty &&  in i915_gem_object_prepare_read()
670 	    !(obj->read_domains & I915_GEM_DOMAIN_CPU))  in i915_gem_object_prepare_read()
678 	i915_gem_object_unpin_pages(obj);  in i915_gem_object_prepare_read()
682 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,  in i915_gem_object_prepare_write()  argument
688 	if (!i915_gem_object_has_struct_page(obj))  in i915_gem_object_prepare_write()
691 	assert_object_held(obj);  in i915_gem_object_prepare_write()
693 	ret = i915_gem_object_wait(obj,  in i915_gem_object_prepare_write()
700 	ret = i915_gem_object_pin_pages(obj);  in i915_gem_object_prepare_write()
704 	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||  in i915_gem_object_prepare_write()
706 		ret = i915_gem_object_set_to_cpu_domain(obj, true);  in i915_gem_object_prepare_write()
713 	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);  in i915_gem_object_prepare_write()
720 	if (!obj->cache_dirty) {  in i915_gem_object_prepare_write()
727 		if (!(obj->read_domains & I915_GEM_DOMAIN_CPU))  in i915_gem_object_prepare_write()
732 	i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);  in i915_gem_object_prepare_write()
733 	obj->mm.dirty = true;  in i915_gem_object_prepare_write()
738 	i915_gem_object_unpin_pages(obj);  in i915_gem_object_prepare_write()