22 void i915_gem_object_free(struct drm_i915_gem_object *obj);
24 void i915_gem_object_init(struct drm_i915_gem_object *obj,
35 void __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
39 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align);
42 void i915_gem_free_object(struct drm_gem_object *obj);
47 __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj);
48 void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
71 i915_gem_object_get_rcu(struct drm_i915_gem_object *obj)  in i915_gem_object_get_rcu()  argument
73 	if (obj && !kref_get_unless_zero(&obj->base.refcount))  in i915_gem_object_get_rcu()
74 		obj = NULL;  in i915_gem_object_get_rcu()
76 	return obj;  in i915_gem_object_get_rcu()
82 	struct drm_i915_gem_object *obj;  in i915_gem_object_lookup()  local
85 	obj = i915_gem_object_lookup_rcu(file, handle);  in i915_gem_object_lookup()
86 	obj = i915_gem_object_get_rcu(obj);  in i915_gem_object_lookup()
89 	return obj;  in i915_gem_object_lookup()
98 i915_gem_object_get(struct drm_i915_gem_object *obj)  in i915_gem_object_get()  argument
100 	drm_gem_object_get(&obj->base);  in i915_gem_object_get()
101 	return obj;  in i915_gem_object_get()
106 i915_gem_object_put(struct drm_i915_gem_object *obj)  in i915_gem_object_put()  argument
108 	__drm_gem_object_put(&obj->base);  in i915_gem_object_put()
111 #define assert_object_held(obj) dma_resv_assert_held((obj)->base.resv)  argument
113 static inline int __i915_gem_object_lock(struct drm_i915_gem_object *obj,  in __i915_gem_object_lock()  argument
120 		ret = dma_resv_lock_interruptible(obj->base.resv, ww ? &ww->ctx : NULL);  in __i915_gem_object_lock()
122 		ret = dma_resv_lock(obj->base.resv, ww ? &ww->ctx : NULL);  in __i915_gem_object_lock()
125 		list_add_tail(&obj->obj_link, &ww->obj_list);  in __i915_gem_object_lock()
130 		ww->contended = obj;  in __i915_gem_object_lock()
135 static inline int i915_gem_object_lock(struct drm_i915_gem_object *obj,  in i915_gem_object_lock()  argument
138 	return __i915_gem_object_lock(obj, ww, ww && ww->intr);  in i915_gem_object_lock()
141 static inline int i915_gem_object_lock_interruptible(struct drm_i915_gem_object *obj,  in i915_gem_object_lock_interruptible()  argument
145 	return __i915_gem_object_lock(obj, ww, true);  in i915_gem_object_lock_interruptible()
148 static inline bool i915_gem_object_trylock(struct drm_i915_gem_object *obj)  in i915_gem_object_trylock()  argument
150 	return dma_resv_trylock(obj->base.resv);  in i915_gem_object_trylock()
153 static inline void i915_gem_object_unlock(struct drm_i915_gem_object *obj)  in i915_gem_object_unlock()  argument
155 	dma_resv_unlock(obj->base.resv);  in i915_gem_object_unlock()
159 i915_gem_object_lock_fence(struct drm_i915_gem_object *obj);
160 void i915_gem_object_unlock_fence(struct drm_i915_gem_object *obj,
164 i915_gem_object_set_readonly(struct drm_i915_gem_object *obj)  in i915_gem_object_set_readonly()  argument
166 	obj->flags |= I915_BO_READONLY;  in i915_gem_object_set_readonly()
170 i915_gem_object_is_readonly(const struct drm_i915_gem_object *obj)  in i915_gem_object_is_readonly()  argument
172 	return obj->flags & I915_BO_READONLY;  in i915_gem_object_is_readonly()
176 i915_gem_object_is_contiguous(const struct drm_i915_gem_object *obj)  in i915_gem_object_is_contiguous()  argument
178 	return obj->flags & I915_BO_ALLOC_CONTIGUOUS;  in i915_gem_object_is_contiguous()
182 i915_gem_object_is_volatile(const struct drm_i915_gem_object *obj)  in i915_gem_object_is_volatile()  argument
184 	return obj->flags & I915_BO_ALLOC_VOLATILE;  in i915_gem_object_is_volatile()
188 i915_gem_object_set_volatile(struct drm_i915_gem_object *obj)  in i915_gem_object_set_volatile()  argument
190 	obj->flags |= I915_BO_ALLOC_VOLATILE;  in i915_gem_object_set_volatile()
194 i915_gem_object_type_has(const struct drm_i915_gem_object *obj,  in i915_gem_object_type_has()  argument
197 	return obj->ops->flags & flags;  in i915_gem_object_type_has()
201 i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)  in i915_gem_object_has_struct_page()  argument
203 	return i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_STRUCT_PAGE);  in i915_gem_object_has_struct_page()
207 i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj)  in i915_gem_object_is_shrinkable()  argument
209 	return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_SHRINKABLE);  in i915_gem_object_is_shrinkable()
213 i915_gem_object_is_proxy(const struct drm_i915_gem_object *obj)  in i915_gem_object_is_proxy()  argument
215 	return i915_gem_object_type_has(obj, I915_GEM_OBJECT_IS_PROXY);  in i915_gem_object_is_proxy()
219 i915_gem_object_never_mmap(const struct drm_i915_gem_object *obj)  in i915_gem_object_never_mmap()  argument
221 	return i915_gem_object_type_has(obj, I915_GEM_OBJECT_NO_MMAP);  in i915_gem_object_never_mmap()
225 i915_gem_object_needs_async_cancel(const struct drm_i915_gem_object *obj)  in i915_gem_object_needs_async_cancel()  argument
227 	return i915_gem_object_type_has(obj, I915_GEM_OBJECT_ASYNC_CANCEL);  in i915_gem_object_needs_async_cancel()
231 i915_gem_object_is_framebuffer(const struct drm_i915_gem_object *obj)  in i915_gem_object_is_framebuffer()  argument
233 	return READ_ONCE(obj->frontbuffer);  in i915_gem_object_is_framebuffer()
237 i915_gem_object_get_tiling(const struct drm_i915_gem_object *obj)  in i915_gem_object_get_tiling()  argument
239 	return obj->tiling_and_stride & TILING_MASK;  in i915_gem_object_get_tiling()
243 i915_gem_object_is_tiled(const struct drm_i915_gem_object *obj)  in i915_gem_object_is_tiled()  argument
245 	return i915_gem_object_get_tiling(obj) != I915_TILING_NONE;  in i915_gem_object_is_tiled()
249 i915_gem_object_get_stride(const struct drm_i915_gem_object *obj)  in i915_gem_object_get_stride()  argument
251 	return obj->tiling_and_stride & STRIDE_MASK;  in i915_gem_object_get_stride()
262 i915_gem_object_get_tile_height(const struct drm_i915_gem_object *obj)  in i915_gem_object_get_tile_height()  argument
264 	return i915_gem_tile_height(i915_gem_object_get_tiling(obj));  in i915_gem_object_get_tile_height()
268 i915_gem_object_get_tile_row_size(const struct drm_i915_gem_object *obj)  in i915_gem_object_get_tile_row_size()  argument
270 	return (i915_gem_object_get_stride(obj) *  in i915_gem_object_get_tile_row_size()
271 		i915_gem_object_get_tile_height(obj));  in i915_gem_object_get_tile_row_size()
274 int i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
278 i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
282 i915_gem_object_get_page(struct drm_i915_gem_object *obj,
286 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
290 i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj,
295 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
298 void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
302 int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
303 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
323 i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)  in i915_gem_object_pin_pages()  argument
325 	might_lock_nested(&obj->mm.lock, I915_MM_GET_PAGES);  in i915_gem_object_pin_pages()
327 	if (atomic_inc_not_zero(&obj->mm.pages_pin_count))  in i915_gem_object_pin_pages()
330 	return __i915_gem_object_get_pages(obj);  in i915_gem_object_pin_pages()
334 i915_gem_object_has_pages(struct drm_i915_gem_object *obj)  in i915_gem_object_has_pages()  argument
336 	return !IS_ERR_OR_NULL(READ_ONCE(obj->mm.pages));  in i915_gem_object_has_pages()
340 __i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)  in __i915_gem_object_pin_pages()  argument
342 	GEM_BUG_ON(!i915_gem_object_has_pages(obj));  in __i915_gem_object_pin_pages()
344 	atomic_inc(&obj->mm.pages_pin_count);  in __i915_gem_object_pin_pages()
348 i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj)  in i915_gem_object_has_pinned_pages()  argument
350 	return atomic_read(&obj->mm.pages_pin_count);  in i915_gem_object_has_pinned_pages()
354 __i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)  in __i915_gem_object_unpin_pages()  argument
356 	GEM_BUG_ON(!i915_gem_object_has_pages(obj));  in __i915_gem_object_unpin_pages()
357 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));  in __i915_gem_object_unpin_pages()
359 	atomic_dec(&obj->mm.pages_pin_count);  in __i915_gem_object_unpin_pages()
363 i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)  in i915_gem_object_unpin_pages()  argument
365 	__i915_gem_object_unpin_pages(obj);  in i915_gem_object_unpin_pages()
368 int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
369 void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
370 void i915_gem_object_writeback(struct drm_i915_gem_object *obj);
396 void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
399 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
402 static inline void i915_gem_object_flush_map(struct drm_i915_gem_object *obj)  in i915_gem_object_flush_map()  argument
404 	__i915_gem_object_flush_map(obj, 0, obj->base.size);  in i915_gem_object_flush_map()
416 static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj)  in i915_gem_object_unpin_map()  argument
418 	i915_gem_object_unpin_pages(obj);  in i915_gem_object_unpin_map()
421 void __i915_gem_object_release_map(struct drm_i915_gem_object *obj);
424 i915_gem_object_flush_write_domain(struct drm_i915_gem_object *obj,
427 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,
429 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,
436 i915_gem_object_finish_access(struct drm_i915_gem_object *obj)  in i915_gem_object_finish_access()  argument
438 	i915_gem_object_unpin_pages(obj);  in i915_gem_object_finish_access()
442 i915_gem_object_last_write_engine(struct drm_i915_gem_object *obj)  in i915_gem_object_last_write_engine()  argument
448 	fence = dma_resv_get_excl_rcu(obj->base.resv);  in i915_gem_object_last_write_engine()
458 void i915_gem_object_set_cache_coherency(struct drm_i915_gem_object *obj,
460 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj);
461 void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj);
464 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write);
466 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write);
468 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write);
470 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
476 void i915_gem_object_make_unshrinkable(struct drm_i915_gem_object *obj);
477 void i915_gem_object_make_shrinkable(struct drm_i915_gem_object *obj);
478 void i915_gem_object_make_purgeable(struct drm_i915_gem_object *obj);
480 static inline bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)  in cpu_write_needs_clflush()  argument
482 	if (obj->cache_dirty)  in cpu_write_needs_clflush()
485 	if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))  in cpu_write_needs_clflush()
489 	return i915_gem_object_is_framebuffer(obj);  in cpu_write_needs_clflush()
492 static inline void __start_cpu_write(struct drm_i915_gem_object *obj)  in __start_cpu_write()  argument
494 	obj->read_domains = I915_GEM_DOMAIN_CPU;  in __start_cpu_write()
495 	obj->write_domain = I915_GEM_DOMAIN_CPU;  in __start_cpu_write()
496 	if (cpu_write_needs_clflush(obj))  in __start_cpu_write()
497 		obj->cache_dirty = true;  in __start_cpu_write()
500 int i915_gem_object_wait(struct drm_i915_gem_object *obj,
503 int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
507 void __i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj,
509 void __i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj,
513 i915_gem_object_flush_frontbuffer(struct drm_i915_gem_object *obj,  in i915_gem_object_flush_frontbuffer()  argument
516 	if (unlikely(rcu_access_pointer(obj->frontbuffer)))  in i915_gem_object_flush_frontbuffer()
517 		__i915_gem_object_flush_frontbuffer(obj, origin);  in i915_gem_object_flush_frontbuffer()
521 i915_gem_object_invalidate_frontbuffer(struct drm_i915_gem_object *obj,  in i915_gem_object_invalidate_frontbuffer()  argument
524 	if (unlikely(rcu_access_pointer(obj->frontbuffer)))  in i915_gem_object_invalidate_frontbuffer()
525 		__i915_gem_object_invalidate_frontbuffer(obj, origin);  in i915_gem_object_invalidate_frontbuffer()