1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2014-2018 Intel Corporation
5 */
6
7 #include "gem/i915_gem_object.h"
8
9 #include "i915_drv.h"
10 #include "intel_engine_pm.h"
11 #include "intel_engine_pool.h"
12
to_engine(struct intel_engine_pool * pool)13 static struct intel_engine_cs *to_engine(struct intel_engine_pool *pool)
14 {
15 return container_of(pool, struct intel_engine_cs, pool);
16 }
17
18 static struct list_head *
bucket_for_size(struct intel_engine_pool * pool,size_t sz)19 bucket_for_size(struct intel_engine_pool *pool, size_t sz)
20 {
21 int n;
22
23 /*
24 * Compute a power-of-two bucket, but throw everything greater than
25 * 16KiB into the same bucket: i.e. the buckets hold objects of
26 * (1 page, 2 pages, 4 pages, 8+ pages).
27 */
28 n = fls(sz >> PAGE_SHIFT) - 1;
29 if (n >= ARRAY_SIZE(pool->cache_list))
30 n = ARRAY_SIZE(pool->cache_list) - 1;
31
32 return &pool->cache_list[n];
33 }
34
node_free(struct intel_engine_pool_node * node)35 static void node_free(struct intel_engine_pool_node *node)
36 {
37 i915_gem_object_put(node->obj);
38 i915_active_fini(&node->active);
39 kfree(node);
40 }
41
pool_active(struct i915_active * ref)42 static int pool_active(struct i915_active *ref)
43 {
44 struct intel_engine_pool_node *node =
45 container_of(ref, typeof(*node), active);
46 struct dma_resv *resv = node->obj->base.resv;
47 int err;
48
49 if (dma_resv_trylock(resv)) {
50 dma_resv_add_excl_fence(resv, NULL);
51 dma_resv_unlock(resv);
52 }
53
54 err = i915_gem_object_pin_pages(node->obj);
55 if (err)
56 return err;
57
58 /* Hide this pinned object from the shrinker until retired */
59 i915_gem_object_make_unshrinkable(node->obj);
60
61 return 0;
62 }
63
pool_retire(struct i915_active * ref)64 static void pool_retire(struct i915_active *ref)
65 {
66 struct intel_engine_pool_node *node =
67 container_of(ref, typeof(*node), active);
68 struct intel_engine_pool *pool = node->pool;
69 struct list_head *list = bucket_for_size(pool, node->obj->base.size);
70 unsigned long flags;
71
72 GEM_BUG_ON(!intel_engine_pm_is_awake(to_engine(pool)));
73
74 i915_gem_object_unpin_pages(node->obj);
75
76 /* Return this object to the shrinker pool */
77 i915_gem_object_make_purgeable(node->obj);
78
79 spin_lock_irqsave(&pool->lock, flags);
80 list_add(&node->link, list);
81 spin_unlock_irqrestore(&pool->lock, flags);
82 }
83
84 static struct intel_engine_pool_node *
node_create(struct intel_engine_pool * pool,size_t sz)85 node_create(struct intel_engine_pool *pool, size_t sz)
86 {
87 struct intel_engine_cs *engine = to_engine(pool);
88 struct intel_engine_pool_node *node;
89 struct drm_i915_gem_object *obj;
90
91 node = kmalloc(sizeof(*node),
92 GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
93 if (!node)
94 return ERR_PTR(-ENOMEM);
95
96 node->pool = pool;
97 i915_active_init(engine->i915, &node->active, pool_active, pool_retire);
98
99 obj = i915_gem_object_create_internal(engine->i915, sz);
100 if (IS_ERR(obj)) {
101 i915_active_fini(&node->active);
102 kfree(node);
103 return ERR_CAST(obj);
104 }
105
106 i915_gem_object_set_readonly(obj);
107
108 node->obj = obj;
109 return node;
110 }
111
112 struct intel_engine_pool_node *
intel_engine_pool_get(struct intel_engine_pool * pool,size_t size)113 intel_engine_pool_get(struct intel_engine_pool *pool, size_t size)
114 {
115 struct intel_engine_pool_node *node;
116 struct list_head *list;
117 unsigned long flags;
118 int ret;
119
120 GEM_BUG_ON(!intel_engine_pm_is_awake(to_engine(pool)));
121
122 size = PAGE_ALIGN(size);
123 list = bucket_for_size(pool, size);
124
125 spin_lock_irqsave(&pool->lock, flags);
126 list_for_each_entry(node, list, link) {
127 if (node->obj->base.size < size)
128 continue;
129 list_del(&node->link);
130 break;
131 }
132 spin_unlock_irqrestore(&pool->lock, flags);
133
134 if (&node->link == list) {
135 node = node_create(pool, size);
136 if (IS_ERR(node))
137 return node;
138 }
139
140 ret = i915_active_acquire(&node->active);
141 if (ret) {
142 node_free(node);
143 return ERR_PTR(ret);
144 }
145
146 return node;
147 }
148
intel_engine_pool_init(struct intel_engine_pool * pool)149 void intel_engine_pool_init(struct intel_engine_pool *pool)
150 {
151 int n;
152
153 spin_lock_init(&pool->lock);
154 for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++)
155 INIT_LIST_HEAD(&pool->cache_list[n]);
156 }
157
intel_engine_pool_park(struct intel_engine_pool * pool)158 void intel_engine_pool_park(struct intel_engine_pool *pool)
159 {
160 int n;
161
162 for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) {
163 struct list_head *list = &pool->cache_list[n];
164 struct intel_engine_pool_node *node, *nn;
165
166 list_for_each_entry_safe(node, nn, list, link)
167 node_free(node);
168
169 INIT_LIST_HEAD(list);
170 }
171 }
172
intel_engine_pool_fini(struct intel_engine_pool * pool)173 void intel_engine_pool_fini(struct intel_engine_pool *pool)
174 {
175 int n;
176
177 for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++)
178 GEM_BUG_ON(!list_empty(&pool->cache_list[n]));
179 }
180
