1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2019 Intel Corporation
5 */
6
7 #include <linux/slab.h>
8 #include <linux/workqueue.h>
9
10 #include "i915_active.h"
11 #include "gem/i915_gem_context.h"
12 #include "gem/i915_gem_object.h"
13 #include "i915_globals.h"
14 #include "i915_request.h"
15 #include "i915_scheduler.h"
16 #include "i915_vma.h"
17
18 static LIST_HEAD(globals);
19
20 static atomic_t active;
21 static atomic_t epoch;
22 static struct park_work {
23 struct delayed_work work;
24 struct rcu_head rcu;
25 unsigned long flags;
26 #define PENDING 0
27 int epoch;
28 } park;
29
i915_globals_shrink(void)30 static void i915_globals_shrink(void)
31 {
32 struct i915_global *global;
33
34 /*
35 * kmem_cache_shrink() discards empty slabs and reorders partially
36 * filled slabs to prioritise allocating from the mostly full slabs,
37 * with the aim of reducing fragmentation.
38 */
39 list_for_each_entry(global, &globals, link)
40 global->shrink();
41 }
42
__i915_globals_grace(struct rcu_head * rcu)43 static void __i915_globals_grace(struct rcu_head *rcu)
44 {
45 /* Ratelimit parking as shrinking is quite slow */
46 schedule_delayed_work(&park.work, round_jiffies_up_relative(2 * HZ));
47 }
48
__i915_globals_queue_rcu(void)49 static void __i915_globals_queue_rcu(void)
50 {
51 park.epoch = atomic_inc_return(&epoch);
52 if (!atomic_read(&active)) {
53 init_rcu_head(&park.rcu);
54 call_rcu(&park.rcu, __i915_globals_grace);
55 }
56 }
57
__i915_globals_park(struct work_struct * work)58 static void __i915_globals_park(struct work_struct *work)
59 {
60 destroy_rcu_head(&park.rcu);
61
62 /* Confirm nothing woke up in the last grace period */
63 if (park.epoch != atomic_read(&epoch)) {
64 __i915_globals_queue_rcu();
65 return;
66 }
67
68 clear_bit(PENDING, &park.flags);
69 i915_globals_shrink();
70 }
71
i915_global_register(struct i915_global * global)72 void __init i915_global_register(struct i915_global *global)
73 {
74 GEM_BUG_ON(!global->shrink);
75 GEM_BUG_ON(!global->exit);
76
77 list_add_tail(&global->link, &globals);
78 }
79
__i915_globals_cleanup(void)80 static void __i915_globals_cleanup(void)
81 {
82 struct i915_global *global, *next;
83
84 list_for_each_entry_safe_reverse(global, next, &globals, link)
85 global->exit();
86 }
87
88 static __initconst int (* const initfn[])(void) = {
89 i915_global_active_init,
90 i915_global_buddy_init,
91 i915_global_context_init,
92 i915_global_gem_context_init,
93 i915_global_objects_init,
94 i915_global_request_init,
95 i915_global_scheduler_init,
96 i915_global_vma_init,
97 };
98
i915_globals_init(void)99 int __init i915_globals_init(void)
100 {
101 int i;
102
103 for (i = 0; i < ARRAY_SIZE(initfn); i++) {
104 int err;
105
106 err = initfn[i]();
107 if (err) {
108 __i915_globals_cleanup();
109 return err;
110 }
111 }
112
113 INIT_DELAYED_WORK(&park.work, __i915_globals_park);
114 return 0;
115 }
116
i915_globals_park(void)117 void i915_globals_park(void)
118 {
119 /*
120 * Defer shrinking the global slab caches (and other work) until
121 * after a RCU grace period has completed with no activity. This
122 * is to try and reduce the latency impact on the consumers caused
123 * by us shrinking the caches the same time as they are trying to
124 * allocate, with the assumption being that if we idle long enough
125 * for an RCU grace period to elapse since the last use, it is likely
126 * to be longer until we need the caches again.
127 */
128 if (!atomic_dec_and_test(&active))
129 return;
130
131 /* Queue cleanup after the next RCU grace period has freed slabs */
132 if (!test_and_set_bit(PENDING, &park.flags))
133 __i915_globals_queue_rcu();
134 }
135
i915_globals_unpark(void)136 void i915_globals_unpark(void)
137 {
138 atomic_inc(&epoch);
139 atomic_inc(&active);
140 }
141
__i915_globals_flush(void)142 static void __exit __i915_globals_flush(void)
143 {
144 atomic_inc(&active); /* skip shrinking */
145
146 rcu_barrier(); /* wait for the work to be queued */
147 flush_delayed_work(&park.work);
148
149 atomic_dec(&active);
150 }
151
i915_globals_exit(void)152 void __exit i915_globals_exit(void)
153 {
154 GEM_BUG_ON(atomic_read(&active));
155
156 __i915_globals_flush();
157 __i915_globals_cleanup();
158
159 /* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
160 rcu_barrier();
161 }
162
