1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2019 Intel Corporation
5 */
6
7 #include <linux/slab.h>
8 #include <linux/workqueue.h>
9
10 #include "i915_active.h"
11 #include "gem/i915_gem_context.h"
12 #include "gem/i915_gem_object.h"
13 #include "i915_globals.h"
14 #include "i915_request.h"
15 #include "i915_scheduler.h"
16 #include "i915_vma.h"
17
18 static LIST_HEAD(globals);
19
20 static atomic_t active;
21 static atomic_t epoch;
22 static struct park_work {
23 struct rcu_work work;
24 int epoch;
25 } park;
26
i915_globals_shrink(void)27 static void i915_globals_shrink(void)
28 {
29 struct i915_global *global;
30
31 /*
32 * kmem_cache_shrink() discards empty slabs and reorders partially
33 * filled slabs to prioritise allocating from the mostly full slabs,
34 * with the aim of reducing fragmentation.
35 */
36 list_for_each_entry(global, &globals, link)
37 global->shrink();
38 }
39
__i915_globals_park(struct work_struct * work)40 static void __i915_globals_park(struct work_struct *work)
41 {
42 /* Confirm nothing woke up in the last grace period */
43 if (park.epoch == atomic_read(&epoch))
44 i915_globals_shrink();
45 }
46
i915_global_register(struct i915_global * global)47 void __init i915_global_register(struct i915_global *global)
48 {
49 GEM_BUG_ON(!global->shrink);
50 GEM_BUG_ON(!global->exit);
51
52 list_add_tail(&global->link, &globals);
53 }
54
__i915_globals_cleanup(void)55 static void __i915_globals_cleanup(void)
56 {
57 struct i915_global *global, *next;
58
59 list_for_each_entry_safe_reverse(global, next, &globals, link)
60 global->exit();
61 }
62
63 static __initconst int (* const initfn[])(void) = {
64 i915_global_active_init,
65 i915_global_buddy_init,
66 i915_global_context_init,
67 i915_global_gem_context_init,
68 i915_global_objects_init,
69 i915_global_request_init,
70 i915_global_scheduler_init,
71 i915_global_vma_init,
72 };
73
i915_globals_init(void)74 int __init i915_globals_init(void)
75 {
76 int i;
77
78 for (i = 0; i < ARRAY_SIZE(initfn); i++) {
79 int err;
80
81 err = initfn[i]();
82 if (err) {
83 __i915_globals_cleanup();
84 return err;
85 }
86 }
87
88 INIT_RCU_WORK(&park.work, __i915_globals_park);
89 return 0;
90 }
91
i915_globals_park(void)92 void i915_globals_park(void)
93 {
94 /*
95 * Defer shrinking the global slab caches (and other work) until
96 * after a RCU grace period has completed with no activity. This
97 * is to try and reduce the latency impact on the consumers caused
98 * by us shrinking the caches the same time as they are trying to
99 * allocate, with the assumption being that if we idle long enough
100 * for an RCU grace period to elapse since the last use, it is likely
101 * to be longer until we need the caches again.
102 */
103 if (!atomic_dec_and_test(&active))
104 return;
105
106 park.epoch = atomic_inc_return(&epoch);
107 queue_rcu_work(system_wq, &park.work);
108 }
109
i915_globals_unpark(void)110 void i915_globals_unpark(void)
111 {
112 atomic_inc(&epoch);
113 atomic_inc(&active);
114 }
115
i915_globals_exit(void)116 void __exit i915_globals_exit(void)
117 {
118 /* Flush any residual park_work */
119 atomic_inc(&epoch);
120 flush_rcu_work(&park.work);
121
122 __i915_globals_cleanup();
123
124 /* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
125 rcu_barrier();
126 }
127
