1 /*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #ifndef KFD_DEVICE_QUEUE_MANAGER_H_
25 #define KFD_DEVICE_QUEUE_MANAGER_H_
26
27 #include <linux/rwsem.h>
28 #include <linux/list.h>
29 #include <linux/mutex.h>
30 #include <linux/sched/mm.h>
31 #include "kfd_priv.h"
32 #include "kfd_mqd_manager.h"
33
34 #define KFD_UNMAP_LATENCY_MS (4000)
35 #define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (2 * KFD_UNMAP_LATENCY_MS + 1000)
36 #define KFD_SDMA_QUEUES_PER_ENGINE (2)
37
38 struct device_process_node {
39 struct qcm_process_device *qpd;
40 struct list_head list;
41 };
42
43 /**
44 * struct device_queue_manager_ops
45 *
46 * @create_queue: Queue creation routine.
47 *
48 * @destroy_queue: Queue destruction routine.
49 *
50 * @update_queue: Queue update routine.
51 *
52 * @get_mqd_manager: Returns the mqd manager according to the mqd type.
53 *
54 * @exeute_queues: Dispatches the queues list to the H/W.
55 *
56 * @register_process: This routine associates a specific process with device.
57 *
58 * @unregister_process: destroys the associations between process to device.
59 *
60 * @initialize: Initializes the pipelines and memory module for that device.
61 *
62 * @start: Initializes the resources/modules the the device needs for queues
63 * execution. This function is called on device initialization and after the
64 * system woke up after suspension.
65 *
66 * @stop: This routine stops execution of all the active queue running on the
67 * H/W and basically this function called on system suspend.
68 *
69 * @uninitialize: Destroys all the device queue manager resources allocated in
70 * initialize routine.
71 *
72 * @create_kernel_queue: Creates kernel queue. Used for debug queue.
73 *
74 * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue.
75 *
76 * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the
77 * memory apertures.
78 *
79 * @process_termination: Clears all process queues belongs to that device.
80 *
81 * @evict_process_queues: Evict all active queues of a process
82 *
83 * @restore_process_queues: Restore all evicted queues queues of a process
84 *
85 */
86
87 struct device_queue_manager_ops {
88 int (*create_queue)(struct device_queue_manager *dqm,
89 struct queue *q,
90 struct qcm_process_device *qpd);
91
92 int (*destroy_queue)(struct device_queue_manager *dqm,
93 struct qcm_process_device *qpd,
94 struct queue *q);
95
96 int (*update_queue)(struct device_queue_manager *dqm,
97 struct queue *q);
98
99 struct mqd_manager * (*get_mqd_manager)
100 (struct device_queue_manager *dqm,
101 enum KFD_MQD_TYPE type);
102
103 int (*register_process)(struct device_queue_manager *dqm,
104 struct qcm_process_device *qpd);
105
106 int (*unregister_process)(struct device_queue_manager *dqm,
107 struct qcm_process_device *qpd);
108
109 int (*initialize)(struct device_queue_manager *dqm);
110 int (*start)(struct device_queue_manager *dqm);
111 int (*stop)(struct device_queue_manager *dqm);
112 void (*uninitialize)(struct device_queue_manager *dqm);
113 int (*create_kernel_queue)(struct device_queue_manager *dqm,
114 struct kernel_queue *kq,
115 struct qcm_process_device *qpd);
116
117 void (*destroy_kernel_queue)(struct device_queue_manager *dqm,
118 struct kernel_queue *kq,
119 struct qcm_process_device *qpd);
120
121 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
122 struct qcm_process_device *qpd,
123 enum cache_policy default_policy,
124 enum cache_policy alternate_policy,
125 void __user *alternate_aperture_base,
126 uint64_t alternate_aperture_size);
127
128 int (*set_trap_handler)(struct device_queue_manager *dqm,
129 struct qcm_process_device *qpd,
130 uint64_t tba_addr,
131 uint64_t tma_addr);
132
133 int (*process_termination)(struct device_queue_manager *dqm,
134 struct qcm_process_device *qpd);
135
136 int (*evict_process_queues)(struct device_queue_manager *dqm,
137 struct qcm_process_device *qpd);
138 int (*restore_process_queues)(struct device_queue_manager *dqm,
139 struct qcm_process_device *qpd);
140 };
141
142 struct device_queue_manager_asic_ops {
143 int (*update_qpd)(struct device_queue_manager *dqm,
144 struct qcm_process_device *qpd);
145 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,
146 struct qcm_process_device *qpd,
147 enum cache_policy default_policy,
148 enum cache_policy alternate_policy,
149 void __user *alternate_aperture_base,
150 uint64_t alternate_aperture_size);
151 void (*init_sdma_vm)(struct device_queue_manager *dqm,
152 struct queue *q,
153 struct qcm_process_device *qpd);
154 };
155
156 /**
157 * struct device_queue_manager
158 *
159 * This struct is a base class for the kfd queues scheduler in the
160 * device level. The device base class should expose the basic operations
161 * for queue creation and queue destruction. This base class hides the
162 * scheduling mode of the driver and the specific implementation of the
163 * concrete device. This class is the only class in the queues scheduler
164 * that configures the H/W.
165 *
166 */
167
168 struct device_queue_manager {
169 struct device_queue_manager_ops ops;
170 struct device_queue_manager_asic_ops asic_ops;
171
172 struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX];
173 struct packet_manager packets;
174 struct kfd_dev *dev;
175 struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */
176 struct list_head queues;
177 unsigned int saved_flags;
178 unsigned int processes_count;
179 unsigned int queue_count;
180 unsigned int sdma_queue_count;
181 unsigned int total_queue_count;
182 unsigned int next_pipe_to_allocate;
183 unsigned int *allocated_queues;
184 unsigned int sdma_bitmap;
185 unsigned int vmid_bitmap;
186 uint64_t pipelines_addr;
187 struct kfd_mem_obj *pipeline_mem;
188 uint64_t fence_gpu_addr;
189 unsigned int *fence_addr;
190 struct kfd_mem_obj *fence_mem;
191 bool active_runlist;
192 int sched_policy;
193
194 /* hw exception */
195 bool is_hws_hang;
196 struct work_struct hw_exception_work;
197 };
198
199 void device_queue_manager_init_cik(
200 struct device_queue_manager_asic_ops *asic_ops);
201 void device_queue_manager_init_cik_hawaii(
202 struct device_queue_manager_asic_ops *asic_ops);
203 void device_queue_manager_init_vi(
204 struct device_queue_manager_asic_ops *asic_ops);
205 void device_queue_manager_init_vi_tonga(
206 struct device_queue_manager_asic_ops *asic_ops);
207 void device_queue_manager_init_v9(
208 struct device_queue_manager_asic_ops *asic_ops);
209 void program_sh_mem_settings(struct device_queue_manager *dqm,
210 struct qcm_process_device *qpd);
211 unsigned int get_queues_num(struct device_queue_manager *dqm);
212 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
213 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
214 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm);
215
get_sh_mem_bases_32(struct kfd_process_device * pdd)216 static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
217 {
218 return (pdd->lds_base >> 16) & 0xFF;
219 }
220
221 static inline unsigned int
get_sh_mem_bases_nybble_64(struct kfd_process_device * pdd)222 get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
223 {
224 return (pdd->lds_base >> 60) & 0x0E;
225 }
226
227 /* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS
228 * happens while holding this lock anywhere to prevent deadlocks when
229 * an MMU notifier runs in reclaim-FS context.
230 */
dqm_lock(struct device_queue_manager * dqm)231 static inline void dqm_lock(struct device_queue_manager *dqm)
232 {
233 mutex_lock(&dqm->lock_hidden);
234 dqm->saved_flags = memalloc_nofs_save();
235 }
dqm_unlock(struct device_queue_manager * dqm)236 static inline void dqm_unlock(struct device_queue_manager *dqm)
237 {
238 memalloc_nofs_restore(dqm->saved_flags);
239 mutex_unlock(&dqm->lock_hidden);
240 }
241
242 #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */
243
