1 /*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/slab.h>
25 #include <linux/list.h>
26 #include "kfd_device_queue_manager.h"
27 #include "kfd_priv.h"
28 #include "kfd_kernel_queue.h"
29
get_queue_by_qid(struct process_queue_manager * pqm,unsigned int qid)30 static inline struct process_queue_node *get_queue_by_qid(
31 struct process_queue_manager *pqm, unsigned int qid)
32 {
33 struct process_queue_node *pqn;
34
35 list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
36 if ((pqn->q && pqn->q->properties.queue_id == qid) ||
37 (pqn->kq && pqn->kq->queue->properties.queue_id == qid))
38 return pqn;
39 }
40
41 return NULL;
42 }
43
find_available_queue_slot(struct process_queue_manager * pqm,unsigned int * qid)44 static int find_available_queue_slot(struct process_queue_manager *pqm,
45 unsigned int *qid)
46 {
47 unsigned long found;
48
49 found = find_first_zero_bit(pqm->queue_slot_bitmap,
50 KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
51
52 pr_debug("The new slot id %lu\n", found);
53
54 if (found >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) {
55 pr_info("Cannot open more queues for process with pasid %d\n",
56 pqm->process->pasid);
57 return -ENOMEM;
58 }
59
60 set_bit(found, pqm->queue_slot_bitmap);
61 *qid = found;
62
63 return 0;
64 }
65
kfd_process_dequeue_from_device(struct kfd_process_device * pdd)66 void kfd_process_dequeue_from_device(struct kfd_process_device *pdd)
67 {
68 struct kfd_dev *dev = pdd->dev;
69
70 if (pdd->already_dequeued)
71 return;
72
73 dev->dqm->ops.process_termination(dev->dqm, &pdd->qpd);
74 pdd->already_dequeued = true;
75 }
76
kfd_process_dequeue_from_all_devices(struct kfd_process * p)77 void kfd_process_dequeue_from_all_devices(struct kfd_process *p)
78 {
79 struct kfd_process_device *pdd;
80
81 list_for_each_entry(pdd, &p->per_device_data, per_device_list)
82 kfd_process_dequeue_from_device(pdd);
83 }
84
pqm_init(struct process_queue_manager * pqm,struct kfd_process * p)85 int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p)
86 {
87 INIT_LIST_HEAD(&pqm->queues);
88 pqm->queue_slot_bitmap =
89 kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
90 BITS_PER_BYTE), GFP_KERNEL);
91 if (!pqm->queue_slot_bitmap)
92 return -ENOMEM;
93 pqm->process = p;
94
95 return 0;
96 }
97
pqm_uninit(struct process_queue_manager * pqm)98 void pqm_uninit(struct process_queue_manager *pqm)
99 {
100 struct process_queue_node *pqn, *next;
101
102 list_for_each_entry_safe(pqn, next, &pqm->queues, process_queue_list) {
103 uninit_queue(pqn->q);
104 list_del(&pqn->process_queue_list);
105 kfree(pqn);
106 }
107
108 kfree(pqm->queue_slot_bitmap);
109 pqm->queue_slot_bitmap = NULL;
110 }
111
create_cp_queue(struct process_queue_manager * pqm,struct kfd_dev * dev,struct queue ** q,struct queue_properties * q_properties,struct file * f,unsigned int qid)112 static int create_cp_queue(struct process_queue_manager *pqm,
113 struct kfd_dev *dev, struct queue **q,
114 struct queue_properties *q_properties,
115 struct file *f, unsigned int qid)
116 {
117 int retval;
118
119 /* Doorbell initialized in user space*/
120 q_properties->doorbell_ptr = NULL;
121
122 /* let DQM handle it*/
123 q_properties->vmid = 0;
124 q_properties->queue_id = qid;
125
126 retval = init_queue(q, q_properties);
127 if (retval != 0)
128 return retval;
129
130 (*q)->device = dev;
131 (*q)->process = pqm->process;
132
133 pr_debug("PQM After init queue");
134
135 return retval;
136 }
137
pqm_create_queue(struct process_queue_manager * pqm,struct kfd_dev * dev,struct file * f,struct queue_properties * properties,unsigned int * qid)138 int pqm_create_queue(struct process_queue_manager *pqm,
139 struct kfd_dev *dev,
140 struct file *f,
141 struct queue_properties *properties,
142 unsigned int *qid)
143 {
144 int retval;
145 struct kfd_process_device *pdd;
146 struct queue *q;
147 struct process_queue_node *pqn;
148 struct kernel_queue *kq;
149 enum kfd_queue_type type = properties->type;
150 unsigned int max_queues = 127; /* HWS limit */
151
152 q = NULL;
153 kq = NULL;
154
155 pdd = kfd_get_process_device_data(dev, pqm->process);
156 if (!pdd) {
157 pr_err("Process device data doesn't exist\n");
158 return -1;
159 }
160
161 /*
162 * for debug process, verify that it is within the static queues limit
163 * currently limit is set to half of the total avail HQD slots
164 * If we are just about to create DIQ, the is_debug flag is not set yet
165 * Hence we also check the type as well
166 */
167 if ((pdd->qpd.is_debug) || (type == KFD_QUEUE_TYPE_DIQ))
168 max_queues = dev->device_info->max_no_of_hqd/2;
169
170 if (pdd->qpd.queue_count >= max_queues)
171 return -ENOSPC;
172
173 retval = find_available_queue_slot(pqm, qid);
174 if (retval != 0)
175 return retval;
176
177 if (list_empty(&pdd->qpd.queues_list) &&
178 list_empty(&pdd->qpd.priv_queue_list))
179 dev->dqm->ops.register_process(dev->dqm, &pdd->qpd);
180
181 pqn = kzalloc(sizeof(*pqn), GFP_KERNEL);
182 if (!pqn) {
183 retval = -ENOMEM;
184 goto err_allocate_pqn;
185 }
186
187 switch (type) {
188 case KFD_QUEUE_TYPE_SDMA:
189 if (dev->dqm->queue_count >= get_num_sdma_queues(dev->dqm)) {
190 pr_err("Over-subscription is not allowed for SDMA.\n");
191 retval = -EPERM;
192 goto err_create_queue;
193 }
194
195 retval = create_cp_queue(pqm, dev, &q, properties, f, *qid);
196 if (retval != 0)
197 goto err_create_queue;
198 pqn->q = q;
199 pqn->kq = NULL;
200 retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd);
201 pr_debug("DQM returned %d for create_queue\n", retval);
202 print_queue(q);
203 break;
204
205 case KFD_QUEUE_TYPE_COMPUTE:
206 /* check if there is over subscription */
207 if ((dev->dqm->sched_policy ==
208 KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
209 ((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) ||
210 (dev->dqm->queue_count >= get_queues_num(dev->dqm)))) {
211 pr_debug("Over-subscription is not allowed when amdkfd.sched_policy == 1\n");
212 retval = -EPERM;
213 goto err_create_queue;
214 }
215
216 retval = create_cp_queue(pqm, dev, &q, properties, f, *qid);
217 if (retval != 0)
218 goto err_create_queue;
219 pqn->q = q;
220 pqn->kq = NULL;
221 retval = dev->dqm->ops.create_queue(dev->dqm, q, &pdd->qpd);
222 pr_debug("DQM returned %d for create_queue\n", retval);
223 print_queue(q);
224 break;
225 case KFD_QUEUE_TYPE_DIQ:
226 kq = kernel_queue_init(dev, KFD_QUEUE_TYPE_DIQ);
227 if (!kq) {
228 retval = -ENOMEM;
229 goto err_create_queue;
230 }
231 kq->queue->properties.queue_id = *qid;
232 pqn->kq = kq;
233 pqn->q = NULL;
234 retval = dev->dqm->ops.create_kernel_queue(dev->dqm,
235 kq, &pdd->qpd);
236 break;
237 default:
238 WARN(1, "Invalid queue type %d", type);
239 retval = -EINVAL;
240 }
241
242 if (retval != 0) {
243 pr_err("Pasid %d DQM create queue %d failed. ret %d\n",
244 pqm->process->pasid, type, retval);
245 goto err_create_queue;
246 }
247
248 if (q)
249 /* Return the doorbell offset within the doorbell page
250 * to the caller so it can be passed up to user mode
251 * (in bytes).
252 */
253 properties->doorbell_off =
254 (q->properties.doorbell_off * sizeof(uint32_t)) &
255 (kfd_doorbell_process_slice(dev) - 1);
256
257 pr_debug("PQM After DQM create queue\n");
258
259 list_add(&pqn->process_queue_list, &pqm->queues);
260
261 if (q) {
262 pr_debug("PQM done creating queue\n");
263 print_queue_properties(&q->properties);
264 }
265
266 return retval;
267
268 err_create_queue:
269 kfree(pqn);
270 err_allocate_pqn:
271 /* check if queues list is empty unregister process from device */
272 clear_bit(*qid, pqm->queue_slot_bitmap);
273 if (list_empty(&pdd->qpd.queues_list) &&
274 list_empty(&pdd->qpd.priv_queue_list))
275 dev->dqm->ops.unregister_process(dev->dqm, &pdd->qpd);
276 return retval;
277 }
278
pqm_destroy_queue(struct process_queue_manager * pqm,unsigned int qid)279 int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid)
280 {
281 struct process_queue_node *pqn;
282 struct kfd_process_device *pdd;
283 struct device_queue_manager *dqm;
284 struct kfd_dev *dev;
285 int retval;
286
287 dqm = NULL;
288
289 retval = 0;
290
291 pqn = get_queue_by_qid(pqm, qid);
292 if (!pqn) {
293 pr_err("Queue id does not match any known queue\n");
294 return -EINVAL;
295 }
296
297 dev = NULL;
298 if (pqn->kq)
299 dev = pqn->kq->dev;
300 if (pqn->q)
301 dev = pqn->q->device;
302 if (WARN_ON(!dev))
303 return -ENODEV;
304
305 pdd = kfd_get_process_device_data(dev, pqm->process);
306 if (!pdd) {
307 pr_err("Process device data doesn't exist\n");
308 return -1;
309 }
310
311 if (pqn->kq) {
312 /* destroy kernel queue (DIQ) */
313 dqm = pqn->kq->dev->dqm;
314 dqm->ops.destroy_kernel_queue(dqm, pqn->kq, &pdd->qpd);
315 kernel_queue_uninit(pqn->kq);
316 }
317
318 if (pqn->q) {
319 dqm = pqn->q->device->dqm;
320 retval = dqm->ops.destroy_queue(dqm, &pdd->qpd, pqn->q);
321 if (retval) {
322 pr_err("Pasid %d destroy queue %d failed, ret %d\n",
323 pqm->process->pasid,
324 pqn->q->properties.queue_id, retval);
325 if (retval != -ETIME)
326 goto err_destroy_queue;
327 }
328 kfree(pqn->q->properties.cu_mask);
329 pqn->q->properties.cu_mask = NULL;
330 uninit_queue(pqn->q);
331 }
332
333 list_del(&pqn->process_queue_list);
334 kfree(pqn);
335 clear_bit(qid, pqm->queue_slot_bitmap);
336
337 if (list_empty(&pdd->qpd.queues_list) &&
338 list_empty(&pdd->qpd.priv_queue_list))
339 dqm->ops.unregister_process(dqm, &pdd->qpd);
340
341 err_destroy_queue:
342 return retval;
343 }
344
pqm_update_queue(struct process_queue_manager * pqm,unsigned int qid,struct queue_properties * p)345 int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
346 struct queue_properties *p)
347 {
348 int retval;
349 struct process_queue_node *pqn;
350
351 pqn = get_queue_by_qid(pqm, qid);
352 if (!pqn) {
353 pr_debug("No queue %d exists for update operation\n", qid);
354 return -EFAULT;
355 }
356
357 pqn->q->properties.queue_address = p->queue_address;
358 pqn->q->properties.queue_size = p->queue_size;
359 pqn->q->properties.queue_percent = p->queue_percent;
360 pqn->q->properties.priority = p->priority;
361
362 retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
363 pqn->q);
364 if (retval != 0)
365 return retval;
366
367 return 0;
368 }
369
pqm_set_cu_mask(struct process_queue_manager * pqm,unsigned int qid,struct queue_properties * p)370 int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
371 struct queue_properties *p)
372 {
373 int retval;
374 struct process_queue_node *pqn;
375
376 pqn = get_queue_by_qid(pqm, qid);
377 if (!pqn) {
378 pr_debug("No queue %d exists for update operation\n", qid);
379 return -EFAULT;
380 }
381
382 /* Free the old CU mask memory if it is already allocated, then
383 * allocate memory for the new CU mask.
384 */
385 kfree(pqn->q->properties.cu_mask);
386
387 pqn->q->properties.cu_mask_count = p->cu_mask_count;
388 pqn->q->properties.cu_mask = p->cu_mask;
389
390 retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
391 pqn->q);
392 if (retval != 0)
393 return retval;
394
395 return 0;
396 }
397
pqm_get_kernel_queue(struct process_queue_manager * pqm,unsigned int qid)398 struct kernel_queue *pqm_get_kernel_queue(
399 struct process_queue_manager *pqm,
400 unsigned int qid)
401 {
402 struct process_queue_node *pqn;
403
404 pqn = get_queue_by_qid(pqm, qid);
405 if (pqn && pqn->kq)
406 return pqn->kq;
407
408 return NULL;
409 }
410
411 #if defined(CONFIG_DEBUG_FS)
412
pqm_debugfs_mqds(struct seq_file * m,void * data)413 int pqm_debugfs_mqds(struct seq_file *m, void *data)
414 {
415 struct process_queue_manager *pqm = data;
416 struct process_queue_node *pqn;
417 struct queue *q;
418 enum KFD_MQD_TYPE mqd_type;
419 struct mqd_manager *mqd_mgr;
420 int r = 0;
421
422 list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
423 if (pqn->q) {
424 q = pqn->q;
425 switch (q->properties.type) {
426 case KFD_QUEUE_TYPE_SDMA:
427 seq_printf(m, " SDMA queue on device %x\n",
428 q->device->id);
429 mqd_type = KFD_MQD_TYPE_SDMA;
430 break;
431 case KFD_QUEUE_TYPE_COMPUTE:
432 seq_printf(m, " Compute queue on device %x\n",
433 q->device->id);
434 mqd_type = KFD_MQD_TYPE_CP;
435 break;
436 default:
437 seq_printf(m,
438 " Bad user queue type %d on device %x\n",
439 q->properties.type, q->device->id);
440 continue;
441 }
442 mqd_mgr = q->device->dqm->ops.get_mqd_manager(
443 q->device->dqm, mqd_type);
444 } else if (pqn->kq) {
445 q = pqn->kq->queue;
446 mqd_mgr = pqn->kq->mqd_mgr;
447 switch (q->properties.type) {
448 case KFD_QUEUE_TYPE_DIQ:
449 seq_printf(m, " DIQ on device %x\n",
450 pqn->kq->dev->id);
451 mqd_type = KFD_MQD_TYPE_HIQ;
452 break;
453 default:
454 seq_printf(m,
455 " Bad kernel queue type %d on device %x\n",
456 q->properties.type,
457 pqn->kq->dev->id);
458 continue;
459 }
460 } else {
461 seq_printf(m,
462 " Weird: Queue node with neither kernel nor user queue\n");
463 continue;
464 }
465
466 r = mqd_mgr->debugfs_show_mqd(m, q->mqd);
467 if (r != 0)
468 break;
469 }
470
471 return r;
472 }
473
474 #endif
475
