95 					    struct kfd_event *ev)  in allocate_event_notification_slot()  argument
113 	id = idr_alloc(&p->event_idr, ev, 0, p->signal_mapped_size / 8,  in allocate_event_notification_slot()
118 	ev->event_id = id;  in allocate_event_notification_slot()
153 	struct kfd_event *ev;  in lookup_signaled_event_by_partial_id()  local
171 	for (ev = NULL; id < KFD_SIGNAL_EVENT_LIMIT && !ev; id += 1U << bits) {  in lookup_signaled_event_by_partial_id()
175 		ev = idr_find(&p->event_idr, id);  in lookup_signaled_event_by_partial_id()
178 	return ev;  in lookup_signaled_event_by_partial_id()
183 				struct kfd_event *ev)  in create_signal_event()  argument
196 	ret = allocate_event_notification_slot(p, ev);  in create_signal_event()
204 	ev->user_signal_address = &p->signal_page->user_address[ev->event_id];  in create_signal_event()
206 			p->signal_event_count, ev->event_id,  in create_signal_event()
207 			ev->user_signal_address);  in create_signal_event()
212 static int create_other_event(struct kfd_process *p, struct kfd_event *ev)  in create_other_event()  argument
219 	int id = idr_alloc(&p->event_idr, ev, KFD_FIRST_NONSIGNAL_EVENT_ID,  in create_other_event()
225 	ev->event_id = id;  in create_other_event()
238 static void destroy_event(struct kfd_process *p, struct kfd_event *ev)  in destroy_event()  argument
243 	list_for_each_entry(waiter, &ev->wq.head, wait.entry)  in destroy_event()
245 	wake_up_all(&ev->wq);  in destroy_event()
247 	if (ev->type == KFD_EVENT_TYPE_SIGNAL ||  in destroy_event()
248 	    ev->type == KFD_EVENT_TYPE_DEBUG)  in destroy_event()
251 	idr_remove(&p->event_idr, ev->event_id);  in destroy_event()
252 	kfree(ev);  in destroy_event()
257 	struct kfd_event *ev;  in destroy_events()  local
260 	idr_for_each_entry(&p->event_idr, ev, id)  in destroy_events()
261 		destroy_event(p, ev);  in destroy_events()
287 static bool event_can_be_gpu_signaled(const struct kfd_event *ev)  in event_can_be_gpu_signaled()  argument
289 	return ev->type == KFD_EVENT_TYPE_SIGNAL ||  in event_can_be_gpu_signaled()
290 					ev->type == KFD_EVENT_TYPE_DEBUG;  in event_can_be_gpu_signaled()
293 static bool event_can_be_cpu_signaled(const struct kfd_event *ev)  in event_can_be_cpu_signaled()  argument
295 	return ev->type == KFD_EVENT_TYPE_SIGNAL;  in event_can_be_cpu_signaled()
328 	struct kfd_event *ev = kzalloc(sizeof(*ev), GFP_KERNEL);  in kfd_event_create()  local
330 	if (!ev)  in kfd_event_create()
333 	ev->type = event_type;  in kfd_event_create()
334 	ev->auto_reset = auto_reset;  in kfd_event_create()
335 	ev->signaled = false;  in kfd_event_create()
337 	init_waitqueue_head(&ev->wq);  in kfd_event_create()
346 		ret = create_signal_event(devkfd, p, ev);  in kfd_event_create()
350 			*event_slot_index = ev->event_id;  in kfd_event_create()
354 		ret = create_other_event(p, ev);  in kfd_event_create()
359 		*event_id = ev->event_id;  in kfd_event_create()
360 		*event_trigger_data = ev->event_id;  in kfd_event_create()
362 		kfree(ev);  in kfd_event_create()
373 	struct kfd_event *ev;  in kfd_event_destroy()  local
378 	ev = lookup_event_by_id(p, event_id);  in kfd_event_destroy()
380 	if (ev)  in kfd_event_destroy()
381 		destroy_event(p, ev);  in kfd_event_destroy()
389 static void set_event(struct kfd_event *ev)  in set_event()  argument
398 	ev->signaled = !ev->auto_reset || !waitqueue_active(&ev->wq);  in set_event()
400 	list_for_each_entry(waiter, &ev->wq.head, wait.entry)  in set_event()
403 	wake_up_all(&ev->wq);  in set_event()
410 	struct kfd_event *ev;  in kfd_set_event()  local
414 	ev = lookup_event_by_id(p, event_id);  in kfd_set_event()
416 	if (ev && event_can_be_cpu_signaled(ev))  in kfd_set_event()
417 		set_event(ev);  in kfd_set_event()
425 static void reset_event(struct kfd_event *ev)  in reset_event()  argument
427 	ev->signaled = false;  in reset_event()
434 	struct kfd_event *ev;  in kfd_reset_event()  local
438 	ev = lookup_event_by_id(p, event_id);  in kfd_reset_event()
440 	if (ev && event_can_be_cpu_signaled(ev))  in kfd_reset_event()
441 		reset_event(ev);  in kfd_reset_event()
450 static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev)  in acknowledge_signal()  argument
452 	page_slots(p->signal_page)[ev->event_id] = UNSIGNALED_EVENT_SLOT;  in acknowledge_signal()
456 					struct kfd_event *ev)  in set_event_from_interrupt()  argument
458 	if (ev && event_can_be_gpu_signaled(ev)) {  in set_event_from_interrupt()
459 		acknowledge_signal(p, ev);  in set_event_from_interrupt()
460 		set_event(ev);  in set_event_from_interrupt()
467 	struct kfd_event *ev = NULL;  in kfd_signal_event_interrupt()  local
482 		ev = lookup_signaled_event_by_partial_id(p, partial_id,  in kfd_signal_event_interrupt()
484 	if (ev) {  in kfd_signal_event_interrupt()
485 		set_event_from_interrupt(p, ev);  in kfd_signal_event_interrupt()
503 			idr_for_each_entry(&p->event_idr, ev, id) {  in kfd_signal_event_interrupt()
508 					set_event_from_interrupt(p, ev);  in kfd_signal_event_interrupt()
517 					ev = lookup_event_by_id(p, id);  in kfd_signal_event_interrupt()
518 					set_event_from_interrupt(p, ev);  in kfd_signal_event_interrupt()
548 	struct kfd_event *ev = lookup_event_by_id(p, event_id);  in init_event_waiter_get_status()  local
550 	if (!ev)  in init_event_waiter_get_status()
553 	waiter->event = ev;  in init_event_waiter_get_status()
554 	waiter->activated = ev->signaled;  in init_event_waiter_get_status()
555 	ev->signaled = ev->signaled && !ev->auto_reset;  in init_event_waiter_get_status()
562 	struct kfd_event *ev = waiter->event;  in init_event_waiter_add_to_waitlist()  local
568 		add_wait_queue(&ev->wq, &waiter->wait);  in init_event_waiter_add_to_waitlist()
835 	struct kfd_event *ev;  in lookup_events_by_type_and_signal()  local
842 	idr_for_each_entry_continue(&p->event_idr, ev, id)  in lookup_events_by_type_and_signal()
843 		if (ev->type == type) {  in lookup_events_by_type_and_signal()
847 					ev->event_id, ev->type);  in lookup_events_by_type_and_signal()
848 			set_event(ev);  in lookup_events_by_type_and_signal()
849 			if (ev->type == KFD_EVENT_TYPE_MEMORY && ev_data)  in lookup_events_by_type_and_signal()
850 				ev->memory_exception_data = *ev_data;  in lookup_events_by_type_and_signal()
977 	struct kfd_event *ev;  in kfd_signal_vm_fault_event()  local
1001 	idr_for_each_entry_continue(&p->event_idr, ev, id)  in kfd_signal_vm_fault_event()
1002 		if (ev->type == KFD_EVENT_TYPE_MEMORY) {  in kfd_signal_vm_fault_event()
1003 			ev->memory_exception_data = memory_exception_data;  in kfd_signal_vm_fault_event()
1004 			set_event(ev);  in kfd_signal_vm_fault_event()
1016 	struct kfd_event *ev;  in kfd_signal_reset_event()  local
1038 		idr_for_each_entry_continue(&p->event_idr, ev, id) {  in kfd_signal_reset_event()
1039 			if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) {  in kfd_signal_reset_event()
1040 				ev->hw_exception_data = hw_exception_data;  in kfd_signal_reset_event()
1041 				set_event(ev);  in kfd_signal_reset_event()
1043 			if (ev->type == KFD_EVENT_TYPE_MEMORY &&  in kfd_signal_reset_event()
1045 				ev->memory_exception_data = memory_exception_data;  in kfd_signal_reset_event()
1046 				set_event(ev);  in kfd_signal_reset_event()