19 static void idxd_wq_disable_cleanup(struct idxd_wq *wq);
42 static void free_hw_descs(struct idxd_wq *wq)  in free_hw_descs()  argument
46 	for (i = 0; i < wq->num_descs; i++)  in free_hw_descs()
47 		kfree(wq->hw_descs[i]);  in free_hw_descs()
49 	kfree(wq->hw_descs);  in free_hw_descs()
52 static int alloc_hw_descs(struct idxd_wq *wq, int num)  in alloc_hw_descs()  argument
54 	struct device *dev = &wq->idxd->pdev->dev;  in alloc_hw_descs()
58 	wq->hw_descs = kcalloc_node(num, sizeof(struct dsa_hw_desc *),  in alloc_hw_descs()
60 	if (!wq->hw_descs)  in alloc_hw_descs()
64 		wq->hw_descs[i] = kzalloc_node(sizeof(*wq->hw_descs[i]),  in alloc_hw_descs()
66 		if (!wq->hw_descs[i]) {  in alloc_hw_descs()
67 			free_hw_descs(wq);  in alloc_hw_descs()
75 static void free_descs(struct idxd_wq *wq)  in free_descs()  argument
79 	for (i = 0; i < wq->num_descs; i++)  in free_descs()
80 		kfree(wq->descs[i]);  in free_descs()
82 	kfree(wq->descs);  in free_descs()
85 static int alloc_descs(struct idxd_wq *wq, int num)  in alloc_descs()  argument
87 	struct device *dev = &wq->idxd->pdev->dev;  in alloc_descs()
91 	wq->descs = kcalloc_node(num, sizeof(struct idxd_desc *),  in alloc_descs()
93 	if (!wq->descs)  in alloc_descs()
97 		wq->descs[i] = kzalloc_node(sizeof(*wq->descs[i]),  in alloc_descs()
99 		if (!wq->descs[i]) {  in alloc_descs()
100 			free_descs(wq);  in alloc_descs()
109 int idxd_wq_alloc_resources(struct idxd_wq *wq)  in idxd_wq_alloc_resources()  argument
111 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_alloc_resources()
115 	if (wq->type != IDXD_WQT_KERNEL)  in idxd_wq_alloc_resources()
118 	num_descs = wq_dedicated(wq) ? wq->size : wq->threshold;  in idxd_wq_alloc_resources()
119 	wq->num_descs = num_descs;  in idxd_wq_alloc_resources()
121 	rc = alloc_hw_descs(wq, num_descs);  in idxd_wq_alloc_resources()
125 	wq->compls_size = num_descs * idxd->data->compl_size;  in idxd_wq_alloc_resources()
126 	wq->compls = dma_alloc_coherent(dev, wq->compls_size, &wq->compls_addr, GFP_KERNEL);  in idxd_wq_alloc_resources()
127 	if (!wq->compls) {  in idxd_wq_alloc_resources()
132 	rc = alloc_descs(wq, num_descs);  in idxd_wq_alloc_resources()
136 	rc = sbitmap_queue_init_node(&wq->sbq, num_descs, -1, false, GFP_KERNEL,  in idxd_wq_alloc_resources()
142 		struct idxd_desc *desc = wq->descs[i];  in idxd_wq_alloc_resources()
144 		desc->hw = wq->hw_descs[i];  in idxd_wq_alloc_resources()
146 			desc->completion = &wq->compls[i];  in idxd_wq_alloc_resources()
148 			desc->iax_completion = &wq->iax_compls[i];  in idxd_wq_alloc_resources()
149 		desc->compl_dma = wq->compls_addr + idxd->data->compl_size * i;  in idxd_wq_alloc_resources()
151 		desc->wq = wq;  in idxd_wq_alloc_resources()
158 	free_descs(wq);  in idxd_wq_alloc_resources()
160 	dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);  in idxd_wq_alloc_resources()
162 	free_hw_descs(wq);  in idxd_wq_alloc_resources()
166 void idxd_wq_free_resources(struct idxd_wq *wq)  in idxd_wq_free_resources()  argument
168 	struct device *dev = &wq->idxd->pdev->dev;  in idxd_wq_free_resources()
170 	if (wq->type != IDXD_WQT_KERNEL)  in idxd_wq_free_resources()
173 	free_hw_descs(wq);  in idxd_wq_free_resources()
174 	free_descs(wq);  in idxd_wq_free_resources()
175 	dma_free_coherent(dev, wq->compls_size, wq->compls, wq->compls_addr);  in idxd_wq_free_resources()
176 	sbitmap_queue_free(&wq->sbq);  in idxd_wq_free_resources()
179 int idxd_wq_enable(struct idxd_wq *wq)  in idxd_wq_enable()  argument
181 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_enable()
185 	if (wq->state == IDXD_WQ_ENABLED) {  in idxd_wq_enable()
186 		dev_dbg(dev, "WQ %d already enabled\n", wq->id);  in idxd_wq_enable()
190 	idxd_cmd_exec(idxd, IDXD_CMD_ENABLE_WQ, wq->id, &status);  in idxd_wq_enable()
198 	wq->state = IDXD_WQ_ENABLED;  in idxd_wq_enable()
199 	set_bit(wq->id, idxd->wq_enable_map);  in idxd_wq_enable()
200 	dev_dbg(dev, "WQ %d enabled\n", wq->id);  in idxd_wq_enable()
204 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config)  in idxd_wq_disable()  argument
206 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_disable()
210 	dev_dbg(dev, "Disabling WQ %d\n", wq->id);  in idxd_wq_disable()
212 	if (wq->state != IDXD_WQ_ENABLED) {  in idxd_wq_disable()
213 		dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);  in idxd_wq_disable()
217 	operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);  in idxd_wq_disable()
226 		idxd_wq_disable_cleanup(wq);  in idxd_wq_disable()
227 	clear_bit(wq->id, idxd->wq_enable_map);  in idxd_wq_disable()
228 	wq->state = IDXD_WQ_DISABLED;  in idxd_wq_disable()
229 	dev_dbg(dev, "WQ %d disabled\n", wq->id);  in idxd_wq_disable()
233 void idxd_wq_drain(struct idxd_wq *wq)  in idxd_wq_drain()  argument
235 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_drain()
239 	if (wq->state != IDXD_WQ_ENABLED) {  in idxd_wq_drain()
240 		dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);  in idxd_wq_drain()
244 	dev_dbg(dev, "Draining WQ %d\n", wq->id);  in idxd_wq_drain()
245 	operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);  in idxd_wq_drain()
249 void idxd_wq_reset(struct idxd_wq *wq)  in idxd_wq_reset()  argument
251 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_reset()
255 	if (wq->state != IDXD_WQ_ENABLED) {  in idxd_wq_reset()
256 		dev_dbg(dev, "WQ %d in wrong state: %d\n", wq->id, wq->state);  in idxd_wq_reset()
260 	operand = BIT(wq->id % 16) | ((wq->id / 16) << 16);  in idxd_wq_reset()
262 	idxd_wq_disable_cleanup(wq);  in idxd_wq_reset()
265 int idxd_wq_map_portal(struct idxd_wq *wq)  in idxd_wq_map_portal()  argument
267 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_map_portal()
273 	start += idxd_get_wq_portal_full_offset(wq->id, IDXD_PORTAL_LIMITED);  in idxd_wq_map_portal()
275 	wq->portal = devm_ioremap(dev, start, IDXD_PORTAL_SIZE);  in idxd_wq_map_portal()
276 	if (!wq->portal)  in idxd_wq_map_portal()
282 void idxd_wq_unmap_portal(struct idxd_wq *wq)  in idxd_wq_unmap_portal()  argument
284 	struct device *dev = &wq->idxd->pdev->dev;  in idxd_wq_unmap_portal()
286 	devm_iounmap(dev, wq->portal);  in idxd_wq_unmap_portal()
287 	wq->portal = NULL;  in idxd_wq_unmap_portal()
288 	wq->portal_offset = 0;  in idxd_wq_unmap_portal()
296 		struct idxd_wq *wq = idxd->wqs[i];  in idxd_wqs_unmap_portal()  local
298 		if (wq->portal)  in idxd_wqs_unmap_portal()
299 			idxd_wq_unmap_portal(wq);  in idxd_wqs_unmap_portal()
303 static void __idxd_wq_set_priv_locked(struct idxd_wq *wq, int priv)  in __idxd_wq_set_priv_locked()  argument
305 	struct idxd_device *idxd = wq->idxd;  in __idxd_wq_set_priv_locked()
309 	offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PRIVL_IDX);  in __idxd_wq_set_priv_locked()
313 	wq->wqcfg->bits[WQCFG_PRIVL_IDX] = wqcfg.bits[WQCFG_PRIVL_IDX];  in __idxd_wq_set_priv_locked()
318 static void __idxd_wq_set_pasid_locked(struct idxd_wq *wq, int pasid)  in __idxd_wq_set_pasid_locked()  argument
320 	struct idxd_device *idxd = wq->idxd;  in __idxd_wq_set_pasid_locked()
324 	offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);  in __idxd_wq_set_pasid_locked()
329 	wq->wqcfg->bits[WQCFG_PASID_IDX] = wqcfg.bits[WQCFG_PASID_IDX];  in __idxd_wq_set_pasid_locked()
334 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid)  in idxd_wq_set_pasid()  argument
338 	rc = idxd_wq_disable(wq, false);  in idxd_wq_set_pasid()
342 	__idxd_wq_set_pasid_locked(wq, pasid);  in idxd_wq_set_pasid()
344 	rc = idxd_wq_enable(wq);  in idxd_wq_set_pasid()
351 int idxd_wq_disable_pasid(struct idxd_wq *wq)  in idxd_wq_disable_pasid()  argument
353 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_disable_pasid()
358 	rc = idxd_wq_disable(wq, false);  in idxd_wq_disable_pasid()
362 	offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PASID_IDX);  in idxd_wq_disable_pasid()
370 	rc = idxd_wq_enable(wq);  in idxd_wq_disable_pasid()
377 static void idxd_wq_disable_cleanup(struct idxd_wq *wq)  in idxd_wq_disable_cleanup()  argument
379 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_disable_cleanup()
381 	lockdep_assert_held(&wq->wq_lock);  in idxd_wq_disable_cleanup()
382 	wq->state = IDXD_WQ_DISABLED;  in idxd_wq_disable_cleanup()
383 	memset(wq->wqcfg, 0, idxd->wqcfg_size);  in idxd_wq_disable_cleanup()
384 	wq->type = IDXD_WQT_NONE;  in idxd_wq_disable_cleanup()
385 	wq->threshold = 0;  in idxd_wq_disable_cleanup()
386 	wq->priority = 0;  in idxd_wq_disable_cleanup()
387 	wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES;  in idxd_wq_disable_cleanup()
388 	clear_bit(WQ_FLAG_DEDICATED, &wq->flags);  in idxd_wq_disable_cleanup()
389 	clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags);  in idxd_wq_disable_cleanup()
390 	clear_bit(WQ_FLAG_ATS_DISABLE, &wq->flags);  in idxd_wq_disable_cleanup()
391 	memset(wq->name, 0, WQ_NAME_SIZE);  in idxd_wq_disable_cleanup()
392 	wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER;  in idxd_wq_disable_cleanup()
393 	idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH);  in idxd_wq_disable_cleanup()
394 	if (wq->opcap_bmap)  in idxd_wq_disable_cleanup()
395 		bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS);  in idxd_wq_disable_cleanup()
398 static void idxd_wq_device_reset_cleanup(struct idxd_wq *wq)  in idxd_wq_device_reset_cleanup()  argument
400 	lockdep_assert_held(&wq->wq_lock);  in idxd_wq_device_reset_cleanup()
402 	wq->size = 0;  in idxd_wq_device_reset_cleanup()
403 	wq->group = NULL;  in idxd_wq_device_reset_cleanup()
408 	struct idxd_wq *wq = container_of(ref, struct idxd_wq, wq_active);  in idxd_wq_ref_release()  local
410 	complete(&wq->wq_dead);  in idxd_wq_ref_release()
413 int idxd_wq_init_percpu_ref(struct idxd_wq *wq)  in idxd_wq_init_percpu_ref()  argument
417 	memset(&wq->wq_active, 0, sizeof(wq->wq_active));  in idxd_wq_init_percpu_ref()
418 	rc = percpu_ref_init(&wq->wq_active, idxd_wq_ref_release,  in idxd_wq_init_percpu_ref()
422 	reinit_completion(&wq->wq_dead);  in idxd_wq_init_percpu_ref()
423 	reinit_completion(&wq->wq_resurrect);  in idxd_wq_init_percpu_ref()
427 void __idxd_wq_quiesce(struct idxd_wq *wq)  in __idxd_wq_quiesce()  argument
429 	lockdep_assert_held(&wq->wq_lock);  in __idxd_wq_quiesce()
430 	reinit_completion(&wq->wq_resurrect);  in __idxd_wq_quiesce()
431 	percpu_ref_kill(&wq->wq_active);  in __idxd_wq_quiesce()
432 	complete_all(&wq->wq_resurrect);  in __idxd_wq_quiesce()
433 	wait_for_completion(&wq->wq_dead);  in __idxd_wq_quiesce()
436 void idxd_wq_quiesce(struct idxd_wq *wq)  in idxd_wq_quiesce()  argument
438 	mutex_lock(&wq->wq_lock);  in idxd_wq_quiesce()
439 	__idxd_wq_quiesce(wq);  in idxd_wq_quiesce()
440 	mutex_unlock(&wq->wq_lock);  in idxd_wq_quiesce()
722 		struct idxd_wq *wq = idxd->wqs[i];  in idxd_device_wqs_clear_state()  local
724 		mutex_lock(&wq->wq_lock);  in idxd_device_wqs_clear_state()
725 		idxd_wq_disable_cleanup(wq);  in idxd_device_wqs_clear_state()
726 		idxd_wq_device_reset_cleanup(wq);  in idxd_device_wqs_clear_state()
727 		mutex_unlock(&wq->wq_lock);  in idxd_device_wqs_clear_state()
819 static int idxd_wq_config_write(struct idxd_wq *wq)  in idxd_wq_config_write()  argument
821 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_config_write()
826 	if (!wq->group)  in idxd_wq_config_write()
834 		wq_offset = WQCFG_OFFSET(idxd, wq->id, i);  in idxd_wq_config_write()
835 		wq->wqcfg->bits[i] |= ioread32(idxd->reg_base + wq_offset);  in idxd_wq_config_write()
838 	if (wq->size == 0 && wq->type != IDXD_WQT_NONE)  in idxd_wq_config_write()
839 		wq->size = WQ_DEFAULT_QUEUE_DEPTH;  in idxd_wq_config_write()
842 	wq->wqcfg->wq_size = wq->size;  in idxd_wq_config_write()
845 	wq->wqcfg->wq_thresh = wq->threshold;  in idxd_wq_config_write()
848 	if (wq_dedicated(wq))  in idxd_wq_config_write()
849 		wq->wqcfg->mode = 1;  in idxd_wq_config_write()
862 	if (wq_dedicated(wq) && wq->wqcfg->pasid_en &&  in idxd_wq_config_write()
864 	    wq->type == IDXD_WQT_KERNEL) {  in idxd_wq_config_write()
869 	wq->wqcfg->priority = wq->priority;  in idxd_wq_config_write()
872 	    test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags))  in idxd_wq_config_write()
873 		wq->wqcfg->bof = 1;  in idxd_wq_config_write()
876 		wq->wqcfg->wq_ats_disable = test_bit(WQ_FLAG_ATS_DISABLE, &wq->flags);  in idxd_wq_config_write()
879 	wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes);  in idxd_wq_config_write()
880 	idxd_wqcfg_set_max_batch_shift(idxd->data->type, wq->wqcfg, ilog2(wq->max_batch_size));  in idxd_wq_config_write()
883 	if (idxd->hw.wq_cap.op_config && wq->opcap_bmap) {  in idxd_wq_config_write()
884 		memset(wq->wqcfg->op_config, 0, IDXD_MAX_OPCAP_BITS / 8);  in idxd_wq_config_write()
885 		for_each_set_bit(n, wq->opcap_bmap, IDXD_MAX_OPCAP_BITS) {  in idxd_wq_config_write()
889 			wq->wqcfg->op_config[idx] |= BIT(pos);  in idxd_wq_config_write()
893 	dev_dbg(dev, "WQ %d CFGs\n", wq->id);  in idxd_wq_config_write()
895 		wq_offset = WQCFG_OFFSET(idxd, wq->id, i);  in idxd_wq_config_write()
896 		iowrite32(wq->wqcfg->bits[i], idxd->reg_base + wq_offset);  in idxd_wq_config_write()
898 			wq->id, i, wq_offset,  in idxd_wq_config_write()
910 		struct idxd_wq *wq = idxd->wqs[i];  in idxd_wqs_config_write()  local
912 		rc = idxd_wq_config_write(wq);  in idxd_wqs_config_write()
978 	struct idxd_wq *wq;  in idxd_wqs_setup()  local
990 		wq = idxd->wqs[i];  in idxd_wqs_setup()
991 		group = wq->group;  in idxd_wqs_setup()
993 		if (!wq->group)  in idxd_wqs_setup()
996 		if (wq_shared(wq) && !wq_shared_supported(wq)) {  in idxd_wqs_setup()
1002 		group->grpcfg.wqs[wq->id / 64] |= BIT(wq->id % 64);  in idxd_wqs_setup()
1040 static int idxd_wq_load_config(struct idxd_wq *wq)  in idxd_wq_load_config()  argument
1042 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_load_config()
1047 	wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, 0);  in idxd_wq_load_config()
1048 	memcpy_fromio(wq->wqcfg, idxd->reg_base + wqcfg_offset, idxd->wqcfg_size);  in idxd_wq_load_config()
1050 	wq->size = wq->wqcfg->wq_size;  in idxd_wq_load_config()
1051 	wq->threshold = wq->wqcfg->wq_thresh;  in idxd_wq_load_config()
1054 	if (wq->wqcfg->mode == 0 || wq->wqcfg->pasid_en)  in idxd_wq_load_config()
1057 	set_bit(WQ_FLAG_DEDICATED, &wq->flags);  in idxd_wq_load_config()
1059 	wq->priority = wq->wqcfg->priority;  in idxd_wq_load_config()
1061 	wq->max_xfer_bytes = 1ULL << wq->wqcfg->max_xfer_shift;  in idxd_wq_load_config()
1062 	idxd_wq_set_max_batch_size(idxd->data->type, wq, 1U << wq->wqcfg->max_batch_shift);  in idxd_wq_load_config()
1065 		wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, i);  in idxd_wq_load_config()
1066 		dev_dbg(dev, "WQ[%d][%d][%#x]: %#x\n", wq->id, i, wqcfg_offset, wq->wqcfg->bits[i]);  in idxd_wq_load_config()
1083 		struct idxd_wq *wq;  in idxd_group_load_config()  local
1103 				wq = idxd->wqs[id];  in idxd_group_load_config()
1104 				wq->group = group;  in idxd_group_load_config()
1147 		struct idxd_wq *wq = idxd->wqs[i];  in idxd_device_load_config()  local
1149 		rc = idxd_wq_load_config(wq);  in idxd_device_load_config()
1202 void idxd_wq_free_irq(struct idxd_wq *wq)  in idxd_wq_free_irq()  argument
1204 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_free_irq()
1205 	struct idxd_irq_entry *ie = &wq->ie;  in idxd_wq_free_irq()
1207 	if (wq->type != IDXD_WQT_KERNEL)  in idxd_wq_free_irq()
1220 int idxd_wq_request_irq(struct idxd_wq *wq)  in idxd_wq_request_irq()  argument
1222 	struct idxd_device *idxd = wq->idxd;  in idxd_wq_request_irq()
1228 	if (wq->type != IDXD_WQT_KERNEL)  in idxd_wq_request_irq()
1231 	ie = &wq->ie;  in idxd_wq_request_irq()
1262 int drv_enable_wq(struct idxd_wq *wq)  in drv_enable_wq()  argument
1264 	struct idxd_device *idxd = wq->idxd;  in drv_enable_wq()
1268 	lockdep_assert_held(&wq->wq_lock);  in drv_enable_wq()
1275 	if (wq->state != IDXD_WQ_DISABLED) {  in drv_enable_wq()
1276 		dev_dbg(dev, "wq %d already enabled.\n", wq->id);  in drv_enable_wq()
1282 	if (!wq->group) {  in drv_enable_wq()
1283 		dev_dbg(dev, "wq %d not attached to group.\n", wq->id);  in drv_enable_wq()
1288 	if (strlen(wq->name) == 0) {  in drv_enable_wq()
1290 		dev_dbg(dev, "wq %d name not set.\n", wq->id);  in drv_enable_wq()
1295 	if (wq_shared(wq)) {  in drv_enable_wq()
1296 		if (!wq_shared_supported(wq)) {  in drv_enable_wq()
1309 		if (wq->threshold == 0) {  in drv_enable_wq()
1326 		if (wq_pasid_enabled(wq)) {  in drv_enable_wq()
1327 			if (is_idxd_wq_kernel(wq) || wq_shared(wq)) {  in drv_enable_wq()
1328 				u32 pasid = wq_dedicated(wq) ? idxd->pasid : 0;  in drv_enable_wq()
1330 				__idxd_wq_set_pasid_locked(wq, pasid);  in drv_enable_wq()
1334 		if (is_idxd_wq_kernel(wq))  in drv_enable_wq()
1336 		__idxd_wq_set_priv_locked(wq, priv);  in drv_enable_wq()
1345 		dev_dbg(dev, "Writing wq %d config failed: %d\n", wq->id, rc);  in drv_enable_wq()
1349 	rc = idxd_wq_enable(wq);  in drv_enable_wq()
1351 		dev_dbg(dev, "wq %d enabling failed: %d\n", wq->id, rc);  in drv_enable_wq()
1355 	rc = idxd_wq_map_portal(wq);  in drv_enable_wq()
1358 		dev_dbg(dev, "wq %d portal mapping failed: %d\n", wq->id, rc);  in drv_enable_wq()
1362 	wq->client_count = 0;  in drv_enable_wq()
1364 	rc = idxd_wq_request_irq(wq);  in drv_enable_wq()
1367 		dev_dbg(dev, "WQ %d irq setup failed: %d\n", wq->id, rc);  in drv_enable_wq()
1371 	rc = idxd_wq_alloc_resources(wq);  in drv_enable_wq()
1378 	rc = idxd_wq_init_percpu_ref(wq);  in drv_enable_wq()
1388 	idxd_wq_free_resources(wq);  in drv_enable_wq()
1390 	idxd_wq_free_irq(wq);  in drv_enable_wq()
1392 	idxd_wq_unmap_portal(wq);  in drv_enable_wq()
1394 	rc = idxd_wq_disable(wq, false);  in drv_enable_wq()
1396 		dev_dbg(dev, "wq %s disable failed\n", dev_name(wq_confdev(wq)));  in drv_enable_wq()
1401 void drv_disable_wq(struct idxd_wq *wq)  in drv_disable_wq()  argument
1403 	struct idxd_device *idxd = wq->idxd;  in drv_disable_wq()
1406 	lockdep_assert_held(&wq->wq_lock);  in drv_disable_wq()
1408 	if (idxd_wq_refcount(wq))  in drv_disable_wq()
1410 			 wq->id, idxd_wq_refcount(wq));  in drv_disable_wq()
1412 	idxd_wq_free_resources(wq);  in drv_disable_wq()
1413 	idxd_wq_unmap_portal(wq);  in drv_disable_wq()
1414 	idxd_wq_drain(wq);  in drv_disable_wq()
1415 	idxd_wq_free_irq(wq);  in drv_disable_wq()
1416 	idxd_wq_reset(wq);  in drv_disable_wq()
1417 	percpu_ref_exit(&wq->wq_active);  in drv_disable_wq()
1418 	wq->type = IDXD_WQT_NONE;  in drv_disable_wq()
1419 	wq->client_count = 0;  in drv_disable_wq()
1469 		struct idxd_wq *wq = idxd->wqs[i];  in idxd_device_drv_remove()  local
1470 		struct device *wq_dev = wq_confdev(wq);  in idxd_device_drv_remove()
1472 		if (wq->state == IDXD_WQ_DISABLED)  in idxd_device_drv_remove()