Lines Matching refs:hwq
159 struct hwq *hwq = get_hwq(afu, cmd->hwq_index); in cmd_complete() local
161 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); in cmd_complete()
163 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); in cmd_complete()
191 static void flush_pending_cmds(struct hwq *hwq) in flush_pending_cmds() argument
193 struct cxlflash_cfg *cfg = hwq->afu->parent; in flush_pending_cmds()
198 list_for_each_entry_safe(cmd, tmp, &hwq->pending_cmds, list) { in flush_pending_cmds()
235 static int context_reset(struct hwq *hwq, __be64 __iomem *reset_reg) in context_reset() argument
237 struct cxlflash_cfg *cfg = hwq->afu->parent; in context_reset()
244 dev_dbg(dev, "%s: hwq=%p\n", __func__, hwq); in context_reset()
246 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); in context_reset()
261 flush_pending_cmds(hwq); in context_reset()
263 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); in context_reset()
276 static int context_reset_ioarrin(struct hwq *hwq) in context_reset_ioarrin() argument
278 return context_reset(hwq, &hwq->host_map->ioarrin); in context_reset_ioarrin()
287 static int context_reset_sq(struct hwq *hwq) in context_reset_sq() argument
289 return context_reset(hwq, &hwq->host_map->sq_ctx_reset); in context_reset_sq()
304 struct hwq *hwq = get_hwq(afu, cmd->hwq_index); in send_cmd_ioarrin() local
313 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); in send_cmd_ioarrin()
314 if (--hwq->room < 0) { in send_cmd_ioarrin()
315 room = readq_be(&hwq->host_map->cmd_room); in send_cmd_ioarrin()
320 hwq->room = 0; in send_cmd_ioarrin()
324 hwq->room = room - 1; in send_cmd_ioarrin()
327 list_add(&cmd->list, &hwq->pending_cmds); in send_cmd_ioarrin()
328 writeq_be((u64)&cmd->rcb, &hwq->host_map->ioarrin); in send_cmd_ioarrin()
330 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); in send_cmd_ioarrin()
348 struct hwq *hwq = get_hwq(afu, cmd->hwq_index); in send_cmd_sq() local
353 newval = atomic_dec_if_positive(&hwq->hsq_credits); in send_cmd_sq()
361 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); in send_cmd_sq()
363 *hwq->hsq_curr = cmd->rcb; in send_cmd_sq()
364 if (hwq->hsq_curr < hwq->hsq_end) in send_cmd_sq()
365 hwq->hsq_curr++; in send_cmd_sq()
367 hwq->hsq_curr = hwq->hsq_start; in send_cmd_sq()
369 list_add(&cmd->list, &hwq->pending_cmds); in send_cmd_sq()
370 writeq_be((u64)hwq->hsq_curr, &hwq->host_map->sq_tail); in send_cmd_sq()
372 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); in send_cmd_sq()
376 cmd->rcb.data_ea, cmd->rcb.ioasa, rc, hwq->hsq_curr, in send_cmd_sq()
377 readq_be(&hwq->host_map->sq_head), in send_cmd_sq()
378 readq_be(&hwq->host_map->sq_tail)); in send_cmd_sq()
426 u32 hwq = 0; in cmd_to_target_hwq() local
433 hwq = afu->hwq_rr_count++ % afu->num_hwqs; in cmd_to_target_hwq()
437 hwq = blk_mq_unique_tag_to_hwq(tag); in cmd_to_target_hwq()
440 hwq = smp_processor_id() % afu->num_hwqs; in cmd_to_target_hwq()
446 return hwq; in cmd_to_target_hwq()
464 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ); in send_tmf() local
492 cmd->hwq_index = hwq->index; in send_tmf()
494 cmd->rcb.ctx_id = hwq->ctx_hndl; in send_tmf()
533 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); in send_tmf()
535 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); in send_tmf()
568 struct hwq *hwq = get_hwq(afu, hwq_index); in cxlflash_queuecommand() local
622 cmd->rcb.ctx_id = hwq->ctx_hndl; in cxlflash_queuecommand()
693 struct hwq *hwq; in stop_afu() local
706 hwq = get_hwq(afu, i); in stop_afu()
708 irq_poll_disable(&hwq->irqpoll); in stop_afu()
732 struct hwq *hwq; in term_intr() local
739 hwq = get_hwq(afu, index); in term_intr()
741 if (!hwq->ctx_cookie) { in term_intr()
750 cfg->ops->unmap_afu_irq(hwq->ctx_cookie, 3, hwq); in term_intr()
753 cfg->ops->unmap_afu_irq(hwq->ctx_cookie, 2, hwq); in term_intr()
756 cfg->ops->unmap_afu_irq(hwq->ctx_cookie, 1, hwq); in term_intr()
759 cfg->ops->free_afu_irqs(hwq->ctx_cookie); in term_intr()
778 struct hwq *hwq; in term_mc() local
786 hwq = get_hwq(afu, index); in term_mc()
788 if (!hwq->ctx_cookie) { in term_mc()
793 WARN_ON(cfg->ops->stop_context(hwq->ctx_cookie)); in term_mc()
795 WARN_ON(cfg->ops->release_context(hwq->ctx_cookie)); in term_mc()
796 hwq->ctx_cookie = NULL; in term_mc()
798 spin_lock_irqsave(&hwq->hrrq_slock, lock_flags); in term_mc()
799 hwq->hrrq_online = false; in term_mc()
800 spin_unlock_irqrestore(&hwq->hrrq_slock, lock_flags); in term_mc()
802 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); in term_mc()
803 flush_pending_cmds(hwq); in term_mc()
804 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); in term_mc()
1265 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ); in afu_err_intr_init() local
1278 reg = ((u64) (((hwq->ctx_hndl << 8) | SISL_MSI_ASYNC_ERROR)) << 40); in afu_err_intr_init()
1315 hwq = get_hwq(afu, i); in afu_err_intr_init()
1317 reg = readq_be(&hwq->host_map->ctx_ctrl); in afu_err_intr_init()
1320 writeq_be(reg, &hwq->host_map->ctx_ctrl); in afu_err_intr_init()
1321 writeq_be(SISL_ISTATUS_MASK, &hwq->host_map->intr_mask); in afu_err_intr_init()
1334 struct hwq *hwq = (struct hwq *)data; in cxlflash_sync_err_irq() local
1335 struct cxlflash_cfg *cfg = hwq->afu->parent; in cxlflash_sync_err_irq()
1340 reg = readq_be(&hwq->host_map->intr_status); in cxlflash_sync_err_irq()
1352 writeq_be(reg_unmasked, &hwq->host_map->intr_clear); in cxlflash_sync_err_irq()
1368 static int process_hrrq(struct hwq *hwq, struct list_head *doneq, int budget) in process_hrrq() argument
1370 struct afu *afu = hwq->afu; in process_hrrq()
1374 bool toggle = hwq->toggle; in process_hrrq()
1377 *hrrq_start = hwq->hrrq_start, in process_hrrq()
1378 *hrrq_end = hwq->hrrq_end, in process_hrrq()
1379 *hrrq_curr = hwq->hrrq_curr; in process_hrrq()
1408 atomic_inc(&hwq->hsq_credits); in process_hrrq()
1415 hwq->hrrq_curr = hrrq_curr; in process_hrrq()
1416 hwq->toggle = toggle; in process_hrrq()
1446 struct hwq *hwq = container_of(irqpoll, struct hwq, irqpoll); in cxlflash_irqpoll() local
1451 spin_lock_irqsave(&hwq->hrrq_slock, hrrq_flags); in cxlflash_irqpoll()
1453 num_entries = process_hrrq(hwq, &doneq, budget); in cxlflash_irqpoll()
1457 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags); in cxlflash_irqpoll()
1472 struct hwq *hwq = (struct hwq *)data; in cxlflash_rrq_irq() local
1473 struct afu *afu = hwq->afu; in cxlflash_rrq_irq()
1478 spin_lock_irqsave(&hwq->hrrq_slock, hrrq_flags); in cxlflash_rrq_irq()
1481 if (!hwq->hrrq_online) { in cxlflash_rrq_irq()
1482 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags); in cxlflash_rrq_irq()
1487 irq_poll_sched(&hwq->irqpoll); in cxlflash_rrq_irq()
1488 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags); in cxlflash_rrq_irq()
1492 num_entries = process_hrrq(hwq, &doneq, -1); in cxlflash_rrq_irq()
1493 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags); in cxlflash_rrq_irq()
1540 struct hwq *hwq = (struct hwq *)data; in cxlflash_async_err_irq() local
1541 struct afu *afu = hwq->afu; in cxlflash_async_err_irq()
1734 struct hwq *hwq; in init_pcr() local
1749 hwq = get_hwq(afu, i); in init_pcr()
1750 cookie = hwq->ctx_cookie; in init_pcr()
1752 hwq->ctx_hndl = (u16) cfg->ops->process_element(cookie); in init_pcr()
1753 hwq->host_map = &afu->afu_map->hosts[hwq->ctx_hndl].host; in init_pcr()
1754 hwq->ctrl_map = &afu->afu_map->ctrls[hwq->ctx_hndl].ctrl; in init_pcr()
1757 writeq_be(SISL_ENDIAN_CTRL, &hwq->host_map->endian_ctrl); in init_pcr()
1769 struct hwq *hwq; in init_global() local
1787 hwq = get_hwq(afu, i); in init_global()
1788 hmap = hwq->host_map; in init_global()
1790 writeq_be((u64) hwq->hrrq_start, &hmap->rrq_start); in init_global()
1791 writeq_be((u64) hwq->hrrq_end, &hmap->rrq_end); in init_global()
1792 hwq->hrrq_online = true; in init_global()
1795 writeq_be((u64)hwq->hsq_start, &hmap->sq_start); in init_global()
1796 writeq_be((u64)hwq->hsq_end, &hmap->sq_end); in init_global()
1841 hwq = get_hwq(afu, i); in init_global()
1842 ctx = hwq->ctx_cookie; in init_global()
1844 for (j = 0; j < hwq->num_irqs; j++) { in init_global()
1846 writeq_be(reg, &hwq->ctrl_map->lisn_ea[j]); in init_global()
1849 reg = hwq->ctx_hndl; in init_global()
1851 &hwq->ctrl_map->lisn_pasid[0]); in init_global()
1853 &hwq->ctrl_map->lisn_pasid[1]); in init_global()
1861 hwq = get_hwq(afu, i); in init_global()
1863 (void)readq_be(&hwq->ctrl_map->mbox_r); /* unlock ctx_cap */ in init_global()
1867 &hwq->ctrl_map->ctx_cap); in init_global()
1876 hwq = get_hwq(afu, PRIMARY_HWQ); in init_global()
1877 reg = readq_be(&hwq->host_map->ctx_ctrl); in init_global()
1895 struct hwq *hwq; in start_afu() local
1903 hwq = get_hwq(afu, i); in start_afu()
1906 memset(&hwq->rrq_entry, 0, sizeof(hwq->rrq_entry)); in start_afu()
1909 hwq->hrrq_start = &hwq->rrq_entry[0]; in start_afu()
1910 hwq->hrrq_end = &hwq->rrq_entry[NUM_RRQ_ENTRY - 1]; in start_afu()
1911 hwq->hrrq_curr = hwq->hrrq_start; in start_afu()
1912 hwq->toggle = 1; in start_afu()
1915 spin_lock_init(&hwq->hrrq_slock); in start_afu()
1916 spin_lock_init(&hwq->hsq_slock); in start_afu()
1920 memset(&hwq->sq, 0, sizeof(hwq->sq)); in start_afu()
1921 hwq->hsq_start = &hwq->sq[0]; in start_afu()
1922 hwq->hsq_end = &hwq->sq[NUM_SQ_ENTRY - 1]; in start_afu()
1923 hwq->hsq_curr = hwq->hsq_start; in start_afu()
1925 atomic_set(&hwq->hsq_credits, NUM_SQ_ENTRY - 1); in start_afu()
1930 irq_poll_init(&hwq->irqpoll, afu->irqpoll_weight, in start_afu()
1949 struct hwq *hwq) in init_intr() argument
1952 void *ctx = hwq->ctx_cookie; in init_intr()
1955 bool is_primary_hwq = (hwq->index == PRIMARY_HWQ); in init_intr()
1956 int num_irqs = hwq->num_irqs; in init_intr()
1966 rc = cfg->ops->map_afu_irq(ctx, 1, cxlflash_sync_err_irq, hwq, in init_intr()
1974 rc = cfg->ops->map_afu_irq(ctx, 2, cxlflash_rrq_irq, hwq, in init_intr()
1986 rc = cfg->ops->map_afu_irq(ctx, 3, cxlflash_async_err_irq, hwq, in init_intr()
2008 struct hwq *hwq = get_hwq(cfg->afu, index); in init_mc() local
2013 hwq->afu = cfg->afu; in init_mc()
2014 hwq->index = index; in init_mc()
2015 INIT_LIST_HEAD(&hwq->pending_cmds); in init_mc()
2029 WARN_ON(hwq->ctx_cookie); in init_mc()
2030 hwq->ctx_cookie = ctx; in init_mc()
2031 hwq->num_irqs = num_irqs; in init_mc()
2046 level = init_intr(cfg, hwq); in init_mc()
2053 rc = cfg->ops->start_context(hwq->ctx_cookie); in init_mc()
2068 hwq->ctx_cookie = NULL; in init_mc()
2120 struct hwq *hwq; in init_afu() local
2137 hwq = get_hwq(afu, PRIMARY_HWQ); in init_afu()
2138 afu->afu_map = cfg->ops->psa_map(hwq->ctx_cookie); in init_afu()
2179 hwq = get_hwq(afu, i); in init_afu()
2181 hwq->room = readq_be(&hwq->host_map->cmd_room); in init_afu()
2304 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ); in send_afu_cmd() local
2333 cmd->hwq_index = hwq->index; in send_afu_cmd()
2334 cmd->rcb.ctx_id = hwq->ctx_hndl; in send_afu_cmd()
2348 rc = afu->context_reset(hwq); in send_afu_cmd()
2351 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); in send_afu_cmd()
2353 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); in send_afu_cmd()
2432 struct hwq *hwq = get_hwq(afu, cmd->hwq_index); in cxlflash_eh_abort_handler() local
2451 rc = afu->context_reset(hwq); in cxlflash_eh_abort_handler()
2909 struct hwq *hwq; in irqpoll_weight_store() local
2931 hwq = get_hwq(afu, i); in irqpoll_weight_store()
2933 irq_poll_disable(&hwq->irqpoll); in irqpoll_weight_store()
2941 hwq = get_hwq(afu, i); in irqpoll_weight_store()
2943 irq_poll_init(&hwq->irqpoll, weight, cxlflash_irqpoll); in irqpoll_weight_store()