338 	return &smmu->cmdq;  in arm_smmu_get_cmdq()
417 	__arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq.q);  in arm_smmu_cmdq_skip_err()
432 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)  in arm_smmu_cmdq_shared_lock()  argument
442 	if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)  in arm_smmu_cmdq_shared_lock()
446 		val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);  in arm_smmu_cmdq_shared_lock()
447 	} while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);  in arm_smmu_cmdq_shared_lock()
450 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)  in arm_smmu_cmdq_shared_unlock()  argument
452 	(void)atomic_dec_return_release(&cmdq->lock);  in arm_smmu_cmdq_shared_unlock()
455 static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq)  in arm_smmu_cmdq_shared_tryunlock()  argument
457 	if (atomic_read(&cmdq->lock) == 1)  in arm_smmu_cmdq_shared_tryunlock()
460 	arm_smmu_cmdq_shared_unlock(cmdq);  in arm_smmu_cmdq_shared_tryunlock()
464 #define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)		\  argument
468 	__ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN);	\
474 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags)		\  argument
476 	atomic_set_release(&cmdq->lock, 0);				\
520 static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq,  in __arm_smmu_cmdq_poll_set_valid_map()  argument
525 		.max_n_shift	= cmdq->q.llq.max_n_shift,  in __arm_smmu_cmdq_poll_set_valid_map()
540 		ptr = &cmdq->valid_map[swidx];  in __arm_smmu_cmdq_poll_set_valid_map()
567 static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq,  in arm_smmu_cmdq_set_valid_map()  argument
570 	__arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true);  in arm_smmu_cmdq_set_valid_map()
574 static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq,  in arm_smmu_cmdq_poll_valid_map()  argument
577 	__arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false);  in arm_smmu_cmdq_poll_valid_map()
586 	struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);  in arm_smmu_cmdq_poll_until_not_full()  local
593 	if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) {  in arm_smmu_cmdq_poll_until_not_full()
594 		WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));  in arm_smmu_cmdq_poll_until_not_full()
595 		arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags);  in arm_smmu_cmdq_poll_until_not_full()
596 		llq->val = READ_ONCE(cmdq->q.llq.val);  in arm_smmu_cmdq_poll_until_not_full()
602 		llq->val = READ_ONCE(cmdq->q.llq.val);  in arm_smmu_cmdq_poll_until_not_full()
621 	struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);  in __arm_smmu_cmdq_poll_until_msi()  local
622 	u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));  in __arm_smmu_cmdq_poll_until_msi()
644 	struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);  in __arm_smmu_cmdq_poll_until_consumed()  local
649 	llq->val = READ_ONCE(cmdq->q.llq.val);  in __arm_smmu_cmdq_poll_until_consumed()
684 		llq->cons = readl(cmdq->q.cons_reg);  in __arm_smmu_cmdq_poll_until_consumed()
699 static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,  in arm_smmu_cmdq_write_entries()  argument
704 		.max_n_shift	= cmdq->q.llq.max_n_shift,  in arm_smmu_cmdq_write_entries()
712 		queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);  in arm_smmu_cmdq_write_entries()
739 	struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);  in arm_smmu_cmdq_issue_cmdlist()  local
743 	llq.max_n_shift = cmdq->q.llq.max_n_shift;  in arm_smmu_cmdq_issue_cmdlist()
747 	llq.val = READ_ONCE(cmdq->q.llq.val);  in arm_smmu_cmdq_issue_cmdlist()
762 		old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);  in arm_smmu_cmdq_issue_cmdlist()
776 	arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);  in arm_smmu_cmdq_issue_cmdlist()
779 		arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, &cmdq->q, prod);  in arm_smmu_cmdq_issue_cmdlist()
780 		queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);  in arm_smmu_cmdq_issue_cmdlist()
788 		arm_smmu_cmdq_shared_lock(cmdq);  in arm_smmu_cmdq_issue_cmdlist()
793 	arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);  in arm_smmu_cmdq_issue_cmdlist()
798 		atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);  in arm_smmu_cmdq_issue_cmdlist()
802 						   &cmdq->q.llq.atomic.prod);  in arm_smmu_cmdq_issue_cmdlist()
810 		arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);  in arm_smmu_cmdq_issue_cmdlist()
816 		writel_relaxed(prod, cmdq->q.prod_reg);  in arm_smmu_cmdq_issue_cmdlist()
823 		atomic_set_release(&cmdq->owner_prod, prod);  in arm_smmu_cmdq_issue_cmdlist()
834 					    readl_relaxed(cmdq->q.prod_reg),  in arm_smmu_cmdq_issue_cmdlist()
835 					    readl_relaxed(cmdq->q.cons_reg));  in arm_smmu_cmdq_issue_cmdlist()
842 		if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) {  in arm_smmu_cmdq_issue_cmdlist()
843 			WRITE_ONCE(cmdq->q.llq.cons, llq.cons);  in arm_smmu_cmdq_issue_cmdlist()
844 			arm_smmu_cmdq_shared_unlock(cmdq);  in arm_smmu_cmdq_issue_cmdlist()
2916 	struct arm_smmu_cmdq *cmdq = &smmu->cmdq;  in arm_smmu_cmdq_init()  local
2917 	unsigned int nents = 1 << cmdq->q.llq.max_n_shift;  in arm_smmu_cmdq_init()
2919 	atomic_set(&cmdq->owner_prod, 0);  in arm_smmu_cmdq_init()
2920 	atomic_set(&cmdq->lock, 0);  in arm_smmu_cmdq_init()
2922 	cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents,  in arm_smmu_cmdq_init()
2924 	if (!cmdq->valid_map)  in arm_smmu_cmdq_init()
2935 	ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base,  in arm_smmu_init_queues()
3320 	writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);  in arm_smmu_device_reset()
3321 	writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);  in arm_smmu_device_reset()
3322 	writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);  in arm_smmu_device_reset()
3524 	smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,  in arm_smmu_device_hw_probe()
3526 	if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {  in arm_smmu_device_hw_probe()