200 	return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);  in __io_cqring_events()
205 	return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head);  in __io_cqring_events_user()
365 	struct io_rings *r = ctx->rings;  in io_account_cq_overflow()
584 	if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)  in io_eventfd_signal()
719 		atomic_andnot(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);  in __io_cqring_overflow_flush()
813 		atomic_or(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);  in io_cqring_event_overflow()
842 	struct io_rings *rings = ctx->rings;  in io_cqe_cache_refill()  local
867 	ctx->cqe_cached = &rings->cqes[off];  in io_cqe_cache_refill()
1170 		atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);  in ctx_flush_and_put()
1348 			atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);  in io_req_local_work_add()
1375 		atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);  in io_req_normal_work_add()
1417 		atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);  in __io_run_local_work()
2357 	struct io_rings *rings = ctx->rings;  in io_commit_sqring()  local
2364 	smp_store_release(&rings->sq.head, ctx->cached_sq_head);  in io_commit_sqring()
2387 			WRITE_ONCE(ctx->rings->sq_dropped,  in io_get_sqe()
2388 				   READ_ONCE(ctx->rings->sq_dropped) + 1);  in io_get_sqe()
2476 	int dist = READ_ONCE(ctx->rings->cq.tail) - (int) iowq->cq_tail;  in io_should_wake()
2566 	struct io_rings *rings = ctx->rings;  in io_cqring_wait()  local
2597 	iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;  in io_cqring_wait()
2613 			int nr_wait = (int) iowq.cq_tail - READ_ONCE(ctx->rings->cq.tail);  in io_cqring_wait()
2659 	return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;  in io_cqring_wait()
2759 		io_mem_free(ctx->rings);  in io_rings_free()
2761 		ctx->rings = NULL;  in io_rings_free()
2785 	struct io_rings *rings;  in rings_size()  local
2788 	off = struct_size(rings, cqes, cq_entries);  in rings_size()
3183 	if (ctx->rings)  in io_ring_ctx_wait_and_kill()
3191 	if (ctx->rings)  in io_ring_ctx_wait_and_kill()
3295 	if (!ctx->rings)  in io_uring_try_cancel_requests()
3443 		ptr = ctx->rings;  in io_uring_validate_mmap_request()
3758 	struct io_rings *rings;  in io_allocate_scq_urings()  local
3771 		rings = io_mem_alloc(size);  in io_allocate_scq_urings()
3773 		rings = io_rings_map(ctx, p->cq_off.user_addr, size);  in io_allocate_scq_urings()
3775 	if (IS_ERR(rings))  in io_allocate_scq_urings()
3776 		return PTR_ERR(rings);  in io_allocate_scq_urings()
3778 	ctx->rings = rings;  in io_allocate_scq_urings()
3780 		ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);  in io_allocate_scq_urings()
3781 	rings->sq_ring_mask = p->sq_entries - 1;  in io_allocate_scq_urings()
3782 	rings->cq_ring_mask = p->cq_entries - 1;  in io_allocate_scq_urings()
3783 	rings->sq_ring_entries = p->sq_entries;  in io_allocate_scq_urings()
3784 	rings->cq_ring_entries = p->cq_entries;  in io_allocate_scq_urings()
3992 		p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;  in io_uring_create()