225 	struct io_rings	*rings;  member
471 	struct io_rings *rings = ctx->rings;  in __io_commit_cqring()  local
473 	if (ctx->cached_cq_tail != READ_ONCE(rings->cq.tail)) {  in __io_commit_cqring()
475 		smp_store_release(&rings->cq.tail, ctx->cached_cq_tail);  in __io_commit_cqring()
546 	struct io_rings *rings = ctx->rings;  in io_get_cqring()  local
555 	if (tail - READ_ONCE(rings->cq.head) == rings->cq_ring_entries)  in io_get_cqring()
559 	return &rings->cqes[tail & ctx->cq_mask];  in io_get_cqring()
578 		WRITE_ONCE(ctx->rings->cq_overflow,  in io_cqring_fill_event()
738 static unsigned io_cqring_events(struct io_rings *rings)  in io_cqring_events()  argument
742 	return READ_ONCE(rings->cq.tail) - READ_ONCE(rings->cq.head);  in io_cqring_events()
747 	struct io_rings *rings = ctx->rings;  in io_sqring_entries()  local
750 	return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;  in io_sqring_entries()
895 		if (io_cqring_events(ctx->rings))  in __io_iopoll_check()
2544 	struct io_rings *rings = ctx->rings;  in io_commit_sqring()  local
2546 	if (ctx->cached_sq_head != READ_ONCE(rings->sq.head)) {  in io_commit_sqring()
2552 		smp_store_release(&rings->sq.head, ctx->cached_sq_head);  in io_commit_sqring()
2566 	struct io_rings *rings = ctx->rings;  in io_get_sqring()  local
2580 	if (head == smp_load_acquire(&rings->sq.tail))  in io_get_sqring()
2595 	WRITE_ONCE(rings->sq_dropped, ctx->cached_sq_dropped);  in io_get_sqring()
2740 			ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;  in io_sq_thread()
2755 				ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;  in io_sq_thread()
2760 			ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;  in io_sq_thread()
2866 	return io_cqring_events(ctx->rings) >= iowq->to_wait ||  in io_should_wake()
2898 	struct io_rings *rings = ctx->rings;  in io_cqring_wait()  local
2901 	if (io_cqring_events(rings) >= min_events)  in io_cqring_wait()
2936 	return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;  in io_cqring_wait()
3282 	struct io_rings *rings;  in rings_size()  local
3285 	off = struct_size(rings, cqes, cq_entries);  in rings_size()
3562 	io_mem_free(ctx->rings);  in io_ring_ctx_free()
3584 	if (READ_ONCE(ctx->rings->sq.tail) - ctx->cached_sq_head !=  in io_uring_poll()
3585 	    ctx->rings->sq_ring_entries)  in io_uring_poll()
3587 	if (READ_ONCE(ctx->rings->cq.head) != ctx->cached_cq_tail)  in io_uring_poll()
3634 		ptr = ctx->rings;  in io_uring_mmap()
3721 	struct io_rings *rings;  in io_allocate_scq_urings()  local
3728 	rings = io_mem_alloc(size);  in io_allocate_scq_urings()
3729 	if (!rings)  in io_allocate_scq_urings()
3732 	ctx->rings = rings;  in io_allocate_scq_urings()
3733 	ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);  in io_allocate_scq_urings()
3734 	rings->sq_ring_mask = p->sq_entries - 1;  in io_allocate_scq_urings()
3735 	rings->cq_ring_mask = p->cq_entries - 1;  in io_allocate_scq_urings()
3736 	rings->sq_ring_entries = p->sq_entries;  in io_allocate_scq_urings()
3737 	rings->cq_ring_entries = p->cq_entries;  in io_allocate_scq_urings()
3738 	ctx->sq_mask = rings->sq_ring_mask;  in io_allocate_scq_urings()
3739 	ctx->cq_mask = rings->cq_ring_mask;  in io_allocate_scq_urings()
3740 	ctx->sq_entries = rings->sq_ring_entries;  in io_allocate_scq_urings()
3741 	ctx->cq_entries = rings->cq_ring_entries;  in io_allocate_scq_urings()
3856 	p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;  in io_uring_create()