22  *   - Kernel space indices are always masked off to q->index_mask
29  *   - By passing the type in the parameter list separate from q
33  *   paths just q->type is passed.
64 void rxe_queue_reset(struct rxe_queue *q);
69 int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,
79 static inline int next_index(struct rxe_queue *q, int index)  in next_index()  argument
81 	return (index + 1) & q->buf->index_mask;  in next_index()
84 static inline int queue_empty(struct rxe_queue *q, enum queue_type type)  in queue_empty()  argument
92 		prod = smp_load_acquire(&q->buf->producer_index);  in queue_empty()
93 		cons = q->index;  in queue_empty()
96 		prod = q->index;  in queue_empty()
98 		cons = smp_load_acquire(&q->buf->consumer_index);  in queue_empty()
101 		prod = q->buf->producer_index;  in queue_empty()
102 		cons = q->buf->consumer_index;  in queue_empty()
106 	return ((prod - cons) & q->index_mask) == 0;  in queue_empty()
109 static inline int queue_full(struct rxe_queue *q, enum queue_type type)  in queue_full()  argument
117 		prod = smp_load_acquire(&q->buf->producer_index);  in queue_full()
118 		cons = q->index;  in queue_full()
121 		prod = q->index;  in queue_full()
123 		cons = smp_load_acquire(&q->buf->consumer_index);  in queue_full()
126 		prod = q->buf->producer_index;  in queue_full()
127 		cons = q->buf->consumer_index;  in queue_full()
131 	return ((prod + 1 - cons) & q->index_mask) == 0;  in queue_full()
134 static inline unsigned int queue_count(const struct rxe_queue *q,  in queue_count()  argument
143 		prod = smp_load_acquire(&q->buf->producer_index);  in queue_count()
144 		cons = q->index;  in queue_count()
147 		prod = q->index;  in queue_count()
149 		cons = smp_load_acquire(&q->buf->consumer_index);  in queue_count()
152 		prod = q->buf->producer_index;  in queue_count()
153 		cons = q->buf->consumer_index;  in queue_count()
157 	return (prod - cons) & q->index_mask;  in queue_count()
160 static inline void advance_producer(struct rxe_queue *q, enum queue_type type)  in advance_producer()  argument
168 		prod = smp_load_acquire(&q->buf->producer_index);  in advance_producer()
169 		prod = (prod + 1) & q->index_mask;  in advance_producer()
171 		smp_store_release(&q->buf->producer_index, prod);  in advance_producer()
174 		prod = q->index;  in advance_producer()
175 		q->index = (prod + 1) & q->index_mask;  in advance_producer()
176 		q->buf->producer_index = q->index;  in advance_producer()
179 		prod = q->buf->producer_index;  in advance_producer()
180 		q->buf->producer_index = (prod + 1) & q->index_mask;  in advance_producer()
185 static inline void advance_consumer(struct rxe_queue *q, enum queue_type type)  in advance_consumer()  argument
191 		cons = q->index;  in advance_consumer()
192 		q->index = (cons + 1) & q->index_mask;  in advance_consumer()
193 		q->buf->consumer_index = q->index;  in advance_consumer()
198 		cons = smp_load_acquire(&q->buf->consumer_index);  in advance_consumer()
199 		cons = (cons + 1) & q->index_mask;  in advance_consumer()
201 		smp_store_release(&q->buf->consumer_index, cons);  in advance_consumer()
204 		cons = q->buf->consumer_index;  in advance_consumer()
205 		q->buf->consumer_index = (cons + 1) & q->index_mask;  in advance_consumer()
210 static inline void *producer_addr(struct rxe_queue *q, enum queue_type type)  in producer_addr()  argument
217 		prod = smp_load_acquire(&q->buf->producer_index);  in producer_addr()
218 		prod &= q->index_mask;  in producer_addr()
221 		prod = q->index;  in producer_addr()
224 		prod = q->buf->producer_index;  in producer_addr()
228 	return q->buf->data + (prod << q->log2_elem_size);  in producer_addr()
231 static inline void *consumer_addr(struct rxe_queue *q, enum queue_type type)  in consumer_addr()  argument
237 		cons = q->index;  in consumer_addr()
241 		cons = smp_load_acquire(&q->buf->consumer_index);  in consumer_addr()
242 		cons &= q->index_mask;  in consumer_addr()
245 		cons = q->buf->consumer_index;  in consumer_addr()
249 	return q->buf->data + (cons << q->log2_elem_size);  in consumer_addr()
252 static inline unsigned int producer_index(struct rxe_queue *q,  in producer_index()  argument
260 		prod = smp_load_acquire(&q->buf->producer_index);  in producer_index()
261 		prod &= q->index_mask;  in producer_index()
264 		prod = q->index;  in producer_index()
267 		prod = q->buf->producer_index;  in producer_index()
274 static inline unsigned int consumer_index(struct rxe_queue *q,  in consumer_index()  argument
281 		cons = q->index;  in consumer_index()
285 		cons = smp_load_acquire(&q->buf->consumer_index);  in consumer_index()
286 		cons &= q->index_mask;  in consumer_index()
289 		cons = q->buf->consumer_index;  in consumer_index()
296 static inline void *addr_from_index(struct rxe_queue *q,  in addr_from_index()  argument
299 	return q->buf->data + ((index & q->index_mask)  in addr_from_index()
300 				<< q->buf->log2_elem_size);  in addr_from_index()
303 static inline unsigned int index_from_addr(const struct rxe_queue *q,  in index_from_addr()  argument
306 	return (((u8 *)addr - q->buf->data) >> q->log2_elem_size)  in index_from_addr()
307 				& q->index_mask;  in index_from_addr()
310 static inline void *queue_head(struct rxe_queue *q, enum queue_type type)  in queue_head()  argument
312 	return queue_empty(q, type) ? NULL : consumer_addr(q, type);  in queue_head()