1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6 #include <linux/vmalloc.h>
7 #include "rxe.h"
8 #include "rxe_loc.h"
9 #include "rxe_queue.h"
10
rxe_cq_chk_attr(struct rxe_dev * rxe,struct rxe_cq * cq,int cqe,int comp_vector)11 int rxe_cq_chk_attr(struct rxe_dev *rxe, struct rxe_cq *cq,
12 int cqe, int comp_vector)
13 {
14 int count;
15
16 if (cqe <= 0) {
17 pr_warn("cqe(%d) <= 0\n", cqe);
18 goto err1;
19 }
20
21 if (cqe > rxe->attr.max_cqe) {
22 pr_warn("cqe(%d) > max_cqe(%d)\n",
23 cqe, rxe->attr.max_cqe);
24 goto err1;
25 }
26
27 if (cq) {
28 if (cq->is_user)
29 count = queue_count(cq->queue, QUEUE_TYPE_TO_USER);
30 else
31 count = queue_count(cq->queue, QUEUE_TYPE_KERNEL);
32
33 if (cqe < count) {
34 pr_warn("cqe(%d) < current # elements in queue (%d)",
35 cqe, count);
36 goto err1;
37 }
38 }
39
40 return 0;
41
42 err1:
43 return -EINVAL;
44 }
45
rxe_send_complete(struct tasklet_struct * t)46 static void rxe_send_complete(struct tasklet_struct *t)
47 {
48 struct rxe_cq *cq = from_tasklet(cq, t, comp_task);
49 unsigned long flags;
50
51 spin_lock_irqsave(&cq->cq_lock, flags);
52 if (cq->is_dying) {
53 spin_unlock_irqrestore(&cq->cq_lock, flags);
54 return;
55 }
56 spin_unlock_irqrestore(&cq->cq_lock, flags);
57
58 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
59 }
60
rxe_cq_from_init(struct rxe_dev * rxe,struct rxe_cq * cq,int cqe,int comp_vector,struct ib_udata * udata,struct rxe_create_cq_resp __user * uresp)61 int rxe_cq_from_init(struct rxe_dev *rxe, struct rxe_cq *cq, int cqe,
62 int comp_vector, struct ib_udata *udata,
63 struct rxe_create_cq_resp __user *uresp)
64 {
65 int err;
66 enum queue_type type;
67
68 type = uresp ? QUEUE_TYPE_TO_USER : QUEUE_TYPE_KERNEL;
69 cq->queue = rxe_queue_init(rxe, &cqe,
70 sizeof(struct rxe_cqe), type);
71 if (!cq->queue) {
72 pr_warn("unable to create cq\n");
73 return -ENOMEM;
74 }
75
76 err = do_mmap_info(rxe, uresp ? &uresp->mi : NULL, udata,
77 cq->queue->buf, cq->queue->buf_size, &cq->queue->ip);
78 if (err) {
79 vfree(cq->queue->buf);
80 kfree(cq->queue);
81 return err;
82 }
83
84 if (uresp)
85 cq->is_user = 1;
86
87 cq->is_dying = false;
88
89 tasklet_setup(&cq->comp_task, rxe_send_complete);
90
91 spin_lock_init(&cq->cq_lock);
92 cq->ibcq.cqe = cqe;
93 return 0;
94 }
95
rxe_cq_resize_queue(struct rxe_cq * cq,int cqe,struct rxe_resize_cq_resp __user * uresp,struct ib_udata * udata)96 int rxe_cq_resize_queue(struct rxe_cq *cq, int cqe,
97 struct rxe_resize_cq_resp __user *uresp,
98 struct ib_udata *udata)
99 {
100 int err;
101
102 err = rxe_queue_resize(cq->queue, (unsigned int *)&cqe,
103 sizeof(struct rxe_cqe), udata,
104 uresp ? &uresp->mi : NULL, NULL, &cq->cq_lock);
105 if (!err)
106 cq->ibcq.cqe = cqe;
107
108 return err;
109 }
110
rxe_cq_post(struct rxe_cq * cq,struct rxe_cqe * cqe,int solicited)111 int rxe_cq_post(struct rxe_cq *cq, struct rxe_cqe *cqe, int solicited)
112 {
113 struct ib_event ev;
114 unsigned long flags;
115 int full;
116 void *addr;
117
118 spin_lock_irqsave(&cq->cq_lock, flags);
119
120 if (cq->is_user)
121 full = queue_full(cq->queue, QUEUE_TYPE_TO_USER);
122 else
123 full = queue_full(cq->queue, QUEUE_TYPE_KERNEL);
124
125 if (unlikely(full)) {
126 spin_unlock_irqrestore(&cq->cq_lock, flags);
127 if (cq->ibcq.event_handler) {
128 ev.device = cq->ibcq.device;
129 ev.element.cq = &cq->ibcq;
130 ev.event = IB_EVENT_CQ_ERR;
131 cq->ibcq.event_handler(&ev, cq->ibcq.cq_context);
132 }
133
134 return -EBUSY;
135 }
136
137 if (cq->is_user)
138 addr = producer_addr(cq->queue, QUEUE_TYPE_TO_USER);
139 else
140 addr = producer_addr(cq->queue, QUEUE_TYPE_KERNEL);
141
142 memcpy(addr, cqe, sizeof(*cqe));
143
144 if (cq->is_user)
145 advance_producer(cq->queue, QUEUE_TYPE_TO_USER);
146 else
147 advance_producer(cq->queue, QUEUE_TYPE_KERNEL);
148
149 spin_unlock_irqrestore(&cq->cq_lock, flags);
150
151 if ((cq->notify == IB_CQ_NEXT_COMP) ||
152 (cq->notify == IB_CQ_SOLICITED && solicited)) {
153 cq->notify = 0;
154 tasklet_schedule(&cq->comp_task);
155 }
156
157 return 0;
158 }
159
rxe_cq_disable(struct rxe_cq * cq)160 void rxe_cq_disable(struct rxe_cq *cq)
161 {
162 unsigned long flags;
163
164 spin_lock_irqsave(&cq->cq_lock, flags);
165 cq->is_dying = true;
166 spin_unlock_irqrestore(&cq->cq_lock, flags);
167 }
168
rxe_cq_cleanup(struct rxe_pool_entry * arg)169 void rxe_cq_cleanup(struct rxe_pool_entry *arg)
170 {
171 struct rxe_cq *cq = container_of(arg, typeof(*cq), pelem);
172
173 if (cq->queue)
174 rxe_queue_cleanup(cq->queue);
175 }
176
