1 /*
2 * Copyright (c) 2013-2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/highmem.h>
34 #include <linux/errno.h>
35 #include <linux/pci.h>
36 #include <linux/dma-mapping.h>
37 #include <linux/slab.h>
38 #include <linux/delay.h>
39 #include <linux/random.h>
40 #include <linux/io-mapping.h>
41 #include <linux/mlx5/driver.h>
42 #include <linux/mlx5/eq.h>
43 #include <linux/debugfs.h>
44
45 #include "mlx5_core.h"
46 #include "lib/eq.h"
47 #include "lib/tout.h"
48 #define CREATE_TRACE_POINTS
49 #include "diag/cmd_tracepoint.h"
50
51 enum {
52 CMD_IF_REV = 5,
53 };
54
55 enum {
56 CMD_MODE_POLLING,
57 CMD_MODE_EVENTS
58 };
59
60 enum {
61 MLX5_CMD_DELIVERY_STAT_OK = 0x0,
62 MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR = 0x1,
63 MLX5_CMD_DELIVERY_STAT_TOK_ERR = 0x2,
64 MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR = 0x3,
65 MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR = 0x4,
66 MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR = 0x5,
67 MLX5_CMD_DELIVERY_STAT_FW_ERR = 0x6,
68 MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR = 0x7,
69 MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR = 0x8,
70 MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR = 0x9,
71 MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR = 0x10,
72 };
73
74 static struct mlx5_cmd_work_ent *
cmd_alloc_ent(struct mlx5_cmd * cmd,struct mlx5_cmd_msg * in,struct mlx5_cmd_msg * out,void * uout,int uout_size,mlx5_cmd_cbk_t cbk,void * context,int page_queue)75 cmd_alloc_ent(struct mlx5_cmd *cmd, struct mlx5_cmd_msg *in,
76 struct mlx5_cmd_msg *out, void *uout, int uout_size,
77 mlx5_cmd_cbk_t cbk, void *context, int page_queue)
78 {
79 gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL;
80 struct mlx5_cmd_work_ent *ent;
81
82 ent = kzalloc(sizeof(*ent), alloc_flags);
83 if (!ent)
84 return ERR_PTR(-ENOMEM);
85
86 ent->idx = -EINVAL;
87 ent->in = in;
88 ent->out = out;
89 ent->uout = uout;
90 ent->uout_size = uout_size;
91 ent->callback = cbk;
92 ent->context = context;
93 ent->cmd = cmd;
94 ent->page_queue = page_queue;
95 refcount_set(&ent->refcnt, 1);
96
97 return ent;
98 }
99
cmd_free_ent(struct mlx5_cmd_work_ent * ent)100 static void cmd_free_ent(struct mlx5_cmd_work_ent *ent)
101 {
102 kfree(ent);
103 }
104
alloc_token(struct mlx5_cmd * cmd)105 static u8 alloc_token(struct mlx5_cmd *cmd)
106 {
107 u8 token;
108
109 spin_lock(&cmd->token_lock);
110 cmd->token++;
111 if (cmd->token == 0)
112 cmd->token++;
113 token = cmd->token;
114 spin_unlock(&cmd->token_lock);
115
116 return token;
117 }
118
cmd_alloc_index(struct mlx5_cmd * cmd)119 static int cmd_alloc_index(struct mlx5_cmd *cmd)
120 {
121 unsigned long flags;
122 int ret;
123
124 spin_lock_irqsave(&cmd->alloc_lock, flags);
125 ret = find_first_bit(&cmd->bitmask, cmd->max_reg_cmds);
126 if (ret < cmd->max_reg_cmds)
127 clear_bit(ret, &cmd->bitmask);
128 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
129
130 return ret < cmd->max_reg_cmds ? ret : -ENOMEM;
131 }
132
cmd_free_index(struct mlx5_cmd * cmd,int idx)133 static void cmd_free_index(struct mlx5_cmd *cmd, int idx)
134 {
135 lockdep_assert_held(&cmd->alloc_lock);
136 set_bit(idx, &cmd->bitmask);
137 }
138
cmd_ent_get(struct mlx5_cmd_work_ent * ent)139 static void cmd_ent_get(struct mlx5_cmd_work_ent *ent)
140 {
141 refcount_inc(&ent->refcnt);
142 }
143
cmd_ent_put(struct mlx5_cmd_work_ent * ent)144 static void cmd_ent_put(struct mlx5_cmd_work_ent *ent)
145 {
146 struct mlx5_cmd *cmd = ent->cmd;
147 unsigned long flags;
148
149 spin_lock_irqsave(&cmd->alloc_lock, flags);
150 if (!refcount_dec_and_test(&ent->refcnt))
151 goto out;
152
153 if (ent->idx >= 0) {
154 cmd_free_index(cmd, ent->idx);
155 up(ent->page_queue ? &cmd->pages_sem : &cmd->sem);
156 }
157
158 cmd_free_ent(ent);
159 out:
160 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
161 }
162
get_inst(struct mlx5_cmd * cmd,int idx)163 static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx)
164 {
165 return cmd->cmd_buf + (idx << cmd->log_stride);
166 }
167
mlx5_calc_cmd_blocks(struct mlx5_cmd_msg * msg)168 static int mlx5_calc_cmd_blocks(struct mlx5_cmd_msg *msg)
169 {
170 int size = msg->len;
171 int blen = size - min_t(int, sizeof(msg->first.data), size);
172
173 return DIV_ROUND_UP(blen, MLX5_CMD_DATA_BLOCK_SIZE);
174 }
175
xor8_buf(void * buf,size_t offset,int len)176 static u8 xor8_buf(void *buf, size_t offset, int len)
177 {
178 u8 *ptr = buf;
179 u8 sum = 0;
180 int i;
181 int end = len + offset;
182
183 for (i = offset; i < end; i++)
184 sum ^= ptr[i];
185
186 return sum;
187 }
188
verify_block_sig(struct mlx5_cmd_prot_block * block)189 static int verify_block_sig(struct mlx5_cmd_prot_block *block)
190 {
191 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);
192 int xor_len = sizeof(*block) - sizeof(block->data) - 1;
193
194 if (xor8_buf(block, rsvd0_off, xor_len) != 0xff)
195 return -EHWPOISON;
196
197 if (xor8_buf(block, 0, sizeof(*block)) != 0xff)
198 return -EHWPOISON;
199
200 return 0;
201 }
202
calc_block_sig(struct mlx5_cmd_prot_block * block)203 static void calc_block_sig(struct mlx5_cmd_prot_block *block)
204 {
205 int ctrl_xor_len = sizeof(*block) - sizeof(block->data) - 2;
206 size_t rsvd0_off = offsetof(struct mlx5_cmd_prot_block, rsvd0);
207
208 block->ctrl_sig = ~xor8_buf(block, rsvd0_off, ctrl_xor_len);
209 block->sig = ~xor8_buf(block, 0, sizeof(*block) - 1);
210 }
211
calc_chain_sig(struct mlx5_cmd_msg * msg)212 static void calc_chain_sig(struct mlx5_cmd_msg *msg)
213 {
214 struct mlx5_cmd_mailbox *next = msg->next;
215 int n = mlx5_calc_cmd_blocks(msg);
216 int i = 0;
217
218 for (i = 0; i < n && next; i++) {
219 calc_block_sig(next->buf);
220 next = next->next;
221 }
222 }
223
set_signature(struct mlx5_cmd_work_ent * ent,int csum)224 static void set_signature(struct mlx5_cmd_work_ent *ent, int csum)
225 {
226 ent->lay->sig = ~xor8_buf(ent->lay, 0, sizeof(*ent->lay));
227 if (csum) {
228 calc_chain_sig(ent->in);
229 calc_chain_sig(ent->out);
230 }
231 }
232
poll_timeout(struct mlx5_cmd_work_ent * ent)233 static void poll_timeout(struct mlx5_cmd_work_ent *ent)
234 {
235 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev, cmd);
236 u64 cmd_to_ms = mlx5_tout_ms(dev, CMD);
237 unsigned long poll_end;
238 u8 own;
239
240 poll_end = jiffies + msecs_to_jiffies(cmd_to_ms + 1000);
241
242 do {
243 own = READ_ONCE(ent->lay->status_own);
244 if (!(own & CMD_OWNER_HW)) {
245 ent->ret = 0;
246 return;
247 }
248 cond_resched();
249 } while (time_before(jiffies, poll_end));
250
251 ent->ret = -ETIMEDOUT;
252 }
253
verify_signature(struct mlx5_cmd_work_ent * ent)254 static int verify_signature(struct mlx5_cmd_work_ent *ent)
255 {
256 struct mlx5_cmd_mailbox *next = ent->out->next;
257 int n = mlx5_calc_cmd_blocks(ent->out);
258 int err;
259 u8 sig;
260 int i = 0;
261
262 sig = xor8_buf(ent->lay, 0, sizeof(*ent->lay));
263 if (sig != 0xff)
264 return -EHWPOISON;
265
266 for (i = 0; i < n && next; i++) {
267 err = verify_block_sig(next->buf);
268 if (err)
269 return -EHWPOISON;
270
271 next = next->next;
272 }
273
274 return 0;
275 }
276
dump_buf(void * buf,int size,int data_only,int offset,int idx)277 static void dump_buf(void *buf, int size, int data_only, int offset, int idx)
278 {
279 __be32 *p = buf;
280 int i;
281
282 for (i = 0; i < size; i += 16) {
283 pr_debug("cmd[%d]: %03x: %08x %08x %08x %08x\n", idx, offset,
284 be32_to_cpu(p[0]), be32_to_cpu(p[1]),
285 be32_to_cpu(p[2]), be32_to_cpu(p[3]));
286 p += 4;
287 offset += 16;
288 }
289 if (!data_only)
290 pr_debug("\n");
291 }
292
mlx5_internal_err_ret_value(struct mlx5_core_dev * dev,u16 op,u32 * synd,u8 * status)293 static int mlx5_internal_err_ret_value(struct mlx5_core_dev *dev, u16 op,
294 u32 *synd, u8 *status)
295 {
296 *synd = 0;
297 *status = 0;
298
299 switch (op) {
300 case MLX5_CMD_OP_TEARDOWN_HCA:
301 case MLX5_CMD_OP_DISABLE_HCA:
302 case MLX5_CMD_OP_MANAGE_PAGES:
303 case MLX5_CMD_OP_DESTROY_MKEY:
304 case MLX5_CMD_OP_DESTROY_EQ:
305 case MLX5_CMD_OP_DESTROY_CQ:
306 case MLX5_CMD_OP_DESTROY_QP:
307 case MLX5_CMD_OP_DESTROY_PSV:
308 case MLX5_CMD_OP_DESTROY_SRQ:
309 case MLX5_CMD_OP_DESTROY_XRC_SRQ:
310 case MLX5_CMD_OP_DESTROY_XRQ:
311 case MLX5_CMD_OP_DESTROY_DCT:
312 case MLX5_CMD_OP_DEALLOC_Q_COUNTER:
313 case MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT:
314 case MLX5_CMD_OP_DESTROY_QOS_PARA_VPORT:
315 case MLX5_CMD_OP_DEALLOC_PD:
316 case MLX5_CMD_OP_DEALLOC_UAR:
317 case MLX5_CMD_OP_DETACH_FROM_MCG:
318 case MLX5_CMD_OP_DEALLOC_XRCD:
319 case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN:
320 case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT:
321 case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY:
322 case MLX5_CMD_OP_DESTROY_LAG:
323 case MLX5_CMD_OP_DESTROY_VPORT_LAG:
324 case MLX5_CMD_OP_DESTROY_TIR:
325 case MLX5_CMD_OP_DESTROY_SQ:
326 case MLX5_CMD_OP_DESTROY_RQ:
327 case MLX5_CMD_OP_DESTROY_RMP:
328 case MLX5_CMD_OP_DESTROY_TIS:
329 case MLX5_CMD_OP_DESTROY_RQT:
330 case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
331 case MLX5_CMD_OP_DESTROY_FLOW_GROUP:
332 case MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY:
333 case MLX5_CMD_OP_DEALLOC_FLOW_COUNTER:
334 case MLX5_CMD_OP_2ERR_QP:
335 case MLX5_CMD_OP_2RST_QP:
336 case MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT:
337 case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
338 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
339 case MLX5_CMD_OP_SET_FLOW_TABLE_ROOT:
340 case MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT:
341 case MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT:
342 case MLX5_CMD_OP_FPGA_DESTROY_QP:
343 case MLX5_CMD_OP_DESTROY_GENERAL_OBJECT:
344 case MLX5_CMD_OP_DEALLOC_MEMIC:
345 case MLX5_CMD_OP_PAGE_FAULT_RESUME:
346 case MLX5_CMD_OP_QUERY_ESW_FUNCTIONS:
347 case MLX5_CMD_OP_DEALLOC_SF:
348 case MLX5_CMD_OP_DESTROY_UCTX:
349 case MLX5_CMD_OP_DESTROY_UMEM:
350 case MLX5_CMD_OP_MODIFY_RQT:
351 return MLX5_CMD_STAT_OK;
352
353 case MLX5_CMD_OP_QUERY_HCA_CAP:
354 case MLX5_CMD_OP_QUERY_ADAPTER:
355 case MLX5_CMD_OP_INIT_HCA:
356 case MLX5_CMD_OP_ENABLE_HCA:
357 case MLX5_CMD_OP_QUERY_PAGES:
358 case MLX5_CMD_OP_SET_HCA_CAP:
359 case MLX5_CMD_OP_QUERY_ISSI:
360 case MLX5_CMD_OP_SET_ISSI:
361 case MLX5_CMD_OP_CREATE_MKEY:
362 case MLX5_CMD_OP_QUERY_MKEY:
363 case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS:
364 case MLX5_CMD_OP_CREATE_EQ:
365 case MLX5_CMD_OP_QUERY_EQ:
366 case MLX5_CMD_OP_GEN_EQE:
367 case MLX5_CMD_OP_CREATE_CQ:
368 case MLX5_CMD_OP_QUERY_CQ:
369 case MLX5_CMD_OP_MODIFY_CQ:
370 case MLX5_CMD_OP_CREATE_QP:
371 case MLX5_CMD_OP_RST2INIT_QP:
372 case MLX5_CMD_OP_INIT2RTR_QP:
373 case MLX5_CMD_OP_RTR2RTS_QP:
374 case MLX5_CMD_OP_RTS2RTS_QP:
375 case MLX5_CMD_OP_SQERR2RTS_QP:
376 case MLX5_CMD_OP_QUERY_QP:
377 case MLX5_CMD_OP_SQD_RTS_QP:
378 case MLX5_CMD_OP_INIT2INIT_QP:
379 case MLX5_CMD_OP_CREATE_PSV:
380 case MLX5_CMD_OP_CREATE_SRQ:
381 case MLX5_CMD_OP_QUERY_SRQ:
382 case MLX5_CMD_OP_ARM_RQ:
383 case MLX5_CMD_OP_CREATE_XRC_SRQ:
384 case MLX5_CMD_OP_QUERY_XRC_SRQ:
385 case MLX5_CMD_OP_ARM_XRC_SRQ:
386 case MLX5_CMD_OP_CREATE_XRQ:
387 case MLX5_CMD_OP_QUERY_XRQ:
388 case MLX5_CMD_OP_ARM_XRQ:
389 case MLX5_CMD_OP_CREATE_DCT:
390 case MLX5_CMD_OP_DRAIN_DCT:
391 case MLX5_CMD_OP_QUERY_DCT:
392 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
393 case MLX5_CMD_OP_QUERY_VPORT_STATE:
394 case MLX5_CMD_OP_MODIFY_VPORT_STATE:
395 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
396 case MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT:
397 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
398 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
399 case MLX5_CMD_OP_SET_ROCE_ADDRESS:
400 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
401 case MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT:
402 case MLX5_CMD_OP_QUERY_HCA_VPORT_GID:
403 case MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY:
404 case MLX5_CMD_OP_QUERY_VNIC_ENV:
405 case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
406 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
407 case MLX5_CMD_OP_QUERY_Q_COUNTER:
408 case MLX5_CMD_OP_SET_MONITOR_COUNTER:
409 case MLX5_CMD_OP_ARM_MONITOR_COUNTER:
410 case MLX5_CMD_OP_SET_PP_RATE_LIMIT:
411 case MLX5_CMD_OP_QUERY_RATE_LIMIT:
412 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
413 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
414 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
415 case MLX5_CMD_OP_CREATE_QOS_PARA_VPORT:
416 case MLX5_CMD_OP_ALLOC_PD:
417 case MLX5_CMD_OP_ALLOC_UAR:
418 case MLX5_CMD_OP_CONFIG_INT_MODERATION:
419 case MLX5_CMD_OP_ACCESS_REG:
420 case MLX5_CMD_OP_ATTACH_TO_MCG:
421 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
422 case MLX5_CMD_OP_MAD_IFC:
423 case MLX5_CMD_OP_QUERY_MAD_DEMUX:
424 case MLX5_CMD_OP_SET_MAD_DEMUX:
425 case MLX5_CMD_OP_NOP:
426 case MLX5_CMD_OP_ALLOC_XRCD:
427 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
428 case MLX5_CMD_OP_QUERY_CONG_STATUS:
429 case MLX5_CMD_OP_MODIFY_CONG_STATUS:
430 case MLX5_CMD_OP_QUERY_CONG_PARAMS:
431 case MLX5_CMD_OP_MODIFY_CONG_PARAMS:
432 case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
433 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
434 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
435 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
436 case MLX5_CMD_OP_CREATE_LAG:
437 case MLX5_CMD_OP_MODIFY_LAG:
438 case MLX5_CMD_OP_QUERY_LAG:
439 case MLX5_CMD_OP_CREATE_VPORT_LAG:
440 case MLX5_CMD_OP_CREATE_TIR:
441 case MLX5_CMD_OP_MODIFY_TIR:
442 case MLX5_CMD_OP_QUERY_TIR:
443 case MLX5_CMD_OP_CREATE_SQ:
444 case MLX5_CMD_OP_MODIFY_SQ:
445 case MLX5_CMD_OP_QUERY_SQ:
446 case MLX5_CMD_OP_CREATE_RQ:
447 case MLX5_CMD_OP_MODIFY_RQ:
448 case MLX5_CMD_OP_QUERY_RQ:
449 case MLX5_CMD_OP_CREATE_RMP:
450 case MLX5_CMD_OP_MODIFY_RMP:
451 case MLX5_CMD_OP_QUERY_RMP:
452 case MLX5_CMD_OP_CREATE_TIS:
453 case MLX5_CMD_OP_MODIFY_TIS:
454 case MLX5_CMD_OP_QUERY_TIS:
455 case MLX5_CMD_OP_CREATE_RQT:
456 case MLX5_CMD_OP_QUERY_RQT:
457
458 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
459 case MLX5_CMD_OP_QUERY_FLOW_TABLE:
460 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
461 case MLX5_CMD_OP_QUERY_FLOW_GROUP:
462 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
463 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
464 case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
465 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
466 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
467 case MLX5_CMD_OP_FPGA_CREATE_QP:
468 case MLX5_CMD_OP_FPGA_MODIFY_QP:
469 case MLX5_CMD_OP_FPGA_QUERY_QP:
470 case MLX5_CMD_OP_FPGA_QUERY_QP_COUNTERS:
471 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
472 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
473 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
474 case MLX5_CMD_OP_CREATE_UCTX:
475 case MLX5_CMD_OP_CREATE_UMEM:
476 case MLX5_CMD_OP_ALLOC_MEMIC:
477 case MLX5_CMD_OP_MODIFY_XRQ:
478 case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
479 case MLX5_CMD_OP_QUERY_VHCA_STATE:
480 case MLX5_CMD_OP_MODIFY_VHCA_STATE:
481 case MLX5_CMD_OP_ALLOC_SF:
482 case MLX5_CMD_OP_SUSPEND_VHCA:
483 case MLX5_CMD_OP_RESUME_VHCA:
484 case MLX5_CMD_OP_QUERY_VHCA_MIGRATION_STATE:
485 case MLX5_CMD_OP_SAVE_VHCA_STATE:
486 case MLX5_CMD_OP_LOAD_VHCA_STATE:
487 *status = MLX5_DRIVER_STATUS_ABORTED;
488 *synd = MLX5_DRIVER_SYND;
489 return -ENOLINK;
490 default:
491 mlx5_core_err(dev, "Unknown FW command (%d)\n", op);
492 return -EINVAL;
493 }
494 }
495
mlx5_command_str(int command)496 const char *mlx5_command_str(int command)
497 {
498 #define MLX5_COMMAND_STR_CASE(__cmd) case MLX5_CMD_OP_ ## __cmd: return #__cmd
499
500 switch (command) {
501 MLX5_COMMAND_STR_CASE(QUERY_HCA_CAP);
502 MLX5_COMMAND_STR_CASE(QUERY_ADAPTER);
503 MLX5_COMMAND_STR_CASE(INIT_HCA);
504 MLX5_COMMAND_STR_CASE(TEARDOWN_HCA);
505 MLX5_COMMAND_STR_CASE(ENABLE_HCA);
506 MLX5_COMMAND_STR_CASE(DISABLE_HCA);
507 MLX5_COMMAND_STR_CASE(QUERY_PAGES);
508 MLX5_COMMAND_STR_CASE(MANAGE_PAGES);
509 MLX5_COMMAND_STR_CASE(SET_HCA_CAP);
510 MLX5_COMMAND_STR_CASE(QUERY_ISSI);
511 MLX5_COMMAND_STR_CASE(SET_ISSI);
512 MLX5_COMMAND_STR_CASE(SET_DRIVER_VERSION);
513 MLX5_COMMAND_STR_CASE(CREATE_MKEY);
514 MLX5_COMMAND_STR_CASE(QUERY_MKEY);
515 MLX5_COMMAND_STR_CASE(DESTROY_MKEY);
516 MLX5_COMMAND_STR_CASE(QUERY_SPECIAL_CONTEXTS);
517 MLX5_COMMAND_STR_CASE(PAGE_FAULT_RESUME);
518 MLX5_COMMAND_STR_CASE(CREATE_EQ);
519 MLX5_COMMAND_STR_CASE(DESTROY_EQ);
520 MLX5_COMMAND_STR_CASE(QUERY_EQ);
521 MLX5_COMMAND_STR_CASE(GEN_EQE);
522 MLX5_COMMAND_STR_CASE(CREATE_CQ);
523 MLX5_COMMAND_STR_CASE(DESTROY_CQ);
524 MLX5_COMMAND_STR_CASE(QUERY_CQ);
525 MLX5_COMMAND_STR_CASE(MODIFY_CQ);
526 MLX5_COMMAND_STR_CASE(CREATE_QP);
527 MLX5_COMMAND_STR_CASE(DESTROY_QP);
528 MLX5_COMMAND_STR_CASE(RST2INIT_QP);
529 MLX5_COMMAND_STR_CASE(INIT2RTR_QP);
530 MLX5_COMMAND_STR_CASE(RTR2RTS_QP);
531 MLX5_COMMAND_STR_CASE(RTS2RTS_QP);
532 MLX5_COMMAND_STR_CASE(SQERR2RTS_QP);
533 MLX5_COMMAND_STR_CASE(2ERR_QP);
534 MLX5_COMMAND_STR_CASE(2RST_QP);
535 MLX5_COMMAND_STR_CASE(QUERY_QP);
536 MLX5_COMMAND_STR_CASE(SQD_RTS_QP);
537 MLX5_COMMAND_STR_CASE(INIT2INIT_QP);
538 MLX5_COMMAND_STR_CASE(CREATE_PSV);
539 MLX5_COMMAND_STR_CASE(DESTROY_PSV);
540 MLX5_COMMAND_STR_CASE(CREATE_SRQ);
541 MLX5_COMMAND_STR_CASE(DESTROY_SRQ);
542 MLX5_COMMAND_STR_CASE(QUERY_SRQ);
543 MLX5_COMMAND_STR_CASE(ARM_RQ);
544 MLX5_COMMAND_STR_CASE(CREATE_XRC_SRQ);
545 MLX5_COMMAND_STR_CASE(DESTROY_XRC_SRQ);
546 MLX5_COMMAND_STR_CASE(QUERY_XRC_SRQ);
547 MLX5_COMMAND_STR_CASE(ARM_XRC_SRQ);
548 MLX5_COMMAND_STR_CASE(CREATE_DCT);
549 MLX5_COMMAND_STR_CASE(DESTROY_DCT);
550 MLX5_COMMAND_STR_CASE(DRAIN_DCT);
551 MLX5_COMMAND_STR_CASE(QUERY_DCT);
552 MLX5_COMMAND_STR_CASE(ARM_DCT_FOR_KEY_VIOLATION);
553 MLX5_COMMAND_STR_CASE(QUERY_VPORT_STATE);
554 MLX5_COMMAND_STR_CASE(MODIFY_VPORT_STATE);
555 MLX5_COMMAND_STR_CASE(QUERY_ESW_VPORT_CONTEXT);
556 MLX5_COMMAND_STR_CASE(MODIFY_ESW_VPORT_CONTEXT);
557 MLX5_COMMAND_STR_CASE(QUERY_NIC_VPORT_CONTEXT);
558 MLX5_COMMAND_STR_CASE(MODIFY_NIC_VPORT_CONTEXT);
559 MLX5_COMMAND_STR_CASE(QUERY_ROCE_ADDRESS);
560 MLX5_COMMAND_STR_CASE(SET_ROCE_ADDRESS);
561 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_CONTEXT);
562 MLX5_COMMAND_STR_CASE(MODIFY_HCA_VPORT_CONTEXT);
563 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_GID);
564 MLX5_COMMAND_STR_CASE(QUERY_HCA_VPORT_PKEY);
565 MLX5_COMMAND_STR_CASE(QUERY_VNIC_ENV);
566 MLX5_COMMAND_STR_CASE(QUERY_VPORT_COUNTER);
567 MLX5_COMMAND_STR_CASE(ALLOC_Q_COUNTER);
568 MLX5_COMMAND_STR_CASE(DEALLOC_Q_COUNTER);
569 MLX5_COMMAND_STR_CASE(QUERY_Q_COUNTER);
570 MLX5_COMMAND_STR_CASE(SET_MONITOR_COUNTER);
571 MLX5_COMMAND_STR_CASE(ARM_MONITOR_COUNTER);
572 MLX5_COMMAND_STR_CASE(SET_PP_RATE_LIMIT);
573 MLX5_COMMAND_STR_CASE(QUERY_RATE_LIMIT);
574 MLX5_COMMAND_STR_CASE(CREATE_SCHEDULING_ELEMENT);
575 MLX5_COMMAND_STR_CASE(DESTROY_SCHEDULING_ELEMENT);
576 MLX5_COMMAND_STR_CASE(QUERY_SCHEDULING_ELEMENT);
577 MLX5_COMMAND_STR_CASE(MODIFY_SCHEDULING_ELEMENT);
578 MLX5_COMMAND_STR_CASE(CREATE_QOS_PARA_VPORT);
579 MLX5_COMMAND_STR_CASE(DESTROY_QOS_PARA_VPORT);
580 MLX5_COMMAND_STR_CASE(ALLOC_PD);
581 MLX5_COMMAND_STR_CASE(DEALLOC_PD);
582 MLX5_COMMAND_STR_CASE(ALLOC_UAR);
583 MLX5_COMMAND_STR_CASE(DEALLOC_UAR);
584 MLX5_COMMAND_STR_CASE(CONFIG_INT_MODERATION);
585 MLX5_COMMAND_STR_CASE(ACCESS_REG);
586 MLX5_COMMAND_STR_CASE(ATTACH_TO_MCG);
587 MLX5_COMMAND_STR_CASE(DETACH_FROM_MCG);
588 MLX5_COMMAND_STR_CASE(GET_DROPPED_PACKET_LOG);
589 MLX5_COMMAND_STR_CASE(MAD_IFC);
590 MLX5_COMMAND_STR_CASE(QUERY_MAD_DEMUX);
591 MLX5_COMMAND_STR_CASE(SET_MAD_DEMUX);
592 MLX5_COMMAND_STR_CASE(NOP);
593 MLX5_COMMAND_STR_CASE(ALLOC_XRCD);
594 MLX5_COMMAND_STR_CASE(DEALLOC_XRCD);
595 MLX5_COMMAND_STR_CASE(ALLOC_TRANSPORT_DOMAIN);
596 MLX5_COMMAND_STR_CASE(DEALLOC_TRANSPORT_DOMAIN);
597 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATUS);
598 MLX5_COMMAND_STR_CASE(MODIFY_CONG_STATUS);
599 MLX5_COMMAND_STR_CASE(QUERY_CONG_PARAMS);
600 MLX5_COMMAND_STR_CASE(MODIFY_CONG_PARAMS);
601 MLX5_COMMAND_STR_CASE(QUERY_CONG_STATISTICS);
602 MLX5_COMMAND_STR_CASE(ADD_VXLAN_UDP_DPORT);
603 MLX5_COMMAND_STR_CASE(DELETE_VXLAN_UDP_DPORT);
604 MLX5_COMMAND_STR_CASE(SET_L2_TABLE_ENTRY);
605 MLX5_COMMAND_STR_CASE(QUERY_L2_TABLE_ENTRY);
606 MLX5_COMMAND_STR_CASE(DELETE_L2_TABLE_ENTRY);
607 MLX5_COMMAND_STR_CASE(SET_WOL_ROL);
608 MLX5_COMMAND_STR_CASE(QUERY_WOL_ROL);
609 MLX5_COMMAND_STR_CASE(CREATE_LAG);
610 MLX5_COMMAND_STR_CASE(MODIFY_LAG);
611 MLX5_COMMAND_STR_CASE(QUERY_LAG);
612 MLX5_COMMAND_STR_CASE(DESTROY_LAG);
613 MLX5_COMMAND_STR_CASE(CREATE_VPORT_LAG);
614 MLX5_COMMAND_STR_CASE(DESTROY_VPORT_LAG);
615 MLX5_COMMAND_STR_CASE(CREATE_TIR);
616 MLX5_COMMAND_STR_CASE(MODIFY_TIR);
617 MLX5_COMMAND_STR_CASE(DESTROY_TIR);
618 MLX5_COMMAND_STR_CASE(QUERY_TIR);
619 MLX5_COMMAND_STR_CASE(CREATE_SQ);
620 MLX5_COMMAND_STR_CASE(MODIFY_SQ);
621 MLX5_COMMAND_STR_CASE(DESTROY_SQ);
622 MLX5_COMMAND_STR_CASE(QUERY_SQ);
623 MLX5_COMMAND_STR_CASE(CREATE_RQ);
624 MLX5_COMMAND_STR_CASE(MODIFY_RQ);
625 MLX5_COMMAND_STR_CASE(DESTROY_RQ);
626 MLX5_COMMAND_STR_CASE(QUERY_RQ);
627 MLX5_COMMAND_STR_CASE(CREATE_RMP);
628 MLX5_COMMAND_STR_CASE(MODIFY_RMP);
629 MLX5_COMMAND_STR_CASE(DESTROY_RMP);
630 MLX5_COMMAND_STR_CASE(QUERY_RMP);
631 MLX5_COMMAND_STR_CASE(CREATE_TIS);
632 MLX5_COMMAND_STR_CASE(MODIFY_TIS);
633 MLX5_COMMAND_STR_CASE(DESTROY_TIS);
634 MLX5_COMMAND_STR_CASE(QUERY_TIS);
635 MLX5_COMMAND_STR_CASE(CREATE_RQT);
636 MLX5_COMMAND_STR_CASE(MODIFY_RQT);
637 MLX5_COMMAND_STR_CASE(DESTROY_RQT);
638 MLX5_COMMAND_STR_CASE(QUERY_RQT);
639 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ROOT);
640 MLX5_COMMAND_STR_CASE(CREATE_FLOW_TABLE);
641 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_TABLE);
642 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE);
643 MLX5_COMMAND_STR_CASE(CREATE_FLOW_GROUP);
644 MLX5_COMMAND_STR_CASE(DESTROY_FLOW_GROUP);
645 MLX5_COMMAND_STR_CASE(QUERY_FLOW_GROUP);
646 MLX5_COMMAND_STR_CASE(SET_FLOW_TABLE_ENTRY);
647 MLX5_COMMAND_STR_CASE(QUERY_FLOW_TABLE_ENTRY);
648 MLX5_COMMAND_STR_CASE(DELETE_FLOW_TABLE_ENTRY);
649 MLX5_COMMAND_STR_CASE(ALLOC_FLOW_COUNTER);
650 MLX5_COMMAND_STR_CASE(DEALLOC_FLOW_COUNTER);
651 MLX5_COMMAND_STR_CASE(QUERY_FLOW_COUNTER);
652 MLX5_COMMAND_STR_CASE(MODIFY_FLOW_TABLE);
653 MLX5_COMMAND_STR_CASE(ALLOC_PACKET_REFORMAT_CONTEXT);
654 MLX5_COMMAND_STR_CASE(DEALLOC_PACKET_REFORMAT_CONTEXT);
655 MLX5_COMMAND_STR_CASE(ALLOC_MODIFY_HEADER_CONTEXT);
656 MLX5_COMMAND_STR_CASE(DEALLOC_MODIFY_HEADER_CONTEXT);
657 MLX5_COMMAND_STR_CASE(FPGA_CREATE_QP);
658 MLX5_COMMAND_STR_CASE(FPGA_MODIFY_QP);
659 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP);
660 MLX5_COMMAND_STR_CASE(FPGA_QUERY_QP_COUNTERS);
661 MLX5_COMMAND_STR_CASE(FPGA_DESTROY_QP);
662 MLX5_COMMAND_STR_CASE(CREATE_XRQ);
663 MLX5_COMMAND_STR_CASE(DESTROY_XRQ);
664 MLX5_COMMAND_STR_CASE(QUERY_XRQ);
665 MLX5_COMMAND_STR_CASE(ARM_XRQ);
666 MLX5_COMMAND_STR_CASE(CREATE_GENERAL_OBJECT);
667 MLX5_COMMAND_STR_CASE(DESTROY_GENERAL_OBJECT);
668 MLX5_COMMAND_STR_CASE(MODIFY_GENERAL_OBJECT);
669 MLX5_COMMAND_STR_CASE(QUERY_GENERAL_OBJECT);
670 MLX5_COMMAND_STR_CASE(QUERY_MODIFY_HEADER_CONTEXT);
671 MLX5_COMMAND_STR_CASE(ALLOC_MEMIC);
672 MLX5_COMMAND_STR_CASE(DEALLOC_MEMIC);
673 MLX5_COMMAND_STR_CASE(QUERY_ESW_FUNCTIONS);
674 MLX5_COMMAND_STR_CASE(CREATE_UCTX);
675 MLX5_COMMAND_STR_CASE(DESTROY_UCTX);
676 MLX5_COMMAND_STR_CASE(CREATE_UMEM);
677 MLX5_COMMAND_STR_CASE(DESTROY_UMEM);
678 MLX5_COMMAND_STR_CASE(RELEASE_XRQ_ERROR);
679 MLX5_COMMAND_STR_CASE(MODIFY_XRQ);
680 MLX5_COMMAND_STR_CASE(QUERY_VHCA_STATE);
681 MLX5_COMMAND_STR_CASE(MODIFY_VHCA_STATE);
682 MLX5_COMMAND_STR_CASE(ALLOC_SF);
683 MLX5_COMMAND_STR_CASE(DEALLOC_SF);
684 MLX5_COMMAND_STR_CASE(SUSPEND_VHCA);
685 MLX5_COMMAND_STR_CASE(RESUME_VHCA);
686 MLX5_COMMAND_STR_CASE(QUERY_VHCA_MIGRATION_STATE);
687 MLX5_COMMAND_STR_CASE(SAVE_VHCA_STATE);
688 MLX5_COMMAND_STR_CASE(LOAD_VHCA_STATE);
689 default: return "unknown command opcode";
690 }
691 }
692
cmd_status_str(u8 status)693 static const char *cmd_status_str(u8 status)
694 {
695 switch (status) {
696 case MLX5_CMD_STAT_OK:
697 return "OK";
698 case MLX5_CMD_STAT_INT_ERR:
699 return "internal error";
700 case MLX5_CMD_STAT_BAD_OP_ERR:
701 return "bad operation";
702 case MLX5_CMD_STAT_BAD_PARAM_ERR:
703 return "bad parameter";
704 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR:
705 return "bad system state";
706 case MLX5_CMD_STAT_BAD_RES_ERR:
707 return "bad resource";
708 case MLX5_CMD_STAT_RES_BUSY:
709 return "resource busy";
710 case MLX5_CMD_STAT_LIM_ERR:
711 return "limits exceeded";
712 case MLX5_CMD_STAT_BAD_RES_STATE_ERR:
713 return "bad resource state";
714 case MLX5_CMD_STAT_IX_ERR:
715 return "bad index";
716 case MLX5_CMD_STAT_NO_RES_ERR:
717 return "no resources";
718 case MLX5_CMD_STAT_BAD_INP_LEN_ERR:
719 return "bad input length";
720 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR:
721 return "bad output length";
722 case MLX5_CMD_STAT_BAD_QP_STATE_ERR:
723 return "bad QP state";
724 case MLX5_CMD_STAT_BAD_PKT_ERR:
725 return "bad packet (discarded)";
726 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR:
727 return "bad size too many outstanding CQEs";
728 default:
729 return "unknown status";
730 }
731 }
732
cmd_status_to_err(u8 status)733 static int cmd_status_to_err(u8 status)
734 {
735 switch (status) {
736 case MLX5_CMD_STAT_OK: return 0;
737 case MLX5_CMD_STAT_INT_ERR: return -EIO;
738 case MLX5_CMD_STAT_BAD_OP_ERR: return -EINVAL;
739 case MLX5_CMD_STAT_BAD_PARAM_ERR: return -EINVAL;
740 case MLX5_CMD_STAT_BAD_SYS_STATE_ERR: return -EIO;
741 case MLX5_CMD_STAT_BAD_RES_ERR: return -EINVAL;
742 case MLX5_CMD_STAT_RES_BUSY: return -EBUSY;
743 case MLX5_CMD_STAT_LIM_ERR: return -ENOMEM;
744 case MLX5_CMD_STAT_BAD_RES_STATE_ERR: return -EINVAL;
745 case MLX5_CMD_STAT_IX_ERR: return -EINVAL;
746 case MLX5_CMD_STAT_NO_RES_ERR: return -EAGAIN;
747 case MLX5_CMD_STAT_BAD_INP_LEN_ERR: return -EIO;
748 case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR: return -EIO;
749 case MLX5_CMD_STAT_BAD_QP_STATE_ERR: return -EINVAL;
750 case MLX5_CMD_STAT_BAD_PKT_ERR: return -EINVAL;
751 case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR: return -EINVAL;
752 default: return -EIO;
753 }
754 }
755
756 struct mlx5_ifc_mbox_out_bits {
757 u8 status[0x8];
758 u8 reserved_at_8[0x18];
759
760 u8 syndrome[0x20];
761
762 u8 reserved_at_40[0x40];
763 };
764
765 struct mlx5_ifc_mbox_in_bits {
766 u8 opcode[0x10];
767 u8 uid[0x10];
768
769 u8 reserved_at_20[0x10];
770 u8 op_mod[0x10];
771
772 u8 reserved_at_40[0x40];
773 };
774
mlx5_cmd_out_err(struct mlx5_core_dev * dev,u16 opcode,u16 op_mod,void * out)775 void mlx5_cmd_out_err(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out)
776 {
777 u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
778 u8 status = MLX5_GET(mbox_out, out, status);
779
780 mlx5_core_err_rl(dev,
781 "%s(0x%x) op_mod(0x%x) failed, status %s(0x%x), syndrome (0x%x), err(%d)\n",
782 mlx5_command_str(opcode), opcode, op_mod,
783 cmd_status_str(status), status, syndrome, cmd_status_to_err(status));
784 }
785 EXPORT_SYMBOL(mlx5_cmd_out_err);
786
cmd_status_print(struct mlx5_core_dev * dev,void * in,void * out)787 static void cmd_status_print(struct mlx5_core_dev *dev, void *in, void *out)
788 {
789 u16 opcode, op_mod;
790 u16 uid;
791
792 opcode = MLX5_GET(mbox_in, in, opcode);
793 op_mod = MLX5_GET(mbox_in, in, op_mod);
794 uid = MLX5_GET(mbox_in, in, uid);
795
796 if (!uid && opcode != MLX5_CMD_OP_DESTROY_MKEY)
797 mlx5_cmd_out_err(dev, opcode, op_mod, out);
798 }
799
mlx5_cmd_check(struct mlx5_core_dev * dev,int err,void * in,void * out)800 int mlx5_cmd_check(struct mlx5_core_dev *dev, int err, void *in, void *out)
801 {
802 /* aborted due to PCI error or via reset flow mlx5_cmd_trigger_completions() */
803 if (err == -ENXIO) {
804 u16 opcode = MLX5_GET(mbox_in, in, opcode);
805 u32 syndrome;
806 u8 status;
807
808 /* PCI Error, emulate command return status, for smooth reset */
809 err = mlx5_internal_err_ret_value(dev, opcode, &syndrome, &status);
810 MLX5_SET(mbox_out, out, status, status);
811 MLX5_SET(mbox_out, out, syndrome, syndrome);
812 if (!err)
813 return 0;
814 }
815
816 /* driver or FW delivery error */
817 if (err != -EREMOTEIO && err)
818 return err;
819
820 /* check outbox status */
821 err = cmd_status_to_err(MLX5_GET(mbox_out, out, status));
822 if (err)
823 cmd_status_print(dev, in, out);
824
825 return err;
826 }
827 EXPORT_SYMBOL(mlx5_cmd_check);
828
dump_command(struct mlx5_core_dev * dev,struct mlx5_cmd_work_ent * ent,int input)829 static void dump_command(struct mlx5_core_dev *dev,
830 struct mlx5_cmd_work_ent *ent, int input)
831 {
832 struct mlx5_cmd_msg *msg = input ? ent->in : ent->out;
833 u16 op = MLX5_GET(mbox_in, ent->lay->in, opcode);
834 struct mlx5_cmd_mailbox *next = msg->next;
835 int n = mlx5_calc_cmd_blocks(msg);
836 int data_only;
837 u32 offset = 0;
838 int dump_len;
839 int i;
840
841 mlx5_core_dbg(dev, "cmd[%d]: start dump\n", ent->idx);
842 data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA));
843
844 if (data_only)
845 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_DATA,
846 "cmd[%d]: dump command data %s(0x%x) %s\n",
847 ent->idx, mlx5_command_str(op), op,
848 input ? "INPUT" : "OUTPUT");
849 else
850 mlx5_core_dbg(dev, "cmd[%d]: dump command %s(0x%x) %s\n",
851 ent->idx, mlx5_command_str(op), op,
852 input ? "INPUT" : "OUTPUT");
853
854 if (data_only) {
855 if (input) {
856 dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset, ent->idx);
857 offset += sizeof(ent->lay->in);
858 } else {
859 dump_buf(ent->lay->out, sizeof(ent->lay->out), 1, offset, ent->idx);
860 offset += sizeof(ent->lay->out);
861 }
862 } else {
863 dump_buf(ent->lay, sizeof(*ent->lay), 0, offset, ent->idx);
864 offset += sizeof(*ent->lay);
865 }
866
867 for (i = 0; i < n && next; i++) {
868 if (data_only) {
869 dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg->len - offset);
870 dump_buf(next->buf, dump_len, 1, offset, ent->idx);
871 offset += MLX5_CMD_DATA_BLOCK_SIZE;
872 } else {
873 mlx5_core_dbg(dev, "cmd[%d]: command block:\n", ent->idx);
874 dump_buf(next->buf, sizeof(struct mlx5_cmd_prot_block), 0, offset,
875 ent->idx);
876 offset += sizeof(struct mlx5_cmd_prot_block);
877 }
878 next = next->next;
879 }
880
881 if (data_only)
882 pr_debug("\n");
883
884 mlx5_core_dbg(dev, "cmd[%d]: end dump\n", ent->idx);
885 }
886
msg_to_opcode(struct mlx5_cmd_msg * in)887 static u16 msg_to_opcode(struct mlx5_cmd_msg *in)
888 {
889 return MLX5_GET(mbox_in, in->first.data, opcode);
890 }
891
892 static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced);
893
cb_timeout_handler(struct work_struct * work)894 static void cb_timeout_handler(struct work_struct *work)
895 {
896 struct delayed_work *dwork = container_of(work, struct delayed_work,
897 work);
898 struct mlx5_cmd_work_ent *ent = container_of(dwork,
899 struct mlx5_cmd_work_ent,
900 cb_timeout_work);
901 struct mlx5_core_dev *dev = container_of(ent->cmd, struct mlx5_core_dev,
902 cmd);
903
904 mlx5_cmd_eq_recover(dev);
905
906 /* Maybe got handled by eq recover ? */
907 if (!test_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state)) {
908 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, recovered after timeout\n", ent->idx,
909 mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in));
910 goto out; /* phew, already handled */
911 }
912
913 ent->ret = -ETIMEDOUT;
914 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) Async, timeout. Will cause a leak of a command resource\n",
915 ent->idx, mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in));
916 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
917
918 out:
919 cmd_ent_put(ent); /* for the cmd_ent_get() took on schedule delayed work */
920 }
921
922 static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg);
923 static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,
924 struct mlx5_cmd_msg *msg);
925
opcode_allowed(struct mlx5_cmd * cmd,u16 opcode)926 static bool opcode_allowed(struct mlx5_cmd *cmd, u16 opcode)
927 {
928 if (cmd->allowed_opcode == CMD_ALLOWED_OPCODE_ALL)
929 return true;
930
931 return cmd->allowed_opcode == opcode;
932 }
933
mlx5_cmd_is_down(struct mlx5_core_dev * dev)934 bool mlx5_cmd_is_down(struct mlx5_core_dev *dev)
935 {
936 return pci_channel_offline(dev->pdev) ||
937 dev->cmd.state != MLX5_CMDIF_STATE_UP ||
938 dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR;
939 }
940
cmd_work_handler(struct work_struct * work)941 static void cmd_work_handler(struct work_struct *work)
942 {
943 struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work);
944 struct mlx5_cmd *cmd = ent->cmd;
945 bool poll_cmd = ent->polling;
946 struct mlx5_cmd_layout *lay;
947 struct mlx5_core_dev *dev;
948 unsigned long cb_timeout;
949 struct semaphore *sem;
950 unsigned long flags;
951 int alloc_ret;
952 int cmd_mode;
953
954 dev = container_of(cmd, struct mlx5_core_dev, cmd);
955 cb_timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD));
956
957 complete(&ent->handling);
958 sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem;
959 down(sem);
960 if (!ent->page_queue) {
961 alloc_ret = cmd_alloc_index(cmd);
962 if (alloc_ret < 0) {
963 mlx5_core_err_rl(dev, "failed to allocate command entry\n");
964 if (ent->callback) {
965 ent->callback(-EAGAIN, ent->context);
966 mlx5_free_cmd_msg(dev, ent->out);
967 free_msg(dev, ent->in);
968 cmd_ent_put(ent);
969 } else {
970 ent->ret = -EAGAIN;
971 complete(&ent->done);
972 }
973 up(sem);
974 return;
975 }
976 ent->idx = alloc_ret;
977 } else {
978 ent->idx = cmd->max_reg_cmds;
979 spin_lock_irqsave(&cmd->alloc_lock, flags);
980 clear_bit(ent->idx, &cmd->bitmask);
981 spin_unlock_irqrestore(&cmd->alloc_lock, flags);
982 }
983
984 cmd->ent_arr[ent->idx] = ent;
985 lay = get_inst(cmd, ent->idx);
986 ent->lay = lay;
987 memset(lay, 0, sizeof(*lay));
988 memcpy(lay->in, ent->in->first.data, sizeof(lay->in));
989 ent->op = be32_to_cpu(lay->in[0]) >> 16;
990 if (ent->in->next)
991 lay->in_ptr = cpu_to_be64(ent->in->next->dma);
992 lay->inlen = cpu_to_be32(ent->in->len);
993 if (ent->out->next)
994 lay->out_ptr = cpu_to_be64(ent->out->next->dma);
995 lay->outlen = cpu_to_be32(ent->out->len);
996 lay->type = MLX5_PCI_CMD_XPORT;
997 lay->token = ent->token;
998 lay->status_own = CMD_OWNER_HW;
999 set_signature(ent, !cmd->checksum_disabled);
1000 dump_command(dev, ent, 1);
1001 ent->ts1 = ktime_get_ns();
1002 cmd_mode = cmd->mode;
1003
1004 if (ent->callback && schedule_delayed_work(&ent->cb_timeout_work, cb_timeout))
1005 cmd_ent_get(ent);
1006 set_bit(MLX5_CMD_ENT_STATE_PENDING_COMP, &ent->state);
1007
1008 cmd_ent_get(ent); /* for the _real_ FW event on completion */
1009 /* Skip sending command to fw if internal error */
1010 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, ent->op)) {
1011 ent->ret = -ENXIO;
1012 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
1013 return;
1014 }
1015
1016 /* ring doorbell after the descriptor is valid */
1017 mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx);
1018 wmb();
1019 iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell);
1020 /* if not in polling don't use ent after this point */
1021 if (cmd_mode == CMD_MODE_POLLING || poll_cmd) {
1022 poll_timeout(ent);
1023 /* make sure we read the descriptor after ownership is SW */
1024 rmb();
1025 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, (ent->ret == -ETIMEDOUT));
1026 }
1027 }
1028
deliv_status_to_err(u8 status)1029 static int deliv_status_to_err(u8 status)
1030 {
1031 switch (status) {
1032 case MLX5_CMD_DELIVERY_STAT_OK:
1033 case MLX5_DRIVER_STATUS_ABORTED:
1034 return 0;
1035 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:
1036 case MLX5_CMD_DELIVERY_STAT_TOK_ERR:
1037 return -EBADR;
1038 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:
1039 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:
1040 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:
1041 return -EFAULT; /* Bad address */
1042 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:
1043 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:
1044 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:
1045 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:
1046 return -ENOMSG;
1047 case MLX5_CMD_DELIVERY_STAT_FW_ERR:
1048 return -EIO;
1049 default:
1050 return -EINVAL;
1051 }
1052 }
1053
deliv_status_to_str(u8 status)1054 static const char *deliv_status_to_str(u8 status)
1055 {
1056 switch (status) {
1057 case MLX5_CMD_DELIVERY_STAT_OK:
1058 return "no errors";
1059 case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:
1060 return "signature error";
1061 case MLX5_CMD_DELIVERY_STAT_TOK_ERR:
1062 return "token error";
1063 case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:
1064 return "bad block number";
1065 case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:
1066 return "output pointer not aligned to block size";
1067 case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:
1068 return "input pointer not aligned to block size";
1069 case MLX5_CMD_DELIVERY_STAT_FW_ERR:
1070 return "firmware internal error";
1071 case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:
1072 return "command input length error";
1073 case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:
1074 return "command output length error";
1075 case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:
1076 return "reserved fields not cleared";
1077 case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:
1078 return "bad command descriptor type";
1079 default:
1080 return "unknown status code";
1081 }
1082 }
1083
1084 enum {
1085 MLX5_CMD_TIMEOUT_RECOVER_MSEC = 5 * 1000,
1086 };
1087
wait_func_handle_exec_timeout(struct mlx5_core_dev * dev,struct mlx5_cmd_work_ent * ent)1088 static void wait_func_handle_exec_timeout(struct mlx5_core_dev *dev,
1089 struct mlx5_cmd_work_ent *ent)
1090 {
1091 unsigned long timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_RECOVER_MSEC);
1092
1093 mlx5_cmd_eq_recover(dev);
1094
1095 /* Re-wait on the ent->done after executing the recovery flow. If the
1096 * recovery flow (or any other recovery flow running simultaneously)
1097 * has recovered an EQE, it should cause the entry to be completed by
1098 * the command interface.
1099 */
1100 if (wait_for_completion_timeout(&ent->done, timeout)) {
1101 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) recovered after timeout\n", ent->idx,
1102 mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in));
1103 return;
1104 }
1105
1106 mlx5_core_warn(dev, "cmd[%d]: %s(0x%x) No done completion\n", ent->idx,
1107 mlx5_command_str(msg_to_opcode(ent->in)), msg_to_opcode(ent->in));
1108
1109 ent->ret = -ETIMEDOUT;
1110 mlx5_cmd_comp_handler(dev, 1ULL << ent->idx, true);
1111 }
1112
wait_func(struct mlx5_core_dev * dev,struct mlx5_cmd_work_ent * ent)1113 static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent)
1114 {
1115 unsigned long timeout = msecs_to_jiffies(mlx5_tout_ms(dev, CMD));
1116 struct mlx5_cmd *cmd = &dev->cmd;
1117 int err;
1118
1119 if (!wait_for_completion_timeout(&ent->handling, timeout) &&
1120 cancel_work_sync(&ent->work)) {
1121 ent->ret = -ECANCELED;
1122 goto out_err;
1123 }
1124 if (cmd->mode == CMD_MODE_POLLING || ent->polling)
1125 wait_for_completion(&ent->done);
1126 else if (!wait_for_completion_timeout(&ent->done, timeout))
1127 wait_func_handle_exec_timeout(dev, ent);
1128
1129 out_err:
1130 err = ent->ret;
1131
1132 if (err == -ETIMEDOUT) {
1133 mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n",
1134 mlx5_command_str(msg_to_opcode(ent->in)),
1135 msg_to_opcode(ent->in));
1136 } else if (err == -ECANCELED) {
1137 mlx5_core_warn(dev, "%s(0x%x) canceled on out of queue timeout.\n",
1138 mlx5_command_str(msg_to_opcode(ent->in)),
1139 msg_to_opcode(ent->in));
1140 }
1141 mlx5_core_dbg(dev, "err %d, delivery status %s(%d)\n",
1142 err, deliv_status_to_str(ent->status), ent->status);
1143
1144 return err;
1145 }
1146
1147 /* Notes:
1148 * 1. Callback functions may not sleep
1149 * 2. page queue commands do not support asynchrous completion
1150 *
1151 * return value in case (!callback):
1152 * ret < 0 : Command execution couldn't be submitted by driver
1153 * ret > 0 : Command execution couldn't be performed by firmware
1154 * ret == 0: Command was executed by FW, Caller must check FW outbox status.
1155 *
1156 * return value in case (callback):
1157 * ret < 0 : Command execution couldn't be submitted by driver
1158 * ret == 0: Command will be submitted to FW for execution
1159 * and the callback will be called for further status updates
1160 */
mlx5_cmd_invoke(struct mlx5_core_dev * dev,struct mlx5_cmd_msg * in,struct mlx5_cmd_msg * out,void * uout,int uout_size,mlx5_cmd_cbk_t callback,void * context,int page_queue,u8 token,bool force_polling)1161 static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,
1162 struct mlx5_cmd_msg *out, void *uout, int uout_size,
1163 mlx5_cmd_cbk_t callback,
1164 void *context, int page_queue,
1165 u8 token, bool force_polling)
1166 {
1167 struct mlx5_cmd *cmd = &dev->cmd;
1168 struct mlx5_cmd_work_ent *ent;
1169 struct mlx5_cmd_stats *stats;
1170 u8 status = 0;
1171 int err = 0;
1172 s64 ds;
1173 u16 op;
1174
1175 if (callback && page_queue)
1176 return -EINVAL;
1177
1178 ent = cmd_alloc_ent(cmd, in, out, uout, uout_size,
1179 callback, context, page_queue);
1180 if (IS_ERR(ent))
1181 return PTR_ERR(ent);
1182
1183 /* put for this ent is when consumed, depending on the use case
1184 * 1) (!callback) blocking flow: by caller after wait_func completes
1185 * 2) (callback) flow: by mlx5_cmd_comp_handler() when ent is handled
1186 */
1187
1188 ent->token = token;
1189 ent->polling = force_polling;
1190
1191 init_completion(&ent->handling);
1192 if (!callback)
1193 init_completion(&ent->done);
1194
1195 INIT_DELAYED_WORK(&ent->cb_timeout_work, cb_timeout_handler);
1196 INIT_WORK(&ent->work, cmd_work_handler);
1197 if (page_queue) {
1198 cmd_work_handler(&ent->work);
1199 } else if (!queue_work(cmd->wq, &ent->work)) {
1200 mlx5_core_warn(dev, "failed to queue work\n");
1201 err = -EALREADY;
1202 goto out_free;
1203 }
1204
1205 if (callback)
1206 return 0; /* mlx5_cmd_comp_handler() will put(ent) */
1207
1208 err = wait_func(dev, ent);
1209 if (err == -ETIMEDOUT || err == -ECANCELED)
1210 goto out_free;
1211
1212 ds = ent->ts2 - ent->ts1;
1213 op = MLX5_GET(mbox_in, in->first.data, opcode);
1214 if (op < MLX5_CMD_OP_MAX) {
1215 stats = &cmd->stats[op];
1216 spin_lock_irq(&stats->lock);
1217 stats->sum += ds;
1218 ++stats->n;
1219 spin_unlock_irq(&stats->lock);
1220 }
1221 mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_TIME,
1222 "fw exec time for %s is %lld nsec\n",
1223 mlx5_command_str(op), ds);
1224
1225 out_free:
1226 status = ent->status;
1227 cmd_ent_put(ent);
1228 return err ? : status;
1229 }
1230
dbg_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)1231 static ssize_t dbg_write(struct file *filp, const char __user *buf,
1232 size_t count, loff_t *pos)
1233 {
1234 struct mlx5_core_dev *dev = filp->private_data;
1235 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1236 char lbuf[3];
1237 int err;
1238
1239 if (!dbg->in_msg || !dbg->out_msg)
1240 return -ENOMEM;
1241
1242 if (count < sizeof(lbuf) - 1)
1243 return -EINVAL;
1244
1245 if (copy_from_user(lbuf, buf, sizeof(lbuf) - 1))
1246 return -EFAULT;
1247
1248 lbuf[sizeof(lbuf) - 1] = 0;
1249
1250 if (strcmp(lbuf, "go"))
1251 return -EINVAL;
1252
1253 err = mlx5_cmd_exec(dev, dbg->in_msg, dbg->inlen, dbg->out_msg, dbg->outlen);
1254
1255 return err ? err : count;
1256 }
1257
1258 static const struct file_operations fops = {
1259 .owner = THIS_MODULE,
1260 .open = simple_open,
1261 .write = dbg_write,
1262 };
1263
mlx5_copy_to_msg(struct mlx5_cmd_msg * to,void * from,int size,u8 token)1264 static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size,
1265 u8 token)
1266 {
1267 struct mlx5_cmd_prot_block *block;
1268 struct mlx5_cmd_mailbox *next;
1269 int copy;
1270
1271 if (!to || !from)
1272 return -ENOMEM;
1273
1274 copy = min_t(int, size, sizeof(to->first.data));
1275 memcpy(to->first.data, from, copy);
1276 size -= copy;
1277 from += copy;
1278
1279 next = to->next;
1280 while (size) {
1281 if (!next) {
1282 /* this is a BUG */
1283 return -ENOMEM;
1284 }
1285
1286 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
1287 block = next->buf;
1288 memcpy(block->data, from, copy);
1289 from += copy;
1290 size -= copy;
1291 block->token = token;
1292 next = next->next;
1293 }
1294
1295 return 0;
1296 }
1297
mlx5_copy_from_msg(void * to,struct mlx5_cmd_msg * from,int size)1298 static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size)
1299 {
1300 struct mlx5_cmd_prot_block *block;
1301 struct mlx5_cmd_mailbox *next;
1302 int copy;
1303
1304 if (!to || !from)
1305 return -ENOMEM;
1306
1307 copy = min_t(int, size, sizeof(from->first.data));
1308 memcpy(to, from->first.data, copy);
1309 size -= copy;
1310 to += copy;
1311
1312 next = from->next;
1313 while (size) {
1314 if (!next) {
1315 /* this is a BUG */
1316 return -ENOMEM;
1317 }
1318
1319 copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
1320 block = next->buf;
1321
1322 memcpy(to, block->data, copy);
1323 to += copy;
1324 size -= copy;
1325 next = next->next;
1326 }
1327
1328 return 0;
1329 }
1330
alloc_cmd_box(struct mlx5_core_dev * dev,gfp_t flags)1331 static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev,
1332 gfp_t flags)
1333 {
1334 struct mlx5_cmd_mailbox *mailbox;
1335
1336 mailbox = kmalloc(sizeof(*mailbox), flags);
1337 if (!mailbox)
1338 return ERR_PTR(-ENOMEM);
1339
1340 mailbox->buf = dma_pool_zalloc(dev->cmd.pool, flags,
1341 &mailbox->dma);
1342 if (!mailbox->buf) {
1343 mlx5_core_dbg(dev, "failed allocation\n");
1344 kfree(mailbox);
1345 return ERR_PTR(-ENOMEM);
1346 }
1347 mailbox->next = NULL;
1348
1349 return mailbox;
1350 }
1351
free_cmd_box(struct mlx5_core_dev * dev,struct mlx5_cmd_mailbox * mailbox)1352 static void free_cmd_box(struct mlx5_core_dev *dev,
1353 struct mlx5_cmd_mailbox *mailbox)
1354 {
1355 dma_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma);
1356 kfree(mailbox);
1357 }
1358
mlx5_alloc_cmd_msg(struct mlx5_core_dev * dev,gfp_t flags,int size,u8 token)1359 static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev,
1360 gfp_t flags, int size,
1361 u8 token)
1362 {
1363 struct mlx5_cmd_mailbox *tmp, *head = NULL;
1364 struct mlx5_cmd_prot_block *block;
1365 struct mlx5_cmd_msg *msg;
1366 int err;
1367 int n;
1368 int i;
1369
1370 msg = kzalloc(sizeof(*msg), flags);
1371 if (!msg)
1372 return ERR_PTR(-ENOMEM);
1373
1374 msg->len = size;
1375 n = mlx5_calc_cmd_blocks(msg);
1376
1377 for (i = 0; i < n; i++) {
1378 tmp = alloc_cmd_box(dev, flags);
1379 if (IS_ERR(tmp)) {
1380 mlx5_core_warn(dev, "failed allocating block\n");
1381 err = PTR_ERR(tmp);
1382 goto err_alloc;
1383 }
1384
1385 block = tmp->buf;
1386 tmp->next = head;
1387 block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0);
1388 block->block_num = cpu_to_be32(n - i - 1);
1389 block->token = token;
1390 head = tmp;
1391 }
1392 msg->next = head;
1393 return msg;
1394
1395 err_alloc:
1396 while (head) {
1397 tmp = head->next;
1398 free_cmd_box(dev, head);
1399 head = tmp;
1400 }
1401 kfree(msg);
1402
1403 return ERR_PTR(err);
1404 }
1405
mlx5_free_cmd_msg(struct mlx5_core_dev * dev,struct mlx5_cmd_msg * msg)1406 static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,
1407 struct mlx5_cmd_msg *msg)
1408 {
1409 struct mlx5_cmd_mailbox *head = msg->next;
1410 struct mlx5_cmd_mailbox *next;
1411
1412 while (head) {
1413 next = head->next;
1414 free_cmd_box(dev, head);
1415 head = next;
1416 }
1417 kfree(msg);
1418 }
1419
data_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)1420 static ssize_t data_write(struct file *filp, const char __user *buf,
1421 size_t count, loff_t *pos)
1422 {
1423 struct mlx5_core_dev *dev = filp->private_data;
1424 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1425 void *ptr;
1426
1427 if (*pos != 0)
1428 return -EINVAL;
1429
1430 kfree(dbg->in_msg);
1431 dbg->in_msg = NULL;
1432 dbg->inlen = 0;
1433 ptr = memdup_user(buf, count);
1434 if (IS_ERR(ptr))
1435 return PTR_ERR(ptr);
1436 dbg->in_msg = ptr;
1437 dbg->inlen = count;
1438
1439 *pos = count;
1440
1441 return count;
1442 }
1443
data_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)1444 static ssize_t data_read(struct file *filp, char __user *buf, size_t count,
1445 loff_t *pos)
1446 {
1447 struct mlx5_core_dev *dev = filp->private_data;
1448 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1449
1450 if (!dbg->out_msg)
1451 return -ENOMEM;
1452
1453 return simple_read_from_buffer(buf, count, pos, dbg->out_msg,
1454 dbg->outlen);
1455 }
1456
1457 static const struct file_operations dfops = {
1458 .owner = THIS_MODULE,
1459 .open = simple_open,
1460 .write = data_write,
1461 .read = data_read,
1462 };
1463
outlen_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)1464 static ssize_t outlen_read(struct file *filp, char __user *buf, size_t count,
1465 loff_t *pos)
1466 {
1467 struct mlx5_core_dev *dev = filp->private_data;
1468 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1469 char outlen[8];
1470 int err;
1471
1472 err = snprintf(outlen, sizeof(outlen), "%d", dbg->outlen);
1473 if (err < 0)
1474 return err;
1475
1476 return simple_read_from_buffer(buf, count, pos, outlen, err);
1477 }
1478
outlen_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)1479 static ssize_t outlen_write(struct file *filp, const char __user *buf,
1480 size_t count, loff_t *pos)
1481 {
1482 struct mlx5_core_dev *dev = filp->private_data;
1483 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1484 char outlen_str[8] = {0};
1485 int outlen;
1486 void *ptr;
1487 int err;
1488
1489 if (*pos != 0 || count > 6)
1490 return -EINVAL;
1491
1492 kfree(dbg->out_msg);
1493 dbg->out_msg = NULL;
1494 dbg->outlen = 0;
1495
1496 if (copy_from_user(outlen_str, buf, count))
1497 return -EFAULT;
1498
1499 err = sscanf(outlen_str, "%d", &outlen);
1500 if (err != 1)
1501 return -EINVAL;
1502
1503 ptr = kzalloc(outlen, GFP_KERNEL);
1504 if (!ptr)
1505 return -ENOMEM;
1506
1507 dbg->out_msg = ptr;
1508 dbg->outlen = outlen;
1509
1510 *pos = count;
1511
1512 return count;
1513 }
1514
1515 static const struct file_operations olfops = {
1516 .owner = THIS_MODULE,
1517 .open = simple_open,
1518 .write = outlen_write,
1519 .read = outlen_read,
1520 };
1521
set_wqname(struct mlx5_core_dev * dev)1522 static void set_wqname(struct mlx5_core_dev *dev)
1523 {
1524 struct mlx5_cmd *cmd = &dev->cmd;
1525
1526 snprintf(cmd->wq_name, sizeof(cmd->wq_name), "mlx5_cmd_%s",
1527 dev_name(dev->device));
1528 }
1529
clean_debug_files(struct mlx5_core_dev * dev)1530 static void clean_debug_files(struct mlx5_core_dev *dev)
1531 {
1532 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1533
1534 if (!mlx5_debugfs_root)
1535 return;
1536
1537 mlx5_cmdif_debugfs_cleanup(dev);
1538 debugfs_remove_recursive(dbg->dbg_root);
1539 }
1540
create_debugfs_files(struct mlx5_core_dev * dev)1541 static void create_debugfs_files(struct mlx5_core_dev *dev)
1542 {
1543 struct mlx5_cmd_debug *dbg = &dev->cmd.dbg;
1544
1545 dbg->dbg_root = debugfs_create_dir("cmd", mlx5_debugfs_get_dev_root(dev));
1546
1547 debugfs_create_file("in", 0400, dbg->dbg_root, dev, &dfops);
1548 debugfs_create_file("out", 0200, dbg->dbg_root, dev, &dfops);
1549 debugfs_create_file("out_len", 0600, dbg->dbg_root, dev, &olfops);
1550 debugfs_create_u8("status", 0600, dbg->dbg_root, &dbg->status);
1551 debugfs_create_file("run", 0200, dbg->dbg_root, dev, &fops);
1552
1553 mlx5_cmdif_debugfs_init(dev);
1554 }
1555
mlx5_cmd_allowed_opcode(struct mlx5_core_dev * dev,u16 opcode)1556 void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode)
1557 {
1558 struct mlx5_cmd *cmd = &dev->cmd;
1559 int i;
1560
1561 for (i = 0; i < cmd->max_reg_cmds; i++)
1562 down(&cmd->sem);
1563 down(&cmd->pages_sem);
1564
1565 cmd->allowed_opcode = opcode;
1566
1567 up(&cmd->pages_sem);
1568 for (i = 0; i < cmd->max_reg_cmds; i++)
1569 up(&cmd->sem);
1570 }
1571
mlx5_cmd_change_mod(struct mlx5_core_dev * dev,int mode)1572 static void mlx5_cmd_change_mod(struct mlx5_core_dev *dev, int mode)
1573 {
1574 struct mlx5_cmd *cmd = &dev->cmd;
1575 int i;
1576
1577 for (i = 0; i < cmd->max_reg_cmds; i++)
1578 down(&cmd->sem);
1579 down(&cmd->pages_sem);
1580
1581 cmd->mode = mode;
1582
1583 up(&cmd->pages_sem);
1584 for (i = 0; i < cmd->max_reg_cmds; i++)
1585 up(&cmd->sem);
1586 }
1587
cmd_comp_notifier(struct notifier_block * nb,unsigned long type,void * data)1588 static int cmd_comp_notifier(struct notifier_block *nb,
1589 unsigned long type, void *data)
1590 {
1591 struct mlx5_core_dev *dev;
1592 struct mlx5_cmd *cmd;
1593 struct mlx5_eqe *eqe;
1594
1595 cmd = mlx5_nb_cof(nb, struct mlx5_cmd, nb);
1596 dev = container_of(cmd, struct mlx5_core_dev, cmd);
1597 eqe = data;
1598
1599 mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector), false);
1600
1601 return NOTIFY_OK;
1602 }
mlx5_cmd_use_events(struct mlx5_core_dev * dev)1603 void mlx5_cmd_use_events(struct mlx5_core_dev *dev)
1604 {
1605 MLX5_NB_INIT(&dev->cmd.nb, cmd_comp_notifier, CMD);
1606 mlx5_eq_notifier_register(dev, &dev->cmd.nb);
1607 mlx5_cmd_change_mod(dev, CMD_MODE_EVENTS);
1608 }
1609
mlx5_cmd_use_polling(struct mlx5_core_dev * dev)1610 void mlx5_cmd_use_polling(struct mlx5_core_dev *dev)
1611 {
1612 mlx5_cmd_change_mod(dev, CMD_MODE_POLLING);
1613 mlx5_eq_notifier_unregister(dev, &dev->cmd.nb);
1614 }
1615
free_msg(struct mlx5_core_dev * dev,struct mlx5_cmd_msg * msg)1616 static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg)
1617 {
1618 unsigned long flags;
1619
1620 if (msg->parent) {
1621 spin_lock_irqsave(&msg->parent->lock, flags);
1622 list_add_tail(&msg->list, &msg->parent->head);
1623 spin_unlock_irqrestore(&msg->parent->lock, flags);
1624 } else {
1625 mlx5_free_cmd_msg(dev, msg);
1626 }
1627 }
1628
mlx5_cmd_comp_handler(struct mlx5_core_dev * dev,u64 vec,bool forced)1629 static void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, u64 vec, bool forced)
1630 {
1631 struct mlx5_cmd *cmd = &dev->cmd;
1632 struct mlx5_cmd_work_ent *ent;
1633 mlx5_cmd_cbk_t callback;
1634 void *context;
1635 int err;
1636 int i;
1637 s64 ds;
1638 struct mlx5_cmd_stats *stats;
1639 unsigned long flags;
1640 unsigned long vector;
1641
1642 /* there can be at most 32 command queues */
1643 vector = vec & 0xffffffff;
1644 for (i = 0; i < (1 << cmd->log_sz); i++) {
1645 if (test_bit(i, &vector)) {
1646 ent = cmd->ent_arr[i];
1647
1648 /* if we already completed the command, ignore it */
1649 if (!test_and_clear_bit(MLX5_CMD_ENT_STATE_PENDING_COMP,
1650 &ent->state)) {
1651 /* only real completion can free the cmd slot */
1652 if (!forced) {
1653 mlx5_core_err(dev, "Command completion arrived after timeout (entry idx = %d).\n",
1654 ent->idx);
1655 cmd_ent_put(ent);
1656 }
1657 continue;
1658 }
1659
1660 if (ent->callback && cancel_delayed_work(&ent->cb_timeout_work))
1661 cmd_ent_put(ent); /* timeout work was canceled */
1662
1663 if (!forced || /* Real FW completion */
1664 mlx5_cmd_is_down(dev) || /* No real FW completion is expected */
1665 !opcode_allowed(cmd, ent->op))
1666 cmd_ent_put(ent);
1667
1668 ent->ts2 = ktime_get_ns();
1669 memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out));
1670 dump_command(dev, ent, 0);
1671
1672 if (vec & MLX5_TRIGGERED_CMD_COMP)
1673 ent->ret = -ENXIO;
1674
1675 if (!ent->ret) { /* Command completed by FW */
1676 if (!cmd->checksum_disabled)
1677 ent->ret = verify_signature(ent);
1678
1679 ent->status = ent->lay->status_own >> 1;
1680
1681 mlx5_core_dbg(dev, "command completed. ret 0x%x, delivery status %s(0x%x)\n",
1682 ent->ret, deliv_status_to_str(ent->status), ent->status);
1683 }
1684
1685 if (ent->callback) {
1686 ds = ent->ts2 - ent->ts1;
1687 if (ent->op < MLX5_CMD_OP_MAX) {
1688 stats = &cmd->stats[ent->op];
1689 spin_lock_irqsave(&stats->lock, flags);
1690 stats->sum += ds;
1691 ++stats->n;
1692 spin_unlock_irqrestore(&stats->lock, flags);
1693 }
1694
1695 callback = ent->callback;
1696 context = ent->context;
1697 err = ent->ret ? : ent->status;
1698 if (err > 0) /* Failed in FW, command didn't execute */
1699 err = deliv_status_to_err(err);
1700
1701 if (!err)
1702 err = mlx5_copy_from_msg(ent->uout,
1703 ent->out,
1704 ent->uout_size);
1705
1706 mlx5_free_cmd_msg(dev, ent->out);
1707 free_msg(dev, ent->in);
1708
1709 /* final consumer is done, release ent */
1710 cmd_ent_put(ent);
1711 callback(err, context);
1712 } else {
1713 /* release wait_func() so mlx5_cmd_invoke()
1714 * can make the final ent_put()
1715 */
1716 complete(&ent->done);
1717 }
1718 }
1719 }
1720 }
1721
mlx5_cmd_trigger_completions(struct mlx5_core_dev * dev)1722 static void mlx5_cmd_trigger_completions(struct mlx5_core_dev *dev)
1723 {
1724 struct mlx5_cmd *cmd = &dev->cmd;
1725 unsigned long bitmask;
1726 unsigned long flags;
1727 u64 vector;
1728 int i;
1729
1730 /* wait for pending handlers to complete */
1731 mlx5_eq_synchronize_cmd_irq(dev);
1732 spin_lock_irqsave(&dev->cmd.alloc_lock, flags);
1733 vector = ~dev->cmd.bitmask & ((1ul << (1 << dev->cmd.log_sz)) - 1);
1734 if (!vector)
1735 goto no_trig;
1736
1737 bitmask = vector;
1738 /* we must increment the allocated entries refcount before triggering the completions
1739 * to guarantee pending commands will not get freed in the meanwhile.
1740 * For that reason, it also has to be done inside the alloc_lock.
1741 */
1742 for_each_set_bit(i, &bitmask, (1 << cmd->log_sz))
1743 cmd_ent_get(cmd->ent_arr[i]);
1744 vector |= MLX5_TRIGGERED_CMD_COMP;
1745 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);
1746
1747 mlx5_core_dbg(dev, "vector 0x%llx\n", vector);
1748 mlx5_cmd_comp_handler(dev, vector, true);
1749 for_each_set_bit(i, &bitmask, (1 << cmd->log_sz))
1750 cmd_ent_put(cmd->ent_arr[i]);
1751 return;
1752
1753 no_trig:
1754 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags);
1755 }
1756
mlx5_cmd_flush(struct mlx5_core_dev * dev)1757 void mlx5_cmd_flush(struct mlx5_core_dev *dev)
1758 {
1759 struct mlx5_cmd *cmd = &dev->cmd;
1760 int i;
1761
1762 for (i = 0; i < cmd->max_reg_cmds; i++) {
1763 while (down_trylock(&cmd->sem)) {
1764 mlx5_cmd_trigger_completions(dev);
1765 cond_resched();
1766 }
1767 }
1768
1769 while (down_trylock(&cmd->pages_sem)) {
1770 mlx5_cmd_trigger_completions(dev);
1771 cond_resched();
1772 }
1773
1774 /* Unlock cmdif */
1775 up(&cmd->pages_sem);
1776 for (i = 0; i < cmd->max_reg_cmds; i++)
1777 up(&cmd->sem);
1778 }
1779
alloc_msg(struct mlx5_core_dev * dev,int in_size,gfp_t gfp)1780 static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size,
1781 gfp_t gfp)
1782 {
1783 struct mlx5_cmd_msg *msg = ERR_PTR(-ENOMEM);
1784 struct cmd_msg_cache *ch = NULL;
1785 struct mlx5_cmd *cmd = &dev->cmd;
1786 int i;
1787
1788 if (in_size <= 16)
1789 goto cache_miss;
1790
1791 for (i = 0; i < MLX5_NUM_COMMAND_CACHES; i++) {
1792 ch = &cmd->cache[i];
1793 if (in_size > ch->max_inbox_size)
1794 continue;
1795 spin_lock_irq(&ch->lock);
1796 if (list_empty(&ch->head)) {
1797 spin_unlock_irq(&ch->lock);
1798 continue;
1799 }
1800 msg = list_entry(ch->head.next, typeof(*msg), list);
1801 /* For cached lists, we must explicitly state what is
1802 * the real size
1803 */
1804 msg->len = in_size;
1805 list_del(&msg->list);
1806 spin_unlock_irq(&ch->lock);
1807 break;
1808 }
1809
1810 if (!IS_ERR(msg))
1811 return msg;
1812
1813 cache_miss:
1814 msg = mlx5_alloc_cmd_msg(dev, gfp, in_size, 0);
1815 return msg;
1816 }
1817
is_manage_pages(void * in)1818 static int is_manage_pages(void *in)
1819 {
1820 return MLX5_GET(mbox_in, in, opcode) == MLX5_CMD_OP_MANAGE_PAGES;
1821 }
1822
1823 /* Notes:
1824 * 1. Callback functions may not sleep
1825 * 2. Page queue commands do not support asynchrous completion
1826 */
cmd_exec(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size,mlx5_cmd_cbk_t callback,void * context,bool force_polling)1827 static int cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
1828 int out_size, mlx5_cmd_cbk_t callback, void *context,
1829 bool force_polling)
1830 {
1831 u16 opcode = MLX5_GET(mbox_in, in, opcode);
1832 struct mlx5_cmd_msg *inb, *outb;
1833 int pages_queue;
1834 gfp_t gfp;
1835 u8 token;
1836 int err;
1837
1838 if (mlx5_cmd_is_down(dev) || !opcode_allowed(&dev->cmd, opcode))
1839 return -ENXIO;
1840
1841 pages_queue = is_manage_pages(in);
1842 gfp = callback ? GFP_ATOMIC : GFP_KERNEL;
1843
1844 inb = alloc_msg(dev, in_size, gfp);
1845 if (IS_ERR(inb)) {
1846 err = PTR_ERR(inb);
1847 return err;
1848 }
1849
1850 token = alloc_token(&dev->cmd);
1851
1852 err = mlx5_copy_to_msg(inb, in, in_size, token);
1853 if (err) {
1854 mlx5_core_warn(dev, "err %d\n", err);
1855 goto out_in;
1856 }
1857
1858 outb = mlx5_alloc_cmd_msg(dev, gfp, out_size, token);
1859 if (IS_ERR(outb)) {
1860 err = PTR_ERR(outb);
1861 goto out_in;
1862 }
1863
1864 err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context,
1865 pages_queue, token, force_polling);
1866 if (callback)
1867 return err;
1868
1869 if (err > 0) /* Failed in FW, command didn't execute */
1870 err = deliv_status_to_err(err);
1871
1872 if (err)
1873 goto out_out;
1874
1875 /* command completed by FW */
1876 err = mlx5_copy_from_msg(out, outb, out_size);
1877 out_out:
1878 mlx5_free_cmd_msg(dev, outb);
1879 out_in:
1880 free_msg(dev, inb);
1881 return err;
1882 }
1883
mlx5_cmd_err_trace(struct mlx5_core_dev * dev,u16 opcode,u16 op_mod,void * out)1884 static void mlx5_cmd_err_trace(struct mlx5_core_dev *dev, u16 opcode, u16 op_mod, void *out)
1885 {
1886 u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
1887 u8 status = MLX5_GET(mbox_out, out, status);
1888
1889 trace_mlx5_cmd(mlx5_command_str(opcode), opcode, op_mod,
1890 cmd_status_str(status), status, syndrome,
1891 cmd_status_to_err(status));
1892 }
1893
cmd_status_log(struct mlx5_core_dev * dev,u16 opcode,u8 status,u32 syndrome,int err)1894 static void cmd_status_log(struct mlx5_core_dev *dev, u16 opcode, u8 status,
1895 u32 syndrome, int err)
1896 {
1897 struct mlx5_cmd_stats *stats;
1898
1899 if (!err)
1900 return;
1901
1902 stats = &dev->cmd.stats[opcode];
1903 spin_lock_irq(&stats->lock);
1904 stats->failed++;
1905 if (err < 0)
1906 stats->last_failed_errno = -err;
1907 if (err == -EREMOTEIO) {
1908 stats->failed_mbox_status++;
1909 stats->last_failed_mbox_status = status;
1910 stats->last_failed_syndrome = syndrome;
1911 }
1912 spin_unlock_irq(&stats->lock);
1913 }
1914
1915 /* preserve -EREMOTEIO for outbox.status != OK, otherwise return err as is */
cmd_status_err(struct mlx5_core_dev * dev,int err,u16 opcode,u16 op_mod,void * out)1916 static int cmd_status_err(struct mlx5_core_dev *dev, int err, u16 opcode, u16 op_mod, void *out)
1917 {
1918 u32 syndrome = MLX5_GET(mbox_out, out, syndrome);
1919 u8 status = MLX5_GET(mbox_out, out, status);
1920
1921 if (err == -EREMOTEIO) /* -EREMOTEIO is preserved */
1922 err = -EIO;
1923
1924 if (!err && status != MLX5_CMD_STAT_OK) {
1925 err = -EREMOTEIO;
1926 mlx5_cmd_err_trace(dev, opcode, op_mod, out);
1927 }
1928
1929 cmd_status_log(dev, opcode, status, syndrome, err);
1930 return err;
1931 }
1932
1933 /**
1934 * mlx5_cmd_do - Executes a fw command, wait for completion.
1935 * Unlike mlx5_cmd_exec, this function will not translate or intercept
1936 * outbox.status and will return -EREMOTEIO when
1937 * outbox.status != MLX5_CMD_STAT_OK
1938 *
1939 * @dev: mlx5 core device
1940 * @in: inbox mlx5_ifc command buffer
1941 * @in_size: inbox buffer size
1942 * @out: outbox mlx5_ifc buffer
1943 * @out_size: outbox size
1944 *
1945 * @return:
1946 * -EREMOTEIO : Command executed by FW, outbox.status != MLX5_CMD_STAT_OK.
1947 * Caller must check FW outbox status.
1948 * 0 : Command execution successful, outbox.status == MLX5_CMD_STAT_OK.
1949 * < 0 : Command execution couldn't be performed by firmware or driver
1950 */
mlx5_cmd_do(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size)1951 int mlx5_cmd_do(struct mlx5_core_dev *dev, void *in, int in_size, void *out, int out_size)
1952 {
1953 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, false);
1954 u16 opcode = MLX5_GET(mbox_in, in, opcode);
1955 u16 op_mod = MLX5_GET(mbox_in, in, op_mod);
1956
1957 return cmd_status_err(dev, err, opcode, op_mod, out);
1958 }
1959 EXPORT_SYMBOL(mlx5_cmd_do);
1960
1961 /**
1962 * mlx5_cmd_exec - Executes a fw command, wait for completion
1963 *
1964 * @dev: mlx5 core device
1965 * @in: inbox mlx5_ifc command buffer
1966 * @in_size: inbox buffer size
1967 * @out: outbox mlx5_ifc buffer
1968 * @out_size: outbox size
1969 *
1970 * @return: 0 if no error, FW command execution was successful
1971 * and outbox status is ok.
1972 */
mlx5_cmd_exec(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size)1973 int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
1974 int out_size)
1975 {
1976 int err = mlx5_cmd_do(dev, in, in_size, out, out_size);
1977
1978 return mlx5_cmd_check(dev, err, in, out);
1979 }
1980 EXPORT_SYMBOL(mlx5_cmd_exec);
1981
1982 /**
1983 * mlx5_cmd_exec_polling - Executes a fw command, poll for completion
1984 * Needed for driver force teardown, when command completion EQ
1985 * will not be available to complete the command
1986 *
1987 * @dev: mlx5 core device
1988 * @in: inbox mlx5_ifc command buffer
1989 * @in_size: inbox buffer size
1990 * @out: outbox mlx5_ifc buffer
1991 * @out_size: outbox size
1992 *
1993 * @return: 0 if no error, FW command execution was successful
1994 * and outbox status is ok.
1995 */
mlx5_cmd_exec_polling(struct mlx5_core_dev * dev,void * in,int in_size,void * out,int out_size)1996 int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size,
1997 void *out, int out_size)
1998 {
1999 int err = cmd_exec(dev, in, in_size, out, out_size, NULL, NULL, true);
2000 u16 opcode = MLX5_GET(mbox_in, in, opcode);
2001 u16 op_mod = MLX5_GET(mbox_in, in, op_mod);
2002
2003 err = cmd_status_err(dev, err, opcode, op_mod, out);
2004 return mlx5_cmd_check(dev, err, in, out);
2005 }
2006 EXPORT_SYMBOL(mlx5_cmd_exec_polling);
2007
mlx5_cmd_init_async_ctx(struct mlx5_core_dev * dev,struct mlx5_async_ctx * ctx)2008 void mlx5_cmd_init_async_ctx(struct mlx5_core_dev *dev,
2009 struct mlx5_async_ctx *ctx)
2010 {
2011 ctx->dev = dev;
2012 /* Starts at 1 to avoid doing wake_up if we are not cleaning up */
2013 atomic_set(&ctx->num_inflight, 1);
2014 init_completion(&ctx->inflight_done);
2015 }
2016 EXPORT_SYMBOL(mlx5_cmd_init_async_ctx);
2017
2018 /**
2019 * mlx5_cmd_cleanup_async_ctx - Clean up an async_ctx
2020 * @ctx: The ctx to clean
2021 *
2022 * Upon return all callbacks given to mlx5_cmd_exec_cb() have been called. The
2023 * caller must ensure that mlx5_cmd_exec_cb() is not called during or after
2024 * the call mlx5_cleanup_async_ctx().
2025 */
mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx * ctx)2026 void mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx *ctx)
2027 {
2028 if (!atomic_dec_and_test(&ctx->num_inflight))
2029 wait_for_completion(&ctx->inflight_done);
2030 }
2031 EXPORT_SYMBOL(mlx5_cmd_cleanup_async_ctx);
2032
mlx5_cmd_exec_cb_handler(int status,void * _work)2033 static void mlx5_cmd_exec_cb_handler(int status, void *_work)
2034 {
2035 struct mlx5_async_work *work = _work;
2036 struct mlx5_async_ctx *ctx;
2037
2038 ctx = work->ctx;
2039 status = cmd_status_err(ctx->dev, status, work->opcode, work->op_mod, work->out);
2040 work->user_callback(status, work);
2041 if (atomic_dec_and_test(&ctx->num_inflight))
2042 complete(&ctx->inflight_done);
2043 }
2044
mlx5_cmd_exec_cb(struct mlx5_async_ctx * ctx,void * in,int in_size,void * out,int out_size,mlx5_async_cbk_t callback,struct mlx5_async_work * work)2045 int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size,
2046 void *out, int out_size, mlx5_async_cbk_t callback,
2047 struct mlx5_async_work *work)
2048 {
2049 int ret;
2050
2051 work->ctx = ctx;
2052 work->user_callback = callback;
2053 work->opcode = MLX5_GET(mbox_in, in, opcode);
2054 work->op_mod = MLX5_GET(mbox_in, in, op_mod);
2055 work->out = out;
2056 if (WARN_ON(!atomic_inc_not_zero(&ctx->num_inflight)))
2057 return -EIO;
2058 ret = cmd_exec(ctx->dev, in, in_size, out, out_size,
2059 mlx5_cmd_exec_cb_handler, work, false);
2060 if (ret && atomic_dec_and_test(&ctx->num_inflight))
2061 complete(&ctx->inflight_done);
2062
2063 return ret;
2064 }
2065 EXPORT_SYMBOL(mlx5_cmd_exec_cb);
2066
destroy_msg_cache(struct mlx5_core_dev * dev)2067 static void destroy_msg_cache(struct mlx5_core_dev *dev)
2068 {
2069 struct cmd_msg_cache *ch;
2070 struct mlx5_cmd_msg *msg;
2071 struct mlx5_cmd_msg *n;
2072 int i;
2073
2074 for (i = 0; i < MLX5_NUM_COMMAND_CACHES; i++) {
2075 ch = &dev->cmd.cache[i];
2076 list_for_each_entry_safe(msg, n, &ch->head, list) {
2077 list_del(&msg->list);
2078 mlx5_free_cmd_msg(dev, msg);
2079 }
2080 }
2081 }
2082
2083 static unsigned cmd_cache_num_ent[MLX5_NUM_COMMAND_CACHES] = {
2084 512, 32, 16, 8, 2
2085 };
2086
2087 static unsigned cmd_cache_ent_size[MLX5_NUM_COMMAND_CACHES] = {
2088 16 + MLX5_CMD_DATA_BLOCK_SIZE,
2089 16 + MLX5_CMD_DATA_BLOCK_SIZE * 2,
2090 16 + MLX5_CMD_DATA_BLOCK_SIZE * 16,
2091 16 + MLX5_CMD_DATA_BLOCK_SIZE * 256,
2092 16 + MLX5_CMD_DATA_BLOCK_SIZE * 512,
2093 };
2094
create_msg_cache(struct mlx5_core_dev * dev)2095 static void create_msg_cache(struct mlx5_core_dev *dev)
2096 {
2097 struct mlx5_cmd *cmd = &dev->cmd;
2098 struct cmd_msg_cache *ch;
2099 struct mlx5_cmd_msg *msg;
2100 int i;
2101 int k;
2102
2103 /* Initialize and fill the caches with initial entries */
2104 for (k = 0; k < MLX5_NUM_COMMAND_CACHES; k++) {
2105 ch = &cmd->cache[k];
2106 spin_lock_init(&ch->lock);
2107 INIT_LIST_HEAD(&ch->head);
2108 ch->num_ent = cmd_cache_num_ent[k];
2109 ch->max_inbox_size = cmd_cache_ent_size[k];
2110 for (i = 0; i < ch->num_ent; i++) {
2111 msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL | __GFP_NOWARN,
2112 ch->max_inbox_size, 0);
2113 if (IS_ERR(msg))
2114 break;
2115 msg->parent = ch;
2116 list_add_tail(&msg->list, &ch->head);
2117 }
2118 }
2119 }
2120
alloc_cmd_page(struct mlx5_core_dev * dev,struct mlx5_cmd * cmd)2121 static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
2122 {
2123 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE,
2124 &cmd->alloc_dma, GFP_KERNEL);
2125 if (!cmd->cmd_alloc_buf)
2126 return -ENOMEM;
2127
2128 /* make sure it is aligned to 4K */
2129 if (!((uintptr_t)cmd->cmd_alloc_buf & (MLX5_ADAPTER_PAGE_SIZE - 1))) {
2130 cmd->cmd_buf = cmd->cmd_alloc_buf;
2131 cmd->dma = cmd->alloc_dma;
2132 cmd->alloc_size = MLX5_ADAPTER_PAGE_SIZE;
2133 return 0;
2134 }
2135
2136 dma_free_coherent(mlx5_core_dma_dev(dev), MLX5_ADAPTER_PAGE_SIZE, cmd->cmd_alloc_buf,
2137 cmd->alloc_dma);
2138 cmd->cmd_alloc_buf = dma_alloc_coherent(mlx5_core_dma_dev(dev),
2139 2 * MLX5_ADAPTER_PAGE_SIZE - 1,
2140 &cmd->alloc_dma, GFP_KERNEL);
2141 if (!cmd->cmd_alloc_buf)
2142 return -ENOMEM;
2143
2144 cmd->cmd_buf = PTR_ALIGN(cmd->cmd_alloc_buf, MLX5_ADAPTER_PAGE_SIZE);
2145 cmd->dma = ALIGN(cmd->alloc_dma, MLX5_ADAPTER_PAGE_SIZE);
2146 cmd->alloc_size = 2 * MLX5_ADAPTER_PAGE_SIZE - 1;
2147 return 0;
2148 }
2149
free_cmd_page(struct mlx5_core_dev * dev,struct mlx5_cmd * cmd)2150 static void free_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
2151 {
2152 dma_free_coherent(mlx5_core_dma_dev(dev), cmd->alloc_size, cmd->cmd_alloc_buf,
2153 cmd->alloc_dma);
2154 }
2155
cmdif_rev(struct mlx5_core_dev * dev)2156 static u16 cmdif_rev(struct mlx5_core_dev *dev)
2157 {
2158 return ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16;
2159 }
2160
mlx5_cmd_init(struct mlx5_core_dev * dev)2161 int mlx5_cmd_init(struct mlx5_core_dev *dev)
2162 {
2163 int size = sizeof(struct mlx5_cmd_prot_block);
2164 int align = roundup_pow_of_two(size);
2165 struct mlx5_cmd *cmd = &dev->cmd;
2166 u32 cmd_h, cmd_l;
2167 u16 cmd_if_rev;
2168 int err;
2169 int i;
2170
2171 memset(cmd, 0, sizeof(*cmd));
2172 cmd_if_rev = cmdif_rev(dev);
2173 if (cmd_if_rev != CMD_IF_REV) {
2174 mlx5_core_err(dev,
2175 "Driver cmdif rev(%d) differs from firmware's(%d)\n",
2176 CMD_IF_REV, cmd_if_rev);
2177 return -EINVAL;
2178 }
2179
2180 cmd->stats = kvcalloc(MLX5_CMD_OP_MAX, sizeof(*cmd->stats), GFP_KERNEL);
2181 if (!cmd->stats)
2182 return -ENOMEM;
2183
2184 cmd->pool = dma_pool_create("mlx5_cmd", mlx5_core_dma_dev(dev), size, align, 0);
2185 if (!cmd->pool) {
2186 err = -ENOMEM;
2187 goto dma_pool_err;
2188 }
2189
2190 err = alloc_cmd_page(dev, cmd);
2191 if (err)
2192 goto err_free_pool;
2193
2194 cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff;
2195 cmd->log_sz = cmd_l >> 4 & 0xf;
2196 cmd->log_stride = cmd_l & 0xf;
2197 if (1 << cmd->log_sz > MLX5_MAX_COMMANDS) {
2198 mlx5_core_err(dev, "firmware reports too many outstanding commands %d\n",
2199 1 << cmd->log_sz);
2200 err = -EINVAL;
2201 goto err_free_page;
2202 }
2203
2204 if (cmd->log_sz + cmd->log_stride > MLX5_ADAPTER_PAGE_SHIFT) {
2205 mlx5_core_err(dev, "command queue size overflow\n");
2206 err = -EINVAL;
2207 goto err_free_page;
2208 }
2209
2210 cmd->state = MLX5_CMDIF_STATE_DOWN;
2211 cmd->checksum_disabled = 1;
2212 cmd->max_reg_cmds = (1 << cmd->log_sz) - 1;
2213 cmd->bitmask = (1UL << cmd->max_reg_cmds) - 1;
2214
2215 cmd->cmdif_rev = ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16;
2216 if (cmd->cmdif_rev > CMD_IF_REV) {
2217 mlx5_core_err(dev, "driver does not support command interface version. driver %d, firmware %d\n",
2218 CMD_IF_REV, cmd->cmdif_rev);
2219 err = -EOPNOTSUPP;
2220 goto err_free_page;
2221 }
2222
2223 spin_lock_init(&cmd->alloc_lock);
2224 spin_lock_init(&cmd->token_lock);
2225 for (i = 0; i < MLX5_CMD_OP_MAX; i++)
2226 spin_lock_init(&cmd->stats[i].lock);
2227
2228 sema_init(&cmd->sem, cmd->max_reg_cmds);
2229 sema_init(&cmd->pages_sem, 1);
2230
2231 cmd_h = (u32)((u64)(cmd->dma) >> 32);
2232 cmd_l = (u32)(cmd->dma);
2233 if (cmd_l & 0xfff) {
2234 mlx5_core_err(dev, "invalid command queue address\n");
2235 err = -ENOMEM;
2236 goto err_free_page;
2237 }
2238
2239 iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h);
2240 iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz);
2241
2242 /* Make sure firmware sees the complete address before we proceed */
2243 wmb();
2244
2245 mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma));
2246
2247 cmd->mode = CMD_MODE_POLLING;
2248 cmd->allowed_opcode = CMD_ALLOWED_OPCODE_ALL;
2249
2250 create_msg_cache(dev);
2251
2252 set_wqname(dev);
2253 cmd->wq = create_singlethread_workqueue(cmd->wq_name);
2254 if (!cmd->wq) {
2255 mlx5_core_err(dev, "failed to create command workqueue\n");
2256 err = -ENOMEM;
2257 goto err_cache;
2258 }
2259
2260 create_debugfs_files(dev);
2261
2262 return 0;
2263
2264 err_cache:
2265 destroy_msg_cache(dev);
2266
2267 err_free_page:
2268 free_cmd_page(dev, cmd);
2269
2270 err_free_pool:
2271 dma_pool_destroy(cmd->pool);
2272 dma_pool_err:
2273 kvfree(cmd->stats);
2274 return err;
2275 }
2276
mlx5_cmd_cleanup(struct mlx5_core_dev * dev)2277 void mlx5_cmd_cleanup(struct mlx5_core_dev *dev)
2278 {
2279 struct mlx5_cmd *cmd = &dev->cmd;
2280
2281 clean_debug_files(dev);
2282 destroy_workqueue(cmd->wq);
2283 destroy_msg_cache(dev);
2284 free_cmd_page(dev, cmd);
2285 dma_pool_destroy(cmd->pool);
2286 kvfree(cmd->stats);
2287 }
2288
mlx5_cmd_set_state(struct mlx5_core_dev * dev,enum mlx5_cmdif_state cmdif_state)2289 void mlx5_cmd_set_state(struct mlx5_core_dev *dev,
2290 enum mlx5_cmdif_state cmdif_state)
2291 {
2292 dev->cmd.state = cmdif_state;
2293 }
2294
