445 				      struct mlx5_pagefault *pfault,  in mlx5_ib_page_fault_resume()  argument
448 	int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ?  in mlx5_ib_page_fault_resume()
449 		     pfault->wqe.wq_num : pfault->token;  in mlx5_ib_page_fault_resume()
454 	MLX5_SET(page_fault_resume_in, in, page_fault_type, pfault->type);  in mlx5_ib_page_fault_resume()
455 	MLX5_SET(page_fault_resume_in, in, token, pfault->token);  in mlx5_ib_page_fault_resume()
1056 				   struct mlx5_pagefault *pfault,  in pagefault_data_segments()  argument
1097 					pfault->bytes_committed);  in pagefault_data_segments()
1104 		if (inline_segment || bcnt <= pfault->bytes_committed) {  in pagefault_data_segments()
1105 			pfault->bytes_committed -=  in pagefault_data_segments()
1107 				      pfault->bytes_committed);  in pagefault_data_segments()
1113 						    &pfault->bytes_committed,  in pagefault_data_segments()
1128 	struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault,  in mlx5_ib_mr_initiator_pfault_handler()  argument
1132 	u16 wqe_index = pfault->wqe.wqe_index;  in mlx5_ib_mr_initiator_pfault_handler()
1267 					  struct mlx5_pagefault *pfault)  in mlx5_ib_mr_wqe_pfault_handler()  argument
1269 	bool sq = pfault->type & MLX5_PFAULT_REQUESTOR;  in mlx5_ib_mr_wqe_pfault_handler()
1270 	u16 wqe_index = pfault->wqe.wqe_index;  in mlx5_ib_mr_wqe_pfault_handler()
1279 	res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type);  in mlx5_ib_mr_wqe_pfault_handler()
1281 		mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num);  in mlx5_ib_mr_wqe_pfault_handler()
1288 			    pfault->type);  in mlx5_ib_mr_wqe_pfault_handler()
1306 			dev, pfault, qp, &wqe, &wqe_end, bytes_copied);  in mlx5_ib_mr_wqe_pfault_handler()
1328 	ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, &bytes_mapped,  in mlx5_ib_mr_wqe_pfault_handler()
1345 			ret, wqe_index, pfault->token);  in mlx5_ib_mr_wqe_pfault_handler()
1348 	mlx5_ib_page_fault_resume(dev, pfault, resume_with_error);  in mlx5_ib_mr_wqe_pfault_handler()
1350 		    pfault->wqe.wq_num, resume_with_error,  in mlx5_ib_mr_wqe_pfault_handler()
1351 		    pfault->type);  in mlx5_ib_mr_wqe_pfault_handler()
1363 					   struct mlx5_pagefault *pfault)  in mlx5_ib_mr_rdma_pfault_handler()  argument
1367 	u32 prefetch_len = pfault->bytes_committed;  in mlx5_ib_mr_rdma_pfault_handler()
1369 	u32 rkey = pfault->rdma.r_key;  in mlx5_ib_mr_rdma_pfault_handler()
1378 	pfault->rdma.rdma_va += pfault->bytes_committed;  in mlx5_ib_mr_rdma_pfault_handler()
1379 	pfault->rdma.rdma_op_len -= min(pfault->bytes_committed,  in mlx5_ib_mr_rdma_pfault_handler()
1380 					 pfault->rdma.rdma_op_len);  in mlx5_ib_mr_rdma_pfault_handler()
1381 	pfault->bytes_committed = 0;  in mlx5_ib_mr_rdma_pfault_handler()
1383 	address = pfault->rdma.rdma_va;  in mlx5_ib_mr_rdma_pfault_handler()
1384 	length  = pfault->rdma.rdma_op_len;  in mlx5_ib_mr_rdma_pfault_handler()
1391 		length = pfault->rdma.packet_size;  in mlx5_ib_mr_rdma_pfault_handler()
1396 					    &pfault->bytes_committed, NULL);  in mlx5_ib_mr_rdma_pfault_handler()
1401 		mlx5_ib_page_fault_resume(dev, pfault, 1);  in mlx5_ib_mr_rdma_pfault_handler()
1404 				    ret, pfault->token, pfault->type);  in mlx5_ib_mr_rdma_pfault_handler()
1408 	mlx5_ib_page_fault_resume(dev, pfault, 0);  in mlx5_ib_mr_rdma_pfault_handler()
1410 		    pfault->token, pfault->type,  in mlx5_ib_mr_rdma_pfault_handler()
1426 				    ret, pfault->token, address, prefetch_len);  in mlx5_ib_mr_rdma_pfault_handler()
1431 static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault)  in mlx5_ib_pfault()  argument
1433 	u8 event_subtype = pfault->event_subtype;  in mlx5_ib_pfault()
1437 		mlx5_ib_mr_wqe_pfault_handler(dev, pfault);  in mlx5_ib_pfault()
1440 		mlx5_ib_mr_rdma_pfault_handler(dev, pfault);  in mlx5_ib_pfault()
1445 		mlx5_ib_page_fault_resume(dev, pfault, 1);  in mlx5_ib_pfault()
1451 	struct mlx5_pagefault *pfault = container_of(work,  in mlx5_ib_eqe_pf_action()  local
1454 	struct mlx5_ib_pf_eq *eq = pfault->eq;  in mlx5_ib_eqe_pf_action()
1456 	mlx5_ib_pfault(eq->dev, pfault);  in mlx5_ib_eqe_pf_action()
1457 	mempool_free(pfault, eq->pool);  in mlx5_ib_eqe_pf_action()
1463 	struct mlx5_pagefault *pfault;  in mlx5_ib_eq_pf_process()  local
1468 		pfault = mempool_alloc(eq->pool, GFP_ATOMIC);  in mlx5_ib_eq_pf_process()
1469 		if (!pfault) {  in mlx5_ib_eq_pf_process()
1475 		pfault->event_subtype = eqe->sub_type;  in mlx5_ib_eq_pf_process()
1476 		pfault->bytes_committed = be32_to_cpu(pf_eqe->bytes_committed);  in mlx5_ib_eq_pf_process()
1480 			    eqe->sub_type, pfault->bytes_committed);  in mlx5_ib_eq_pf_process()
1485 			pfault->type =  in mlx5_ib_eq_pf_process()
1487 			pfault->token =  in mlx5_ib_eq_pf_process()
1490 			pfault->rdma.r_key =  in mlx5_ib_eq_pf_process()
1492 			pfault->rdma.packet_size =  in mlx5_ib_eq_pf_process()
1494 			pfault->rdma.rdma_op_len =  in mlx5_ib_eq_pf_process()
1496 			pfault->rdma.rdma_va =  in mlx5_ib_eq_pf_process()
1500 				    pfault->type, pfault->token,  in mlx5_ib_eq_pf_process()
1501 				    pfault->rdma.r_key);  in mlx5_ib_eq_pf_process()
1504 				    pfault->rdma.rdma_op_len,  in mlx5_ib_eq_pf_process()
1505 				    pfault->rdma.rdma_va);  in mlx5_ib_eq_pf_process()
1510 			pfault->type =  in mlx5_ib_eq_pf_process()
1512 			pfault->token =  in mlx5_ib_eq_pf_process()
1514 			pfault->wqe.wq_num =  in mlx5_ib_eq_pf_process()
1517 			pfault->wqe.wqe_index =  in mlx5_ib_eq_pf_process()
1519 			pfault->wqe.packet_size =  in mlx5_ib_eq_pf_process()
1523 				    pfault->type, pfault->token,  in mlx5_ib_eq_pf_process()
1524 				    pfault->wqe.wq_num,  in mlx5_ib_eq_pf_process()
1525 				    pfault->wqe.wqe_index);  in mlx5_ib_eq_pf_process()
1537 		pfault->eq = eq;  in mlx5_ib_eq_pf_process()
1538 		INIT_WORK(&pfault->work, mlx5_ib_eqe_pf_action);  in mlx5_ib_eq_pf_process()
1539 		queue_work(eq->wq, &pfault->work);  in mlx5_ib_eq_pf_process()