387 				      struct mlx5_pagefault *pfault,  in mlx5_ib_page_fault_resume()  argument
390 	int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ?  in mlx5_ib_page_fault_resume()
391 		     pfault->wqe.wq_num : pfault->token;  in mlx5_ib_page_fault_resume()
396 	MLX5_SET(page_fault_resume_in, in, page_fault_type, pfault->type);  in mlx5_ib_page_fault_resume()
397 	MLX5_SET(page_fault_resume_in, in, token, pfault->token);  in mlx5_ib_page_fault_resume()
998 				   struct mlx5_pagefault *pfault,  in pagefault_data_segments()  argument
1039 					pfault->bytes_committed);  in pagefault_data_segments()
1046 		if (inline_segment || bcnt <= pfault->bytes_committed) {  in pagefault_data_segments()
1047 			pfault->bytes_committed -=  in pagefault_data_segments()
1049 				      pfault->bytes_committed);  in pagefault_data_segments()
1055 						    &pfault->bytes_committed,  in pagefault_data_segments()
1070 	struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault,  in mlx5_ib_mr_initiator_pfault_handler()  argument
1074 	u16 wqe_index = pfault->wqe.wqe_index;  in mlx5_ib_mr_initiator_pfault_handler()
1209 					  struct mlx5_pagefault *pfault)  in mlx5_ib_mr_wqe_pfault_handler()  argument
1211 	bool sq = pfault->type & MLX5_PFAULT_REQUESTOR;  in mlx5_ib_mr_wqe_pfault_handler()
1212 	u16 wqe_index = pfault->wqe.wqe_index;  in mlx5_ib_mr_wqe_pfault_handler()
1221 	res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type);  in mlx5_ib_mr_wqe_pfault_handler()
1223 		mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num);  in mlx5_ib_mr_wqe_pfault_handler()
1230 			    pfault->type);  in mlx5_ib_mr_wqe_pfault_handler()
1248 			dev, pfault, qp, &wqe, &wqe_end, bytes_copied);  in mlx5_ib_mr_wqe_pfault_handler()
1270 	ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, &bytes_mapped,  in mlx5_ib_mr_wqe_pfault_handler()
1287 			ret, wqe_index, pfault->token);  in mlx5_ib_mr_wqe_pfault_handler()
1290 	mlx5_ib_page_fault_resume(dev, pfault, resume_with_error);  in mlx5_ib_mr_wqe_pfault_handler()
1292 		    pfault->wqe.wq_num, resume_with_error,  in mlx5_ib_mr_wqe_pfault_handler()
1293 		    pfault->type);  in mlx5_ib_mr_wqe_pfault_handler()
1305 					   struct mlx5_pagefault *pfault)  in mlx5_ib_mr_rdma_pfault_handler()  argument
1309 	u32 prefetch_len = pfault->bytes_committed;  in mlx5_ib_mr_rdma_pfault_handler()
1311 	u32 rkey = pfault->rdma.r_key;  in mlx5_ib_mr_rdma_pfault_handler()
1320 	pfault->rdma.rdma_va += pfault->bytes_committed;  in mlx5_ib_mr_rdma_pfault_handler()
1321 	pfault->rdma.rdma_op_len -= min(pfault->bytes_committed,  in mlx5_ib_mr_rdma_pfault_handler()
1322 					 pfault->rdma.rdma_op_len);  in mlx5_ib_mr_rdma_pfault_handler()
1323 	pfault->bytes_committed = 0;  in mlx5_ib_mr_rdma_pfault_handler()
1325 	address = pfault->rdma.rdma_va;  in mlx5_ib_mr_rdma_pfault_handler()
1326 	length  = pfault->rdma.rdma_op_len;  in mlx5_ib_mr_rdma_pfault_handler()
1333 		length = pfault->rdma.packet_size;  in mlx5_ib_mr_rdma_pfault_handler()
1338 					    &pfault->bytes_committed, NULL);  in mlx5_ib_mr_rdma_pfault_handler()
1343 		mlx5_ib_page_fault_resume(dev, pfault, 1);  in mlx5_ib_mr_rdma_pfault_handler()
1346 				    ret, pfault->token, pfault->type);  in mlx5_ib_mr_rdma_pfault_handler()
1350 	mlx5_ib_page_fault_resume(dev, pfault, 0);  in mlx5_ib_mr_rdma_pfault_handler()
1352 		    pfault->token, pfault->type,  in mlx5_ib_mr_rdma_pfault_handler()
1368 				    ret, pfault->token, address, prefetch_len);  in mlx5_ib_mr_rdma_pfault_handler()
1373 static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault)  in mlx5_ib_pfault()  argument
1375 	u8 event_subtype = pfault->event_subtype;  in mlx5_ib_pfault()
1379 		mlx5_ib_mr_wqe_pfault_handler(dev, pfault);  in mlx5_ib_pfault()
1382 		mlx5_ib_mr_rdma_pfault_handler(dev, pfault);  in mlx5_ib_pfault()
1387 		mlx5_ib_page_fault_resume(dev, pfault, 1);  in mlx5_ib_pfault()
1393 	struct mlx5_pagefault *pfault = container_of(work,  in mlx5_ib_eqe_pf_action()  local
1396 	struct mlx5_ib_pf_eq *eq = pfault->eq;  in mlx5_ib_eqe_pf_action()
1398 	mlx5_ib_pfault(eq->dev, pfault);  in mlx5_ib_eqe_pf_action()
1399 	mempool_free(pfault, eq->pool);  in mlx5_ib_eqe_pf_action()
1405 	struct mlx5_pagefault *pfault;  in mlx5_ib_eq_pf_process()  local
1410 		pfault = mempool_alloc(eq->pool, GFP_ATOMIC);  in mlx5_ib_eq_pf_process()
1411 		if (!pfault) {  in mlx5_ib_eq_pf_process()
1417 		pfault->event_subtype = eqe->sub_type;  in mlx5_ib_eq_pf_process()
1418 		pfault->bytes_committed = be32_to_cpu(pf_eqe->bytes_committed);  in mlx5_ib_eq_pf_process()
1422 			    eqe->sub_type, pfault->bytes_committed);  in mlx5_ib_eq_pf_process()
1427 			pfault->type =  in mlx5_ib_eq_pf_process()
1429 			pfault->token =  in mlx5_ib_eq_pf_process()
1432 			pfault->rdma.r_key =  in mlx5_ib_eq_pf_process()
1434 			pfault->rdma.packet_size =  in mlx5_ib_eq_pf_process()
1436 			pfault->rdma.rdma_op_len =  in mlx5_ib_eq_pf_process()
1438 			pfault->rdma.rdma_va =  in mlx5_ib_eq_pf_process()
1442 				    pfault->type, pfault->token,  in mlx5_ib_eq_pf_process()
1443 				    pfault->rdma.r_key);  in mlx5_ib_eq_pf_process()
1446 				    pfault->rdma.rdma_op_len,  in mlx5_ib_eq_pf_process()
1447 				    pfault->rdma.rdma_va);  in mlx5_ib_eq_pf_process()
1452 			pfault->type =  in mlx5_ib_eq_pf_process()
1454 			pfault->token =  in mlx5_ib_eq_pf_process()
1456 			pfault->wqe.wq_num =  in mlx5_ib_eq_pf_process()
1459 			pfault->wqe.wqe_index =  in mlx5_ib_eq_pf_process()
1461 			pfault->wqe.packet_size =  in mlx5_ib_eq_pf_process()
1465 				    pfault->type, pfault->token,  in mlx5_ib_eq_pf_process()
1466 				    pfault->wqe.wq_num,  in mlx5_ib_eq_pf_process()
1467 				    pfault->wqe.wqe_index);  in mlx5_ib_eq_pf_process()
1479 		pfault->eq = eq;  in mlx5_ib_eq_pf_process()
1480 		INIT_WORK(&pfault->work, mlx5_ib_eqe_pf_action);  in mlx5_ib_eq_pf_process()
1481 		queue_work(eq->wq, &pfault->work);  in mlx5_ib_eq_pf_process()