398 				      struct mlx5_pagefault *pfault,  in mlx5_ib_page_fault_resume()  argument
401 	int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ?  in mlx5_ib_page_fault_resume()
402 		     pfault->wqe.wq_num : pfault->token;  in mlx5_ib_page_fault_resume()
408 	MLX5_SET(page_fault_resume_in, in, page_fault_type, pfault->type);  in mlx5_ib_page_fault_resume()
409 	MLX5_SET(page_fault_resume_in, in, token, pfault->token);  in mlx5_ib_page_fault_resume()
954 				   struct mlx5_pagefault *pfault,  in pagefault_data_segments()  argument
995 					pfault->bytes_committed);  in pagefault_data_segments()
1002 		if (inline_segment || bcnt <= pfault->bytes_committed) {  in pagefault_data_segments()
1003 			pfault->bytes_committed -=  in pagefault_data_segments()
1005 				      pfault->bytes_committed);  in pagefault_data_segments()
1011 						    &pfault->bytes_committed,  in pagefault_data_segments()
1026 	struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault,  in mlx5_ib_mr_initiator_pfault_handler()  argument
1030 	u16 wqe_index = pfault->wqe.wqe_index;  in mlx5_ib_mr_initiator_pfault_handler()
1166 					  struct mlx5_pagefault *pfault)  in mlx5_ib_mr_wqe_pfault_handler()  argument
1168 	bool sq = pfault->type & MLX5_PFAULT_REQUESTOR;  in mlx5_ib_mr_wqe_pfault_handler()
1169 	u16 wqe_index = pfault->wqe.wqe_index;  in mlx5_ib_mr_wqe_pfault_handler()
1178 	res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type);  in mlx5_ib_mr_wqe_pfault_handler()
1180 		mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num);  in mlx5_ib_mr_wqe_pfault_handler()
1187 			    pfault->type);  in mlx5_ib_mr_wqe_pfault_handler()
1205 			dev, pfault, qp, &wqe, &wqe_end, bytes_copied);  in mlx5_ib_mr_wqe_pfault_handler()
1227 	ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, &bytes_mapped,  in mlx5_ib_mr_wqe_pfault_handler()
1244 			ret, wqe_index, pfault->token);  in mlx5_ib_mr_wqe_pfault_handler()
1247 	mlx5_ib_page_fault_resume(dev, pfault, resume_with_error);  in mlx5_ib_mr_wqe_pfault_handler()
1249 		    pfault->wqe.wq_num, resume_with_error,  in mlx5_ib_mr_wqe_pfault_handler()
1250 		    pfault->type);  in mlx5_ib_mr_wqe_pfault_handler()
1262 					   struct mlx5_pagefault *pfault)  in mlx5_ib_mr_rdma_pfault_handler()  argument
1266 	u32 prefetch_len = pfault->bytes_committed;  in mlx5_ib_mr_rdma_pfault_handler()
1268 	u32 rkey = pfault->rdma.r_key;  in mlx5_ib_mr_rdma_pfault_handler()
1277 	pfault->rdma.rdma_va += pfault->bytes_committed;  in mlx5_ib_mr_rdma_pfault_handler()
1278 	pfault->rdma.rdma_op_len -= min(pfault->bytes_committed,  in mlx5_ib_mr_rdma_pfault_handler()
1279 					 pfault->rdma.rdma_op_len);  in mlx5_ib_mr_rdma_pfault_handler()
1280 	pfault->bytes_committed = 0;  in mlx5_ib_mr_rdma_pfault_handler()
1282 	address = pfault->rdma.rdma_va;  in mlx5_ib_mr_rdma_pfault_handler()
1283 	length  = pfault->rdma.rdma_op_len;  in mlx5_ib_mr_rdma_pfault_handler()
1290 		length = pfault->rdma.packet_size;  in mlx5_ib_mr_rdma_pfault_handler()
1295 					    &pfault->bytes_committed, NULL,  in mlx5_ib_mr_rdma_pfault_handler()
1301 		mlx5_ib_page_fault_resume(dev, pfault, 1);  in mlx5_ib_mr_rdma_pfault_handler()
1304 				    ret, pfault->token, pfault->type);  in mlx5_ib_mr_rdma_pfault_handler()
1308 	mlx5_ib_page_fault_resume(dev, pfault, 0);  in mlx5_ib_mr_rdma_pfault_handler()
1310 		    pfault->token, pfault->type,  in mlx5_ib_mr_rdma_pfault_handler()
1327 				    ret, pfault->token, address, prefetch_len);  in mlx5_ib_mr_rdma_pfault_handler()
1332 static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault)  in mlx5_ib_pfault()  argument
1334 	u8 event_subtype = pfault->event_subtype;  in mlx5_ib_pfault()
1338 		mlx5_ib_mr_wqe_pfault_handler(dev, pfault);  in mlx5_ib_pfault()
1341 		mlx5_ib_mr_rdma_pfault_handler(dev, pfault);  in mlx5_ib_pfault()
1346 		mlx5_ib_page_fault_resume(dev, pfault, 1);  in mlx5_ib_pfault()
1352 	struct mlx5_pagefault *pfault = container_of(work,  in mlx5_ib_eqe_pf_action()  local
1355 	struct mlx5_ib_pf_eq *eq = pfault->eq;  in mlx5_ib_eqe_pf_action()
1357 	mlx5_ib_pfault(eq->dev, pfault);  in mlx5_ib_eqe_pf_action()
1358 	mempool_free(pfault, eq->pool);  in mlx5_ib_eqe_pf_action()
1364 	struct mlx5_pagefault *pfault;  in mlx5_ib_eq_pf_process()  local
1369 		pfault = mempool_alloc(eq->pool, GFP_ATOMIC);  in mlx5_ib_eq_pf_process()
1370 		if (!pfault) {  in mlx5_ib_eq_pf_process()
1376 		pfault->event_subtype = eqe->sub_type;  in mlx5_ib_eq_pf_process()
1377 		pfault->bytes_committed = be32_to_cpu(pf_eqe->bytes_committed);  in mlx5_ib_eq_pf_process()
1381 			    eqe->sub_type, pfault->bytes_committed);  in mlx5_ib_eq_pf_process()
1386 			pfault->type =  in mlx5_ib_eq_pf_process()
1388 			pfault->token =  in mlx5_ib_eq_pf_process()
1391 			pfault->rdma.r_key =  in mlx5_ib_eq_pf_process()
1393 			pfault->rdma.packet_size =  in mlx5_ib_eq_pf_process()
1395 			pfault->rdma.rdma_op_len =  in mlx5_ib_eq_pf_process()
1397 			pfault->rdma.rdma_va =  in mlx5_ib_eq_pf_process()
1401 				    pfault->type, pfault->token,  in mlx5_ib_eq_pf_process()
1402 				    pfault->rdma.r_key);  in mlx5_ib_eq_pf_process()
1405 				    pfault->rdma.rdma_op_len,  in mlx5_ib_eq_pf_process()
1406 				    pfault->rdma.rdma_va);  in mlx5_ib_eq_pf_process()
1411 			pfault->type =  in mlx5_ib_eq_pf_process()
1413 			pfault->token =  in mlx5_ib_eq_pf_process()
1415 			pfault->wqe.wq_num =  in mlx5_ib_eq_pf_process()
1418 			pfault->wqe.wqe_index =  in mlx5_ib_eq_pf_process()
1420 			pfault->wqe.packet_size =  in mlx5_ib_eq_pf_process()
1424 				    pfault->type, pfault->token,  in mlx5_ib_eq_pf_process()
1425 				    pfault->wqe.wq_num,  in mlx5_ib_eq_pf_process()
1426 				    pfault->wqe.wqe_index);  in mlx5_ib_eq_pf_process()
1438 		pfault->eq = eq;  in mlx5_ib_eq_pf_process()
1439 		INIT_WORK(&pfault->work, mlx5_ib_eqe_pf_action);  in mlx5_ib_eq_pf_process()
1440 		queue_work(eq->wq, &pfault->work);  in mlx5_ib_eq_pf_process()