386 				      struct mlx5_pagefault *pfault,  in mlx5_ib_page_fault_resume()  argument
389 	int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ?  in mlx5_ib_page_fault_resume()
390 		     pfault->wqe.wq_num : pfault->token;  in mlx5_ib_page_fault_resume()
395 	MLX5_SET(page_fault_resume_in, in, page_fault_type, pfault->type);  in mlx5_ib_page_fault_resume()
396 	MLX5_SET(page_fault_resume_in, in, token, pfault->token);  in mlx5_ib_page_fault_resume()
983 				   struct mlx5_pagefault *pfault,  in pagefault_data_segments()  argument
1024 					pfault->bytes_committed);  in pagefault_data_segments()
1031 		if (inline_segment || bcnt <= pfault->bytes_committed) {  in pagefault_data_segments()
1032 			pfault->bytes_committed -=  in pagefault_data_segments()
1034 				      pfault->bytes_committed);  in pagefault_data_segments()
1040 						    &pfault->bytes_committed,  in pagefault_data_segments()
1055 	struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault,  in mlx5_ib_mr_initiator_pfault_handler()  argument
1059 	u16 wqe_index = pfault->wqe.wqe_index;  in mlx5_ib_mr_initiator_pfault_handler()
1194 					  struct mlx5_pagefault *pfault)  in mlx5_ib_mr_wqe_pfault_handler()  argument
1196 	bool sq = pfault->type & MLX5_PFAULT_REQUESTOR;  in mlx5_ib_mr_wqe_pfault_handler()
1197 	u16 wqe_index = pfault->wqe.wqe_index;  in mlx5_ib_mr_wqe_pfault_handler()
1206 	res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type);  in mlx5_ib_mr_wqe_pfault_handler()
1208 		mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num);  in mlx5_ib_mr_wqe_pfault_handler()
1215 			    pfault->type);  in mlx5_ib_mr_wqe_pfault_handler()
1233 			dev, pfault, qp, &wqe, &wqe_end, bytes_copied);  in mlx5_ib_mr_wqe_pfault_handler()
1255 	ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, &bytes_mapped,  in mlx5_ib_mr_wqe_pfault_handler()
1272 			ret, wqe_index, pfault->token);  in mlx5_ib_mr_wqe_pfault_handler()
1275 	mlx5_ib_page_fault_resume(dev, pfault, resume_with_error);  in mlx5_ib_mr_wqe_pfault_handler()
1277 		    pfault->wqe.wq_num, resume_with_error,  in mlx5_ib_mr_wqe_pfault_handler()
1278 		    pfault->type);  in mlx5_ib_mr_wqe_pfault_handler()
1290 					   struct mlx5_pagefault *pfault)  in mlx5_ib_mr_rdma_pfault_handler()  argument
1294 	u32 prefetch_len = pfault->bytes_committed;  in mlx5_ib_mr_rdma_pfault_handler()
1296 	u32 rkey = pfault->rdma.r_key;  in mlx5_ib_mr_rdma_pfault_handler()
1305 	pfault->rdma.rdma_va += pfault->bytes_committed;  in mlx5_ib_mr_rdma_pfault_handler()
1306 	pfault->rdma.rdma_op_len -= min(pfault->bytes_committed,  in mlx5_ib_mr_rdma_pfault_handler()
1307 					 pfault->rdma.rdma_op_len);  in mlx5_ib_mr_rdma_pfault_handler()
1308 	pfault->bytes_committed = 0;  in mlx5_ib_mr_rdma_pfault_handler()
1310 	address = pfault->rdma.rdma_va;  in mlx5_ib_mr_rdma_pfault_handler()
1311 	length  = pfault->rdma.rdma_op_len;  in mlx5_ib_mr_rdma_pfault_handler()
1318 		length = pfault->rdma.packet_size;  in mlx5_ib_mr_rdma_pfault_handler()
1323 					    &pfault->bytes_committed, NULL);  in mlx5_ib_mr_rdma_pfault_handler()
1328 		mlx5_ib_page_fault_resume(dev, pfault, 1);  in mlx5_ib_mr_rdma_pfault_handler()
1331 				    ret, pfault->token, pfault->type);  in mlx5_ib_mr_rdma_pfault_handler()
1335 	mlx5_ib_page_fault_resume(dev, pfault, 0);  in mlx5_ib_mr_rdma_pfault_handler()
1337 		    pfault->token, pfault->type,  in mlx5_ib_mr_rdma_pfault_handler()
1353 				    ret, pfault->token, address, prefetch_len);  in mlx5_ib_mr_rdma_pfault_handler()
1358 static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault)  in mlx5_ib_pfault()  argument
1360 	u8 event_subtype = pfault->event_subtype;  in mlx5_ib_pfault()
1364 		mlx5_ib_mr_wqe_pfault_handler(dev, pfault);  in mlx5_ib_pfault()
1367 		mlx5_ib_mr_rdma_pfault_handler(dev, pfault);  in mlx5_ib_pfault()
1372 		mlx5_ib_page_fault_resume(dev, pfault, 1);  in mlx5_ib_pfault()
1378 	struct mlx5_pagefault *pfault = container_of(work,  in mlx5_ib_eqe_pf_action()  local
1381 	struct mlx5_ib_pf_eq *eq = pfault->eq;  in mlx5_ib_eqe_pf_action()
1383 	mlx5_ib_pfault(eq->dev, pfault);  in mlx5_ib_eqe_pf_action()
1384 	mempool_free(pfault, eq->pool);  in mlx5_ib_eqe_pf_action()
1390 	struct mlx5_pagefault *pfault;  in mlx5_ib_eq_pf_process()  local
1395 		pfault = mempool_alloc(eq->pool, GFP_ATOMIC);  in mlx5_ib_eq_pf_process()
1396 		if (!pfault) {  in mlx5_ib_eq_pf_process()
1402 		pfault->event_subtype = eqe->sub_type;  in mlx5_ib_eq_pf_process()
1403 		pfault->bytes_committed = be32_to_cpu(pf_eqe->bytes_committed);  in mlx5_ib_eq_pf_process()
1407 			    eqe->sub_type, pfault->bytes_committed);  in mlx5_ib_eq_pf_process()
1412 			pfault->type =  in mlx5_ib_eq_pf_process()
1414 			pfault->token =  in mlx5_ib_eq_pf_process()
1417 			pfault->rdma.r_key =  in mlx5_ib_eq_pf_process()
1419 			pfault->rdma.packet_size =  in mlx5_ib_eq_pf_process()
1421 			pfault->rdma.rdma_op_len =  in mlx5_ib_eq_pf_process()
1423 			pfault->rdma.rdma_va =  in mlx5_ib_eq_pf_process()
1427 				    pfault->type, pfault->token,  in mlx5_ib_eq_pf_process()
1428 				    pfault->rdma.r_key);  in mlx5_ib_eq_pf_process()
1431 				    pfault->rdma.rdma_op_len,  in mlx5_ib_eq_pf_process()
1432 				    pfault->rdma.rdma_va);  in mlx5_ib_eq_pf_process()
1437 			pfault->type =  in mlx5_ib_eq_pf_process()
1439 			pfault->token =  in mlx5_ib_eq_pf_process()
1441 			pfault->wqe.wq_num =  in mlx5_ib_eq_pf_process()
1444 			pfault->wqe.wqe_index =  in mlx5_ib_eq_pf_process()
1446 			pfault->wqe.packet_size =  in mlx5_ib_eq_pf_process()
1450 				    pfault->type, pfault->token,  in mlx5_ib_eq_pf_process()
1451 				    pfault->wqe.wq_num,  in mlx5_ib_eq_pf_process()
1452 				    pfault->wqe.wqe_index);  in mlx5_ib_eq_pf_process()
1464 		pfault->eq = eq;  in mlx5_ib_eq_pf_process()
1465 		INIT_WORK(&pfault->work, mlx5_ib_eqe_pf_action);  in mlx5_ib_eq_pf_process()
1466 		queue_work(eq->wq, &pfault->work);  in mlx5_ib_eq_pf_process()