422 	struct hns_roce_mtr *mtr = &mr->pbl_mtr;  in hns_roce_map_mr_sg()  local
438 	mtr->hem_cfg.region[0].offset = 0;  in hns_roce_map_mr_sg()
439 	mtr->hem_cfg.region[0].count = mr->npages;  in hns_roce_map_mr_sg()
440 	mtr->hem_cfg.region[0].hopnum = mr->pbl_hop_num;  in hns_roce_map_mr_sg()
441 	mtr->hem_cfg.region_count = 1;  in hns_roce_map_mr_sg()
442 	ret = hns_roce_mtr_map(hr_dev, mtr, mr->page_list, mr->npages);  in hns_roce_map_mr_sg()
572 static int mtr_map_region(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in mtr_map_region()  argument
587 		mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,  in mtr_map_region()
648 static void mtr_free_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)  in mtr_free_bufs()  argument
651 	if (mtr->umem) {  in mtr_free_bufs()
652 		ib_umem_release(mtr->umem);  in mtr_free_bufs()
653 		mtr->umem = NULL;  in mtr_free_bufs()
657 	if (mtr->kmem) {  in mtr_free_bufs()
658 		hns_roce_buf_free(hr_dev, mtr->kmem);  in mtr_free_bufs()
659 		mtr->kmem = NULL;  in mtr_free_bufs()
663 static int mtr_alloc_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in mtr_alloc_bufs()  argument
673 		mtr->kmem = NULL;  in mtr_alloc_bufs()
674 		mtr->umem = ib_umem_get(ibdev, user_addr, total_size,  in mtr_alloc_bufs()
676 		if (IS_ERR_OR_NULL(mtr->umem)) {  in mtr_alloc_bufs()
678 				  PTR_ERR(mtr->umem));  in mtr_alloc_bufs()
682 		mtr->umem = NULL;  in mtr_alloc_bufs()
683 		mtr->kmem = hns_roce_buf_alloc(hr_dev, total_size,  in mtr_alloc_bufs()
685 					       mtr->hem_cfg.is_direct ?  in mtr_alloc_bufs()
687 		if (IS_ERR(mtr->kmem)) {  in mtr_alloc_bufs()
689 				  PTR_ERR(mtr->kmem));  in mtr_alloc_bufs()
690 			return PTR_ERR(mtr->kmem);  in mtr_alloc_bufs()
697 static int mtr_map_bufs(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in mtr_map_bufs()  argument
710 	if (mtr->umem)  in mtr_map_bufs()
712 					       mtr->umem, page_shift);  in mtr_map_bufs()
715 					       mtr->kmem, page_shift);  in mtr_map_bufs()
724 	if (mtr->hem_cfg.is_direct && npage > 1) {  in mtr_map_bufs()
728 				  mtr->umem ? "umtr" : "kmtr", ret, npage);  in mtr_map_bufs()
734 	ret = hns_roce_mtr_map(hr_dev, mtr, pages, page_count);  in mtr_map_bufs()
744 int hns_roce_mtr_map(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in hns_roce_mtr_map()  argument
756 	if (mtr->hem_cfg.is_direct) {  in hns_roce_mtr_map()
757 		mtr->hem_cfg.root_ba = pages[0];  in hns_roce_mtr_map()
761 	for (i = 0, mapped_cnt = 0; i < mtr->hem_cfg.region_count &&  in hns_roce_mtr_map()
763 		r = &mtr->hem_cfg.region[i];  in hns_roce_mtr_map()
778 		ret = mtr_map_region(hr_dev, mtr, r, &pages[r->offset],  in hns_roce_mtr_map()
799 int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in hns_roce_mtr_find()  argument
802 	struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg;  in hns_roce_mtr_find()
835 		mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list,  in hns_roce_mtr_find()
912 static int mtr_alloc_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in mtr_alloc_mtt()  argument
915 	struct hns_roce_hem_cfg *cfg = &mtr->hem_cfg;  in mtr_alloc_mtt()
918 	hns_roce_hem_list_init(&mtr->hem_list);  in mtr_alloc_mtt()
920 		ret = hns_roce_hem_list_request(hr_dev, &mtr->hem_list,  in mtr_alloc_mtt()
925 		cfg->root_ba = mtr->hem_list.root_ba;  in mtr_alloc_mtt()
934 static void mtr_free_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)  in mtr_free_mtt()  argument
936 	hns_roce_hem_list_release(hr_dev, &mtr->hem_list);  in mtr_free_mtt()
949 int hns_roce_mtr_create(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,  in hns_roce_mtr_create()  argument
959 	buf_page_cnt = mtr_init_buf_cfg(hr_dev, buf_attr, &mtr->hem_cfg,  in hns_roce_mtr_create()
968 	ret = mtr_alloc_mtt(hr_dev, mtr, ba_page_shift);  in hns_roce_mtr_create()
978 		mtr->umem = NULL;  in hns_roce_mtr_create()
979 		mtr->kmem = NULL;  in hns_roce_mtr_create()
983 	ret = mtr_alloc_bufs(hr_dev, mtr, buf_attr, udata, user_addr);  in hns_roce_mtr_create()
990 	ret = mtr_map_bufs(hr_dev, mtr, buf_page_cnt, buf_page_shift);  in hns_roce_mtr_create()
996 	mtr_free_bufs(hr_dev, mtr);  in hns_roce_mtr_create()
998 	mtr_free_mtt(hr_dev, mtr);  in hns_roce_mtr_create()
1002 void hns_roce_mtr_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr)  in hns_roce_mtr_destroy()  argument
1005 	hns_roce_hem_list_release(hr_dev, &mtr->hem_list);  in hns_roce_mtr_destroy()
1008 	mtr_free_bufs(hr_dev, mtr);  in hns_roce_mtr_destroy()