53 			 struct smc_buf_desc *buf_desc);
1107 static void smcr_buf_unuse(struct smc_buf_desc *buf_desc, bool is_rmb,  in smcr_buf_unuse()  argument
1113 	if (is_rmb && buf_desc->is_conf_rkey && !list_empty(&lgr->list)) {  in smcr_buf_unuse()
1119 			smc_llc_do_delete_rkey(lgr, buf_desc);  in smcr_buf_unuse()
1120 			buf_desc->is_conf_rkey = false;  in smcr_buf_unuse()
1126 	if (buf_desc->is_reg_err) {  in smcr_buf_unuse()
1131 		list_del(&buf_desc->list);  in smcr_buf_unuse()
1134 		smc_buf_free(lgr, is_rmb, buf_desc);  in smcr_buf_unuse()
1137 		memzero_explicit(buf_desc->cpu_addr, buf_desc->len);  in smcr_buf_unuse()
1138 		WRITE_ONCE(buf_desc->used, 0);  in smcr_buf_unuse()
1205 static void smcr_buf_unmap_link(struct smc_buf_desc *buf_desc, bool is_rmb,  in smcr_buf_unmap_link()  argument
1208 	if (is_rmb || buf_desc->is_vm)  in smcr_buf_unmap_link()
1209 		buf_desc->is_reg_mr[lnk->link_idx] = false;  in smcr_buf_unmap_link()
1210 	if (!buf_desc->is_map_ib[lnk->link_idx])  in smcr_buf_unmap_link()
1213 	if ((is_rmb || buf_desc->is_vm) &&  in smcr_buf_unmap_link()
1214 	    buf_desc->mr[lnk->link_idx]) {  in smcr_buf_unmap_link()
1215 		smc_ib_put_memory_region(buf_desc->mr[lnk->link_idx]);  in smcr_buf_unmap_link()
1216 		buf_desc->mr[lnk->link_idx] = NULL;  in smcr_buf_unmap_link()
1219 		smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_FROM_DEVICE);  in smcr_buf_unmap_link()
1221 		smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_TO_DEVICE);  in smcr_buf_unmap_link()
1223 	sg_free_table(&buf_desc->sgt[lnk->link_idx]);  in smcr_buf_unmap_link()
1224 	buf_desc->is_map_ib[lnk->link_idx] = false;  in smcr_buf_unmap_link()
1231 	struct smc_buf_desc *buf_desc, *bf;  in smcr_buf_unmap_lgr()  local
1236 		list_for_each_entry_safe(buf_desc, bf, &lgr->rmbs[i], list)  in smcr_buf_unmap_lgr()
1237 			smcr_buf_unmap_link(buf_desc, true, lnk);  in smcr_buf_unmap_lgr()
1241 		list_for_each_entry_safe(buf_desc, bf, &lgr->sndbufs[i],  in smcr_buf_unmap_lgr()
1243 			smcr_buf_unmap_link(buf_desc, false, lnk);  in smcr_buf_unmap_lgr()
1305 			  struct smc_buf_desc *buf_desc)  in smcr_buf_free()  argument
1310 		smcr_buf_unmap_link(buf_desc, is_rmb, &lgr->lnk[i]);  in smcr_buf_free()
1312 	if (!buf_desc->is_vm && buf_desc->pages)  in smcr_buf_free()
1313 		__free_pages(buf_desc->pages, buf_desc->order);  in smcr_buf_free()
1314 	else if (buf_desc->is_vm && buf_desc->cpu_addr)  in smcr_buf_free()
1315 		vfree(buf_desc->cpu_addr);  in smcr_buf_free()
1316 	kfree(buf_desc);  in smcr_buf_free()
1320 			  struct smc_buf_desc *buf_desc)  in smcd_buf_free()  argument
1324 		buf_desc->len += sizeof(struct smcd_cdc_msg);  in smcd_buf_free()
1325 		smc_ism_unregister_dmb(lgr->smcd, buf_desc);  in smcd_buf_free()
1327 		kfree(buf_desc->cpu_addr);  in smcd_buf_free()
1329 	kfree(buf_desc);  in smcd_buf_free()
1333 			 struct smc_buf_desc *buf_desc)  in smc_buf_free()  argument
1336 		smcd_buf_free(lgr, is_rmb, buf_desc);  in smc_buf_free()
1338 		smcr_buf_free(lgr, is_rmb, buf_desc);  in smc_buf_free()
1343 	struct smc_buf_desc *buf_desc, *bf_desc;  in __smc_lgr_free_bufs()  local
1352 		list_for_each_entry_safe(buf_desc, bf_desc, buf_list,  in __smc_lgr_free_bufs()
1354 			list_del(&buf_desc->list);  in __smc_lgr_free_bufs()
1355 			smc_buf_free(lgr, is_rmb, buf_desc);  in __smc_lgr_free_bufs()
2033 static int smcr_buf_map_link(struct smc_buf_desc *buf_desc, bool is_rmb,  in smcr_buf_map_link()  argument
2040 	if (buf_desc->is_map_ib[lnk->link_idx])  in smcr_buf_map_link()
2043 	if (buf_desc->is_vm) {  in smcr_buf_map_link()
2044 		buf = buf_desc->cpu_addr;  in smcr_buf_map_link()
2045 		buf_size = buf_desc->len;  in smcr_buf_map_link()
2046 		offset = offset_in_page(buf_desc->cpu_addr);  in smcr_buf_map_link()
2052 	rc = sg_alloc_table(&buf_desc->sgt[lnk->link_idx], nents, GFP_KERNEL);  in smcr_buf_map_link()
2056 	if (buf_desc->is_vm) {  in smcr_buf_map_link()
2058 		for_each_sg(buf_desc->sgt[lnk->link_idx].sgl, sg, nents, i) {  in smcr_buf_map_link()
2067 		sg_set_buf(buf_desc->sgt[lnk->link_idx].sgl,  in smcr_buf_map_link()
2068 			   buf_desc->cpu_addr, buf_desc->len);  in smcr_buf_map_link()
2072 	rc = smc_ib_buf_map_sg(lnk, buf_desc,  in smcr_buf_map_link()
2080 	buf_desc->is_dma_need_sync |=  in smcr_buf_map_link()
2081 		smc_ib_is_sg_need_sync(lnk, buf_desc) << lnk->link_idx;  in smcr_buf_map_link()
2083 	if (is_rmb || buf_desc->is_vm) {  in smcr_buf_map_link()
2090 					      buf_desc, lnk->link_idx);  in smcr_buf_map_link()
2093 		smc_ib_sync_sg_for_device(lnk, buf_desc,  in smcr_buf_map_link()
2096 	buf_desc->is_map_ib[lnk->link_idx] = true;  in smcr_buf_map_link()
2100 	smc_ib_buf_unmap_sg(lnk, buf_desc,  in smcr_buf_map_link()
2103 	sg_free_table(&buf_desc->sgt[lnk->link_idx]);  in smcr_buf_map_link()
2110 int smcr_link_reg_buf(struct smc_link *link, struct smc_buf_desc *buf_desc)  in smcr_link_reg_buf()  argument
2114 	if (!buf_desc->is_reg_mr[link->link_idx]) {  in smcr_link_reg_buf()
2116 		if (buf_desc->is_vm)  in smcr_link_reg_buf()
2117 			buf_desc->mr[link->link_idx]->iova =  in smcr_link_reg_buf()
2118 				(uintptr_t)buf_desc->cpu_addr;  in smcr_link_reg_buf()
2119 		if (smc_wr_reg_send(link, buf_desc->mr[link->link_idx])) {  in smcr_link_reg_buf()
2120 			buf_desc->is_reg_err = true;  in smcr_link_reg_buf()
2123 		buf_desc->is_reg_mr[link->link_idx] = true;  in smcr_link_reg_buf()
2131 	struct smc_buf_desc *buf_desc, *bf;  in _smcr_buf_map_lgr()  local
2135 	list_for_each_entry_safe(buf_desc, bf, lst, list) {  in _smcr_buf_map_lgr()
2136 		if (!buf_desc->used)  in _smcr_buf_map_lgr()
2138 		rc = smcr_buf_map_link(buf_desc, is_rmb, lnk);  in _smcr_buf_map_lgr()
2172 	struct smc_buf_desc *buf_desc, *bf;  in smcr_buf_reg_lgr()  local
2178 		list_for_each_entry_safe(buf_desc, bf, &lgr->rmbs[i], list) {  in smcr_buf_reg_lgr()
2179 			if (!buf_desc->used)  in smcr_buf_reg_lgr()
2181 			rc = smcr_link_reg_buf(lnk, buf_desc);  in smcr_buf_reg_lgr()
2196 		list_for_each_entry_safe(buf_desc, bf, &lgr->sndbufs[i], list) {  in smcr_buf_reg_lgr()
2197 			if (!buf_desc->used || !buf_desc->is_vm)  in smcr_buf_reg_lgr()
2199 			rc = smcr_link_reg_buf(lnk, buf_desc);  in smcr_buf_reg_lgr()
2213 	struct smc_buf_desc *buf_desc;  in smcr_new_buf_create()  local
2216 	buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL);  in smcr_new_buf_create()
2217 	if (!buf_desc)  in smcr_new_buf_create()
2223 		buf_desc->order = get_order(bufsize);  in smcr_new_buf_create()
2224 		buf_desc->pages = alloc_pages(GFP_KERNEL | __GFP_NOWARN |  in smcr_new_buf_create()
2227 					      buf_desc->order);  in smcr_new_buf_create()
2228 		if (buf_desc->pages) {  in smcr_new_buf_create()
2229 			buf_desc->cpu_addr =  in smcr_new_buf_create()
2230 				(void *)page_address(buf_desc->pages);  in smcr_new_buf_create()
2231 			buf_desc->len = bufsize;  in smcr_new_buf_create()
2232 			buf_desc->is_vm = false;  in smcr_new_buf_create()
2239 		buf_desc->order = get_order(bufsize);  in smcr_new_buf_create()
2240 		buf_desc->cpu_addr = vzalloc(PAGE_SIZE << buf_desc->order);  in smcr_new_buf_create()
2241 		if (!buf_desc->cpu_addr)  in smcr_new_buf_create()
2243 		buf_desc->pages = NULL;  in smcr_new_buf_create()
2244 		buf_desc->len = bufsize;  in smcr_new_buf_create()
2245 		buf_desc->is_vm = true;  in smcr_new_buf_create()
2248 	return buf_desc;  in smcr_new_buf_create()
2251 	kfree(buf_desc);  in smcr_new_buf_create()
2259 				     struct smc_buf_desc *buf_desc, bool is_rmb)  in smcr_buf_map_usable_links()  argument
2270 		if (smcr_buf_map_link(buf_desc, is_rmb, lnk)) {  in smcr_buf_map_usable_links()
2286 	struct smc_buf_desc *buf_desc;  in smcd_new_buf_create()  local
2290 	buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL);  in smcd_new_buf_create()
2291 	if (!buf_desc)  in smcd_new_buf_create()
2294 		rc = smc_ism_register_dmb(lgr, bufsize, buf_desc);  in smcd_new_buf_create()
2296 			kfree(buf_desc);  in smcd_new_buf_create()
2303 		buf_desc->pages = virt_to_page(buf_desc->cpu_addr);  in smcd_new_buf_create()
2305 		buf_desc->len = bufsize - sizeof(struct smcd_cdc_msg);  in smcd_new_buf_create()
2307 		buf_desc->cpu_addr = kzalloc(bufsize, GFP_KERNEL |  in smcd_new_buf_create()
2310 		if (!buf_desc->cpu_addr) {  in smcd_new_buf_create()
2311 			kfree(buf_desc);  in smcd_new_buf_create()
2314 		buf_desc->len = bufsize;  in smcd_new_buf_create()
2316 	return buf_desc;  in smcd_new_buf_create()
2321 	struct smc_buf_desc *buf_desc = ERR_PTR(-ENOMEM);  in __smc_buf_create()  local
2348 		buf_desc = smc_buf_get_slot(bufsize_comp, lock, buf_list);  in __smc_buf_create()
2349 		if (buf_desc) {  in __smc_buf_create()
2350 			buf_desc->is_dma_need_sync = 0;  in __smc_buf_create()
2357 			buf_desc = smcd_new_buf_create(lgr, is_rmb, bufsize);  in __smc_buf_create()
2359 			buf_desc = smcr_new_buf_create(lgr, is_rmb, bufsize);  in __smc_buf_create()
2361 		if (PTR_ERR(buf_desc) == -ENOMEM)  in __smc_buf_create()
2363 		if (IS_ERR(buf_desc)) {  in __smc_buf_create()
2373 		buf_desc->used = 1;  in __smc_buf_create()
2375 		list_add(&buf_desc->list, buf_list);  in __smc_buf_create()
2380 	if (IS_ERR(buf_desc))  in __smc_buf_create()
2381 		return PTR_ERR(buf_desc);  in __smc_buf_create()
2384 		if (smcr_buf_map_usable_links(lgr, buf_desc, is_rmb)) {  in __smc_buf_create()
2385 			smcr_buf_unuse(buf_desc, is_rmb, lgr);  in __smc_buf_create()
2391 		conn->rmb_desc = buf_desc;  in __smc_buf_create()
2396 			smc_rmb_wnd_update_limit(buf_desc->len);  in __smc_buf_create()
2400 		conn->sndbuf_desc = buf_desc;  in __smc_buf_create()