58 static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr,  in create_mad_addr_info()  argument
62 	struct ib_ud_wr *wr = &mad_send_wr->send_wr;  in create_mad_addr_info()
594 				  struct ib_mad_send_wr_private *mad_send_wr)  in handle_outgoing_dr_smp()  argument
597 	struct ib_smp *smp = mad_send_wr->send_buf.mad;  in handle_outgoing_dr_smp()
607 	struct ib_ud_wr *send_wr = &mad_send_wr->send_wr;  in handle_outgoing_dr_smp()
694 		mad_wc.byte_len = mad_send_wr->send_buf.hdr_len  in handle_outgoing_dr_smp()
695 					+ mad_send_wr->send_buf.data_len  in handle_outgoing_dr_smp()
748 	local->mad_send_wr = mad_send_wr;  in handle_outgoing_dr_smp()
750 		local->mad_send_wr->send_wr.pkey_index = out_mad_pkey_index;  in handle_outgoing_dr_smp()
778 static void free_send_rmpp_list(struct ib_mad_send_wr_private *mad_send_wr)  in free_send_rmpp_list()  argument
782 	list_for_each_entry_safe(s, t, &mad_send_wr->rmpp_list, list) {  in free_send_rmpp_list()
840 	struct ib_mad_send_wr_private *mad_send_wr;  in ib_create_send_mad()  local
867 	buf = kzalloc(sizeof *mad_send_wr + size, gfp_mask);  in ib_create_send_mad()
871 	mad_send_wr = buf + size;  in ib_create_send_mad()
872 	INIT_LIST_HEAD(&mad_send_wr->rmpp_list);  in ib_create_send_mad()
873 	mad_send_wr->send_buf.mad = buf;  in ib_create_send_mad()
874 	mad_send_wr->send_buf.hdr_len = hdr_len;  in ib_create_send_mad()
875 	mad_send_wr->send_buf.data_len = data_len;  in ib_create_send_mad()
876 	mad_send_wr->pad = pad;  in ib_create_send_mad()
878 	mad_send_wr->mad_agent_priv = mad_agent_priv;  in ib_create_send_mad()
879 	mad_send_wr->sg_list[0].length = hdr_len;  in ib_create_send_mad()
880 	mad_send_wr->sg_list[0].lkey = mad_agent->qp->pd->local_dma_lkey;  in ib_create_send_mad()
885 		mad_send_wr->sg_list[1].length = data_len;  in ib_create_send_mad()
887 		mad_send_wr->sg_list[1].length = mad_size - hdr_len;  in ib_create_send_mad()
889 	mad_send_wr->sg_list[1].lkey = mad_agent->qp->pd->local_dma_lkey;  in ib_create_send_mad()
891 	mad_send_wr->mad_list.cqe.done = ib_mad_send_done;  in ib_create_send_mad()
893 	mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe;  in ib_create_send_mad()
894 	mad_send_wr->send_wr.wr.sg_list = mad_send_wr->sg_list;  in ib_create_send_mad()
895 	mad_send_wr->send_wr.wr.num_sge = 2;  in ib_create_send_mad()
896 	mad_send_wr->send_wr.wr.opcode = IB_WR_SEND;  in ib_create_send_mad()
897 	mad_send_wr->send_wr.wr.send_flags = IB_SEND_SIGNALED;  in ib_create_send_mad()
898 	mad_send_wr->send_wr.remote_qpn = remote_qpn;  in ib_create_send_mad()
899 	mad_send_wr->send_wr.remote_qkey = IB_QP_SET_QKEY;  in ib_create_send_mad()
900 	mad_send_wr->send_wr.pkey_index = pkey_index;  in ib_create_send_mad()
903 		ret = alloc_send_rmpp_list(mad_send_wr, mad_size, gfp_mask);  in ib_create_send_mad()
910 	mad_send_wr->send_buf.mad_agent = mad_agent;  in ib_create_send_mad()
912 	return &mad_send_wr->send_buf;  in ib_create_send_mad()
947 	struct ib_mad_send_wr_private *mad_send_wr;  in ib_get_rmpp_segment()  local
950 	mad_send_wr = container_of(send_buf, struct ib_mad_send_wr_private,  in ib_get_rmpp_segment()
952 	list = &mad_send_wr->cur_seg->list;  in ib_get_rmpp_segment()
954 	if (mad_send_wr->cur_seg->num < seg_num) {  in ib_get_rmpp_segment()
955 		list_for_each_entry(mad_send_wr->cur_seg, list, list)  in ib_get_rmpp_segment()
956 			if (mad_send_wr->cur_seg->num == seg_num)  in ib_get_rmpp_segment()
958 	} else if (mad_send_wr->cur_seg->num > seg_num) {  in ib_get_rmpp_segment()
959 		list_for_each_entry_reverse(mad_send_wr->cur_seg, list, list)  in ib_get_rmpp_segment()
960 			if (mad_send_wr->cur_seg->num == seg_num)  in ib_get_rmpp_segment()
963 	return mad_send_wr->cur_seg->data;  in ib_get_rmpp_segment()
967 static inline void *ib_get_payload(struct ib_mad_send_wr_private *mad_send_wr)  in ib_get_payload()  argument
969 	if (mad_send_wr->send_buf.seg_count)  in ib_get_payload()
970 		return ib_get_rmpp_segment(&mad_send_wr->send_buf,  in ib_get_payload()
971 					   mad_send_wr->seg_num);  in ib_get_payload()
973 		return mad_send_wr->send_buf.mad +  in ib_get_payload()
974 		       mad_send_wr->send_buf.hdr_len;  in ib_get_payload()
980 	struct ib_mad_send_wr_private *mad_send_wr;  in ib_free_send_mad()  local
984 	mad_send_wr = container_of(send_buf, struct ib_mad_send_wr_private,  in ib_free_send_mad()
987 	free_send_rmpp_list(mad_send_wr);  in ib_free_send_mad()
993 int ib_send_mad(struct ib_mad_send_wr_private *mad_send_wr)  in ib_send_mad()  argument
1003 	qp_info = mad_send_wr->mad_agent_priv->qp_info;  in ib_send_mad()
1004 	mad_send_wr->mad_list.mad_queue = &qp_info->send_queue;  in ib_send_mad()
1005 	mad_send_wr->mad_list.cqe.done = ib_mad_send_done;  in ib_send_mad()
1006 	mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe;  in ib_send_mad()
1008 	mad_agent = mad_send_wr->send_buf.mad_agent;  in ib_send_mad()
1009 	sge = mad_send_wr->sg_list;  in ib_send_mad()
1011 					mad_send_wr->send_buf.mad,  in ib_send_mad()
1017 	mad_send_wr->header_mapping = sge[0].addr;  in ib_send_mad()
1020 					ib_get_payload(mad_send_wr),  in ib_send_mad()
1025 				    mad_send_wr->header_mapping,  in ib_send_mad()
1029 	mad_send_wr->payload_mapping = sge[1].addr;  in ib_send_mad()
1033 		trace_ib_mad_ib_send_mad(mad_send_wr, qp_info);  in ib_send_mad()
1034 		ret = ib_post_send(mad_agent->qp, &mad_send_wr->send_wr.wr,  in ib_send_mad()
1044 		list_add_tail(&mad_send_wr->mad_list.list, list);  in ib_send_mad()
1049 				    mad_send_wr->header_mapping,  in ib_send_mad()
1052 				    mad_send_wr->payload_mapping,  in ib_send_mad()
1067 	struct ib_mad_send_wr_private *mad_send_wr;  in ib_post_send_mad()  local
1073 		mad_send_wr = container_of(send_buf,  in ib_post_send_mad()
1076 		mad_agent_priv = mad_send_wr->mad_agent_priv;  in ib_post_send_mad()
1079 					      mad_send_wr->send_wr.pkey_index);  in ib_post_send_mad()
1103 		mad_send_wr->send_wr.ah = send_buf->ah;  in ib_post_send_mad()
1108 						     mad_send_wr);  in ib_post_send_mad()
1115 		mad_send_wr->tid = ((struct ib_mad_hdr *) send_buf->mad)->tid;  in ib_post_send_mad()
1117 		mad_send_wr->timeout = msecs_to_jiffies(send_buf->timeout_ms);  in ib_post_send_mad()
1118 		mad_send_wr->max_retries = send_buf->retries;  in ib_post_send_mad()
1119 		mad_send_wr->retries_left = send_buf->retries;  in ib_post_send_mad()
1122 		mad_send_wr->refcount = 1 + (mad_send_wr->timeout > 0);  in ib_post_send_mad()
1123 		mad_send_wr->status = IB_WC_SUCCESS;  in ib_post_send_mad()
1128 		list_add_tail(&mad_send_wr->agent_list,  in ib_post_send_mad()
1133 			ret = ib_send_rmpp_mad(mad_send_wr);  in ib_post_send_mad()
1135 				ret = ib_send_mad(mad_send_wr);  in ib_post_send_mad()
1137 			ret = ib_send_mad(mad_send_wr);  in ib_post_send_mad()
1141 			list_del(&mad_send_wr->agent_list);  in ib_post_send_mad()
1773 void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr)  in ib_mark_mad_done()  argument
1775 	mad_send_wr->timeout = 0;  in ib_mark_mad_done()
1776 	if (mad_send_wr->refcount == 1)  in ib_mark_mad_done()
1777 		list_move_tail(&mad_send_wr->agent_list,  in ib_mark_mad_done()
1778 			      &mad_send_wr->mad_agent_priv->done_list);  in ib_mark_mad_done()
1784 	struct ib_mad_send_wr_private *mad_send_wr;  in ib_mad_complete_recv()  local
1811 		mad_send_wr = ib_find_send_mad(mad_agent_priv, mad_recv_wc);  in ib_mad_complete_recv()
1812 		if (!mad_send_wr) {  in ib_mad_complete_recv()
1834 			ib_mark_mad_done(mad_send_wr);  in ib_mad_complete_recv()
1840 					&mad_send_wr->send_buf,  in ib_mad_complete_recv()
1846 			mad_send_wc.send_buf = &mad_send_wr->send_buf;  in ib_mad_complete_recv()
1847 			ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc);  in ib_mad_complete_recv()
2150 	struct ib_mad_send_wr_private *mad_send_wr;  in adjust_timeout()  local
2156 		mad_send_wr = list_entry(mad_agent_priv->wait_list.next,  in adjust_timeout()
2161 			       mad_send_wr->timeout)) {  in adjust_timeout()
2162 			mad_agent_priv->timeout = mad_send_wr->timeout;  in adjust_timeout()
2163 			delay = mad_send_wr->timeout - jiffies;  in adjust_timeout()
2172 static void wait_for_response(struct ib_mad_send_wr_private *mad_send_wr)  in wait_for_response()  argument
2179 	mad_agent_priv = mad_send_wr->mad_agent_priv;  in wait_for_response()
2180 	list_del(&mad_send_wr->agent_list);  in wait_for_response()
2182 	delay = mad_send_wr->timeout;  in wait_for_response()
2183 	mad_send_wr->timeout += jiffies;  in wait_for_response()
2190 			if (time_after(mad_send_wr->timeout,  in wait_for_response()
2198 	list_add(&mad_send_wr->agent_list, list_item);  in wait_for_response()
2201 	if (mad_agent_priv->wait_list.next == &mad_send_wr->agent_list)  in wait_for_response()
2206 void ib_reset_mad_timeout(struct ib_mad_send_wr_private *mad_send_wr,  in ib_reset_mad_timeout()  argument
2209 	mad_send_wr->timeout = msecs_to_jiffies(timeout_ms);  in ib_reset_mad_timeout()
2210 	wait_for_response(mad_send_wr);  in ib_reset_mad_timeout()
2216 void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,  in ib_mad_complete_send_wr()  argument
2223 	mad_agent_priv = mad_send_wr->mad_agent_priv;  in ib_mad_complete_send_wr()
2226 		ret = ib_process_rmpp_send_wc(mad_send_wr, mad_send_wc);  in ib_mad_complete_send_wr()
2233 	    mad_send_wr->status == IB_WC_SUCCESS) {  in ib_mad_complete_send_wr()
2234 		mad_send_wr->status = mad_send_wc->status;  in ib_mad_complete_send_wr()
2235 		mad_send_wr->refcount -= (mad_send_wr->timeout > 0);  in ib_mad_complete_send_wr()
2238 	if (--mad_send_wr->refcount > 0) {  in ib_mad_complete_send_wr()
2239 		if (mad_send_wr->refcount == 1 && mad_send_wr->timeout &&  in ib_mad_complete_send_wr()
2240 		    mad_send_wr->status == IB_WC_SUCCESS) {  in ib_mad_complete_send_wr()
2241 			wait_for_response(mad_send_wr);  in ib_mad_complete_send_wr()
2247 	list_del(&mad_send_wr->agent_list);  in ib_mad_complete_send_wr()
2251 	if (mad_send_wr->status != IB_WC_SUCCESS)  in ib_mad_complete_send_wr()
2252 		mad_send_wc->status = mad_send_wr->status;  in ib_mad_complete_send_wr()
2271 	struct ib_mad_send_wr_private	*mad_send_wr, *queued_send_wr;  in ib_mad_send_done()  local
2286 	mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private,  in ib_mad_send_done()
2291 	trace_ib_mad_send_done_agent(mad_send_wr->mad_agent_priv);  in ib_mad_send_done()
2292 	trace_ib_mad_send_done_handler(mad_send_wr, wc);  in ib_mad_send_done()
2295 	ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device,  in ib_mad_send_done()
2296 			    mad_send_wr->header_mapping,  in ib_mad_send_done()
2297 			    mad_send_wr->sg_list[0].length, DMA_TO_DEVICE);  in ib_mad_send_done()
2298 	ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device,  in ib_mad_send_done()
2299 			    mad_send_wr->payload_mapping,  in ib_mad_send_done()
2300 			    mad_send_wr->sg_list[1].length, DMA_TO_DEVICE);  in ib_mad_send_done()
2316 	mad_send_wc.send_buf = &mad_send_wr->send_buf;  in ib_mad_send_done()
2319 	ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc);  in ib_mad_send_done()
2328 			mad_send_wr = queued_send_wr;  in ib_mad_send_done()
2337 	struct ib_mad_send_wr_private *mad_send_wr;  in mark_sends_for_retry()  local
2343 		mad_send_wr = container_of(mad_list,  in mark_sends_for_retry()
2346 		mad_send_wr->retry = 1;  in mark_sends_for_retry()
2357 	struct ib_mad_send_wr_private *mad_send_wr;  in ib_mad_send_error()  local
2364 	mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private,  in ib_mad_send_error()
2367 		if (mad_send_wr->retry) {  in ib_mad_send_error()
2369 			mad_send_wr->retry = 0;  in ib_mad_send_error()
2370 			trace_ib_mad_error_handler(mad_send_wr, qp_info);  in ib_mad_send_error()
2371 			ret = ib_post_send(qp_info->qp, &mad_send_wr->send_wr.wr,  in ib_mad_send_error()
2402 	struct ib_mad_send_wr_private *mad_send_wr, *temp_mad_send_wr;  in cancel_mads()  local
2409 	list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,  in cancel_mads()
2411 		if (mad_send_wr->status == IB_WC_SUCCESS) {  in cancel_mads()
2412 			mad_send_wr->status = IB_WC_WR_FLUSH_ERR;  in cancel_mads()
2413 			mad_send_wr->refcount -= (mad_send_wr->timeout > 0);  in cancel_mads()
2425 	list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,  in cancel_mads()
2427 		mad_send_wc.send_buf = &mad_send_wr->send_buf;  in cancel_mads()
2428 		list_del(&mad_send_wr->agent_list);  in cancel_mads()
2439 	struct ib_mad_send_wr_private *mad_send_wr;  in find_send_wr()  local
2441 	list_for_each_entry(mad_send_wr, &mad_agent_priv->wait_list,  in find_send_wr()
2443 		if (&mad_send_wr->send_buf == send_buf)  in find_send_wr()
2444 			return mad_send_wr;  in find_send_wr()
2447 	list_for_each_entry(mad_send_wr, &mad_agent_priv->send_list,  in find_send_wr()
2450 				     mad_send_wr->send_buf.mad) &&  in find_send_wr()
2451 		    &mad_send_wr->send_buf == send_buf)  in find_send_wr()
2452 			return mad_send_wr;  in find_send_wr()
2460 	struct ib_mad_send_wr_private *mad_send_wr;  in ib_modify_mad()  local
2470 	mad_send_wr = find_send_wr(mad_agent_priv, send_buf);  in ib_modify_mad()
2471 	if (!mad_send_wr || mad_send_wr->status != IB_WC_SUCCESS) {  in ib_modify_mad()
2476 	active = (!mad_send_wr->timeout || mad_send_wr->refcount > 1);  in ib_modify_mad()
2478 		mad_send_wr->status = IB_WC_WR_FLUSH_ERR;  in ib_modify_mad()
2479 		mad_send_wr->refcount -= (mad_send_wr->timeout > 0);  in ib_modify_mad()
2482 	mad_send_wr->send_buf.timeout_ms = timeout_ms;  in ib_modify_mad()
2484 		mad_send_wr->timeout = msecs_to_jiffies(timeout_ms);  in ib_modify_mad()
2486 		ib_reset_mad_timeout(mad_send_wr, timeout_ms);  in ib_modify_mad()
2533 				     local->mad_send_wr->send_wr.wr.wr_cqe,  in local_completions()
2535 				     local->mad_send_wr->send_wr.pkey_index,  in local_completions()
2557 						&local->mad_send_wr->send_buf,  in local_completions()
2568 		mad_send_wc.send_buf = &local->mad_send_wr->send_buf;  in local_completions()
2581 static int retry_send(struct ib_mad_send_wr_private *mad_send_wr)  in retry_send()  argument
2585 	if (!mad_send_wr->retries_left)  in retry_send()
2588 	mad_send_wr->retries_left--;  in retry_send()
2589 	mad_send_wr->send_buf.retries++;  in retry_send()
2591 	mad_send_wr->timeout = msecs_to_jiffies(mad_send_wr->send_buf.timeout_ms);  in retry_send()
2593 	if (ib_mad_kernel_rmpp_agent(&mad_send_wr->mad_agent_priv->agent)) {  in retry_send()
2594 		ret = ib_retry_rmpp(mad_send_wr);  in retry_send()
2597 			ret = ib_send_mad(mad_send_wr);  in retry_send()
2607 		ret = ib_send_mad(mad_send_wr);  in retry_send()
2610 		mad_send_wr->refcount++;  in retry_send()
2611 		list_add_tail(&mad_send_wr->agent_list,  in retry_send()
2612 			      &mad_send_wr->mad_agent_priv->send_list);  in retry_send()
2620 	struct ib_mad_send_wr_private *mad_send_wr;  in timeout_sends()  local
2630 		mad_send_wr = list_entry(mad_agent_priv->wait_list.next,  in timeout_sends()
2634 		if (time_after(mad_send_wr->timeout, jiffies)) {  in timeout_sends()
2635 			delay = mad_send_wr->timeout - jiffies;  in timeout_sends()
2644 		list_del(&mad_send_wr->agent_list);  in timeout_sends()
2645 		if (mad_send_wr->status == IB_WC_SUCCESS &&  in timeout_sends()
2646 		    !retry_send(mad_send_wr))  in timeout_sends()
2651 		if (mad_send_wr->status == IB_WC_SUCCESS)  in timeout_sends()
2654 			mad_send_wc.status = mad_send_wr->status;  in timeout_sends()
2655 		mad_send_wc.send_buf = &mad_send_wr->send_buf;  in timeout_sends()