1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2009, Microsoft Corporation.
4  *
5  * Authors:
6  *   Haiyang Zhang <haiyangz@microsoft.com>
7  *   Hank Janssen  <hjanssen@microsoft.com>
8  */
9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10 
11 #include <linux/kernel.h>
12 #include <linux/interrupt.h>
13 #include <linux/sched.h>
14 #include <linux/wait.h>
15 #include <linux/mm.h>
16 #include <linux/slab.h>
17 #include <linux/list.h>
18 #include <linux/module.h>
19 #include <linux/completion.h>
20 #include <linux/delay.h>
21 #include <linux/cpu.h>
22 #include <linux/hyperv.h>
23 #include <asm/mshyperv.h>
24 #include <linux/sched/isolation.h>
25 
26 #include "hyperv_vmbus.h"
27 
28 static void init_vp_index(struct vmbus_channel *channel);
29 
30 const struct vmbus_device vmbus_devs[] = {
31 	/* IDE */
32 	{ .dev_type = HV_IDE,
33 	  HV_IDE_GUID,
34 	  .perf_device = true,
35 	  .allowed_in_isolated = false,
36 	},
37 
38 	/* SCSI */
39 	{ .dev_type = HV_SCSI,
40 	  HV_SCSI_GUID,
41 	  .perf_device = true,
42 	  .allowed_in_isolated = true,
43 	},
44 
45 	/* Fibre Channel */
46 	{ .dev_type = HV_FC,
47 	  HV_SYNTHFC_GUID,
48 	  .perf_device = true,
49 	  .allowed_in_isolated = false,
50 	},
51 
52 	/* Synthetic NIC */
53 	{ .dev_type = HV_NIC,
54 	  HV_NIC_GUID,
55 	  .perf_device = true,
56 	  .allowed_in_isolated = true,
57 	},
58 
59 	/* Network Direct */
60 	{ .dev_type = HV_ND,
61 	  HV_ND_GUID,
62 	  .perf_device = true,
63 	  .allowed_in_isolated = false,
64 	},
65 
66 	/* PCIE */
67 	{ .dev_type = HV_PCIE,
68 	  HV_PCIE_GUID,
69 	  .perf_device = false,
70 	  .allowed_in_isolated = false,
71 	},
72 
73 	/* Synthetic Frame Buffer */
74 	{ .dev_type = HV_FB,
75 	  HV_SYNTHVID_GUID,
76 	  .perf_device = false,
77 	  .allowed_in_isolated = false,
78 	},
79 
80 	/* Synthetic Keyboard */
81 	{ .dev_type = HV_KBD,
82 	  HV_KBD_GUID,
83 	  .perf_device = false,
84 	  .allowed_in_isolated = false,
85 	},
86 
87 	/* Synthetic MOUSE */
88 	{ .dev_type = HV_MOUSE,
89 	  HV_MOUSE_GUID,
90 	  .perf_device = false,
91 	  .allowed_in_isolated = false,
92 	},
93 
94 	/* KVP */
95 	{ .dev_type = HV_KVP,
96 	  HV_KVP_GUID,
97 	  .perf_device = false,
98 	  .allowed_in_isolated = false,
99 	},
100 
101 	/* Time Synch */
102 	{ .dev_type = HV_TS,
103 	  HV_TS_GUID,
104 	  .perf_device = false,
105 	  .allowed_in_isolated = true,
106 	},
107 
108 	/* Heartbeat */
109 	{ .dev_type = HV_HB,
110 	  HV_HEART_BEAT_GUID,
111 	  .perf_device = false,
112 	  .allowed_in_isolated = true,
113 	},
114 
115 	/* Shutdown */
116 	{ .dev_type = HV_SHUTDOWN,
117 	  HV_SHUTDOWN_GUID,
118 	  .perf_device = false,
119 	  .allowed_in_isolated = true,
120 	},
121 
122 	/* File copy */
123 	{ .dev_type = HV_FCOPY,
124 	  HV_FCOPY_GUID,
125 	  .perf_device = false,
126 	  .allowed_in_isolated = false,
127 	},
128 
129 	/* Backup */
130 	{ .dev_type = HV_BACKUP,
131 	  HV_VSS_GUID,
132 	  .perf_device = false,
133 	  .allowed_in_isolated = false,
134 	},
135 
136 	/* Dynamic Memory */
137 	{ .dev_type = HV_DM,
138 	  HV_DM_GUID,
139 	  .perf_device = false,
140 	  .allowed_in_isolated = false,
141 	},
142 
143 	/* Unknown GUID */
144 	{ .dev_type = HV_UNKNOWN,
145 	  .perf_device = false,
146 	  .allowed_in_isolated = false,
147 	},
148 };
149 
150 static const struct {
151 	guid_t guid;
152 } vmbus_unsupported_devs[] = {
153 	{ HV_AVMA1_GUID },
154 	{ HV_AVMA2_GUID },
155 	{ HV_RDV_GUID	},
156 	{ HV_IMC_GUID	},
157 };
158 
159 /*
160  * The rescinded channel may be blocked waiting for a response from the host;
161  * take care of that.
162  */
vmbus_rescind_cleanup(struct vmbus_channel * channel)163 static void vmbus_rescind_cleanup(struct vmbus_channel *channel)
164 {
165 	struct vmbus_channel_msginfo *msginfo;
166 	unsigned long flags;
167 
168 
169 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
170 	channel->rescind = true;
171 	list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
172 				msglistentry) {
173 
174 		if (msginfo->waiting_channel == channel) {
175 			complete(&msginfo->waitevent);
176 			break;
177 		}
178 	}
179 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
180 }
181 
is_unsupported_vmbus_devs(const guid_t * guid)182 static bool is_unsupported_vmbus_devs(const guid_t *guid)
183 {
184 	int i;
185 
186 	for (i = 0; i < ARRAY_SIZE(vmbus_unsupported_devs); i++)
187 		if (guid_equal(guid, &vmbus_unsupported_devs[i].guid))
188 			return true;
189 	return false;
190 }
191 
hv_get_dev_type(const struct vmbus_channel * channel)192 static u16 hv_get_dev_type(const struct vmbus_channel *channel)
193 {
194 	const guid_t *guid = &channel->offermsg.offer.if_type;
195 	u16 i;
196 
197 	if (is_hvsock_channel(channel) || is_unsupported_vmbus_devs(guid))
198 		return HV_UNKNOWN;
199 
200 	for (i = HV_IDE; i < HV_UNKNOWN; i++) {
201 		if (guid_equal(guid, &vmbus_devs[i].guid))
202 			return i;
203 	}
204 	pr_info("Unknown GUID: %pUl\n", guid);
205 	return i;
206 }
207 
208 /**
209  * vmbus_prep_negotiate_resp() - Create default response for Negotiate message
210  * @icmsghdrp: Pointer to msg header structure
211  * @buf: Raw buffer channel data
212  * @buflen: Length of the raw buffer channel data.
213  * @fw_version: The framework versions we can support.
214  * @fw_vercnt: The size of @fw_version.
215  * @srv_version: The service versions we can support.
216  * @srv_vercnt: The size of @srv_version.
217  * @nego_fw_version: The selected framework version.
218  * @nego_srv_version: The selected service version.
219  *
220  * Note: Versions are given in decreasing order.
221  *
222  * Set up and fill in default negotiate response message.
223  * Mainly used by Hyper-V drivers.
224  */
vmbus_prep_negotiate_resp(struct icmsg_hdr * icmsghdrp,u8 * buf,u32 buflen,const int * fw_version,int fw_vercnt,const int * srv_version,int srv_vercnt,int * nego_fw_version,int * nego_srv_version)225 bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, u8 *buf,
226 				u32 buflen, const int *fw_version, int fw_vercnt,
227 				const int *srv_version, int srv_vercnt,
228 				int *nego_fw_version, int *nego_srv_version)
229 {
230 	int icframe_major, icframe_minor;
231 	int icmsg_major, icmsg_minor;
232 	int fw_major, fw_minor;
233 	int srv_major, srv_minor;
234 	int i, j;
235 	bool found_match = false;
236 	struct icmsg_negotiate *negop;
237 
238 	/* Check that there's enough space for icframe_vercnt, icmsg_vercnt */
239 	if (buflen < ICMSG_HDR + offsetof(struct icmsg_negotiate, reserved)) {
240 		pr_err_ratelimited("Invalid icmsg negotiate\n");
241 		return false;
242 	}
243 
244 	icmsghdrp->icmsgsize = 0x10;
245 	negop = (struct icmsg_negotiate *)&buf[ICMSG_HDR];
246 
247 	icframe_major = negop->icframe_vercnt;
248 	icframe_minor = 0;
249 
250 	icmsg_major = negop->icmsg_vercnt;
251 	icmsg_minor = 0;
252 
253 	/* Validate negop packet */
254 	if (icframe_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT ||
255 	    icmsg_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT ||
256 	    ICMSG_NEGOTIATE_PKT_SIZE(icframe_major, icmsg_major) > buflen) {
257 		pr_err_ratelimited("Invalid icmsg negotiate - icframe_major: %u, icmsg_major: %u\n",
258 				   icframe_major, icmsg_major);
259 		goto fw_error;
260 	}
261 
262 	/*
263 	 * Select the framework version number we will
264 	 * support.
265 	 */
266 
267 	for (i = 0; i < fw_vercnt; i++) {
268 		fw_major = (fw_version[i] >> 16);
269 		fw_minor = (fw_version[i] & 0xFFFF);
270 
271 		for (j = 0; j < negop->icframe_vercnt; j++) {
272 			if ((negop->icversion_data[j].major == fw_major) &&
273 			    (negop->icversion_data[j].minor == fw_minor)) {
274 				icframe_major = negop->icversion_data[j].major;
275 				icframe_minor = negop->icversion_data[j].minor;
276 				found_match = true;
277 				break;
278 			}
279 		}
280 
281 		if (found_match)
282 			break;
283 	}
284 
285 	if (!found_match)
286 		goto fw_error;
287 
288 	found_match = false;
289 
290 	for (i = 0; i < srv_vercnt; i++) {
291 		srv_major = (srv_version[i] >> 16);
292 		srv_minor = (srv_version[i] & 0xFFFF);
293 
294 		for (j = negop->icframe_vercnt;
295 			(j < negop->icframe_vercnt + negop->icmsg_vercnt);
296 			j++) {
297 
298 			if ((negop->icversion_data[j].major == srv_major) &&
299 				(negop->icversion_data[j].minor == srv_minor)) {
300 
301 				icmsg_major = negop->icversion_data[j].major;
302 				icmsg_minor = negop->icversion_data[j].minor;
303 				found_match = true;
304 				break;
305 			}
306 		}
307 
308 		if (found_match)
309 			break;
310 	}
311 
312 	/*
313 	 * Respond with the framework and service
314 	 * version numbers we can support.
315 	 */
316 
317 fw_error:
318 	if (!found_match) {
319 		negop->icframe_vercnt = 0;
320 		negop->icmsg_vercnt = 0;
321 	} else {
322 		negop->icframe_vercnt = 1;
323 		negop->icmsg_vercnt = 1;
324 	}
325 
326 	if (nego_fw_version)
327 		*nego_fw_version = (icframe_major << 16) | icframe_minor;
328 
329 	if (nego_srv_version)
330 		*nego_srv_version = (icmsg_major << 16) | icmsg_minor;
331 
332 	negop->icversion_data[0].major = icframe_major;
333 	negop->icversion_data[0].minor = icframe_minor;
334 	negop->icversion_data[1].major = icmsg_major;
335 	negop->icversion_data[1].minor = icmsg_minor;
336 	return found_match;
337 }
338 EXPORT_SYMBOL_GPL(vmbus_prep_negotiate_resp);
339 
340 /*
341  * alloc_channel - Allocate and initialize a vmbus channel object
342  */
alloc_channel(void)343 static struct vmbus_channel *alloc_channel(void)
344 {
345 	struct vmbus_channel *channel;
346 
347 	channel = kzalloc(sizeof(*channel), GFP_ATOMIC);
348 	if (!channel)
349 		return NULL;
350 
351 	spin_lock_init(&channel->sched_lock);
352 	init_completion(&channel->rescind_event);
353 
354 	INIT_LIST_HEAD(&channel->sc_list);
355 
356 	tasklet_init(&channel->callback_event,
357 		     vmbus_on_event, (unsigned long)channel);
358 
359 	hv_ringbuffer_pre_init(channel);
360 
361 	return channel;
362 }
363 
364 /*
365  * free_channel - Release the resources used by the vmbus channel object
366  */
free_channel(struct vmbus_channel * channel)367 static void free_channel(struct vmbus_channel *channel)
368 {
369 	tasklet_kill(&channel->callback_event);
370 	vmbus_remove_channel_attr_group(channel);
371 
372 	kobject_put(&channel->kobj);
373 }
374 
vmbus_channel_map_relid(struct vmbus_channel * channel)375 void vmbus_channel_map_relid(struct vmbus_channel *channel)
376 {
377 	if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))
378 		return;
379 	/*
380 	 * The mapping of the channel's relid is visible from the CPUs that
381 	 * execute vmbus_chan_sched() by the time that vmbus_chan_sched() will
382 	 * execute:
383 	 *
384 	 *  (a) In the "normal (i.e., not resuming from hibernation)" path,
385 	 *      the full barrier in virt_store_mb() guarantees that the store
386 	 *      is propagated to all CPUs before the add_channel_work work
387 	 *      is queued.  In turn, add_channel_work is queued before the
388 	 *      channel's ring buffer is allocated/initialized and the
389 	 *      OPENCHANNEL message for the channel is sent in vmbus_open().
390 	 *      Hyper-V won't start sending the interrupts for the channel
391 	 *      before the OPENCHANNEL message is acked.  The memory barrier
392 	 *      in vmbus_chan_sched() -> sync_test_and_clear_bit() ensures
393 	 *      that vmbus_chan_sched() must find the channel's relid in
394 	 *      recv_int_page before retrieving the channel pointer from the
395 	 *      array of channels.
396 	 *
397 	 *  (b) In the "resuming from hibernation" path, the virt_store_mb()
398 	 *      guarantees that the store is propagated to all CPUs before
399 	 *      the VMBus connection is marked as ready for the resume event
400 	 *      (cf. check_ready_for_resume_event()).  The interrupt handler
401 	 *      of the VMBus driver and vmbus_chan_sched() can not run before
402 	 *      vmbus_bus_resume() has completed execution (cf. resume_noirq).
403 	 */
404 	virt_store_mb(
405 		vmbus_connection.channels[channel->offermsg.child_relid],
406 		channel);
407 }
408 
vmbus_channel_unmap_relid(struct vmbus_channel * channel)409 void vmbus_channel_unmap_relid(struct vmbus_channel *channel)
410 {
411 	if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))
412 		return;
413 	WRITE_ONCE(
414 		vmbus_connection.channels[channel->offermsg.child_relid],
415 		NULL);
416 }
417 
vmbus_release_relid(u32 relid)418 static void vmbus_release_relid(u32 relid)
419 {
420 	struct vmbus_channel_relid_released msg;
421 	int ret;
422 
423 	memset(&msg, 0, sizeof(struct vmbus_channel_relid_released));
424 	msg.child_relid = relid;
425 	msg.header.msgtype = CHANNELMSG_RELID_RELEASED;
426 	ret = vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released),
427 			     true);
428 
429 	trace_vmbus_release_relid(&msg, ret);
430 }
431 
hv_process_channel_removal(struct vmbus_channel * channel)432 void hv_process_channel_removal(struct vmbus_channel *channel)
433 {
434 	lockdep_assert_held(&vmbus_connection.channel_mutex);
435 	BUG_ON(!channel->rescind);
436 
437 	/*
438 	 * hv_process_channel_removal() could find INVALID_RELID only for
439 	 * hv_sock channels.  See the inline comments in vmbus_onoffer().
440 	 */
441 	WARN_ON(channel->offermsg.child_relid == INVALID_RELID &&
442 		!is_hvsock_channel(channel));
443 
444 	/*
445 	 * Upon suspend, an in-use hv_sock channel is removed from the array of
446 	 * channels and the relid is invalidated.  After hibernation, when the
447 	 * user-space application destroys the channel, it's unnecessary and
448 	 * unsafe to remove the channel from the array of channels.  See also
449 	 * the inline comments before the call of vmbus_release_relid() below.
450 	 */
451 	if (channel->offermsg.child_relid != INVALID_RELID)
452 		vmbus_channel_unmap_relid(channel);
453 
454 	if (channel->primary_channel == NULL)
455 		list_del(&channel->listentry);
456 	else
457 		list_del(&channel->sc_list);
458 
459 	/*
460 	 * If this is a "perf" channel, updates the hv_numa_map[] masks so that
461 	 * init_vp_index() can (re-)use the CPU.
462 	 */
463 	if (hv_is_perf_channel(channel))
464 		hv_clear_allocated_cpu(channel->target_cpu);
465 
466 	/*
467 	 * Upon suspend, an in-use hv_sock channel is marked as "rescinded" and
468 	 * the relid is invalidated; after hibernation, when the user-space app
469 	 * destroys the channel, the relid is INVALID_RELID, and in this case
470 	 * it's unnecessary and unsafe to release the old relid, since the same
471 	 * relid can refer to a completely different channel now.
472 	 */
473 	if (channel->offermsg.child_relid != INVALID_RELID)
474 		vmbus_release_relid(channel->offermsg.child_relid);
475 
476 	free_channel(channel);
477 }
478 
vmbus_free_channels(void)479 void vmbus_free_channels(void)
480 {
481 	struct vmbus_channel *channel, *tmp;
482 
483 	list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list,
484 		listentry) {
485 		/* hv_process_channel_removal() needs this */
486 		channel->rescind = true;
487 
488 		vmbus_device_unregister(channel->device_obj);
489 	}
490 }
491 
492 /* Note: the function can run concurrently for primary/sub channels. */
vmbus_add_channel_work(struct work_struct * work)493 static void vmbus_add_channel_work(struct work_struct *work)
494 {
495 	struct vmbus_channel *newchannel =
496 		container_of(work, struct vmbus_channel, add_channel_work);
497 	struct vmbus_channel *primary_channel = newchannel->primary_channel;
498 	int ret;
499 
500 	/*
501 	 * This state is used to indicate a successful open
502 	 * so that when we do close the channel normally, we
503 	 * can cleanup properly.
504 	 */
505 	newchannel->state = CHANNEL_OPEN_STATE;
506 
507 	if (primary_channel != NULL) {
508 		/* newchannel is a sub-channel. */
509 		struct hv_device *dev = primary_channel->device_obj;
510 
511 		if (vmbus_add_channel_kobj(dev, newchannel))
512 			goto err_deq_chan;
513 
514 		if (primary_channel->sc_creation_callback != NULL)
515 			primary_channel->sc_creation_callback(newchannel);
516 
517 		newchannel->probe_done = true;
518 		return;
519 	}
520 
521 	/*
522 	 * Start the process of binding the primary channel to the driver
523 	 */
524 	newchannel->device_obj = vmbus_device_create(
525 		&newchannel->offermsg.offer.if_type,
526 		&newchannel->offermsg.offer.if_instance,
527 		newchannel);
528 	if (!newchannel->device_obj)
529 		goto err_deq_chan;
530 
531 	newchannel->device_obj->device_id = newchannel->device_id;
532 	/*
533 	 * Add the new device to the bus. This will kick off device-driver
534 	 * binding which eventually invokes the device driver's AddDevice()
535 	 * method.
536 	 *
537 	 * If vmbus_device_register() fails, the 'device_obj' is freed in
538 	 * vmbus_device_release() as called by device_unregister() in the
539 	 * error path of vmbus_device_register(). In the outside error
540 	 * path, there's no need to free it.
541 	 */
542 	ret = vmbus_device_register(newchannel->device_obj);
543 
544 	if (ret != 0) {
545 		pr_err("unable to add child device object (relid %d)\n",
546 			newchannel->offermsg.child_relid);
547 		goto err_deq_chan;
548 	}
549 
550 	newchannel->probe_done = true;
551 	return;
552 
553 err_deq_chan:
554 	mutex_lock(&vmbus_connection.channel_mutex);
555 
556 	/*
557 	 * We need to set the flag, otherwise
558 	 * vmbus_onoffer_rescind() can be blocked.
559 	 */
560 	newchannel->probe_done = true;
561 
562 	if (primary_channel == NULL)
563 		list_del(&newchannel->listentry);
564 	else
565 		list_del(&newchannel->sc_list);
566 
567 	/* vmbus_process_offer() has mapped the channel. */
568 	vmbus_channel_unmap_relid(newchannel);
569 
570 	mutex_unlock(&vmbus_connection.channel_mutex);
571 
572 	vmbus_release_relid(newchannel->offermsg.child_relid);
573 
574 	free_channel(newchannel);
575 }
576 
577 /*
578  * vmbus_process_offer - Process the offer by creating a channel/device
579  * associated with this offer
580  */
vmbus_process_offer(struct vmbus_channel * newchannel)581 static void vmbus_process_offer(struct vmbus_channel *newchannel)
582 {
583 	struct vmbus_channel *channel;
584 	struct workqueue_struct *wq;
585 	bool fnew = true;
586 
587 	/*
588 	 * Synchronize vmbus_process_offer() and CPU hotplugging:
589 	 *
590 	 * CPU1				CPU2
591 	 *
592 	 * [vmbus_process_offer()]	[Hot removal of the CPU]
593 	 *
594 	 * CPU_READ_LOCK		CPUS_WRITE_LOCK
595 	 * LOAD cpu_online_mask		SEARCH chn_list
596 	 * STORE target_cpu		LOAD target_cpu
597 	 * INSERT chn_list		STORE cpu_online_mask
598 	 * CPUS_READ_UNLOCK		CPUS_WRITE_UNLOCK
599 	 *
600 	 * Forbids: CPU1's LOAD from *not* seing CPU2's STORE &&
601 	 *              CPU2's SEARCH from *not* seeing CPU1's INSERT
602 	 *
603 	 * Forbids: CPU2's SEARCH from seeing CPU1's INSERT &&
604 	 *              CPU2's LOAD from *not* seing CPU1's STORE
605 	 */
606 	cpus_read_lock();
607 
608 	/*
609 	 * Serializes the modifications of the chn_list list as well as
610 	 * the accesses to next_numa_node_id in init_vp_index().
611 	 */
612 	mutex_lock(&vmbus_connection.channel_mutex);
613 
614 	list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {
615 		if (guid_equal(&channel->offermsg.offer.if_type,
616 			       &newchannel->offermsg.offer.if_type) &&
617 		    guid_equal(&channel->offermsg.offer.if_instance,
618 			       &newchannel->offermsg.offer.if_instance)) {
619 			fnew = false;
620 			newchannel->primary_channel = channel;
621 			break;
622 		}
623 	}
624 
625 	init_vp_index(newchannel);
626 
627 	/* Remember the channels that should be cleaned up upon suspend. */
628 	if (is_hvsock_channel(newchannel) || is_sub_channel(newchannel))
629 		atomic_inc(&vmbus_connection.nr_chan_close_on_suspend);
630 
631 	/*
632 	 * Now that we have acquired the channel_mutex,
633 	 * we can release the potentially racing rescind thread.
634 	 */
635 	atomic_dec(&vmbus_connection.offer_in_progress);
636 
637 	if (fnew) {
638 		list_add_tail(&newchannel->listentry,
639 			      &vmbus_connection.chn_list);
640 	} else {
641 		/*
642 		 * Check to see if this is a valid sub-channel.
643 		 */
644 		if (newchannel->offermsg.offer.sub_channel_index == 0) {
645 			mutex_unlock(&vmbus_connection.channel_mutex);
646 			cpus_read_unlock();
647 			/*
648 			 * Don't call free_channel(), because newchannel->kobj
649 			 * is not initialized yet.
650 			 */
651 			kfree(newchannel);
652 			WARN_ON_ONCE(1);
653 			return;
654 		}
655 		/*
656 		 * Process the sub-channel.
657 		 */
658 		list_add_tail(&newchannel->sc_list, &channel->sc_list);
659 	}
660 
661 	vmbus_channel_map_relid(newchannel);
662 
663 	mutex_unlock(&vmbus_connection.channel_mutex);
664 	cpus_read_unlock();
665 
666 	/*
667 	 * vmbus_process_offer() mustn't call channel->sc_creation_callback()
668 	 * directly for sub-channels, because sc_creation_callback() ->
669 	 * vmbus_open() may never get the host's response to the
670 	 * OPEN_CHANNEL message (the host may rescind a channel at any time,
671 	 * e.g. in the case of hot removing a NIC), and vmbus_onoffer_rescind()
672 	 * may not wake up the vmbus_open() as it's blocked due to a non-zero
673 	 * vmbus_connection.offer_in_progress, and finally we have a deadlock.
674 	 *
675 	 * The above is also true for primary channels, if the related device
676 	 * drivers use sync probing mode by default.
677 	 *
678 	 * And, usually the handling of primary channels and sub-channels can
679 	 * depend on each other, so we should offload them to different
680 	 * workqueues to avoid possible deadlock, e.g. in sync-probing mode,
681 	 * NIC1's netvsc_subchan_work() can race with NIC2's netvsc_probe() ->
682 	 * rtnl_lock(), and causes deadlock: the former gets the rtnl_lock
683 	 * and waits for all the sub-channels to appear, but the latter
684 	 * can't get the rtnl_lock and this blocks the handling of
685 	 * sub-channels.
686 	 */
687 	INIT_WORK(&newchannel->add_channel_work, vmbus_add_channel_work);
688 	wq = fnew ? vmbus_connection.handle_primary_chan_wq :
689 		    vmbus_connection.handle_sub_chan_wq;
690 	queue_work(wq, &newchannel->add_channel_work);
691 }
692 
693 /*
694  * Check if CPUs used by other channels of the same device.
695  * It should only be called by init_vp_index().
696  */
hv_cpuself_used(u32 cpu,struct vmbus_channel * chn)697 static bool hv_cpuself_used(u32 cpu, struct vmbus_channel *chn)
698 {
699 	struct vmbus_channel *primary = chn->primary_channel;
700 	struct vmbus_channel *sc;
701 
702 	lockdep_assert_held(&vmbus_connection.channel_mutex);
703 
704 	if (!primary)
705 		return false;
706 
707 	if (primary->target_cpu == cpu)
708 		return true;
709 
710 	list_for_each_entry(sc, &primary->sc_list, sc_list)
711 		if (sc != chn && sc->target_cpu == cpu)
712 			return true;
713 
714 	return false;
715 }
716 
717 /*
718  * We use this state to statically distribute the channel interrupt load.
719  */
720 static int next_numa_node_id;
721 
722 /*
723  * We can statically distribute the incoming channel interrupt load
724  * by binding a channel to VCPU.
725  *
726  * For non-performance critical channels we assign the VMBUS_CONNECT_CPU.
727  * Performance critical channels will be distributed evenly among all
728  * the available NUMA nodes.  Once the node is assigned, we will assign
729  * the CPU based on a simple round robin scheme.
730  */
init_vp_index(struct vmbus_channel * channel)731 static void init_vp_index(struct vmbus_channel *channel)
732 {
733 	bool perf_chn = hv_is_perf_channel(channel);
734 	u32 i, ncpu = num_online_cpus();
735 	cpumask_var_t available_mask;
736 	struct cpumask *allocated_mask;
737 	const struct cpumask *hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);
738 	u32 target_cpu;
739 	int numa_node;
740 
741 	if (!perf_chn ||
742 	    !alloc_cpumask_var(&available_mask, GFP_KERNEL) ||
743 	    cpumask_empty(hk_mask)) {
744 		/*
745 		 * If the channel is not a performance critical
746 		 * channel, bind it to VMBUS_CONNECT_CPU.
747 		 * In case alloc_cpumask_var() fails, bind it to
748 		 * VMBUS_CONNECT_CPU.
749 		 * If all the cpus are isolated, bind it to
750 		 * VMBUS_CONNECT_CPU.
751 		 */
752 		channel->target_cpu = VMBUS_CONNECT_CPU;
753 		if (perf_chn)
754 			hv_set_allocated_cpu(VMBUS_CONNECT_CPU);
755 		return;
756 	}
757 
758 	for (i = 1; i <= ncpu + 1; i++) {
759 		while (true) {
760 			numa_node = next_numa_node_id++;
761 			if (numa_node == nr_node_ids) {
762 				next_numa_node_id = 0;
763 				continue;
764 			}
765 			if (cpumask_empty(cpumask_of_node(numa_node)))
766 				continue;
767 			break;
768 		}
769 		allocated_mask = &hv_context.hv_numa_map[numa_node];
770 
771 retry:
772 		cpumask_xor(available_mask, allocated_mask, cpumask_of_node(numa_node));
773 		cpumask_and(available_mask, available_mask, hk_mask);
774 
775 		if (cpumask_empty(available_mask)) {
776 			/*
777 			 * We have cycled through all the CPUs in the node;
778 			 * reset the allocated map.
779 			 */
780 			cpumask_clear(allocated_mask);
781 			goto retry;
782 		}
783 
784 		target_cpu = cpumask_first(available_mask);
785 		cpumask_set_cpu(target_cpu, allocated_mask);
786 
787 		if (channel->offermsg.offer.sub_channel_index >= ncpu ||
788 		    i > ncpu || !hv_cpuself_used(target_cpu, channel))
789 			break;
790 	}
791 
792 	channel->target_cpu = target_cpu;
793 
794 	free_cpumask_var(available_mask);
795 }
796 
797 #define UNLOAD_DELAY_UNIT_MS	10		/* 10 milliseconds */
798 #define UNLOAD_WAIT_MS		(100*1000)	/* 100 seconds */
799 #define UNLOAD_WAIT_LOOPS	(UNLOAD_WAIT_MS/UNLOAD_DELAY_UNIT_MS)
800 #define UNLOAD_MSG_MS		(5*1000)	/* Every 5 seconds */
801 #define UNLOAD_MSG_LOOPS	(UNLOAD_MSG_MS/UNLOAD_DELAY_UNIT_MS)
802 
vmbus_wait_for_unload(void)803 static void vmbus_wait_for_unload(void)
804 {
805 	int cpu;
806 	void *page_addr;
807 	struct hv_message *msg;
808 	struct vmbus_channel_message_header *hdr;
809 	u32 message_type, i;
810 
811 	/*
812 	 * CHANNELMSG_UNLOAD_RESPONSE is always delivered to the CPU which was
813 	 * used for initial contact or to CPU0 depending on host version. When
814 	 * we're crashing on a different CPU let's hope that IRQ handler on
815 	 * the cpu which receives CHANNELMSG_UNLOAD_RESPONSE is still
816 	 * functional and vmbus_unload_response() will complete
817 	 * vmbus_connection.unload_event. If not, the last thing we can do is
818 	 * read message pages for all CPUs directly.
819 	 *
820 	 * Wait up to 100 seconds since an Azure host must writeback any dirty
821 	 * data in its disk cache before the VMbus UNLOAD request will
822 	 * complete. This flushing has been empirically observed to take up
823 	 * to 50 seconds in cases with a lot of dirty data, so allow additional
824 	 * leeway and for inaccuracies in mdelay(). But eventually time out so
825 	 * that the panic path can't get hung forever in case the response
826 	 * message isn't seen.
827 	 */
828 	for (i = 1; i <= UNLOAD_WAIT_LOOPS; i++) {
829 		if (completion_done(&vmbus_connection.unload_event))
830 			goto completed;
831 
832 		for_each_online_cpu(cpu) {
833 			struct hv_per_cpu_context *hv_cpu
834 				= per_cpu_ptr(hv_context.cpu_context, cpu);
835 
836 			page_addr = hv_cpu->synic_message_page;
837 			msg = (struct hv_message *)page_addr
838 				+ VMBUS_MESSAGE_SINT;
839 
840 			message_type = READ_ONCE(msg->header.message_type);
841 			if (message_type == HVMSG_NONE)
842 				continue;
843 
844 			hdr = (struct vmbus_channel_message_header *)
845 				msg->u.payload;
846 
847 			if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE)
848 				complete(&vmbus_connection.unload_event);
849 
850 			vmbus_signal_eom(msg, message_type);
851 		}
852 
853 		/*
854 		 * Give a notice periodically so someone watching the
855 		 * serial output won't think it is completely hung.
856 		 */
857 		if (!(i % UNLOAD_MSG_LOOPS))
858 			pr_notice("Waiting for VMBus UNLOAD to complete\n");
859 
860 		mdelay(UNLOAD_DELAY_UNIT_MS);
861 	}
862 	pr_err("Continuing even though VMBus UNLOAD did not complete\n");
863 
864 completed:
865 	/*
866 	 * We're crashing and already got the UNLOAD_RESPONSE, cleanup all
867 	 * maybe-pending messages on all CPUs to be able to receive new
868 	 * messages after we reconnect.
869 	 */
870 	for_each_online_cpu(cpu) {
871 		struct hv_per_cpu_context *hv_cpu
872 			= per_cpu_ptr(hv_context.cpu_context, cpu);
873 
874 		page_addr = hv_cpu->synic_message_page;
875 		msg = (struct hv_message *)page_addr + VMBUS_MESSAGE_SINT;
876 		msg->header.message_type = HVMSG_NONE;
877 	}
878 }
879 
880 /*
881  * vmbus_unload_response - Handler for the unload response.
882  */
vmbus_unload_response(struct vmbus_channel_message_header * hdr)883 static void vmbus_unload_response(struct vmbus_channel_message_header *hdr)
884 {
885 	/*
886 	 * This is a global event; just wakeup the waiting thread.
887 	 * Once we successfully unload, we can cleanup the monitor state.
888 	 *
889 	 * NB.  A malicious or compromised Hyper-V could send a spurious
890 	 * message of type CHANNELMSG_UNLOAD_RESPONSE, and trigger a call
891 	 * of the complete() below.  Make sure that unload_event has been
892 	 * initialized by the time this complete() is executed.
893 	 */
894 	complete(&vmbus_connection.unload_event);
895 }
896 
vmbus_initiate_unload(bool crash)897 void vmbus_initiate_unload(bool crash)
898 {
899 	struct vmbus_channel_message_header hdr;
900 
901 	if (xchg(&vmbus_connection.conn_state, DISCONNECTED) == DISCONNECTED)
902 		return;
903 
904 	/* Pre-Win2012R2 hosts don't support reconnect */
905 	if (vmbus_proto_version < VERSION_WIN8_1)
906 		return;
907 
908 	reinit_completion(&vmbus_connection.unload_event);
909 	memset(&hdr, 0, sizeof(struct vmbus_channel_message_header));
910 	hdr.msgtype = CHANNELMSG_UNLOAD;
911 	vmbus_post_msg(&hdr, sizeof(struct vmbus_channel_message_header),
912 		       !crash);
913 
914 	/*
915 	 * vmbus_initiate_unload() is also called on crash and the crash can be
916 	 * happening in an interrupt context, where scheduling is impossible.
917 	 */
918 	if (!crash)
919 		wait_for_completion(&vmbus_connection.unload_event);
920 	else
921 		vmbus_wait_for_unload();
922 }
923 
check_ready_for_resume_event(void)924 static void check_ready_for_resume_event(void)
925 {
926 	/*
927 	 * If all the old primary channels have been fixed up, then it's safe
928 	 * to resume.
929 	 */
930 	if (atomic_dec_and_test(&vmbus_connection.nr_chan_fixup_on_resume))
931 		complete(&vmbus_connection.ready_for_resume_event);
932 }
933 
vmbus_setup_channel_state(struct vmbus_channel * channel,struct vmbus_channel_offer_channel * offer)934 static void vmbus_setup_channel_state(struct vmbus_channel *channel,
935 				      struct vmbus_channel_offer_channel *offer)
936 {
937 	/*
938 	 * Setup state for signalling the host.
939 	 */
940 	channel->sig_event = VMBUS_EVENT_CONNECTION_ID;
941 
942 	channel->is_dedicated_interrupt =
943 			(offer->is_dedicated_interrupt != 0);
944 	channel->sig_event = offer->connection_id;
945 
946 	memcpy(&channel->offermsg, offer,
947 	       sizeof(struct vmbus_channel_offer_channel));
948 	channel->monitor_grp = (u8)offer->monitorid / 32;
949 	channel->monitor_bit = (u8)offer->monitorid % 32;
950 	channel->device_id = hv_get_dev_type(channel);
951 }
952 
953 /*
954  * find_primary_channel_by_offer - Get the channel object given the new offer.
955  * This is only used in the resume path of hibernation.
956  */
957 static struct vmbus_channel *
find_primary_channel_by_offer(const struct vmbus_channel_offer_channel * offer)958 find_primary_channel_by_offer(const struct vmbus_channel_offer_channel *offer)
959 {
960 	struct vmbus_channel *channel = NULL, *iter;
961 	const guid_t *inst1, *inst2;
962 
963 	/* Ignore sub-channel offers. */
964 	if (offer->offer.sub_channel_index != 0)
965 		return NULL;
966 
967 	mutex_lock(&vmbus_connection.channel_mutex);
968 
969 	list_for_each_entry(iter, &vmbus_connection.chn_list, listentry) {
970 		inst1 = &iter->offermsg.offer.if_instance;
971 		inst2 = &offer->offer.if_instance;
972 
973 		if (guid_equal(inst1, inst2)) {
974 			channel = iter;
975 			break;
976 		}
977 	}
978 
979 	mutex_unlock(&vmbus_connection.channel_mutex);
980 
981 	return channel;
982 }
983 
vmbus_is_valid_offer(const struct vmbus_channel_offer_channel * offer)984 static bool vmbus_is_valid_offer(const struct vmbus_channel_offer_channel *offer)
985 {
986 	const guid_t *guid = &offer->offer.if_type;
987 	u16 i;
988 
989 	if (!hv_is_isolation_supported())
990 		return true;
991 
992 	if (is_hvsock_offer(offer))
993 		return true;
994 
995 	for (i = 0; i < ARRAY_SIZE(vmbus_devs); i++) {
996 		if (guid_equal(guid, &vmbus_devs[i].guid))
997 			return vmbus_devs[i].allowed_in_isolated;
998 	}
999 	return false;
1000 }
1001 
1002 /*
1003  * vmbus_onoffer - Handler for channel offers from vmbus in parent partition.
1004  *
1005  */
vmbus_onoffer(struct vmbus_channel_message_header * hdr)1006 static void vmbus_onoffer(struct vmbus_channel_message_header *hdr)
1007 {
1008 	struct vmbus_channel_offer_channel *offer;
1009 	struct vmbus_channel *oldchannel, *newchannel;
1010 	size_t offer_sz;
1011 
1012 	offer = (struct vmbus_channel_offer_channel *)hdr;
1013 
1014 	trace_vmbus_onoffer(offer);
1015 
1016 	if (!vmbus_is_valid_offer(offer)) {
1017 		pr_err_ratelimited("Invalid offer %d from the host supporting isolation\n",
1018 				   offer->child_relid);
1019 		atomic_dec(&vmbus_connection.offer_in_progress);
1020 		return;
1021 	}
1022 
1023 	oldchannel = find_primary_channel_by_offer(offer);
1024 
1025 	if (oldchannel != NULL) {
1026 		/*
1027 		 * We're resuming from hibernation: all the sub-channel and
1028 		 * hv_sock channels we had before the hibernation should have
1029 		 * been cleaned up, and now we must be seeing a re-offered
1030 		 * primary channel that we had before the hibernation.
1031 		 */
1032 
1033 		/*
1034 		 * { Initially: channel relid = INVALID_RELID,
1035 		 *		channels[valid_relid] = NULL }
1036 		 *
1037 		 * CPU1					CPU2
1038 		 *
1039 		 * [vmbus_onoffer()]			[vmbus_device_release()]
1040 		 *
1041 		 * LOCK channel_mutex			LOCK channel_mutex
1042 		 * STORE channel relid = valid_relid	LOAD r1 = channel relid
1043 		 * MAP_RELID channel			if (r1 != INVALID_RELID)
1044 		 * UNLOCK channel_mutex			  UNMAP_RELID channel
1045 		 *					UNLOCK channel_mutex
1046 		 *
1047 		 * Forbids: r1 == valid_relid &&
1048 		 *              channels[valid_relid] == channel
1049 		 *
1050 		 * Note.  r1 can be INVALID_RELID only for an hv_sock channel.
1051 		 * None of the hv_sock channels which were present before the
1052 		 * suspend are re-offered upon the resume.  See the WARN_ON()
1053 		 * in hv_process_channel_removal().
1054 		 */
1055 		mutex_lock(&vmbus_connection.channel_mutex);
1056 
1057 		atomic_dec(&vmbus_connection.offer_in_progress);
1058 
1059 		WARN_ON(oldchannel->offermsg.child_relid != INVALID_RELID);
1060 		/* Fix up the relid. */
1061 		oldchannel->offermsg.child_relid = offer->child_relid;
1062 
1063 		offer_sz = sizeof(*offer);
1064 		if (memcmp(offer, &oldchannel->offermsg, offer_sz) != 0) {
1065 			/*
1066 			 * This is not an error, since the host can also change
1067 			 * the other field(s) of the offer, e.g. on WS RS5
1068 			 * (Build 17763), the offer->connection_id of the
1069 			 * Mellanox VF vmbus device can change when the host
1070 			 * reoffers the device upon resume.
1071 			 */
1072 			pr_debug("vmbus offer changed: relid=%d\n",
1073 				 offer->child_relid);
1074 
1075 			print_hex_dump_debug("Old vmbus offer: ",
1076 					     DUMP_PREFIX_OFFSET, 16, 4,
1077 					     &oldchannel->offermsg, offer_sz,
1078 					     false);
1079 			print_hex_dump_debug("New vmbus offer: ",
1080 					     DUMP_PREFIX_OFFSET, 16, 4,
1081 					     offer, offer_sz, false);
1082 
1083 			/* Fix up the old channel. */
1084 			vmbus_setup_channel_state(oldchannel, offer);
1085 		}
1086 
1087 		/* Add the channel back to the array of channels. */
1088 		vmbus_channel_map_relid(oldchannel);
1089 		check_ready_for_resume_event();
1090 
1091 		mutex_unlock(&vmbus_connection.channel_mutex);
1092 		return;
1093 	}
1094 
1095 	/* Allocate the channel object and save this offer. */
1096 	newchannel = alloc_channel();
1097 	if (!newchannel) {
1098 		vmbus_release_relid(offer->child_relid);
1099 		atomic_dec(&vmbus_connection.offer_in_progress);
1100 		pr_err("Unable to allocate channel object\n");
1101 		return;
1102 	}
1103 
1104 	vmbus_setup_channel_state(newchannel, offer);
1105 
1106 	vmbus_process_offer(newchannel);
1107 }
1108 
check_ready_for_suspend_event(void)1109 static void check_ready_for_suspend_event(void)
1110 {
1111 	/*
1112 	 * If all the sub-channels or hv_sock channels have been cleaned up,
1113 	 * then it's safe to suspend.
1114 	 */
1115 	if (atomic_dec_and_test(&vmbus_connection.nr_chan_close_on_suspend))
1116 		complete(&vmbus_connection.ready_for_suspend_event);
1117 }
1118 
1119 /*
1120  * vmbus_onoffer_rescind - Rescind offer handler.
1121  *
1122  * We queue a work item to process this offer synchronously
1123  */
vmbus_onoffer_rescind(struct vmbus_channel_message_header * hdr)1124 static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr)
1125 {
1126 	struct vmbus_channel_rescind_offer *rescind;
1127 	struct vmbus_channel *channel;
1128 	struct device *dev;
1129 	bool clean_up_chan_for_suspend;
1130 
1131 	rescind = (struct vmbus_channel_rescind_offer *)hdr;
1132 
1133 	trace_vmbus_onoffer_rescind(rescind);
1134 
1135 	/*
1136 	 * The offer msg and the corresponding rescind msg
1137 	 * from the host are guranteed to be ordered -
1138 	 * offer comes in first and then the rescind.
1139 	 * Since we process these events in work elements,
1140 	 * and with preemption, we may end up processing
1141 	 * the events out of order.  We rely on the synchronization
1142 	 * provided by offer_in_progress and by channel_mutex for
1143 	 * ordering these events:
1144 	 *
1145 	 * { Initially: offer_in_progress = 1 }
1146 	 *
1147 	 * CPU1				CPU2
1148 	 *
1149 	 * [vmbus_onoffer()]		[vmbus_onoffer_rescind()]
1150 	 *
1151 	 * LOCK channel_mutex		WAIT_ON offer_in_progress == 0
1152 	 * DECREMENT offer_in_progress	LOCK channel_mutex
1153 	 * STORE channels[]		LOAD channels[]
1154 	 * UNLOCK channel_mutex		UNLOCK channel_mutex
1155 	 *
1156 	 * Forbids: CPU2's LOAD from *not* seeing CPU1's STORE
1157 	 */
1158 
1159 	while (atomic_read(&vmbus_connection.offer_in_progress) != 0) {
1160 		/*
1161 		 * We wait here until any channel offer is currently
1162 		 * being processed.
1163 		 */
1164 		msleep(1);
1165 	}
1166 
1167 	mutex_lock(&vmbus_connection.channel_mutex);
1168 	channel = relid2channel(rescind->child_relid);
1169 	if (channel != NULL) {
1170 		/*
1171 		 * Guarantee that no other instance of vmbus_onoffer_rescind()
1172 		 * has got a reference to the channel object.  Synchronize on
1173 		 * &vmbus_connection.channel_mutex.
1174 		 */
1175 		if (channel->rescind_ref) {
1176 			mutex_unlock(&vmbus_connection.channel_mutex);
1177 			return;
1178 		}
1179 		channel->rescind_ref = true;
1180 	}
1181 	mutex_unlock(&vmbus_connection.channel_mutex);
1182 
1183 	if (channel == NULL) {
1184 		/*
1185 		 * We failed in processing the offer message;
1186 		 * we would have cleaned up the relid in that
1187 		 * failure path.
1188 		 */
1189 		return;
1190 	}
1191 
1192 	clean_up_chan_for_suspend = is_hvsock_channel(channel) ||
1193 				    is_sub_channel(channel);
1194 	/*
1195 	 * Before setting channel->rescind in vmbus_rescind_cleanup(), we
1196 	 * should make sure the channel callback is not running any more.
1197 	 */
1198 	vmbus_reset_channel_cb(channel);
1199 
1200 	/*
1201 	 * Now wait for offer handling to complete.
1202 	 */
1203 	vmbus_rescind_cleanup(channel);
1204 	while (READ_ONCE(channel->probe_done) == false) {
1205 		/*
1206 		 * We wait here until any channel offer is currently
1207 		 * being processed.
1208 		 */
1209 		msleep(1);
1210 	}
1211 
1212 	/*
1213 	 * At this point, the rescind handling can proceed safely.
1214 	 */
1215 
1216 	if (channel->device_obj) {
1217 		if (channel->chn_rescind_callback) {
1218 			channel->chn_rescind_callback(channel);
1219 
1220 			if (clean_up_chan_for_suspend)
1221 				check_ready_for_suspend_event();
1222 
1223 			return;
1224 		}
1225 		/*
1226 		 * We will have to unregister this device from the
1227 		 * driver core.
1228 		 */
1229 		dev = get_device(&channel->device_obj->device);
1230 		if (dev) {
1231 			vmbus_device_unregister(channel->device_obj);
1232 			put_device(dev);
1233 		}
1234 	} else if (channel->primary_channel != NULL) {
1235 		/*
1236 		 * Sub-channel is being rescinded. Following is the channel
1237 		 * close sequence when initiated from the driveri (refer to
1238 		 * vmbus_close() for details):
1239 		 * 1. Close all sub-channels first
1240 		 * 2. Then close the primary channel.
1241 		 */
1242 		mutex_lock(&vmbus_connection.channel_mutex);
1243 		if (channel->state == CHANNEL_OPEN_STATE) {
1244 			/*
1245 			 * The channel is currently not open;
1246 			 * it is safe for us to cleanup the channel.
1247 			 */
1248 			hv_process_channel_removal(channel);
1249 		} else {
1250 			complete(&channel->rescind_event);
1251 		}
1252 		mutex_unlock(&vmbus_connection.channel_mutex);
1253 	}
1254 
1255 	/* The "channel" may have been freed. Do not access it any longer. */
1256 
1257 	if (clean_up_chan_for_suspend)
1258 		check_ready_for_suspend_event();
1259 }
1260 
vmbus_hvsock_device_unregister(struct vmbus_channel * channel)1261 void vmbus_hvsock_device_unregister(struct vmbus_channel *channel)
1262 {
1263 	BUG_ON(!is_hvsock_channel(channel));
1264 
1265 	/* We always get a rescind msg when a connection is closed. */
1266 	while (!READ_ONCE(channel->probe_done) || !READ_ONCE(channel->rescind))
1267 		msleep(1);
1268 
1269 	vmbus_device_unregister(channel->device_obj);
1270 }
1271 EXPORT_SYMBOL_GPL(vmbus_hvsock_device_unregister);
1272 
1273 
1274 /*
1275  * vmbus_onoffers_delivered -
1276  * This is invoked when all offers have been delivered.
1277  *
1278  * Nothing to do here.
1279  */
vmbus_onoffers_delivered(struct vmbus_channel_message_header * hdr)1280 static void vmbus_onoffers_delivered(
1281 			struct vmbus_channel_message_header *hdr)
1282 {
1283 }
1284 
1285 /*
1286  * vmbus_onopen_result - Open result handler.
1287  *
1288  * This is invoked when we received a response to our channel open request.
1289  * Find the matching request, copy the response and signal the requesting
1290  * thread.
1291  */
vmbus_onopen_result(struct vmbus_channel_message_header * hdr)1292 static void vmbus_onopen_result(struct vmbus_channel_message_header *hdr)
1293 {
1294 	struct vmbus_channel_open_result *result;
1295 	struct vmbus_channel_msginfo *msginfo;
1296 	struct vmbus_channel_message_header *requestheader;
1297 	struct vmbus_channel_open_channel *openmsg;
1298 	unsigned long flags;
1299 
1300 	result = (struct vmbus_channel_open_result *)hdr;
1301 
1302 	trace_vmbus_onopen_result(result);
1303 
1304 	/*
1305 	 * Find the open msg, copy the result and signal/unblock the wait event
1306 	 */
1307 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1308 
1309 	list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1310 				msglistentry) {
1311 		requestheader =
1312 			(struct vmbus_channel_message_header *)msginfo->msg;
1313 
1314 		if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) {
1315 			openmsg =
1316 			(struct vmbus_channel_open_channel *)msginfo->msg;
1317 			if (openmsg->child_relid == result->child_relid &&
1318 			    openmsg->openid == result->openid) {
1319 				memcpy(&msginfo->response.open_result,
1320 				       result,
1321 				       sizeof(
1322 					struct vmbus_channel_open_result));
1323 				complete(&msginfo->waitevent);
1324 				break;
1325 			}
1326 		}
1327 	}
1328 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1329 }
1330 
1331 /*
1332  * vmbus_ongpadl_created - GPADL created handler.
1333  *
1334  * This is invoked when we received a response to our gpadl create request.
1335  * Find the matching request, copy the response and signal the requesting
1336  * thread.
1337  */
vmbus_ongpadl_created(struct vmbus_channel_message_header * hdr)1338 static void vmbus_ongpadl_created(struct vmbus_channel_message_header *hdr)
1339 {
1340 	struct vmbus_channel_gpadl_created *gpadlcreated;
1341 	struct vmbus_channel_msginfo *msginfo;
1342 	struct vmbus_channel_message_header *requestheader;
1343 	struct vmbus_channel_gpadl_header *gpadlheader;
1344 	unsigned long flags;
1345 
1346 	gpadlcreated = (struct vmbus_channel_gpadl_created *)hdr;
1347 
1348 	trace_vmbus_ongpadl_created(gpadlcreated);
1349 
1350 	/*
1351 	 * Find the establish msg, copy the result and signal/unblock the wait
1352 	 * event
1353 	 */
1354 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1355 
1356 	list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1357 				msglistentry) {
1358 		requestheader =
1359 			(struct vmbus_channel_message_header *)msginfo->msg;
1360 
1361 		if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) {
1362 			gpadlheader =
1363 			(struct vmbus_channel_gpadl_header *)requestheader;
1364 
1365 			if ((gpadlcreated->child_relid ==
1366 			     gpadlheader->child_relid) &&
1367 			    (gpadlcreated->gpadl == gpadlheader->gpadl)) {
1368 				memcpy(&msginfo->response.gpadl_created,
1369 				       gpadlcreated,
1370 				       sizeof(
1371 					struct vmbus_channel_gpadl_created));
1372 				complete(&msginfo->waitevent);
1373 				break;
1374 			}
1375 		}
1376 	}
1377 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1378 }
1379 
1380 /*
1381  * vmbus_onmodifychannel_response - Modify Channel response handler.
1382  *
1383  * This is invoked when we received a response to our channel modify request.
1384  * Find the matching request, copy the response and signal the requesting thread.
1385  */
vmbus_onmodifychannel_response(struct vmbus_channel_message_header * hdr)1386 static void vmbus_onmodifychannel_response(struct vmbus_channel_message_header *hdr)
1387 {
1388 	struct vmbus_channel_modifychannel_response *response;
1389 	struct vmbus_channel_msginfo *msginfo;
1390 	unsigned long flags;
1391 
1392 	response = (struct vmbus_channel_modifychannel_response *)hdr;
1393 
1394 	trace_vmbus_onmodifychannel_response(response);
1395 
1396 	/*
1397 	 * Find the modify msg, copy the response and signal/unblock the wait event.
1398 	 */
1399 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1400 
1401 	list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, msglistentry) {
1402 		struct vmbus_channel_message_header *responseheader =
1403 				(struct vmbus_channel_message_header *)msginfo->msg;
1404 
1405 		if (responseheader->msgtype == CHANNELMSG_MODIFYCHANNEL) {
1406 			struct vmbus_channel_modifychannel *modifymsg;
1407 
1408 			modifymsg = (struct vmbus_channel_modifychannel *)msginfo->msg;
1409 			if (modifymsg->child_relid == response->child_relid) {
1410 				memcpy(&msginfo->response.modify_response, response,
1411 				       sizeof(*response));
1412 				complete(&msginfo->waitevent);
1413 				break;
1414 			}
1415 		}
1416 	}
1417 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1418 }
1419 
1420 /*
1421  * vmbus_ongpadl_torndown - GPADL torndown handler.
1422  *
1423  * This is invoked when we received a response to our gpadl teardown request.
1424  * Find the matching request, copy the response and signal the requesting
1425  * thread.
1426  */
vmbus_ongpadl_torndown(struct vmbus_channel_message_header * hdr)1427 static void vmbus_ongpadl_torndown(
1428 			struct vmbus_channel_message_header *hdr)
1429 {
1430 	struct vmbus_channel_gpadl_torndown *gpadl_torndown;
1431 	struct vmbus_channel_msginfo *msginfo;
1432 	struct vmbus_channel_message_header *requestheader;
1433 	struct vmbus_channel_gpadl_teardown *gpadl_teardown;
1434 	unsigned long flags;
1435 
1436 	gpadl_torndown = (struct vmbus_channel_gpadl_torndown *)hdr;
1437 
1438 	trace_vmbus_ongpadl_torndown(gpadl_torndown);
1439 
1440 	/*
1441 	 * Find the open msg, copy the result and signal/unblock the wait event
1442 	 */
1443 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1444 
1445 	list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1446 				msglistentry) {
1447 		requestheader =
1448 			(struct vmbus_channel_message_header *)msginfo->msg;
1449 
1450 		if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) {
1451 			gpadl_teardown =
1452 			(struct vmbus_channel_gpadl_teardown *)requestheader;
1453 
1454 			if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) {
1455 				memcpy(&msginfo->response.gpadl_torndown,
1456 				       gpadl_torndown,
1457 				       sizeof(
1458 					struct vmbus_channel_gpadl_torndown));
1459 				complete(&msginfo->waitevent);
1460 				break;
1461 			}
1462 		}
1463 	}
1464 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1465 }
1466 
1467 /*
1468  * vmbus_onversion_response - Version response handler
1469  *
1470  * This is invoked when we received a response to our initiate contact request.
1471  * Find the matching request, copy the response and signal the requesting
1472  * thread.
1473  */
vmbus_onversion_response(struct vmbus_channel_message_header * hdr)1474 static void vmbus_onversion_response(
1475 		struct vmbus_channel_message_header *hdr)
1476 {
1477 	struct vmbus_channel_msginfo *msginfo;
1478 	struct vmbus_channel_message_header *requestheader;
1479 	struct vmbus_channel_version_response *version_response;
1480 	unsigned long flags;
1481 
1482 	version_response = (struct vmbus_channel_version_response *)hdr;
1483 
1484 	trace_vmbus_onversion_response(version_response);
1485 
1486 	spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);
1487 
1488 	list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,
1489 				msglistentry) {
1490 		requestheader =
1491 			(struct vmbus_channel_message_header *)msginfo->msg;
1492 
1493 		if (requestheader->msgtype ==
1494 		    CHANNELMSG_INITIATE_CONTACT) {
1495 			memcpy(&msginfo->response.version_response,
1496 			      version_response,
1497 			      sizeof(struct vmbus_channel_version_response));
1498 			complete(&msginfo->waitevent);
1499 		}
1500 	}
1501 	spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);
1502 }
1503 
1504 /* Channel message dispatch table */
1505 const struct vmbus_channel_message_table_entry
1506 channel_message_table[CHANNELMSG_COUNT] = {
1507 	{ CHANNELMSG_INVALID,			0, NULL, 0},
1508 	{ CHANNELMSG_OFFERCHANNEL,		0, vmbus_onoffer,
1509 		sizeof(struct vmbus_channel_offer_channel)},
1510 	{ CHANNELMSG_RESCIND_CHANNELOFFER,	0, vmbus_onoffer_rescind,
1511 		sizeof(struct vmbus_channel_rescind_offer) },
1512 	{ CHANNELMSG_REQUESTOFFERS,		0, NULL, 0},
1513 	{ CHANNELMSG_ALLOFFERS_DELIVERED,	1, vmbus_onoffers_delivered, 0},
1514 	{ CHANNELMSG_OPENCHANNEL,		0, NULL, 0},
1515 	{ CHANNELMSG_OPENCHANNEL_RESULT,	1, vmbus_onopen_result,
1516 		sizeof(struct vmbus_channel_open_result)},
1517 	{ CHANNELMSG_CLOSECHANNEL,		0, NULL, 0},
1518 	{ CHANNELMSG_GPADL_HEADER,		0, NULL, 0},
1519 	{ CHANNELMSG_GPADL_BODY,		0, NULL, 0},
1520 	{ CHANNELMSG_GPADL_CREATED,		1, vmbus_ongpadl_created,
1521 		sizeof(struct vmbus_channel_gpadl_created)},
1522 	{ CHANNELMSG_GPADL_TEARDOWN,		0, NULL, 0},
1523 	{ CHANNELMSG_GPADL_TORNDOWN,		1, vmbus_ongpadl_torndown,
1524 		sizeof(struct vmbus_channel_gpadl_torndown) },
1525 	{ CHANNELMSG_RELID_RELEASED,		0, NULL, 0},
1526 	{ CHANNELMSG_INITIATE_CONTACT,		0, NULL, 0},
1527 	{ CHANNELMSG_VERSION_RESPONSE,		1, vmbus_onversion_response,
1528 		sizeof(struct vmbus_channel_version_response)},
1529 	{ CHANNELMSG_UNLOAD,			0, NULL, 0},
1530 	{ CHANNELMSG_UNLOAD_RESPONSE,		1, vmbus_unload_response, 0},
1531 	{ CHANNELMSG_18,			0, NULL, 0},
1532 	{ CHANNELMSG_19,			0, NULL, 0},
1533 	{ CHANNELMSG_20,			0, NULL, 0},
1534 	{ CHANNELMSG_TL_CONNECT_REQUEST,	0, NULL, 0},
1535 	{ CHANNELMSG_MODIFYCHANNEL,		0, NULL, 0},
1536 	{ CHANNELMSG_TL_CONNECT_RESULT,		0, NULL, 0},
1537 	{ CHANNELMSG_MODIFYCHANNEL_RESPONSE,	1, vmbus_onmodifychannel_response,
1538 		sizeof(struct vmbus_channel_modifychannel_response)},
1539 };
1540 
1541 /*
1542  * vmbus_onmessage - Handler for channel protocol messages.
1543  *
1544  * This is invoked in the vmbus worker thread context.
1545  */
vmbus_onmessage(struct vmbus_channel_message_header * hdr)1546 void vmbus_onmessage(struct vmbus_channel_message_header *hdr)
1547 {
1548 	trace_vmbus_on_message(hdr);
1549 
1550 	/*
1551 	 * vmbus_on_msg_dpc() makes sure the hdr->msgtype here can not go
1552 	 * out of bound and the message_handler pointer can not be NULL.
1553 	 */
1554 	channel_message_table[hdr->msgtype].message_handler(hdr);
1555 }
1556 
1557 /*
1558  * vmbus_request_offers - Send a request to get all our pending offers.
1559  */
vmbus_request_offers(void)1560 int vmbus_request_offers(void)
1561 {
1562 	struct vmbus_channel_message_header *msg;
1563 	struct vmbus_channel_msginfo *msginfo;
1564 	int ret;
1565 
1566 	msginfo = kzalloc(sizeof(*msginfo) +
1567 			  sizeof(struct vmbus_channel_message_header),
1568 			  GFP_KERNEL);
1569 	if (!msginfo)
1570 		return -ENOMEM;
1571 
1572 	msg = (struct vmbus_channel_message_header *)msginfo->msg;
1573 
1574 	msg->msgtype = CHANNELMSG_REQUESTOFFERS;
1575 
1576 	ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_message_header),
1577 			     true);
1578 
1579 	trace_vmbus_request_offers(ret);
1580 
1581 	if (ret != 0) {
1582 		pr_err("Unable to request offers - %d\n", ret);
1583 
1584 		goto cleanup;
1585 	}
1586 
1587 cleanup:
1588 	kfree(msginfo);
1589 
1590 	return ret;
1591 }
1592 
vmbus_set_sc_create_callback(struct vmbus_channel * primary_channel,void (* sc_cr_cb)(struct vmbus_channel * new_sc))1593 void vmbus_set_sc_create_callback(struct vmbus_channel *primary_channel,
1594 				void (*sc_cr_cb)(struct vmbus_channel *new_sc))
1595 {
1596 	primary_channel->sc_creation_callback = sc_cr_cb;
1597 }
1598 EXPORT_SYMBOL_GPL(vmbus_set_sc_create_callback);
1599 
vmbus_set_chn_rescind_callback(struct vmbus_channel * channel,void (* chn_rescind_cb)(struct vmbus_channel *))1600 void vmbus_set_chn_rescind_callback(struct vmbus_channel *channel,
1601 		void (*chn_rescind_cb)(struct vmbus_channel *))
1602 {
1603 	channel->chn_rescind_callback = chn_rescind_cb;
1604 }
1605 EXPORT_SYMBOL_GPL(vmbus_set_chn_rescind_callback);
1606