1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/smp.h>
3 #include <linux/cpu.h>
4 #include <linux/slab.h>
5 #include <linux/cpumask.h>
6 #include <linux/percpu.h>
7 
8 #include <xen/events.h>
9 
10 #include <xen/hvc-console.h>
11 #include "xen-ops.h"
12 #include "smp.h"
13 
14 static DEFINE_PER_CPU(struct xen_common_irq, xen_resched_irq) = { .irq = -1 };
15 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfunc_irq) = { .irq = -1 };
16 static DEFINE_PER_CPU(struct xen_common_irq, xen_callfuncsingle_irq) = { .irq = -1 };
17 static DEFINE_PER_CPU(struct xen_common_irq, xen_debug_irq) = { .irq = -1 };
18 
19 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id);
20 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id);
21 
22 /*
23  * Reschedule call back.
24  */
xen_reschedule_interrupt(int irq,void * dev_id)25 static irqreturn_t xen_reschedule_interrupt(int irq, void *dev_id)
26 {
27 	inc_irq_stat(irq_resched_count);
28 	scheduler_ipi();
29 
30 	return IRQ_HANDLED;
31 }
32 
xen_smp_intr_free(unsigned int cpu)33 void xen_smp_intr_free(unsigned int cpu)
34 {
35 	if (per_cpu(xen_resched_irq, cpu).irq >= 0) {
36 		unbind_from_irqhandler(per_cpu(xen_resched_irq, cpu).irq, NULL);
37 		per_cpu(xen_resched_irq, cpu).irq = -1;
38 		kfree(per_cpu(xen_resched_irq, cpu).name);
39 		per_cpu(xen_resched_irq, cpu).name = NULL;
40 	}
41 	if (per_cpu(xen_callfunc_irq, cpu).irq >= 0) {
42 		unbind_from_irqhandler(per_cpu(xen_callfunc_irq, cpu).irq, NULL);
43 		per_cpu(xen_callfunc_irq, cpu).irq = -1;
44 		kfree(per_cpu(xen_callfunc_irq, cpu).name);
45 		per_cpu(xen_callfunc_irq, cpu).name = NULL;
46 	}
47 	if (per_cpu(xen_debug_irq, cpu).irq >= 0) {
48 		unbind_from_irqhandler(per_cpu(xen_debug_irq, cpu).irq, NULL);
49 		per_cpu(xen_debug_irq, cpu).irq = -1;
50 		kfree(per_cpu(xen_debug_irq, cpu).name);
51 		per_cpu(xen_debug_irq, cpu).name = NULL;
52 	}
53 	if (per_cpu(xen_callfuncsingle_irq, cpu).irq >= 0) {
54 		unbind_from_irqhandler(per_cpu(xen_callfuncsingle_irq, cpu).irq,
55 				       NULL);
56 		per_cpu(xen_callfuncsingle_irq, cpu).irq = -1;
57 		kfree(per_cpu(xen_callfuncsingle_irq, cpu).name);
58 		per_cpu(xen_callfuncsingle_irq, cpu).name = NULL;
59 	}
60 }
61 
xen_smp_intr_init(unsigned int cpu)62 int xen_smp_intr_init(unsigned int cpu)
63 {
64 	int rc;
65 	char *resched_name, *callfunc_name, *debug_name;
66 
67 	resched_name = kasprintf(GFP_KERNEL, "resched%d", cpu);
68 	rc = bind_ipi_to_irqhandler(XEN_RESCHEDULE_VECTOR,
69 				    cpu,
70 				    xen_reschedule_interrupt,
71 				    IRQF_PERCPU|IRQF_NOBALANCING,
72 				    resched_name,
73 				    NULL);
74 	if (rc < 0)
75 		goto fail;
76 	per_cpu(xen_resched_irq, cpu).irq = rc;
77 	per_cpu(xen_resched_irq, cpu).name = resched_name;
78 
79 	callfunc_name = kasprintf(GFP_KERNEL, "callfunc%d", cpu);
80 	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_VECTOR,
81 				    cpu,
82 				    xen_call_function_interrupt,
83 				    IRQF_PERCPU|IRQF_NOBALANCING,
84 				    callfunc_name,
85 				    NULL);
86 	if (rc < 0)
87 		goto fail;
88 	per_cpu(xen_callfunc_irq, cpu).irq = rc;
89 	per_cpu(xen_callfunc_irq, cpu).name = callfunc_name;
90 
91 	if (!xen_fifo_events) {
92 		debug_name = kasprintf(GFP_KERNEL, "debug%d", cpu);
93 		rc = bind_virq_to_irqhandler(VIRQ_DEBUG, cpu,
94 					     xen_debug_interrupt,
95 					     IRQF_PERCPU | IRQF_NOBALANCING,
96 					     debug_name, NULL);
97 		if (rc < 0)
98 			goto fail;
99 		per_cpu(xen_debug_irq, cpu).irq = rc;
100 		per_cpu(xen_debug_irq, cpu).name = debug_name;
101 	}
102 
103 	callfunc_name = kasprintf(GFP_KERNEL, "callfuncsingle%d", cpu);
104 	rc = bind_ipi_to_irqhandler(XEN_CALL_FUNCTION_SINGLE_VECTOR,
105 				    cpu,
106 				    xen_call_function_single_interrupt,
107 				    IRQF_PERCPU|IRQF_NOBALANCING,
108 				    callfunc_name,
109 				    NULL);
110 	if (rc < 0)
111 		goto fail;
112 	per_cpu(xen_callfuncsingle_irq, cpu).irq = rc;
113 	per_cpu(xen_callfuncsingle_irq, cpu).name = callfunc_name;
114 
115 	return 0;
116 
117  fail:
118 	xen_smp_intr_free(cpu);
119 	return rc;
120 }
121 
xen_smp_cpus_done(unsigned int max_cpus)122 void __init xen_smp_cpus_done(unsigned int max_cpus)
123 {
124 	if (xen_hvm_domain())
125 		native_smp_cpus_done(max_cpus);
126 	else
127 		calculate_max_logical_packages();
128 }
129 
xen_smp_send_reschedule(int cpu)130 void xen_smp_send_reschedule(int cpu)
131 {
132 	xen_send_IPI_one(cpu, XEN_RESCHEDULE_VECTOR);
133 }
134 
__xen_send_IPI_mask(const struct cpumask * mask,int vector)135 static void __xen_send_IPI_mask(const struct cpumask *mask,
136 			      int vector)
137 {
138 	unsigned cpu;
139 
140 	for_each_cpu_and(cpu, mask, cpu_online_mask)
141 		xen_send_IPI_one(cpu, vector);
142 }
143 
xen_smp_send_call_function_ipi(const struct cpumask * mask)144 void xen_smp_send_call_function_ipi(const struct cpumask *mask)
145 {
146 	int cpu;
147 
148 	__xen_send_IPI_mask(mask, XEN_CALL_FUNCTION_VECTOR);
149 
150 	/* Make sure other vcpus get a chance to run if they need to. */
151 	for_each_cpu(cpu, mask) {
152 		if (xen_vcpu_stolen(cpu)) {
153 			HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
154 			break;
155 		}
156 	}
157 }
158 
xen_smp_send_call_function_single_ipi(int cpu)159 void xen_smp_send_call_function_single_ipi(int cpu)
160 {
161 	__xen_send_IPI_mask(cpumask_of(cpu),
162 			  XEN_CALL_FUNCTION_SINGLE_VECTOR);
163 }
164 
xen_map_vector(int vector)165 static inline int xen_map_vector(int vector)
166 {
167 	int xen_vector;
168 
169 	switch (vector) {
170 	case RESCHEDULE_VECTOR:
171 		xen_vector = XEN_RESCHEDULE_VECTOR;
172 		break;
173 	case CALL_FUNCTION_VECTOR:
174 		xen_vector = XEN_CALL_FUNCTION_VECTOR;
175 		break;
176 	case CALL_FUNCTION_SINGLE_VECTOR:
177 		xen_vector = XEN_CALL_FUNCTION_SINGLE_VECTOR;
178 		break;
179 	case IRQ_WORK_VECTOR:
180 		xen_vector = XEN_IRQ_WORK_VECTOR;
181 		break;
182 #ifdef CONFIG_X86_64
183 	case NMI_VECTOR:
184 	case APIC_DM_NMI: /* Some use that instead of NMI_VECTOR */
185 		xen_vector = XEN_NMI_VECTOR;
186 		break;
187 #endif
188 	default:
189 		xen_vector = -1;
190 		printk(KERN_ERR "xen: vector 0x%x is not implemented\n",
191 			vector);
192 	}
193 
194 	return xen_vector;
195 }
196 
xen_send_IPI_mask(const struct cpumask * mask,int vector)197 void xen_send_IPI_mask(const struct cpumask *mask,
198 			      int vector)
199 {
200 	int xen_vector = xen_map_vector(vector);
201 
202 	if (xen_vector >= 0)
203 		__xen_send_IPI_mask(mask, xen_vector);
204 }
205 
xen_send_IPI_all(int vector)206 void xen_send_IPI_all(int vector)
207 {
208 	int xen_vector = xen_map_vector(vector);
209 
210 	if (xen_vector >= 0)
211 		__xen_send_IPI_mask(cpu_online_mask, xen_vector);
212 }
213 
xen_send_IPI_self(int vector)214 void xen_send_IPI_self(int vector)
215 {
216 	int xen_vector = xen_map_vector(vector);
217 
218 	if (xen_vector >= 0)
219 		xen_send_IPI_one(smp_processor_id(), xen_vector);
220 }
221 
xen_send_IPI_mask_allbutself(const struct cpumask * mask,int vector)222 void xen_send_IPI_mask_allbutself(const struct cpumask *mask,
223 				int vector)
224 {
225 	unsigned cpu;
226 	unsigned int this_cpu = smp_processor_id();
227 	int xen_vector = xen_map_vector(vector);
228 
229 	if (!(num_online_cpus() > 1) || (xen_vector < 0))
230 		return;
231 
232 	for_each_cpu_and(cpu, mask, cpu_online_mask) {
233 		if (this_cpu == cpu)
234 			continue;
235 
236 		xen_send_IPI_one(cpu, xen_vector);
237 	}
238 }
239 
xen_send_IPI_allbutself(int vector)240 void xen_send_IPI_allbutself(int vector)
241 {
242 	xen_send_IPI_mask_allbutself(cpu_online_mask, vector);
243 }
244 
xen_call_function_interrupt(int irq,void * dev_id)245 static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
246 {
247 	generic_smp_call_function_interrupt();
248 	inc_irq_stat(irq_call_count);
249 
250 	return IRQ_HANDLED;
251 }
252 
xen_call_function_single_interrupt(int irq,void * dev_id)253 static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id)
254 {
255 	generic_smp_call_function_single_interrupt();
256 	inc_irq_stat(irq_call_count);
257 
258 	return IRQ_HANDLED;
259 }
260