1 /*
2 * Xen hypercall batching.
3 *
4 * Xen allows multiple hypercalls to be issued at once, using the
5 * multicall interface. This allows the cost of trapping into the
6 * hypervisor to be amortized over several calls.
7 *
8 * This file implements a simple interface for multicalls. There's a
9 * per-cpu buffer of outstanding multicalls. When you want to queue a
10 * multicall for issuing, you can allocate a multicall slot for the
11 * call and its arguments, along with storage for space which is
12 * pointed to by the arguments (for passing pointers to structures,
13 * etc). When the multicall is actually issued, all the space for the
14 * commands and allocated memory is freed for reuse.
15 *
16 * Multicalls are flushed whenever any of the buffers get full, or
17 * when explicitly requested. There's no way to get per-multicall
18 * return results back. It will BUG if any of the multicalls fail.
19 *
20 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
21 */
22 #include <linux/percpu.h>
23 #include <linux/hardirq.h>
24 #include <linux/debugfs.h>
25
26 #include <asm/xen/hypercall.h>
27
28 #include "multicalls.h"
29 #include "debugfs.h"
30
31 #define MC_BATCH 32
32
33 #define MC_DEBUG 1
34
35 #define MC_ARGS (MC_BATCH * 16)
36
37
38 struct mc_buffer {
39 struct multicall_entry entries[MC_BATCH];
40 #if MC_DEBUG
41 struct multicall_entry debug[MC_BATCH];
42 void *caller[MC_BATCH];
43 #endif
44 unsigned char args[MC_ARGS];
45 struct callback {
46 void (*fn)(void *);
47 void *data;
48 } callbacks[MC_BATCH];
49 unsigned mcidx, argidx, cbidx;
50 };
51
52 static DEFINE_PER_CPU(struct mc_buffer, mc_buffer);
53 DEFINE_PER_CPU(unsigned long, xen_mc_irq_flags);
54
55 /* flush reasons 0- slots, 1- args, 2- callbacks */
56 enum flush_reasons
57 {
58 FL_SLOTS,
59 FL_ARGS,
60 FL_CALLBACKS,
61
62 FL_N_REASONS
63 };
64
65 #ifdef CONFIG_XEN_DEBUG_FS
66 #define NHYPERCALLS 40 /* not really */
67
68 static struct {
69 unsigned histo[MC_BATCH+1];
70
71 unsigned issued;
72 unsigned arg_total;
73 unsigned hypercalls;
74 unsigned histo_hypercalls[NHYPERCALLS];
75
76 unsigned flush[FL_N_REASONS];
77 } mc_stats;
78
79 static u8 zero_stats;
80
check_zero(void)81 static inline void check_zero(void)
82 {
83 if (unlikely(zero_stats)) {
84 memset(&mc_stats, 0, sizeof(mc_stats));
85 zero_stats = 0;
86 }
87 }
88
mc_add_stats(const struct mc_buffer * mc)89 static void mc_add_stats(const struct mc_buffer *mc)
90 {
91 int i;
92
93 check_zero();
94
95 mc_stats.issued++;
96 mc_stats.hypercalls += mc->mcidx;
97 mc_stats.arg_total += mc->argidx;
98
99 mc_stats.histo[mc->mcidx]++;
100 for(i = 0; i < mc->mcidx; i++) {
101 unsigned op = mc->entries[i].op;
102 if (op < NHYPERCALLS)
103 mc_stats.histo_hypercalls[op]++;
104 }
105 }
106
mc_stats_flush(enum flush_reasons idx)107 static void mc_stats_flush(enum flush_reasons idx)
108 {
109 check_zero();
110
111 mc_stats.flush[idx]++;
112 }
113
114 #else /* !CONFIG_XEN_DEBUG_FS */
115
mc_add_stats(const struct mc_buffer * mc)116 static inline void mc_add_stats(const struct mc_buffer *mc)
117 {
118 }
119
mc_stats_flush(enum flush_reasons idx)120 static inline void mc_stats_flush(enum flush_reasons idx)
121 {
122 }
123 #endif /* CONFIG_XEN_DEBUG_FS */
124
xen_mc_flush(void)125 void xen_mc_flush(void)
126 {
127 struct mc_buffer *b = &__get_cpu_var(mc_buffer);
128 int ret = 0;
129 unsigned long flags;
130 int i;
131
132 BUG_ON(preemptible());
133
134 /* Disable interrupts in case someone comes in and queues
135 something in the middle */
136 local_irq_save(flags);
137
138 mc_add_stats(b);
139
140 if (b->mcidx) {
141 #if MC_DEBUG
142 memcpy(b->debug, b->entries,
143 b->mcidx * sizeof(struct multicall_entry));
144 #endif
145
146 if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0)
147 BUG();
148 for (i = 0; i < b->mcidx; i++)
149 if (b->entries[i].result < 0)
150 ret++;
151
152 #if MC_DEBUG
153 if (ret) {
154 printk(KERN_ERR "%d multicall(s) failed: cpu %d\n",
155 ret, smp_processor_id());
156 dump_stack();
157 for (i = 0; i < b->mcidx; i++) {
158 printk(KERN_DEBUG " call %2d/%d: op=%lu arg=[%lx] result=%ld\t%pF\n",
159 i+1, b->mcidx,
160 b->debug[i].op,
161 b->debug[i].args[0],
162 b->entries[i].result,
163 b->caller[i]);
164 }
165 }
166 #endif
167
168 b->mcidx = 0;
169 b->argidx = 0;
170 } else
171 BUG_ON(b->argidx != 0);
172
173 for (i = 0; i < b->cbidx; i++) {
174 struct callback *cb = &b->callbacks[i];
175
176 (*cb->fn)(cb->data);
177 }
178 b->cbidx = 0;
179
180 local_irq_restore(flags);
181
182 WARN_ON(ret);
183 }
184
__xen_mc_entry(size_t args)185 struct multicall_space __xen_mc_entry(size_t args)
186 {
187 struct mc_buffer *b = &__get_cpu_var(mc_buffer);
188 struct multicall_space ret;
189 unsigned argidx = roundup(b->argidx, sizeof(u64));
190
191 BUG_ON(preemptible());
192 BUG_ON(b->argidx > MC_ARGS);
193
194 if (b->mcidx == MC_BATCH ||
195 (argidx + args) > MC_ARGS) {
196 mc_stats_flush(b->mcidx == MC_BATCH ? FL_SLOTS : FL_ARGS);
197 xen_mc_flush();
198 argidx = roundup(b->argidx, sizeof(u64));
199 }
200
201 ret.mc = &b->entries[b->mcidx];
202 #ifdef MC_DEBUG
203 b->caller[b->mcidx] = __builtin_return_address(0);
204 #endif
205 b->mcidx++;
206 ret.args = &b->args[argidx];
207 b->argidx = argidx + args;
208
209 BUG_ON(b->argidx > MC_ARGS);
210 return ret;
211 }
212
xen_mc_extend_args(unsigned long op,size_t size)213 struct multicall_space xen_mc_extend_args(unsigned long op, size_t size)
214 {
215 struct mc_buffer *b = &__get_cpu_var(mc_buffer);
216 struct multicall_space ret = { NULL, NULL };
217
218 BUG_ON(preemptible());
219 BUG_ON(b->argidx > MC_ARGS);
220
221 if (b->mcidx == 0)
222 return ret;
223
224 if (b->entries[b->mcidx - 1].op != op)
225 return ret;
226
227 if ((b->argidx + size) > MC_ARGS)
228 return ret;
229
230 ret.mc = &b->entries[b->mcidx - 1];
231 ret.args = &b->args[b->argidx];
232 b->argidx += size;
233
234 BUG_ON(b->argidx > MC_ARGS);
235 return ret;
236 }
237
xen_mc_callback(void (* fn)(void *),void * data)238 void xen_mc_callback(void (*fn)(void *), void *data)
239 {
240 struct mc_buffer *b = &__get_cpu_var(mc_buffer);
241 struct callback *cb;
242
243 if (b->cbidx == MC_BATCH) {
244 mc_stats_flush(FL_CALLBACKS);
245 xen_mc_flush();
246 }
247
248 cb = &b->callbacks[b->cbidx++];
249 cb->fn = fn;
250 cb->data = data;
251 }
252
253 #ifdef CONFIG_XEN_DEBUG_FS
254
255 static struct dentry *d_mc_debug;
256
xen_mc_debugfs(void)257 static int __init xen_mc_debugfs(void)
258 {
259 struct dentry *d_xen = xen_init_debugfs();
260
261 if (d_xen == NULL)
262 return -ENOMEM;
263
264 d_mc_debug = debugfs_create_dir("multicalls", d_xen);
265
266 debugfs_create_u8("zero_stats", 0644, d_mc_debug, &zero_stats);
267
268 debugfs_create_u32("batches", 0444, d_mc_debug, &mc_stats.issued);
269 debugfs_create_u32("hypercalls", 0444, d_mc_debug, &mc_stats.hypercalls);
270 debugfs_create_u32("arg_total", 0444, d_mc_debug, &mc_stats.arg_total);
271
272 xen_debugfs_create_u32_array("batch_histo", 0444, d_mc_debug,
273 mc_stats.histo, MC_BATCH);
274 xen_debugfs_create_u32_array("hypercall_histo", 0444, d_mc_debug,
275 mc_stats.histo_hypercalls, NHYPERCALLS);
276 xen_debugfs_create_u32_array("flush_reasons", 0444, d_mc_debug,
277 mc_stats.flush, FL_N_REASONS);
278
279 return 0;
280 }
281 fs_initcall(xen_mc_debugfs);
282
283 #endif /* CONFIG_XEN_DEBUG_FS */
284