blob: 82f2513e975809908528db6c61f1752696eb5d06 [file] [log] [blame]
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -07001/*
2 * Xen hypercall batching.
3 *
4 * Xen allows multiple hypercalls to be issued at once, using the
5 * multicall interface. This allows the cost of trapping into the
6 * hypervisor to be amortized over several calls.
7 *
8 * This file implements a simple interface for multicalls. There's a
9 * per-cpu buffer of outstanding multicalls. When you want to queue a
10 * multicall for issuing, you can allocate a multicall slot for the
11 * call and its arguments, along with storage for space which is
12 * pointed to by the arguments (for passing pointers to structures,
13 * etc). When the multicall is actually issued, all the space for the
14 * commands and allocated memory is freed for reuse.
15 *
16 * Multicalls are flushed whenever any of the buffers get full, or
17 * when explicitly requested. There's no way to get per-multicall
18 * return results back. It will BUG if any of the multicalls fail.
19 *
20 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
21 */
22#include <linux/percpu.h>
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -070023#include <linux/hardirq.h>
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -070024#include <linux/debugfs.h>
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070025
26#include <asm/xen/hypercall.h>
27
28#include "multicalls.h"
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -070029#include "debugfs.h"
30
31#define MC_BATCH 32
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070032
Jeremy Fitzhardingea122d622007-10-16 11:51:31 -070033#define MC_DEBUG 1
34
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -070035#define MC_ARGS (MC_BATCH * 16)
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070036
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -070037
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070038struct mc_buffer {
39 struct multicall_entry entries[MC_BATCH];
Jeremy Fitzhardingea122d622007-10-16 11:51:31 -070040#if MC_DEBUG
41 struct multicall_entry debug[MC_BATCH];
Ian Campbellb93d51d2009-02-06 13:35:57 -080042 void *caller[MC_BATCH];
Jeremy Fitzhardingea122d622007-10-16 11:51:31 -070043#endif
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -070044 unsigned char args[MC_ARGS];
Jeremy Fitzhardinge91e0c5f2007-10-16 11:51:30 -070045 struct callback {
46 void (*fn)(void *);
47 void *data;
48 } callbacks[MC_BATCH];
49 unsigned mcidx, argidx, cbidx;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -070050};
51
52static DEFINE_PER_CPU(struct mc_buffer, mc_buffer);
53DEFINE_PER_CPU(unsigned long, xen_mc_irq_flags);
54
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -070055/* flush reasons 0- slots, 1- args, 2- callbacks */
56enum flush_reasons
57{
58 FL_SLOTS,
59 FL_ARGS,
60 FL_CALLBACKS,
61
62 FL_N_REASONS
63};
64
65#ifdef CONFIG_XEN_DEBUG_FS
66#define NHYPERCALLS 40 /* not really */
67
68static struct {
69 unsigned histo[MC_BATCH+1];
70
71 unsigned issued;
72 unsigned arg_total;
73 unsigned hypercalls;
74 unsigned histo_hypercalls[NHYPERCALLS];
75
76 unsigned flush[FL_N_REASONS];
77} mc_stats;
78
79static u8 zero_stats;
80
81static inline void check_zero(void)
82{
83 if (unlikely(zero_stats)) {
84 memset(&mc_stats, 0, sizeof(mc_stats));
85 zero_stats = 0;
86 }
87}
88
89static void mc_add_stats(const struct mc_buffer *mc)
90{
91 int i;
92
93 check_zero();
94
95 mc_stats.issued++;
96 mc_stats.hypercalls += mc->mcidx;
97 mc_stats.arg_total += mc->argidx;
98
99 mc_stats.histo[mc->mcidx]++;
100 for(i = 0; i < mc->mcidx; i++) {
101 unsigned op = mc->entries[i].op;
102 if (op < NHYPERCALLS)
103 mc_stats.histo_hypercalls[op]++;
104 }
105}
106
107static void mc_stats_flush(enum flush_reasons idx)
108{
109 check_zero();
110
111 mc_stats.flush[idx]++;
112}
113
114#else /* !CONFIG_XEN_DEBUG_FS */
115
116static inline void mc_add_stats(const struct mc_buffer *mc)
117{
118}
119
120static inline void mc_stats_flush(enum flush_reasons idx)
121{
122}
123#endif /* CONFIG_XEN_DEBUG_FS */
124
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700125void xen_mc_flush(void)
126{
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700127 struct mc_buffer *b = &__get_cpu_var(mc_buffer);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700128 int ret = 0;
129 unsigned long flags;
Jeremy Fitzhardinge91e0c5f2007-10-16 11:51:30 -0700130 int i;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700131
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700132 BUG_ON(preemptible());
133
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700134 /* Disable interrupts in case someone comes in and queues
135 something in the middle */
136 local_irq_save(flags);
137
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700138 mc_add_stats(b);
139
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700140 if (b->mcidx) {
Jeremy Fitzhardingea122d622007-10-16 11:51:31 -0700141#if MC_DEBUG
142 memcpy(b->debug, b->entries,
143 b->mcidx * sizeof(struct multicall_entry));
144#endif
145
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700146 if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0)
147 BUG();
148 for (i = 0; i < b->mcidx; i++)
149 if (b->entries[i].result < 0)
150 ret++;
Jeremy Fitzhardingea122d622007-10-16 11:51:31 -0700151
152#if MC_DEBUG
153 if (ret) {
154 printk(KERN_ERR "%d multicall(s) failed: cpu %d\n",
155 ret, smp_processor_id());
Jeremy Fitzhardinge8ba6c2b2008-07-08 15:06:30 -0700156 dump_stack();
Paolo Ciarrocchi7ebed392008-02-22 23:11:59 +0100157 for (i = 0; i < b->mcidx; i++) {
Ian Campbellb93d51d2009-02-06 13:35:57 -0800158 printk(KERN_DEBUG " call %2d/%d: op=%lu arg=[%lx] result=%ld\t%pF\n",
Jeremy Fitzhardingea122d622007-10-16 11:51:31 -0700159 i+1, b->mcidx,
160 b->debug[i].op,
161 b->debug[i].args[0],
Ian Campbellb93d51d2009-02-06 13:35:57 -0800162 b->entries[i].result,
163 b->caller[i]);
Jeremy Fitzhardingea122d622007-10-16 11:51:31 -0700164 }
165 }
166#endif
167
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700168 b->mcidx = 0;
169 b->argidx = 0;
170 } else
171 BUG_ON(b->argidx != 0);
172
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700173 local_irq_restore(flags);
174
Paolo Ciarrocchi7ebed392008-02-22 23:11:59 +0100175 for (i = 0; i < b->cbidx; i++) {
Jeremy Fitzhardinge91e0c5f2007-10-16 11:51:30 -0700176 struct callback *cb = &b->callbacks[i];
177
178 (*cb->fn)(cb->data);
179 }
180 b->cbidx = 0;
181
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700182 BUG_ON(ret);
183}
184
185struct multicall_space __xen_mc_entry(size_t args)
186{
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700187 struct mc_buffer *b = &__get_cpu_var(mc_buffer);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700188 struct multicall_space ret;
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700189 unsigned argidx = roundup(b->argidx, sizeof(u64));
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700190
Jeremy Fitzhardingef120f132007-07-17 18:37:06 -0700191 BUG_ON(preemptible());
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700192 BUG_ON(b->argidx > MC_ARGS);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700193
194 if (b->mcidx == MC_BATCH ||
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700195 (argidx + args) > MC_ARGS) {
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700196 mc_stats_flush(b->mcidx == MC_BATCH ? FL_SLOTS : FL_ARGS);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700197 xen_mc_flush();
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700198 argidx = roundup(b->argidx, sizeof(u64));
199 }
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700200
201 ret.mc = &b->entries[b->mcidx];
Ian Campbellb93d51d2009-02-06 13:35:57 -0800202#ifdef MC_DEBUG
203 b->caller[b->mcidx] = __builtin_return_address(0);
204#endif
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700205 b->mcidx++;
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700206 ret.args = &b->args[argidx];
207 b->argidx = argidx + args;
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700208
Jeremy Fitzhardinge400d3492008-06-16 04:30:03 -0700209 BUG_ON(b->argidx > MC_ARGS);
210 return ret;
211}
212
213struct multicall_space xen_mc_extend_args(unsigned long op, size_t size)
214{
215 struct mc_buffer *b = &__get_cpu_var(mc_buffer);
216 struct multicall_space ret = { NULL, NULL };
217
218 BUG_ON(preemptible());
219 BUG_ON(b->argidx > MC_ARGS);
220
221 if (b->mcidx == 0)
222 return ret;
223
224 if (b->entries[b->mcidx - 1].op != op)
225 return ret;
226
227 if ((b->argidx + size) > MC_ARGS)
228 return ret;
229
230 ret.mc = &b->entries[b->mcidx - 1];
231 ret.args = &b->args[b->argidx];
232 b->argidx += size;
233
234 BUG_ON(b->argidx > MC_ARGS);
Jeremy Fitzhardinge5ead97c2007-07-17 18:37:04 -0700235 return ret;
236}
Jeremy Fitzhardinge91e0c5f2007-10-16 11:51:30 -0700237
238void xen_mc_callback(void (*fn)(void *), void *data)
239{
240 struct mc_buffer *b = &__get_cpu_var(mc_buffer);
241 struct callback *cb;
242
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700243 if (b->cbidx == MC_BATCH) {
244 mc_stats_flush(FL_CALLBACKS);
Jeremy Fitzhardinge91e0c5f2007-10-16 11:51:30 -0700245 xen_mc_flush();
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700246 }
Jeremy Fitzhardinge91e0c5f2007-10-16 11:51:30 -0700247
248 cb = &b->callbacks[b->cbidx++];
249 cb->fn = fn;
250 cb->data = data;
251}
Jeremy Fitzhardinge994025c2008-08-20 17:02:19 -0700252
253#ifdef CONFIG_XEN_DEBUG_FS
254
255static struct dentry *d_mc_debug;
256
257static int __init xen_mc_debugfs(void)
258{
259 struct dentry *d_xen = xen_init_debugfs();
260
261 if (d_xen == NULL)
262 return -ENOMEM;
263
264 d_mc_debug = debugfs_create_dir("multicalls", d_xen);
265
266 debugfs_create_u8("zero_stats", 0644, d_mc_debug, &zero_stats);
267
268 debugfs_create_u32("batches", 0444, d_mc_debug, &mc_stats.issued);
269 debugfs_create_u32("hypercalls", 0444, d_mc_debug, &mc_stats.hypercalls);
270 debugfs_create_u32("arg_total", 0444, d_mc_debug, &mc_stats.arg_total);
271
272 xen_debugfs_create_u32_array("batch_histo", 0444, d_mc_debug,
273 mc_stats.histo, MC_BATCH);
274 xen_debugfs_create_u32_array("hypercall_histo", 0444, d_mc_debug,
275 mc_stats.histo_hypercalls, NHYPERCALLS);
276 xen_debugfs_create_u32_array("flush_reasons", 0444, d_mc_debug,
277 mc_stats.flush, FL_N_REASONS);
278
279 return 0;
280}
281fs_initcall(xen_mc_debugfs);
282
283#endif /* CONFIG_XEN_DEBUG_FS */