blob: ee1c6d05c3d343d4d5f4c3039779eba5d75662ae [file] [log] [blame]
Rusty Russellf938d2c2007-07-26 10:41:02 -07001/*P:010
2 * A hypervisor allows multiple Operating Systems to run on a single machine.
3 * To quote David Wheeler: "Any problem in computer science can be solved with
4 * another layer of indirection."
Rusty Russell07ad1572007-07-19 01:49:22 -07005 *
Rusty Russellf938d2c2007-07-26 10:41:02 -07006 * We keep things simple in two ways. First, we start with a normal Linux
7 * kernel and insert a module (lg.ko) which allows us to run other Linux
8 * kernels the same way we'd run processes. We call the first kernel the Host,
9 * and the others the Guests. The program which sets up and configures Guests
10 * (such as the example in Documentation/lguest/lguest.c) is called the
11 * Launcher.
12 *
13 * Secondly, we only run specially modified Guests, not normal kernels. When
14 * you set CONFIG_LGUEST to 'y' or 'm', this automatically sets
15 * CONFIG_LGUEST_GUEST=y, which compiles this file into the kernel so it knows
16 * how to be a Guest. This means that you can use the same kernel you boot
17 * normally (ie. as a Host) as a Guest.
18 *
19 * These Guests know that they cannot do privileged operations, such as disable
20 * interrupts, and that they have to ask the Host to do such things explicitly.
21 * This file consists of all the replacements for such low-level native
22 * hardware operations: these special Guest versions call the Host.
23 *
24 * So how does the kernel know it's a Guest? The Guest starts at a special
25 * entry point marked with a magic string, which sets up a few things then
26 * calls here. We replace the native functions in "struct paravirt_ops"
27 * with our Guest versions, then boot like normal. :*/
28
29/*
Rusty Russell07ad1572007-07-19 01:49:22 -070030 * Copyright (C) 2006, Rusty Russell <rusty@rustcorp.com.au> IBM Corporation.
31 *
32 * This program is free software; you can redistribute it and/or modify
33 * it under the terms of the GNU General Public License as published by
34 * the Free Software Foundation; either version 2 of the License, or
35 * (at your option) any later version.
36 *
37 * This program is distributed in the hope that it will be useful, but
38 * WITHOUT ANY WARRANTY; without even the implied warranty of
39 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
40 * NON INFRINGEMENT. See the GNU General Public License for more
41 * details.
42 *
43 * You should have received a copy of the GNU General Public License
44 * along with this program; if not, write to the Free Software
45 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
46 */
47#include <linux/kernel.h>
48#include <linux/start_kernel.h>
49#include <linux/string.h>
50#include <linux/console.h>
51#include <linux/screen_info.h>
52#include <linux/irq.h>
53#include <linux/interrupt.h>
Rusty Russelld7e28ff2007-07-19 01:49:23 -070054#include <linux/clocksource.h>
55#include <linux/clockchips.h>
Rusty Russell07ad1572007-07-19 01:49:22 -070056#include <linux/lguest.h>
57#include <linux/lguest_launcher.h>
58#include <linux/lguest_bus.h>
59#include <asm/paravirt.h>
60#include <asm/param.h>
61#include <asm/page.h>
62#include <asm/pgtable.h>
63#include <asm/desc.h>
64#include <asm/setup.h>
65#include <asm/e820.h>
66#include <asm/mce.h>
67#include <asm/io.h>
68
Rusty Russellb2b47c22007-07-26 10:41:02 -070069/*G:010 Welcome to the Guest!
70 *
71 * The Guest in our tale is a simple creature: identical to the Host but
72 * behaving in simplified but equivalent ways. In particular, the Guest is the
73 * same kernel as the Host (or at least, built from the same source code). :*/
74
Rusty Russell07ad1572007-07-19 01:49:22 -070075/* Declarations for definitions in lguest_guest.S */
76extern char lguest_noirq_start[], lguest_noirq_end[];
77extern const char lgstart_cli[], lgend_cli[];
78extern const char lgstart_sti[], lgend_sti[];
79extern const char lgstart_popf[], lgend_popf[];
80extern const char lgstart_pushf[], lgend_pushf[];
81extern const char lgstart_iret[], lgend_iret[];
82extern void lguest_iret(void);
83
84struct lguest_data lguest_data = {
85 .hcall_status = { [0 ... LHCALL_RING_SIZE-1] = 0xFF },
86 .noirq_start = (u32)lguest_noirq_start,
87 .noirq_end = (u32)lguest_noirq_end,
88 .blocked_interrupts = { 1 }, /* Block timer interrupts */
89};
90struct lguest_device_desc *lguest_devices;
Rusty Russell9d1ca6f2007-07-20 22:15:01 +100091static cycle_t clock_base;
Rusty Russell07ad1572007-07-19 01:49:22 -070092
Rusty Russellb2b47c22007-07-26 10:41:02 -070093/*G:035 Notice the lazy_hcall() above, rather than hcall(). This is our first
94 * real optimization trick!
95 *
96 * When lazy_mode is set, it means we're allowed to defer all hypercalls and do
97 * them as a batch when lazy_mode is eventually turned off. Because hypercalls
98 * are reasonably expensive, batching them up makes sense. For example, a
99 * large mmap might update dozens of page table entries: that code calls
100 * lguest_lazy_mode(PARAVIRT_LAZY_MMU), does the dozen updates, then calls
101 * lguest_lazy_mode(PARAVIRT_LAZY_NONE).
102 *
103 * So, when we're in lazy mode, we call async_hypercall() to store the call for
104 * future processing. When lazy mode is turned off we issue a hypercall to
105 * flush the stored calls.
106 *
107 * There's also a hack where "mode" is set to "PARAVIRT_LAZY_FLUSH" which
108 * indicates we're to flush any outstanding calls immediately. This is used
109 * when an interrupt handler does a kmap_atomic(): the page table changes must
110 * happen immediately even if we're in the middle of a batch. Usually we're
111 * not, though, so there's nothing to do. */
112static enum paravirt_lazy_mode lazy_mode; /* Note: not SMP-safe! */
Rusty Russell07ad1572007-07-19 01:49:22 -0700113static void lguest_lazy_mode(enum paravirt_lazy_mode mode)
114{
115 if (mode == PARAVIRT_LAZY_FLUSH) {
116 if (unlikely(lazy_mode != PARAVIRT_LAZY_NONE))
117 hcall(LHCALL_FLUSH_ASYNC, 0, 0, 0);
118 } else {
119 lazy_mode = mode;
120 if (mode == PARAVIRT_LAZY_NONE)
121 hcall(LHCALL_FLUSH_ASYNC, 0, 0, 0);
122 }
123}
124
125static void lazy_hcall(unsigned long call,
126 unsigned long arg1,
127 unsigned long arg2,
128 unsigned long arg3)
129{
130 if (lazy_mode == PARAVIRT_LAZY_NONE)
131 hcall(call, arg1, arg2, arg3);
132 else
133 async_hcall(call, arg1, arg2, arg3);
134}
135
Rusty Russellb2b47c22007-07-26 10:41:02 -0700136/* async_hcall() is pretty simple: I'm quite proud of it really. We have a
137 * ring buffer of stored hypercalls which the Host will run though next time we
138 * do a normal hypercall. Each entry in the ring has 4 slots for the hypercall
139 * arguments, and a "hcall_status" word which is 0 if the call is ready to go,
140 * and 255 once the Host has finished with it.
141 *
142 * If we come around to a slot which hasn't been finished, then the table is
143 * full and we just make the hypercall directly. This has the nice side
144 * effect of causing the Host to run all the stored calls in the ring buffer
145 * which empties it for next time! */
Rusty Russell07ad1572007-07-19 01:49:22 -0700146void async_hcall(unsigned long call,
147 unsigned long arg1, unsigned long arg2, unsigned long arg3)
148{
149 /* Note: This code assumes we're uniprocessor. */
150 static unsigned int next_call;
151 unsigned long flags;
152
Rusty Russellb2b47c22007-07-26 10:41:02 -0700153 /* Disable interrupts if not already disabled: we don't want an
154 * interrupt handler making a hypercall while we're already doing
155 * one! */
Rusty Russell07ad1572007-07-19 01:49:22 -0700156 local_irq_save(flags);
157 if (lguest_data.hcall_status[next_call] != 0xFF) {
158 /* Table full, so do normal hcall which will flush table. */
159 hcall(call, arg1, arg2, arg3);
160 } else {
161 lguest_data.hcalls[next_call].eax = call;
162 lguest_data.hcalls[next_call].edx = arg1;
163 lguest_data.hcalls[next_call].ebx = arg2;
164 lguest_data.hcalls[next_call].ecx = arg3;
Rusty Russellb2b47c22007-07-26 10:41:02 -0700165 /* Arguments must all be written before we mark it to go */
Rusty Russell07ad1572007-07-19 01:49:22 -0700166 wmb();
167 lguest_data.hcall_status[next_call] = 0;
168 if (++next_call == LHCALL_RING_SIZE)
169 next_call = 0;
170 }
171 local_irq_restore(flags);
172}
Rusty Russellb2b47c22007-07-26 10:41:02 -0700173/*:*/
Rusty Russell07ad1572007-07-19 01:49:22 -0700174
Rusty Russellb2b47c22007-07-26 10:41:02 -0700175/* Wrappers for the SEND_DMA and BIND_DMA hypercalls. This is mainly because
176 * Jeff Garzik complained that __pa() should never appear in drivers, and this
177 * helps remove most of them. But also, it wraps some ugliness. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700178void lguest_send_dma(unsigned long key, struct lguest_dma *dma)
179{
Rusty Russellb2b47c22007-07-26 10:41:02 -0700180 /* The hcall might not write this if something goes wrong */
Rusty Russell07ad1572007-07-19 01:49:22 -0700181 dma->used_len = 0;
182 hcall(LHCALL_SEND_DMA, key, __pa(dma), 0);
183}
184
185int lguest_bind_dma(unsigned long key, struct lguest_dma *dmas,
186 unsigned int num, u8 irq)
187{
Rusty Russellb2b47c22007-07-26 10:41:02 -0700188 /* This is the only hypercall which actually wants 5 arguments, and we
189 * only support 4. Fortunately the interrupt number is always less
190 * than 256, so we can pack it with the number of dmas in the final
191 * argument. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700192 if (!hcall(LHCALL_BIND_DMA, key, __pa(dmas), (num << 8) | irq))
193 return -ENOMEM;
194 return 0;
195}
196
Rusty Russellb2b47c22007-07-26 10:41:02 -0700197/* Unbinding is the same hypercall as binding, but with 0 num & irq. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700198void lguest_unbind_dma(unsigned long key, struct lguest_dma *dmas)
199{
200 hcall(LHCALL_BIND_DMA, key, __pa(dmas), 0);
201}
202
203/* For guests, device memory can be used as normal memory, so we cast away the
204 * __iomem to quieten sparse. */
205void *lguest_map(unsigned long phys_addr, unsigned long pages)
206{
207 return (__force void *)ioremap(phys_addr, PAGE_SIZE*pages);
208}
209
210void lguest_unmap(void *addr)
211{
212 iounmap((__force void __iomem *)addr);
213}
214
Rusty Russellb2b47c22007-07-26 10:41:02 -0700215/*G:033
216 * Here are our first native-instruction replacements: four functions for
217 * interrupt control.
218 *
219 * The simplest way of implementing these would be to have "turn interrupts
220 * off" and "turn interrupts on" hypercalls. Unfortunately, this is too slow:
221 * these are by far the most commonly called functions of those we override.
222 *
223 * So instead we keep an "irq_enabled" field inside our "struct lguest_data",
224 * which the Guest can update with a single instruction. The Host knows to
225 * check there when it wants to deliver an interrupt.
226 */
227
228/* save_flags() is expected to return the processor state (ie. "eflags"). The
229 * eflags word contains all kind of stuff, but in practice Linux only cares
230 * about the interrupt flag. Our "save_flags()" just returns that. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700231static unsigned long save_fl(void)
232{
233 return lguest_data.irq_enabled;
234}
235
Rusty Russellb2b47c22007-07-26 10:41:02 -0700236/* "restore_flags" just sets the flags back to the value given. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700237static void restore_fl(unsigned long flags)
238{
Rusty Russell07ad1572007-07-19 01:49:22 -0700239 lguest_data.irq_enabled = flags;
240}
241
Rusty Russellb2b47c22007-07-26 10:41:02 -0700242/* Interrupts go off... */
Rusty Russell07ad1572007-07-19 01:49:22 -0700243static void irq_disable(void)
244{
245 lguest_data.irq_enabled = 0;
246}
247
Rusty Russellb2b47c22007-07-26 10:41:02 -0700248/* Interrupts go on... */
Rusty Russell07ad1572007-07-19 01:49:22 -0700249static void irq_enable(void)
250{
Rusty Russell07ad1572007-07-19 01:49:22 -0700251 lguest_data.irq_enabled = X86_EFLAGS_IF;
252}
Rusty Russellf56a3842007-07-26 10:41:05 -0700253/*:*/
254/*M:003 Note that we don't check for outstanding interrupts when we re-enable
255 * them (or when we unmask an interrupt). This seems to work for the moment,
256 * since interrupts are rare and we'll just get the interrupt on the next timer
257 * tick, but when we turn on CONFIG_NO_HZ, we should revisit this. One way
258 * would be to put the "irq_enabled" field in a page by itself, and have the
259 * Host write-protect it when an interrupt comes in when irqs are disabled.
260 * There will then be a page fault as soon as interrupts are re-enabled. :*/
Rusty Russell07ad1572007-07-19 01:49:22 -0700261
Rusty Russellb2b47c22007-07-26 10:41:02 -0700262/*G:034
263 * The Interrupt Descriptor Table (IDT).
264 *
265 * The IDT tells the processor what to do when an interrupt comes in. Each
266 * entry in the table is a 64-bit descriptor: this holds the privilege level,
267 * address of the handler, and... well, who cares? The Guest just asks the
268 * Host to make the change anyway, because the Host controls the real IDT.
269 */
Rusty Russell07ad1572007-07-19 01:49:22 -0700270static void lguest_write_idt_entry(struct desc_struct *dt,
271 int entrynum, u32 low, u32 high)
272{
Rusty Russellb2b47c22007-07-26 10:41:02 -0700273 /* Keep the local copy up to date. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700274 write_dt_entry(dt, entrynum, low, high);
Rusty Russellb2b47c22007-07-26 10:41:02 -0700275 /* Tell Host about this new entry. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700276 hcall(LHCALL_LOAD_IDT_ENTRY, entrynum, low, high);
277}
278
Rusty Russellb2b47c22007-07-26 10:41:02 -0700279/* Changing to a different IDT is very rare: we keep the IDT up-to-date every
280 * time it is written, so we can simply loop through all entries and tell the
281 * Host about them. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700282static void lguest_load_idt(const struct Xgt_desc_struct *desc)
283{
284 unsigned int i;
285 struct desc_struct *idt = (void *)desc->address;
286
287 for (i = 0; i < (desc->size+1)/8; i++)
288 hcall(LHCALL_LOAD_IDT_ENTRY, i, idt[i].a, idt[i].b);
289}
290
Rusty Russellb2b47c22007-07-26 10:41:02 -0700291/*
292 * The Global Descriptor Table.
293 *
294 * The Intel architecture defines another table, called the Global Descriptor
295 * Table (GDT). You tell the CPU where it is (and its size) using the "lgdt"
296 * instruction, and then several other instructions refer to entries in the
297 * table. There are three entries which the Switcher needs, so the Host simply
298 * controls the entire thing and the Guest asks it to make changes using the
299 * LOAD_GDT hypercall.
300 *
301 * This is the opposite of the IDT code where we have a LOAD_IDT_ENTRY
302 * hypercall and use that repeatedly to load a new IDT. I don't think it
303 * really matters, but wouldn't it be nice if they were the same?
304 */
Rusty Russell07ad1572007-07-19 01:49:22 -0700305static void lguest_load_gdt(const struct Xgt_desc_struct *desc)
306{
307 BUG_ON((desc->size+1)/8 != GDT_ENTRIES);
308 hcall(LHCALL_LOAD_GDT, __pa(desc->address), GDT_ENTRIES, 0);
309}
310
Rusty Russellb2b47c22007-07-26 10:41:02 -0700311/* For a single GDT entry which changes, we do the lazy thing: alter our GDT,
312 * then tell the Host to reload the entire thing. This operation is so rare
313 * that this naive implementation is reasonable. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700314static void lguest_write_gdt_entry(struct desc_struct *dt,
315 int entrynum, u32 low, u32 high)
316{
317 write_dt_entry(dt, entrynum, low, high);
318 hcall(LHCALL_LOAD_GDT, __pa(dt), GDT_ENTRIES, 0);
319}
320
Rusty Russellb2b47c22007-07-26 10:41:02 -0700321/* OK, I lied. There are three "thread local storage" GDT entries which change
322 * on every context switch (these three entries are how glibc implements
323 * __thread variables). So we have a hypercall specifically for this case. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700324static void lguest_load_tls(struct thread_struct *t, unsigned int cpu)
325{
Rusty Russell0d027c02007-08-09 20:57:13 +1000326 /* There's one problem which normal hardware doesn't have: the Host
327 * can't handle us removing entries we're currently using. So we clear
328 * the GS register here: if it's needed it'll be reloaded anyway. */
329 loadsegment(gs, 0);
Rusty Russell07ad1572007-07-19 01:49:22 -0700330 lazy_hcall(LHCALL_LOAD_TLS, __pa(&t->tls_array), cpu, 0);
331}
332
Rusty Russellb2b47c22007-07-26 10:41:02 -0700333/*G:038 That's enough excitement for now, back to ploughing through each of
334 * the paravirt_ops (we're about 1/3 of the way through).
335 *
336 * This is the Local Descriptor Table, another weird Intel thingy. Linux only
337 * uses this for some strange applications like Wine. We don't do anything
338 * here, so they'll get an informative and friendly Segmentation Fault. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700339static void lguest_set_ldt(const void *addr, unsigned entries)
340{
341}
342
Rusty Russellb2b47c22007-07-26 10:41:02 -0700343/* This loads a GDT entry into the "Task Register": that entry points to a
344 * structure called the Task State Segment. Some comments scattered though the
345 * kernel code indicate that this used for task switching in ages past, along
346 * with blood sacrifice and astrology.
347 *
348 * Now there's nothing interesting in here that we don't get told elsewhere.
349 * But the native version uses the "ltr" instruction, which makes the Host
350 * complain to the Guest about a Segmentation Fault and it'll oops. So we
351 * override the native version with a do-nothing version. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700352static void lguest_load_tr_desc(void)
353{
354}
355
Rusty Russellb2b47c22007-07-26 10:41:02 -0700356/* The "cpuid" instruction is a way of querying both the CPU identity
357 * (manufacturer, model, etc) and its features. It was introduced before the
358 * Pentium in 1993 and keeps getting extended by both Intel and AMD. As you
359 * might imagine, after a decade and a half this treatment, it is now a giant
360 * ball of hair. Its entry in the current Intel manual runs to 28 pages.
361 *
362 * This instruction even it has its own Wikipedia entry. The Wikipedia entry
363 * has been translated into 4 languages. I am not making this up!
364 *
365 * We could get funky here and identify ourselves as "GenuineLguest", but
366 * instead we just use the real "cpuid" instruction. Then I pretty much turned
367 * off feature bits until the Guest booted. (Don't say that: you'll damage
368 * lguest sales!) Shut up, inner voice! (Hey, just pointing out that this is
369 * hardly future proof.) Noone's listening! They don't like you anyway,
370 * parenthetic weirdo!
371 *
372 * Replacing the cpuid so we can turn features off is great for the kernel, but
373 * anyone (including userspace) can just use the raw "cpuid" instruction and
374 * the Host won't even notice since it isn't privileged. So we try not to get
375 * too worked up about it. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700376static void lguest_cpuid(unsigned int *eax, unsigned int *ebx,
377 unsigned int *ecx, unsigned int *edx)
378{
379 int function = *eax;
380
381 native_cpuid(eax, ebx, ecx, edx);
382 switch (function) {
383 case 1: /* Basic feature request. */
384 /* We only allow kernel to see SSE3, CMPXCHG16B and SSSE3 */
385 *ecx &= 0x00002201;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700386 /* SSE, SSE2, FXSR, MMX, CMOV, CMPXCHG8B, FPU. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700387 *edx &= 0x07808101;
Rusty Russellb2b47c22007-07-26 10:41:02 -0700388 /* The Host can do a nice optimization if it knows that the
389 * kernel mappings (addresses above 0xC0000000 or whatever
390 * PAGE_OFFSET is set to) haven't changed. But Linux calls
391 * flush_tlb_user() for both user and kernel mappings unless
392 * the Page Global Enable (PGE) feature bit is set. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700393 *edx |= 0x00002000;
394 break;
395 case 0x80000000:
396 /* Futureproof this a little: if they ask how much extended
Rusty Russellb2b47c22007-07-26 10:41:02 -0700397 * processor information there is, limit it to known fields. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700398 if (*eax > 0x80000008)
399 *eax = 0x80000008;
400 break;
401 }
402}
403
Rusty Russellb2b47c22007-07-26 10:41:02 -0700404/* Intel has four control registers, imaginatively named cr0, cr2, cr3 and cr4.
405 * I assume there's a cr1, but it hasn't bothered us yet, so we'll not bother
406 * it. The Host needs to know when the Guest wants to change them, so we have
407 * a whole series of functions like read_cr0() and write_cr0().
408 *
409 * We start with CR0. CR0 allows you to turn on and off all kinds of basic
410 * features, but Linux only really cares about one: the horrifically-named Task
411 * Switched (TS) bit at bit 3 (ie. 8)
412 *
413 * What does the TS bit do? Well, it causes the CPU to trap (interrupt 7) if
414 * the floating point unit is used. Which allows us to restore FPU state
415 * lazily after a task switch, and Linux uses that gratefully, but wouldn't a
416 * name like "FPUTRAP bit" be a little less cryptic?
417 *
418 * We store cr0 (and cr3) locally, because the Host never changes it. The
419 * Guest sometimes wants to read it and we'd prefer not to bother the Host
420 * unnecessarily. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700421static unsigned long current_cr0, current_cr3;
422static void lguest_write_cr0(unsigned long val)
423{
Rusty Russellb2b47c22007-07-26 10:41:02 -0700424 /* 8 == TS bit. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700425 lazy_hcall(LHCALL_TS, val & 8, 0, 0);
426 current_cr0 = val;
427}
428
429static unsigned long lguest_read_cr0(void)
430{
431 return current_cr0;
432}
433
Rusty Russellb2b47c22007-07-26 10:41:02 -0700434/* Intel provided a special instruction to clear the TS bit for people too cool
435 * to use write_cr0() to do it. This "clts" instruction is faster, because all
436 * the vowels have been optimized out. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700437static void lguest_clts(void)
438{
439 lazy_hcall(LHCALL_TS, 0, 0, 0);
440 current_cr0 &= ~8U;
441}
442
Rusty Russellb2b47c22007-07-26 10:41:02 -0700443/* CR2 is the virtual address of the last page fault, which the Guest only ever
444 * reads. The Host kindly writes this into our "struct lguest_data", so we
445 * just read it out of there. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700446static unsigned long lguest_read_cr2(void)
447{
448 return lguest_data.cr2;
449}
450
Rusty Russellb2b47c22007-07-26 10:41:02 -0700451/* CR3 is the current toplevel pagetable page: the principle is the same as
452 * cr0. Keep a local copy, and tell the Host when it changes. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700453static void lguest_write_cr3(unsigned long cr3)
454{
455 lazy_hcall(LHCALL_NEW_PGTABLE, cr3, 0, 0);
456 current_cr3 = cr3;
457}
458
459static unsigned long lguest_read_cr3(void)
460{
461 return current_cr3;
462}
463
Rusty Russellb2b47c22007-07-26 10:41:02 -0700464/* CR4 is used to enable and disable PGE, but we don't care. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700465static unsigned long lguest_read_cr4(void)
466{
467 return 0;
468}
469
470static void lguest_write_cr4(unsigned long val)
471{
472}
473
Rusty Russellb2b47c22007-07-26 10:41:02 -0700474/*
475 * Page Table Handling.
476 *
477 * Now would be a good time to take a rest and grab a coffee or similarly
478 * relaxing stimulant. The easy parts are behind us, and the trek gradually
479 * winds uphill from here.
480 *
481 * Quick refresher: memory is divided into "pages" of 4096 bytes each. The CPU
482 * maps virtual addresses to physical addresses using "page tables". We could
483 * use one huge index of 1 million entries: each address is 4 bytes, so that's
484 * 1024 pages just to hold the page tables. But since most virtual addresses
485 * are unused, we use a two level index which saves space. The CR3 register
486 * contains the physical address of the top level "page directory" page, which
487 * contains physical addresses of up to 1024 second-level pages. Each of these
488 * second level pages contains up to 1024 physical addresses of actual pages,
489 * or Page Table Entries (PTEs).
490 *
491 * Here's a diagram, where arrows indicate physical addresses:
492 *
493 * CR3 ---> +---------+
494 * | --------->+---------+
495 * | | | PADDR1 |
496 * Top-level | | PADDR2 |
497 * (PMD) page | | |
498 * | | Lower-level |
499 * | | (PTE) page |
500 * | | | |
501 * .... ....
502 *
503 * So to convert a virtual address to a physical address, we look up the top
504 * level, which points us to the second level, which gives us the physical
505 * address of that page. If the top level entry was not present, or the second
506 * level entry was not present, then the virtual address is invalid (we
507 * say "the page was not mapped").
508 *
509 * Put another way, a 32-bit virtual address is divided up like so:
510 *
511 * 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
512 * |<---- 10 bits ---->|<---- 10 bits ---->|<------ 12 bits ------>|
513 * Index into top Index into second Offset within page
514 * page directory page pagetable page
515 *
516 * The kernel spends a lot of time changing both the top-level page directory
517 * and lower-level pagetable pages. The Guest doesn't know physical addresses,
518 * so while it maintains these page tables exactly like normal, it also needs
519 * to keep the Host informed whenever it makes a change: the Host will create
520 * the real page tables based on the Guests'.
521 */
522
523/* The Guest calls this to set a second-level entry (pte), ie. to map a page
524 * into a process' address space. We set the entry then tell the Host the
525 * toplevel and address this corresponds to. The Guest uses one pagetable per
526 * process, so we need to tell the Host which one we're changing (mm->pgd). */
Rusty Russell07ad1572007-07-19 01:49:22 -0700527static void lguest_set_pte_at(struct mm_struct *mm, unsigned long addr,
528 pte_t *ptep, pte_t pteval)
529{
530 *ptep = pteval;
531 lazy_hcall(LHCALL_SET_PTE, __pa(mm->pgd), addr, pteval.pte_low);
532}
533
Rusty Russellb2b47c22007-07-26 10:41:02 -0700534/* The Guest calls this to set a top-level entry. Again, we set the entry then
535 * tell the Host which top-level page we changed, and the index of the entry we
536 * changed. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700537static void lguest_set_pmd(pmd_t *pmdp, pmd_t pmdval)
538{
539 *pmdp = pmdval;
540 lazy_hcall(LHCALL_SET_PMD, __pa(pmdp)&PAGE_MASK,
541 (__pa(pmdp)&(PAGE_SIZE-1))/4, 0);
542}
543
Rusty Russellb2b47c22007-07-26 10:41:02 -0700544/* There are a couple of legacy places where the kernel sets a PTE, but we
545 * don't know the top level any more. This is useless for us, since we don't
546 * know which pagetable is changing or what address, so we just tell the Host
547 * to forget all of them. Fortunately, this is very rare.
548 *
549 * ... except in early boot when the kernel sets up the initial pagetables,
550 * which makes booting astonishingly slow. So we don't even tell the Host
551 * anything changed until we've done the first page table switch.
552 */
Rusty Russell07ad1572007-07-19 01:49:22 -0700553static void lguest_set_pte(pte_t *ptep, pte_t pteval)
554{
555 *ptep = pteval;
556 /* Don't bother with hypercall before initial setup. */
557 if (current_cr3)
558 lazy_hcall(LHCALL_FLUSH_TLB, 1, 0, 0);
559}
560
Rusty Russellb2b47c22007-07-26 10:41:02 -0700561/* Unfortunately for Lguest, the paravirt_ops for page tables were based on
562 * native page table operations. On native hardware you can set a new page
563 * table entry whenever you want, but if you want to remove one you have to do
564 * a TLB flush (a TLB is a little cache of page table entries kept by the CPU).
565 *
566 * So the lguest_set_pte_at() and lguest_set_pmd() functions above are only
567 * called when a valid entry is written, not when it's removed (ie. marked not
568 * present). Instead, this is where we come when the Guest wants to remove a
569 * page table entry: we tell the Host to set that entry to 0 (ie. the present
570 * bit is zero). */
Rusty Russell07ad1572007-07-19 01:49:22 -0700571static void lguest_flush_tlb_single(unsigned long addr)
572{
Rusty Russellb2b47c22007-07-26 10:41:02 -0700573 /* Simply set it to zero: if it was not, it will fault back in. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700574 lazy_hcall(LHCALL_SET_PTE, current_cr3, addr, 0);
575}
576
Rusty Russellb2b47c22007-07-26 10:41:02 -0700577/* This is what happens after the Guest has removed a large number of entries.
578 * This tells the Host that any of the page table entries for userspace might
579 * have changed, ie. virtual addresses below PAGE_OFFSET. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700580static void lguest_flush_tlb_user(void)
581{
582 lazy_hcall(LHCALL_FLUSH_TLB, 0, 0, 0);
583}
584
Rusty Russellb2b47c22007-07-26 10:41:02 -0700585/* This is called when the kernel page tables have changed. That's not very
586 * common (unless the Guest is using highmem, which makes the Guest extremely
587 * slow), so it's worth separating this from the user flushing above. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700588static void lguest_flush_tlb_kernel(void)
589{
590 lazy_hcall(LHCALL_FLUSH_TLB, 1, 0, 0);
591}
592
Rusty Russellb2b47c22007-07-26 10:41:02 -0700593/*
594 * The Unadvanced Programmable Interrupt Controller.
595 *
596 * This is an attempt to implement the simplest possible interrupt controller.
597 * I spent some time looking though routines like set_irq_chip_and_handler,
598 * set_irq_chip_and_handler_name, set_irq_chip_data and set_phasers_to_stun and
599 * I *think* this is as simple as it gets.
600 *
601 * We can tell the Host what interrupts we want blocked ready for using the
602 * lguest_data.interrupts bitmap, so disabling (aka "masking") them is as
603 * simple as setting a bit. We don't actually "ack" interrupts as such, we
604 * just mask and unmask them. I wonder if we should be cleverer?
605 */
Rusty Russell07ad1572007-07-19 01:49:22 -0700606static void disable_lguest_irq(unsigned int irq)
607{
608 set_bit(irq, lguest_data.blocked_interrupts);
609}
610
611static void enable_lguest_irq(unsigned int irq)
612{
613 clear_bit(irq, lguest_data.blocked_interrupts);
Rusty Russell07ad1572007-07-19 01:49:22 -0700614}
615
Rusty Russellb2b47c22007-07-26 10:41:02 -0700616/* This structure describes the lguest IRQ controller. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700617static struct irq_chip lguest_irq_controller = {
618 .name = "lguest",
619 .mask = disable_lguest_irq,
620 .mask_ack = disable_lguest_irq,
621 .unmask = enable_lguest_irq,
622};
623
Rusty Russellb2b47c22007-07-26 10:41:02 -0700624/* This sets up the Interrupt Descriptor Table (IDT) entry for each hardware
625 * interrupt (except 128, which is used for system calls), and then tells the
626 * Linux infrastructure that each interrupt is controlled by our level-based
627 * lguest interrupt controller. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700628static void __init lguest_init_IRQ(void)
629{
630 unsigned int i;
631
632 for (i = 0; i < LGUEST_IRQS; i++) {
633 int vector = FIRST_EXTERNAL_VECTOR + i;
634 if (vector != SYSCALL_VECTOR) {
635 set_intr_gate(vector, interrupt[i]);
636 set_irq_chip_and_handler(i, &lguest_irq_controller,
637 handle_level_irq);
638 }
639 }
Rusty Russellb2b47c22007-07-26 10:41:02 -0700640 /* This call is required to set up for 4k stacks, where we have
641 * separate stacks for hard and soft interrupts. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700642 irq_ctx_init(smp_processor_id());
643}
644
Rusty Russellb2b47c22007-07-26 10:41:02 -0700645/*
646 * Time.
647 *
648 * It would be far better for everyone if the Guest had its own clock, but
Rusty Russell6c8dca52007-07-27 13:42:52 +1000649 * until then the Host gives us the time on every interrupt.
Rusty Russellb2b47c22007-07-26 10:41:02 -0700650 */
Rusty Russell07ad1572007-07-19 01:49:22 -0700651static unsigned long lguest_get_wallclock(void)
652{
Rusty Russell6c8dca52007-07-27 13:42:52 +1000653 return lguest_data.time.tv_sec;
Rusty Russell07ad1572007-07-19 01:49:22 -0700654}
655
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700656static cycle_t lguest_clock_read(void)
Rusty Russell07ad1572007-07-19 01:49:22 -0700657{
Rusty Russell6c8dca52007-07-27 13:42:52 +1000658 unsigned long sec, nsec;
659
660 /* If the Host tells the TSC speed, we can trust that. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700661 if (lguest_data.tsc_khz)
662 return native_read_tsc();
Rusty Russell6c8dca52007-07-27 13:42:52 +1000663
664 /* If we can't use the TSC, we read the time value written by the Host.
665 * Since it's in two parts (seconds and nanoseconds), we risk reading
666 * it just as it's changing from 99 & 0.999999999 to 100 and 0, and
667 * getting 99 and 0. As Linux tends to come apart under the stress of
668 * time travel, we must be careful: */
669 do {
670 /* First we read the seconds part. */
671 sec = lguest_data.time.tv_sec;
672 /* This read memory barrier tells the compiler and the CPU that
673 * this can't be reordered: we have to complete the above
674 * before going on. */
675 rmb();
676 /* Now we read the nanoseconds part. */
677 nsec = lguest_data.time.tv_nsec;
678 /* Make sure we've done that. */
679 rmb();
680 /* Now if the seconds part has changed, try again. */
681 } while (unlikely(lguest_data.time.tv_sec != sec));
682
683 /* Our non-TSC clock is in real nanoseconds. */
684 return sec*1000000000ULL + nsec;
Rusty Russell07ad1572007-07-19 01:49:22 -0700685}
686
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700687/* This is what we tell the kernel is our clocksource. */
688static struct clocksource lguest_clock = {
689 .name = "lguest",
690 .rating = 400,
691 .read = lguest_clock_read,
Rusty Russell6c8dca52007-07-27 13:42:52 +1000692 .mask = CLOCKSOURCE_MASK(64),
Rusty Russell37250092007-08-09 20:52:35 +1000693 .mult = 1 << 22,
694 .shift = 22,
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700695};
696
Rusty Russell6c8dca52007-07-27 13:42:52 +1000697/* The "scheduler clock" is just our real clock, adjusted to start at zero */
Rusty Russell9d1ca6f2007-07-20 22:15:01 +1000698static unsigned long long lguest_sched_clock(void)
699{
700 return cyc2ns(&lguest_clock, lguest_clock_read() - clock_base);
701}
702
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700703/* We also need a "struct clock_event_device": Linux asks us to set it to go
704 * off some time in the future. Actually, James Morris figured all this out, I
705 * just applied the patch. */
706static int lguest_clockevent_set_next_event(unsigned long delta,
707 struct clock_event_device *evt)
708{
709 if (delta < LG_CLOCK_MIN_DELTA) {
710 if (printk_ratelimit())
711 printk(KERN_DEBUG "%s: small delta %lu ns\n",
712 __FUNCTION__, delta);
713 return -ETIME;
714 }
715 hcall(LHCALL_SET_CLOCKEVENT, delta, 0, 0);
716 return 0;
717}
718
719static void lguest_clockevent_set_mode(enum clock_event_mode mode,
720 struct clock_event_device *evt)
721{
722 switch (mode) {
723 case CLOCK_EVT_MODE_UNUSED:
724 case CLOCK_EVT_MODE_SHUTDOWN:
725 /* A 0 argument shuts the clock down. */
726 hcall(LHCALL_SET_CLOCKEVENT, 0, 0, 0);
727 break;
728 case CLOCK_EVT_MODE_ONESHOT:
729 /* This is what we expect. */
730 break;
731 case CLOCK_EVT_MODE_PERIODIC:
732 BUG();
Thomas Gleixner18de5bc2007-07-21 04:37:34 -0700733 case CLOCK_EVT_MODE_RESUME:
734 break;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700735 }
736}
737
738/* This describes our primitive timer chip. */
739static struct clock_event_device lguest_clockevent = {
740 .name = "lguest",
741 .features = CLOCK_EVT_FEAT_ONESHOT,
742 .set_next_event = lguest_clockevent_set_next_event,
743 .set_mode = lguest_clockevent_set_mode,
744 .rating = INT_MAX,
745 .mult = 1,
746 .shift = 0,
747 .min_delta_ns = LG_CLOCK_MIN_DELTA,
748 .max_delta_ns = LG_CLOCK_MAX_DELTA,
749};
750
751/* This is the Guest timer interrupt handler (hardware interrupt 0). We just
752 * call the clockevent infrastructure and it does whatever needs doing. */
753static void lguest_time_irq(unsigned int irq, struct irq_desc *desc)
754{
755 unsigned long flags;
756
757 /* Don't interrupt us while this is running. */
758 local_irq_save(flags);
759 lguest_clockevent.event_handler(&lguest_clockevent);
760 local_irq_restore(flags);
761}
762
Rusty Russellb2b47c22007-07-26 10:41:02 -0700763/* At some point in the boot process, we get asked to set up our timing
764 * infrastructure. The kernel doesn't expect timer interrupts before this, but
765 * we cleverly initialized the "blocked_interrupts" field of "struct
766 * lguest_data" so that timer interrupts were blocked until now. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700767static void lguest_time_init(void)
768{
Rusty Russellb2b47c22007-07-26 10:41:02 -0700769 /* Set up the timer interrupt (0) to go to our simple timer routine */
Rusty Russell07ad1572007-07-19 01:49:22 -0700770 set_irq_handler(0, lguest_time_irq);
Rusty Russell07ad1572007-07-19 01:49:22 -0700771
Rusty Russellb2b47c22007-07-26 10:41:02 -0700772 /* Our clock structure look like arch/i386/kernel/tsc.c if we can use
Rusty Russell6c8dca52007-07-27 13:42:52 +1000773 * the TSC, otherwise it's a dumb nanosecond-resolution clock. Either
774 * way, the "rating" is initialized so high that it's always chosen
775 * over any other clocksource. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700776 if (lguest_data.tsc_khz) {
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700777 lguest_clock.mult = clocksource_khz2mult(lguest_data.tsc_khz,
778 lguest_clock.shift);
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700779 lguest_clock.flags = CLOCK_SOURCE_IS_CONTINUOUS;
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700780 }
Rusty Russell9d1ca6f2007-07-20 22:15:01 +1000781 clock_base = lguest_clock_read();
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700782 clocksource_register(&lguest_clock);
783
Rusty Russell6c8dca52007-07-27 13:42:52 +1000784 /* Now we've set up our clock, we can use it as the scheduler clock */
785 paravirt_ops.sched_clock = lguest_sched_clock;
786
Rusty Russellb2b47c22007-07-26 10:41:02 -0700787 /* We can't set cpumask in the initializer: damn C limitations! Set it
788 * here and register our timer device. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700789 lguest_clockevent.cpumask = cpumask_of_cpu(0);
790 clockevents_register_device(&lguest_clockevent);
791
Rusty Russellb2b47c22007-07-26 10:41:02 -0700792 /* Finally, we unblock the timer interrupt. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700793 enable_lguest_irq(0);
Rusty Russell07ad1572007-07-19 01:49:22 -0700794}
795
Rusty Russellb2b47c22007-07-26 10:41:02 -0700796/*
797 * Miscellaneous bits and pieces.
798 *
799 * Here is an oddball collection of functions which the Guest needs for things
800 * to work. They're pretty simple.
801 */
802
803/* The Guest needs to tell the host what stack it expects traps to use. For
804 * native hardware, this is part of the Task State Segment mentioned above in
805 * lguest_load_tr_desc(), but to help hypervisors there's this special call.
806 *
807 * We tell the Host the segment we want to use (__KERNEL_DS is the kernel data
808 * segment), the privilege level (we're privilege level 1, the Host is 0 and
809 * will not tolerate us trying to use that), the stack pointer, and the number
810 * of pages in the stack. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700811static void lguest_load_esp0(struct tss_struct *tss,
812 struct thread_struct *thread)
813{
814 lazy_hcall(LHCALL_SET_STACK, __KERNEL_DS|0x1, thread->esp0,
815 THREAD_SIZE/PAGE_SIZE);
816}
817
Rusty Russellb2b47c22007-07-26 10:41:02 -0700818/* Let's just say, I wouldn't do debugging under a Guest. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700819static void lguest_set_debugreg(int regno, unsigned long value)
820{
821 /* FIXME: Implement */
822}
823
Rusty Russellb2b47c22007-07-26 10:41:02 -0700824/* There are times when the kernel wants to make sure that no memory writes are
825 * caught in the cache (that they've all reached real hardware devices). This
826 * doesn't matter for the Guest which has virtual hardware.
827 *
828 * On the Pentium 4 and above, cpuid() indicates that the Cache Line Flush
829 * (clflush) instruction is available and the kernel uses that. Otherwise, it
830 * uses the older "Write Back and Invalidate Cache" (wbinvd) instruction.
831 * Unlike clflush, wbinvd can only be run at privilege level 0. So we can
832 * ignore clflush, but replace wbinvd.
833 */
Rusty Russell07ad1572007-07-19 01:49:22 -0700834static void lguest_wbinvd(void)
835{
836}
837
Rusty Russellb2b47c22007-07-26 10:41:02 -0700838/* If the Guest expects to have an Advanced Programmable Interrupt Controller,
839 * we play dumb by ignoring writes and returning 0 for reads. So it's no
840 * longer Programmable nor Controlling anything, and I don't think 8 lines of
841 * code qualifies for Advanced. It will also never interrupt anything. It
842 * does, however, allow us to get through the Linux boot code. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700843#ifdef CONFIG_X86_LOCAL_APIC
844static void lguest_apic_write(unsigned long reg, unsigned long v)
845{
846}
847
848static unsigned long lguest_apic_read(unsigned long reg)
849{
850 return 0;
851}
852#endif
853
Rusty Russellb2b47c22007-07-26 10:41:02 -0700854/* STOP! Until an interrupt comes in. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700855static void lguest_safe_halt(void)
856{
857 hcall(LHCALL_HALT, 0, 0, 0);
858}
859
Rusty Russellb2b47c22007-07-26 10:41:02 -0700860/* Perhaps CRASH isn't the best name for this hypercall, but we use it to get a
861 * message out when we're crashing as well as elegant termination like powering
862 * off.
863 *
864 * Note that the Host always prefers that the Guest speak in physical addresses
865 * rather than virtual addresses, so we use __pa() here. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700866static void lguest_power_off(void)
867{
868 hcall(LHCALL_CRASH, __pa("Power down"), 0, 0);
869}
870
Rusty Russellb2b47c22007-07-26 10:41:02 -0700871/*
872 * Panicing.
873 *
874 * Don't. But if you did, this is what happens.
875 */
Rusty Russell07ad1572007-07-19 01:49:22 -0700876static int lguest_panic(struct notifier_block *nb, unsigned long l, void *p)
877{
878 hcall(LHCALL_CRASH, __pa(p), 0, 0);
Rusty Russellb2b47c22007-07-26 10:41:02 -0700879 /* The hcall won't return, but to keep gcc happy, we're "done". */
Rusty Russell07ad1572007-07-19 01:49:22 -0700880 return NOTIFY_DONE;
881}
882
883static struct notifier_block paniced = {
884 .notifier_call = lguest_panic
885};
886
Rusty Russellb2b47c22007-07-26 10:41:02 -0700887/* Setting up memory is fairly easy. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700888static __init char *lguest_memory_setup(void)
889{
Rusty Russellb2b47c22007-07-26 10:41:02 -0700890 /* We do this here and not earlier because lockcheck barfs if we do it
891 * before start_kernel() */
Rusty Russell07ad1572007-07-19 01:49:22 -0700892 atomic_notifier_chain_register(&panic_notifier_list, &paniced);
893
Rusty Russellb2b47c22007-07-26 10:41:02 -0700894 /* The Linux bootloader header contains an "e820" memory map: the
895 * Launcher populated the first entry with our memory limit. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700896 add_memory_region(E820_MAP->addr, E820_MAP->size, E820_MAP->type);
Rusty Russellb2b47c22007-07-26 10:41:02 -0700897
898 /* This string is for the boot messages. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700899 return "LGUEST";
900}
901
Rusty Russellb2b47c22007-07-26 10:41:02 -0700902/*G:050
903 * Patching (Powerfully Placating Performance Pedants)
904 *
905 * We have already seen that "struct paravirt_ops" lets us replace simple
906 * native instructions with calls to the appropriate back end all throughout
907 * the kernel. This allows the same kernel to run as a Guest and as a native
908 * kernel, but it's slow because of all the indirect branches.
909 *
910 * Remember that David Wheeler quote about "Any problem in computer science can
911 * be solved with another layer of indirection"? The rest of that quote is
912 * "... But that usually will create another problem." This is the first of
913 * those problems.
914 *
915 * Our current solution is to allow the paravirt back end to optionally patch
916 * over the indirect calls to replace them with something more efficient. We
917 * patch the four most commonly called functions: disable interrupts, enable
918 * interrupts, restore interrupts and save interrupts. We usually have 10
919 * bytes to patch into: the Guest versions of these operations are small enough
920 * that we can fit comfortably.
921 *
922 * First we need assembly templates of each of the patchable Guest operations,
923 * and these are in lguest_asm.S. */
924
925/*G:060 We construct a table from the assembler templates: */
Rusty Russell07ad1572007-07-19 01:49:22 -0700926static const struct lguest_insns
927{
928 const char *start, *end;
929} lguest_insns[] = {
930 [PARAVIRT_PATCH(irq_disable)] = { lgstart_cli, lgend_cli },
931 [PARAVIRT_PATCH(irq_enable)] = { lgstart_sti, lgend_sti },
932 [PARAVIRT_PATCH(restore_fl)] = { lgstart_popf, lgend_popf },
933 [PARAVIRT_PATCH(save_fl)] = { lgstart_pushf, lgend_pushf },
934};
Rusty Russellb2b47c22007-07-26 10:41:02 -0700935
936/* Now our patch routine is fairly simple (based on the native one in
937 * paravirt.c). If we have a replacement, we copy it in and return how much of
938 * the available space we used. */
Andi Kleenab144f52007-08-10 22:31:03 +0200939static unsigned lguest_patch(u8 type, u16 clobber, void *ibuf,
940 unsigned long addr, unsigned len)
Rusty Russell07ad1572007-07-19 01:49:22 -0700941{
942 unsigned int insn_len;
943
Rusty Russellb2b47c22007-07-26 10:41:02 -0700944 /* Don't do anything special if we don't have a replacement */
Rusty Russell07ad1572007-07-19 01:49:22 -0700945 if (type >= ARRAY_SIZE(lguest_insns) || !lguest_insns[type].start)
Andi Kleenab144f52007-08-10 22:31:03 +0200946 return paravirt_patch_default(type, clobber, ibuf, addr, len);
Rusty Russell07ad1572007-07-19 01:49:22 -0700947
948 insn_len = lguest_insns[type].end - lguest_insns[type].start;
949
Rusty Russellb2b47c22007-07-26 10:41:02 -0700950 /* Similarly if we can't fit replacement (shouldn't happen, but let's
951 * be thorough). */
Rusty Russell07ad1572007-07-19 01:49:22 -0700952 if (len < insn_len)
Andi Kleenab144f52007-08-10 22:31:03 +0200953 return paravirt_patch_default(type, clobber, ibuf, addr, len);
Rusty Russell07ad1572007-07-19 01:49:22 -0700954
Rusty Russellb2b47c22007-07-26 10:41:02 -0700955 /* Copy in our instructions. */
Andi Kleenab144f52007-08-10 22:31:03 +0200956 memcpy(ibuf, lguest_insns[type].start, insn_len);
Rusty Russell07ad1572007-07-19 01:49:22 -0700957 return insn_len;
958}
959
Rusty Russellb2b47c22007-07-26 10:41:02 -0700960/*G:030 Once we get to lguest_init(), we know we're a Guest. The paravirt_ops
961 * structure in the kernel provides a single point for (almost) every routine
962 * we have to override to avoid privileged instructions. */
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700963__init void lguest_init(void *boot)
Rusty Russell07ad1572007-07-19 01:49:22 -0700964{
Rusty Russellb2b47c22007-07-26 10:41:02 -0700965 /* Copy boot parameters first: the Launcher put the physical location
966 * in %esi, and head.S converted that to a virtual address and handed
Rusty Russellc413fec2007-09-11 17:06:37 +1000967 * it to us. We use "__memcpy" because "memcpy" sometimes tries to do
968 * tricky things to go faster, and we're not ready for that. */
969 __memcpy(&boot_params, boot, PARAM_SIZE);
Rusty Russellb2b47c22007-07-26 10:41:02 -0700970 /* The boot parameters also tell us where the command-line is: save
971 * that, too. */
Rusty Russellc413fec2007-09-11 17:06:37 +1000972 __memcpy(boot_command_line, __va(boot_params.hdr.cmd_line_ptr),
Rusty Russelld7e28ff2007-07-19 01:49:23 -0700973 COMMAND_LINE_SIZE);
974
Rusty Russellb2b47c22007-07-26 10:41:02 -0700975 /* We're under lguest, paravirt is enabled, and we're running at
976 * privilege level 1, not 0 as normal. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700977 paravirt_ops.name = "lguest";
978 paravirt_ops.paravirt_enabled = 1;
979 paravirt_ops.kernel_rpl = 1;
980
Rusty Russellb2b47c22007-07-26 10:41:02 -0700981 /* We set up all the lguest overrides for sensitive operations. These
982 * are detailed with the operations themselves. */
Rusty Russell07ad1572007-07-19 01:49:22 -0700983 paravirt_ops.save_fl = save_fl;
984 paravirt_ops.restore_fl = restore_fl;
985 paravirt_ops.irq_disable = irq_disable;
986 paravirt_ops.irq_enable = irq_enable;
987 paravirt_ops.load_gdt = lguest_load_gdt;
988 paravirt_ops.memory_setup = lguest_memory_setup;
989 paravirt_ops.cpuid = lguest_cpuid;
990 paravirt_ops.write_cr3 = lguest_write_cr3;
991 paravirt_ops.flush_tlb_user = lguest_flush_tlb_user;
992 paravirt_ops.flush_tlb_single = lguest_flush_tlb_single;
993 paravirt_ops.flush_tlb_kernel = lguest_flush_tlb_kernel;
994 paravirt_ops.set_pte = lguest_set_pte;
995 paravirt_ops.set_pte_at = lguest_set_pte_at;
996 paravirt_ops.set_pmd = lguest_set_pmd;
997#ifdef CONFIG_X86_LOCAL_APIC
998 paravirt_ops.apic_write = lguest_apic_write;
999 paravirt_ops.apic_write_atomic = lguest_apic_write;
1000 paravirt_ops.apic_read = lguest_apic_read;
1001#endif
1002 paravirt_ops.load_idt = lguest_load_idt;
1003 paravirt_ops.iret = lguest_iret;
1004 paravirt_ops.load_esp0 = lguest_load_esp0;
1005 paravirt_ops.load_tr_desc = lguest_load_tr_desc;
1006 paravirt_ops.set_ldt = lguest_set_ldt;
1007 paravirt_ops.load_tls = lguest_load_tls;
1008 paravirt_ops.set_debugreg = lguest_set_debugreg;
1009 paravirt_ops.clts = lguest_clts;
1010 paravirt_ops.read_cr0 = lguest_read_cr0;
1011 paravirt_ops.write_cr0 = lguest_write_cr0;
1012 paravirt_ops.init_IRQ = lguest_init_IRQ;
1013 paravirt_ops.read_cr2 = lguest_read_cr2;
1014 paravirt_ops.read_cr3 = lguest_read_cr3;
1015 paravirt_ops.read_cr4 = lguest_read_cr4;
1016 paravirt_ops.write_cr4 = lguest_write_cr4;
1017 paravirt_ops.write_gdt_entry = lguest_write_gdt_entry;
1018 paravirt_ops.write_idt_entry = lguest_write_idt_entry;
1019 paravirt_ops.patch = lguest_patch;
1020 paravirt_ops.safe_halt = lguest_safe_halt;
1021 paravirt_ops.get_wallclock = lguest_get_wallclock;
1022 paravirt_ops.time_init = lguest_time_init;
1023 paravirt_ops.set_lazy_mode = lguest_lazy_mode;
1024 paravirt_ops.wbinvd = lguest_wbinvd;
Rusty Russellb2b47c22007-07-26 10:41:02 -07001025 /* Now is a good time to look at the implementations of these functions
1026 * before returning to the rest of lguest_init(). */
Rusty Russell07ad1572007-07-19 01:49:22 -07001027
Rusty Russellb2b47c22007-07-26 10:41:02 -07001028 /*G:070 Now we've seen all the paravirt_ops, we return to
1029 * lguest_init() where the rest of the fairly chaotic boot setup
1030 * occurs.
1031 *
1032 * The Host expects our first hypercall to tell it where our "struct
1033 * lguest_data" is, so we do that first. */
Rusty Russell07ad1572007-07-19 01:49:22 -07001034 hcall(LHCALL_LGUEST_INIT, __pa(&lguest_data), 0, 0);
Rusty Russell07ad1572007-07-19 01:49:22 -07001035
Rusty Russellb2b47c22007-07-26 10:41:02 -07001036 /* The native boot code sets up initial page tables immediately after
1037 * the kernel itself, and sets init_pg_tables_end so they're not
1038 * clobbered. The Launcher places our initial pagetables somewhere at
1039 * the top of our physical memory, so we don't need extra space: set
1040 * init_pg_tables_end to the end of the kernel. */
Rusty Russell07ad1572007-07-19 01:49:22 -07001041 init_pg_tables_end = __pa(pg0);
1042
Rusty Russellb2b47c22007-07-26 10:41:02 -07001043 /* Load the %fs segment register (the per-cpu segment register) with
1044 * the normal data segment to get through booting. */
Rusty Russell07ad1572007-07-19 01:49:22 -07001045 asm volatile ("mov %0, %%fs" : : "r" (__KERNEL_DS) : "memory");
1046
Rusty Russella8a11f062007-07-27 13:35:43 +10001047 /* Clear the part of the kernel data which is expected to be zero.
1048 * Normally it will be anyway, but if we're loading from a bzImage with
1049 * CONFIG_RELOCATALE=y, the relocations will be sitting here. */
1050 memset(__bss_start, 0, __bss_stop - __bss_start);
1051
Rusty Russellb2b47c22007-07-26 10:41:02 -07001052 /* The Host uses the top of the Guest's virtual address space for the
1053 * Host<->Guest Switcher, and it tells us how much it needs in
1054 * lguest_data.reserve_mem, set up on the LGUEST_INIT hypercall. */
Rusty Russell07ad1572007-07-19 01:49:22 -07001055 reserve_top_address(lguest_data.reserve_mem);
1056
Rusty Russellb2b47c22007-07-26 10:41:02 -07001057 /* If we don't initialize the lock dependency checker now, it crashes
1058 * paravirt_disable_iospace. */
Rusty Russell07ad1572007-07-19 01:49:22 -07001059 lockdep_init();
1060
Rusty Russellb2b47c22007-07-26 10:41:02 -07001061 /* The IDE code spends about 3 seconds probing for disks: if we reserve
1062 * all the I/O ports up front it can't get them and so doesn't probe.
1063 * Other device drivers are similar (but less severe). This cuts the
1064 * kernel boot time on my machine from 4.1 seconds to 0.45 seconds. */
Rusty Russell07ad1572007-07-19 01:49:22 -07001065 paravirt_disable_iospace();
1066
Rusty Russellb2b47c22007-07-26 10:41:02 -07001067 /* This is messy CPU setup stuff which the native boot code does before
1068 * start_kernel, so we have to do, too: */
Rusty Russell07ad1572007-07-19 01:49:22 -07001069 cpu_detect(&new_cpu_data);
1070 /* head.S usually sets up the first capability word, so do it here. */
1071 new_cpu_data.x86_capability[0] = cpuid_edx(1);
1072
1073 /* Math is always hard! */
1074 new_cpu_data.hard_math = 1;
1075
1076#ifdef CONFIG_X86_MCE
1077 mce_disabled = 1;
1078#endif
Rusty Russell07ad1572007-07-19 01:49:22 -07001079#ifdef CONFIG_ACPI
1080 acpi_disabled = 1;
1081 acpi_ht = 0;
1082#endif
1083
Rusty Russellb2b47c22007-07-26 10:41:02 -07001084 /* We set the perferred console to "hvc". This is the "hypervisor
1085 * virtual console" driver written by the PowerPC people, which we also
1086 * adapted for lguest's use. */
Rusty Russell07ad1572007-07-19 01:49:22 -07001087 add_preferred_console("hvc", 0, NULL);
1088
Rusty Russellb2b47c22007-07-26 10:41:02 -07001089 /* Last of all, we set the power management poweroff hook to point to
1090 * the Guest routine to power off. */
Rusty Russell07ad1572007-07-19 01:49:22 -07001091 pm_power_off = lguest_power_off;
Rusty Russellb2b47c22007-07-26 10:41:02 -07001092
1093 /* Now we're set up, call start_kernel() in init/main.c and we proceed
1094 * to boot as normal. It never returns. */
Rusty Russell07ad1572007-07-19 01:49:22 -07001095 start_kernel();
1096}
Rusty Russellb2b47c22007-07-26 10:41:02 -07001097/*
1098 * This marks the end of stage II of our journey, The Guest.
1099 *
1100 * It is now time for us to explore the nooks and crannies of the three Guest
1101 * devices and complete our understanding of the Guest in "make Drivers".
1102 */