xen: avoid hypercalls when updating unpinned pud/pmd
When operating on an unpinned pagetable (ie, one under construction or
destruction), it isn't necessary to use a hypercall to update a
pud/pmd entry. Jan Beulich observed that a similar optimisation
avoided many thousands of hypercalls while doing a kernel build.
One tricky part is that early in the kernel boot there's no page
structure, so we can't check to see if the page is pinned. In that
case, we just always use the hypercall.
Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com>
Cc: Jan Beulich <jbeulich@novell.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 17f374e..4fa0934 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -223,7 +223,14 @@
}
-void xen_set_pmd(pmd_t *ptr, pmd_t val)
+static bool page_pinned(void *ptr)
+{
+ struct page *page = virt_to_page(ptr);
+
+ return PagePinned(page);
+}
+
+void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val)
{
struct multicall_space mcs;
struct mmu_update *u;
@@ -241,6 +248,18 @@
preempt_enable();
}
+void xen_set_pmd(pmd_t *ptr, pmd_t val)
+{
+ /* If page is not pinned, we can just update the entry
+ directly */
+ if (!page_pinned(ptr)) {
+ *ptr = val;
+ return;
+ }
+
+ xen_set_pmd_hyper(ptr, val);
+}
+
/*
* Associate a virtual page frame with a given physical page frame
* and protection flags for that frame.
@@ -348,7 +367,7 @@
return ret;
}
-void xen_set_pud(pud_t *ptr, pud_t val)
+void xen_set_pud_hyper(pud_t *ptr, pud_t val)
{
struct multicall_space mcs;
struct mmu_update *u;
@@ -366,6 +385,18 @@
preempt_enable();
}
+void xen_set_pud(pud_t *ptr, pud_t val)
+{
+ /* If page is not pinned, we can just update the entry
+ directly */
+ if (!page_pinned(ptr)) {
+ *ptr = val;
+ return;
+ }
+
+ xen_set_pud_hyper(ptr, val);
+}
+
void xen_set_pte(pte_t *ptep, pte_t pte)
{
ptep->pte_high = pte.pte_high;
@@ -387,7 +418,7 @@
void xen_pmd_clear(pmd_t *pmdp)
{
- xen_set_pmd(pmdp, __pmd(0));
+ set_pmd(pmdp, __pmd(0));
}
pmd_t xen_make_pmd(pmdval_t pmd)
@@ -758,7 +789,7 @@
spin_lock(&mm->page_table_lock);
/* pgd may not be pinned in the error exit path of execve */
- if (PagePinned(virt_to_page(mm->pgd)))
+ if (page_pinned(mm->pgd))
xen_pgd_unpin(mm->pgd);
spin_unlock(&mm->page_table_lock);