ARM: pgtable: add pud-level code
Add pud_offset() et.al. between the pgd and pmd code in preparation of
using pgtable-nopud.h rather than 4level-fixup.h.
This incorporates a fix from Jamie Iles <jamie@jamieiles.com> for
uaccess_with_memcpy.c.
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
index ca56791..5750704 100644
--- a/arch/arm/include/asm/pgtable.h
+++ b/arch/arm/include/asm/pgtable.h
@@ -301,6 +301,7 @@
#define pgd_present(pgd) (1)
#define pgd_clear(pgdp) do { } while (0)
#define set_pgd(pgd,pgdp) do { } while (0)
+#define set_pud(pud,pudp) do { } while (0)
/* Find an entry in the second-level page table.. */
diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c
index e2d2f2c..8b9b136 100644
--- a/arch/arm/lib/uaccess_with_memcpy.c
+++ b/arch/arm/lib/uaccess_with_memcpy.c
@@ -27,13 +27,18 @@
pgd_t *pgd;
pmd_t *pmd;
pte_t *pte;
+ pud_t *pud;
spinlock_t *ptl;
pgd = pgd_offset(current->mm, addr);
if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd)))
return 0;
- pmd = pmd_offset(pgd, addr);
+ pud = pud_offset(pgd, addr);
+ if (unlikely(pud_none(*pud) || pud_bad(*pud)))
+ return 0;
+
+ pmd = pmd_offset(pud, addr);
if (unlikely(pmd_none(*pmd) || pmd_bad(*pmd)))
return 0;
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index ac6a361..a9bdfcd 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -148,6 +148,7 @@
{
int ret = 0;
pgd_t *pgd;
+ pud_t *pud;
pmd_t *pmd;
pte_t *pte;
int i = 0;
@@ -155,7 +156,15 @@
do {
pgd = pgd_offset(&init_mm, base);
- pmd = pmd_alloc(&init_mm, pgd, base);
+
+ pud = pud_alloc(&init_mm, pgd, base);
+ if (!pud) {
+ printk(KERN_ERR "%s: no pud tables\n", __func__);
+ ret = -ENOMEM;
+ break;
+ }
+
+ pmd = pmd_alloc(&init_mm, pud, base);
if (!pmd) {
printk(KERN_ERR "%s: no pmd tables\n", __func__);
ret = -ENOMEM;
diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c
index 01210db..7cab791 100644
--- a/arch/arm/mm/fault-armv.c
+++ b/arch/arm/mm/fault-armv.c
@@ -95,6 +95,7 @@
{
spinlock_t *ptl;
pgd_t *pgd;
+ pud_t *pud;
pmd_t *pmd;
pte_t *pte;
int ret;
@@ -103,7 +104,11 @@
if (pgd_none_or_clear_bad(pgd))
return 0;
- pmd = pmd_offset(pgd, address);
+ pud = pud_offset(pgd, address);
+ if (pud_none_or_clear_bad(pud))
+ return 0;
+
+ pmd = pmd_offset(pud, address);
if (pmd_none_or_clear_bad(pmd))
return 0;
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index ef0e24f..bc0e1d8 100644
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -80,6 +80,7 @@
addr, (long long)pgd_val(*pgd));
do {
+ pud_t *pud;
pmd_t *pmd;
pte_t *pte;
@@ -91,7 +92,19 @@
break;
}
- pmd = pmd_offset(pgd, addr);
+ pud = pud_offset(pgd, addr);
+ if (PTRS_PER_PUD != 1)
+ printk(", *pud=%08lx", pud_val(*pud));
+
+ if (pud_none(*pud))
+ break;
+
+ if (pud_bad(*pud)) {
+ printk("(bad)");
+ break;
+ }
+
+ pmd = pmd_offset(pud, addr);
if (PTRS_PER_PMD != 1)
printk(", *pmd=%08llx", (long long)pmd_val(*pmd));
@@ -390,6 +403,7 @@
{
unsigned int index;
pgd_t *pgd, *pgd_k;
+ pud_t *pud, *pud_k;
pmd_t *pmd, *pmd_k;
if (addr < TASK_SIZE)
@@ -408,12 +422,19 @@
if (pgd_none(*pgd_k))
goto bad_area;
-
if (!pgd_present(*pgd))
set_pgd(pgd, *pgd_k);
- pmd_k = pmd_offset(pgd_k, addr);
- pmd = pmd_offset(pgd, addr);
+ pud = pud_offset(pgd, addr);
+ pud_k = pud_offset(pgd_k, addr);
+
+ if (pud_none(*pud_k))
+ goto bad_area;
+ if (!pud_present(*pud))
+ set_pud(pud, *pud_k);
+
+ pmd = pmd_offset(pud, addr);
+ pmd_k = pmd_offset(pud_k, addr);
/*
* On ARM one Linux PGD entry contains two hardware entries (see page
diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c
index 5729944..2be9139 100644
--- a/arch/arm/mm/idmap.c
+++ b/arch/arm/mm/idmap.c
@@ -4,10 +4,10 @@
#include <asm/pgalloc.h>
#include <asm/pgtable.h>
-static void idmap_add_pmd(pgd_t *pgd, unsigned long addr, unsigned long end,
+static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end,
unsigned long prot)
{
- pmd_t *pmd = pmd_offset(pgd, addr);
+ pmd_t *pmd = pmd_offset(pud, addr);
addr = (addr & PMD_MASK) | prot;
pmd[0] = __pmd(addr);
@@ -16,6 +16,18 @@
flush_pmd_entry(pmd);
}
+static void idmap_add_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
+ unsigned long prot)
+{
+ pud_t *pud = pud_offset(pgd, addr);
+ unsigned long next;
+
+ do {
+ next = pud_addr_end(addr, end);
+ idmap_add_pmd(pud, addr, next, prot);
+ } while (pud++, addr = next, addr != end);
+}
+
void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end)
{
unsigned long prot, next;
@@ -27,17 +39,28 @@
pgd += pgd_index(addr);
do {
next = pgd_addr_end(addr, end);
- idmap_add_pmd(pgd, addr, next, prot);
+ idmap_add_pud(pgd, addr, next, prot);
} while (pgd++, addr = next, addr != end);
}
#ifdef CONFIG_SMP
-static void idmap_del_pmd(pgd_t *pgd, unsigned long addr, unsigned long end)
+static void idmap_del_pmd(pud_t *pud, unsigned long addr, unsigned long end)
{
- pmd_t *pmd = pmd_offset(pgd, addr);
+ pmd_t *pmd = pmd_offset(pud, addr);
pmd_clear(pmd);
}
+static void idmap_del_pud(pgd_t *pgd, unsigned long addr, unsigned long end)
+{
+ pud_t *pud = pud_offset(pgd, addr);
+ unsigned long next;
+
+ do {
+ next = pud_addr_end(addr, end);
+ idmap_del_pmd(pud, addr, next);
+ } while (pud++, addr = next, addr != end);
+}
+
void identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end)
{
unsigned long next;
@@ -45,7 +68,7 @@
pgd += pgd_index(addr);
do {
next = pgd_addr_end(addr, end);
- idmap_del_pmd(pgd, addr, next);
+ idmap_del_pud(pgd, addr, next);
} while (pgd++, addr = next, addr != end);
}
#endif
diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h
index 36960df..d238410 100644
--- a/arch/arm/mm/mm.h
+++ b/arch/arm/mm/mm.h
@@ -7,7 +7,7 @@
static inline pmd_t *pmd_off(pgd_t *pgd, unsigned long virt)
{
- return pmd_offset(pgd, virt);
+ return pmd_offset(pud_offset(pgd, virt), virt);
}
static inline pmd_t *pmd_off_k(unsigned long virt)
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index 1767490..82ef696 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -550,11 +550,11 @@
} while (pte++, addr += PAGE_SIZE, addr != end);
}
-static void __init alloc_init_section(pgd_t *pgd, unsigned long addr,
+static void __init alloc_init_section(pud_t *pud, unsigned long addr,
unsigned long end, phys_addr_t phys,
const struct mem_type *type)
{
- pmd_t *pmd = pmd_offset(pgd, addr);
+ pmd_t *pmd = pmd_offset(pud, addr);
/*
* Try a section mapping - end, addr and phys must all be aligned
@@ -583,6 +583,19 @@
}
}
+static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
+ unsigned long phys, const struct mem_type *type)
+{
+ pud_t *pud = pud_offset(pgd, addr);
+ unsigned long next;
+
+ do {
+ next = pud_addr_end(addr, end);
+ alloc_init_section(pud, addr, next, phys, type);
+ phys += next - addr;
+ } while (pud++, addr = next, addr != end);
+}
+
static void __init create_36bit_mapping(struct map_desc *md,
const struct mem_type *type)
{
@@ -630,7 +643,8 @@
pgd = pgd_offset_k(addr);
end = addr + length;
do {
- pmd_t *pmd = pmd_offset(pgd, addr);
+ pud_t *pud = pud_offset(pgd, addr);
+ pmd_t *pmd = pmd_offset(pud, addr);
int i;
for (i = 0; i < 16; i++)
@@ -696,7 +710,7 @@
do {
unsigned long next = pgd_addr_end(addr, end);
- alloc_init_section(pgd, addr, next, phys, type);
+ alloc_init_pud(pgd, addr, next, phys, type);
phys += next - addr;
addr = next;
diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
index 93292a1..f7fafb1 100644
--- a/arch/arm/mm/pgd.c
+++ b/arch/arm/mm/pgd.c
@@ -23,6 +23,7 @@
pgd_t *pgd_alloc(struct mm_struct *mm)
{
pgd_t *new_pgd, *init_pgd;
+ pud_t *new_pud, *init_pud;
pmd_t *new_pmd, *init_pmd;
pte_t *new_pte, *init_pte;
@@ -46,7 +47,11 @@
* On ARM, first page must always be allocated since it
* contains the machine vectors.
*/
- new_pmd = pmd_alloc(mm, new_pgd, 0);
+ new_pud = pud_alloc(mm, new_pgd, 0);
+ if (!new_pud)
+ goto no_pud;
+
+ new_pmd = pmd_alloc(mm, new_pud, 0);
if (!new_pmd)
goto no_pmd;
@@ -54,7 +59,8 @@
if (!new_pte)
goto no_pte;
- init_pmd = pmd_offset(init_pgd, 0);
+ init_pud = pud_offset(init_pgd, 0);
+ init_pmd = pmd_offset(init_pud, 0);
init_pte = pte_offset_map(init_pmd, 0);
set_pte_ext(new_pte, *init_pte, 0);
pte_unmap(init_pte);
@@ -66,6 +72,8 @@
no_pte:
pmd_free(mm, new_pmd);
no_pmd:
+ pud_free(mm, new_pud);
+no_pud:
free_pages((unsigned long)new_pgd, 2);
no_pgd:
return NULL;
@@ -74,6 +82,7 @@
void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
{
pgd_t *pgd;
+ pud_t *pud;
pmd_t *pmd;
pgtable_t pte;
@@ -84,7 +93,11 @@
if (pgd_none_or_clear_bad(pgd))
goto no_pgd;
- pmd = pmd_offset(pgd, 0);
+ pud = pud_offset(pgd, 0);
+ if (pud_none_or_clear_bad(pud))
+ goto no_pud;
+
+ pmd = pmd_offset(pud, 0);
if (pmd_none_or_clear_bad(pmd))
goto no_pmd;
@@ -92,8 +105,11 @@
pmd_clear(pmd);
pte_free(mm, pte);
no_pmd:
- pgd_clear(pgd);
+ pud_clear(pud);
pmd_free(mm, pmd);
+no_pud:
+ pgd_clear(pgd);
+ pud_free(mm, pud);
no_pgd:
free_pages((unsigned long) pgd_base, 2);
}