powerpc/book3e-64: use a separate TLB handler when linear map is bolted
On MMUs such as FSL where we can guarantee the entire linear mapping is
bolted, we don't need to worry about linear TLB misses. If on top of
that we do a full table walk, we get rid of all recursive TLB faults, and
can dispense with some state saving. This gains a few percent on
TLB-miss-heavy workloads, and around 50% on a benchmark that had a high
rate of virtual page table faults under the normal handler.
While touching the EX_TLB layout, remove EX_TLB_MMUCR0, EX_TLB_SRR0, and
EX_TLB_SRR1 as they're not used.
[BenH: Fixed build with 64K pages (wsp config)]
Signed-off-by: Scott Wood <scottwood@freescale.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
diff --git a/arch/powerpc/include/asm/exception-64e.h b/arch/powerpc/include/asm/exception-64e.h
index 6d53f31..ac13add 100644
--- a/arch/powerpc/include/asm/exception-64e.h
+++ b/arch/powerpc/include/asm/exception-64e.h
@@ -48,30 +48,33 @@
#define EX_R14 (4 * 8)
#define EX_R15 (5 * 8)
-/* The TLB miss exception uses different slots */
+/*
+ * The TLB miss exception uses different slots.
+ *
+ * The bolted variant uses only the first six fields,
+ * which in combination with pgd and kernel_pgd fits in
+ * one 64-byte cache line.
+ */
#define EX_TLB_R10 ( 0 * 8)
#define EX_TLB_R11 ( 1 * 8)
-#define EX_TLB_R12 ( 2 * 8)
-#define EX_TLB_R13 ( 3 * 8)
-#define EX_TLB_R14 ( 4 * 8)
-#define EX_TLB_R15 ( 5 * 8)
-#define EX_TLB_R16 ( 6 * 8)
-#define EX_TLB_CR ( 7 * 8)
+#define EX_TLB_R14 ( 2 * 8)
+#define EX_TLB_R15 ( 3 * 8)
+#define EX_TLB_R16 ( 4 * 8)
+#define EX_TLB_CR ( 5 * 8)
+#define EX_TLB_R12 ( 6 * 8)
+#define EX_TLB_R13 ( 7 * 8)
#define EX_TLB_DEAR ( 8 * 8) /* Level 0 and 2 only */
#define EX_TLB_ESR ( 9 * 8) /* Level 0 and 2 only */
#define EX_TLB_SRR0 (10 * 8)
#define EX_TLB_SRR1 (11 * 8)
-#define EX_TLB_MMUCR0 (12 * 8) /* Level 0 */
-#define EX_TLB_MAS1 (12 * 8) /* Level 0 */
-#define EX_TLB_MAS2 (13 * 8) /* Level 0 */
#ifdef CONFIG_BOOK3E_MMU_TLB_STATS
-#define EX_TLB_R8 (14 * 8)
-#define EX_TLB_R9 (15 * 8)
-#define EX_TLB_LR (16 * 8)
-#define EX_TLB_SIZE (17 * 8)
+#define EX_TLB_R8 (12 * 8)
+#define EX_TLB_R9 (13 * 8)
+#define EX_TLB_LR (14 * 8)
+#define EX_TLB_SIZE (15 * 8)
#else
-#define EX_TLB_SIZE (14 * 8)
+#define EX_TLB_SIZE (12 * 8)
#endif
#define START_EXCEPTION(label) \
@@ -168,6 +171,16 @@
ld r9,EX_TLB_R9(r12); \
ld r8,EX_TLB_R8(r12); \
mtlr r16;
+#define TLB_MISS_PROLOG_STATS_BOLTED \
+ mflr r10; \
+ std r8,PACA_EXTLB+EX_TLB_R8(r13); \
+ std r9,PACA_EXTLB+EX_TLB_R9(r13); \
+ std r10,PACA_EXTLB+EX_TLB_LR(r13);
+#define TLB_MISS_RESTORE_STATS_BOLTED \
+ ld r16,PACA_EXTLB+EX_TLB_LR(r13); \
+ ld r9,PACA_EXTLB+EX_TLB_R9(r13); \
+ ld r8,PACA_EXTLB+EX_TLB_R8(r13); \
+ mtlr r16;
#define TLB_MISS_STATS_D(name) \
addi r9,r13,MMSTAT_DSTATS+name; \
bl .tlb_stat_inc;
@@ -183,17 +196,20 @@
61: addi r9,r13,MMSTAT_ISTATS+name; \
62: bl .tlb_stat_inc;
#define TLB_MISS_STATS_SAVE_INFO \
- std r14,EX_TLB_ESR(r12); /* save ESR */ \
-
-
+ std r14,EX_TLB_ESR(r12); /* save ESR */
+#define TLB_MISS_STATS_SAVE_INFO_BOLTED \
+ std r14,PACA_EXTLB+EX_TLB_ESR(r13); /* save ESR */
#else
#define TLB_MISS_PROLOG_STATS
#define TLB_MISS_RESTORE_STATS
+#define TLB_MISS_PROLOG_STATS_BOLTED
+#define TLB_MISS_RESTORE_STATS_BOLTED
#define TLB_MISS_STATS_D(name)
#define TLB_MISS_STATS_I(name)
#define TLB_MISS_STATS_X(name)
#define TLB_MISS_STATS_Y(name)
#define TLB_MISS_STATS_SAVE_INFO
+#define TLB_MISS_STATS_SAVE_INFO_BOLTED
#endif
#define SET_IVOR(vector_number, vector_offset) \