mm: adding speculative page fault failure trace events
This patch a set of new trace events to collect the speculative page fault
event failures.
Change-Id: I9dafba293edf40bdad4ae241d105ecdfb42579c1
Signed-off-by: Laurent Dufour <ldufour@linux.vnet.ibm.com>
Patch-mainline: linux-mm @ Tue, 17 Apr 2018 16:33:25
[vinmenon@codeaurora.org: trivial merge conflicts]
Signed-off-by: Vinayak Menon <vinmenon@codeaurora.org>
diff --git a/mm/memory.c b/mm/memory.c
index 4d8b604..303ea71 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -75,6 +75,9 @@
#include "internal.h"
+#define CREATE_TRACE_POINTS
+#include <trace/events/pagefault.h>
+
#if defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS) && !defined(CONFIG_COMPILE_TEST)
#warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
#endif
@@ -1990,8 +1993,10 @@
}
local_irq_disable();
- if (vma_has_changed(fe))
+ if (vma_has_changed(fe)) {
+ trace_spf_vma_changed(_RET_IP_, fe->vma, fe->address);
goto out;
+ }
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
@@ -1999,16 +2004,21 @@
* is not a huge collapse operation in progress in our back.
*/
pmdval = READ_ONCE(*fe->pmd);
- if (!pmd_same(pmdval, fe->orig_pmd))
+ if (!pmd_same(pmdval, fe->orig_pmd)) {
+ trace_spf_pmd_changed(_RET_IP_, fe->vma, fe->address);
goto out;
+ }
#endif
fe->ptl = pte_lockptr(mm, fe->pmd);
- if (unlikely(!spin_trylock(fe->ptl)))
+ if (unlikely(!spin_trylock(fe->ptl))) {
+ trace_spf_pte_lock(_RET_IP_, fe->vma, fe->address);
goto out;
+ }
if (vma_has_changed(fe)) {
spin_unlock(fe->ptl);
+ trace_spf_vma_changed(_RET_IP_, fe->vma, fe->address);
goto out;
}
@@ -2042,8 +2052,10 @@
* block on the PTL and thus we're safe.
*/
local_irq_disable();
- if (vma_has_changed(fe))
+ if (vma_has_changed(fe)) {
+ trace_spf_vma_changed(_RET_IP_, fe->vma, fe->address);
goto out;
+ }
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
@@ -2051,8 +2063,10 @@
* is not a huge collapse operation in progress in our back.
*/
pmdval = READ_ONCE(*fe->pmd);
- if (!pmd_same(pmdval, fe->orig_pmd))
+ if (!pmd_same(pmdval, fe->orig_pmd)) {
+ trace_spf_pmd_changed(_RET_IP_, fe->vma, fe->address);
goto out;
+ }
#endif
/*
@@ -2066,11 +2080,13 @@
pte = pte_offset_map(fe->pmd, fe->address);
if (unlikely(!spin_trylock(ptl))) {
pte_unmap(pte);
+ trace_spf_pte_lock(_RET_IP_, fe->vma, fe->address);
goto out;
}
if (vma_has_changed(fe)) {
pte_unmap_unlock(pte, ptl);
+ trace_spf_vma_changed(_RET_IP_, fe->vma, fe->address);
goto out;
}
@@ -3833,47 +3849,60 @@
/* rmb <-> seqlock,vma_rb_erase() */
seq = raw_read_seqcount(&vma->vm_sequence);
- if (seq & 1)
+ if (seq & 1) {
+ trace_spf_vma_changed(_RET_IP_, vma, address);
goto out_put;
+ }
/*
* Can't call vm_ops service has we don't know what they would do
* with the VMA.
* This include huge page from hugetlbfs.
*/
- if (vma->vm_ops)
+ if (vma->vm_ops) {
+ trace_spf_vma_notsup(_RET_IP_, vma, address);
goto out_put;
+ }
/*
* __anon_vma_prepare() requires the mmap_sem to be held
* because vm_next and vm_prev must be safe. This can't be guaranteed
* in the speculative path.
*/
- if (unlikely(!vma->anon_vma))
+ if (unlikely(!vma->anon_vma)) {
+ trace_spf_vma_notsup(_RET_IP_, vma, address);
goto out_put;
+ }
fe.vma_flags = READ_ONCE(vma->vm_flags);
fe.vma_page_prot = READ_ONCE(vma->vm_page_prot);
/* Can't call userland page fault handler in the speculative path */
- if (unlikely(fe.vma_flags & VM_UFFD_MISSING))
+ if (unlikely(fe.vma_flags & VM_UFFD_MISSING)) {
+ trace_spf_vma_notsup(_RET_IP_, vma, address);
goto out_put;
+ }
- if (fe.vma_flags & VM_GROWSDOWN || fe.vma_flags & VM_GROWSUP)
+ if (fe.vma_flags & VM_GROWSDOWN || fe.vma_flags & VM_GROWSUP) {
/*
* This could be detected by the check address against VMA's
* boundaries but we want to trace it as not supported instead
* of changed.
*/
+ trace_spf_vma_notsup(_RET_IP_, vma, address);
goto out_put;
+ }
if (address < READ_ONCE(vma->vm_start)
- || READ_ONCE(vma->vm_end) <= address)
+ || READ_ONCE(vma->vm_end) <= address) {
+ trace_spf_vma_changed(_RET_IP_, vma, address);
goto out_put;
+ }
if (!arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE,
flags & FAULT_FLAG_INSTRUCTION,
flags & FAULT_FLAG_REMOTE)) {
+ trace_spf_vma_access(_RET_IP_, vma, address);
ret = VM_FAULT_SIGSEGV;
goto out_put;
}
@@ -3885,6 +3914,7 @@
goto out_put;
}
} else if (unlikely(!(fe.vma_flags & (VM_READ|VM_EXEC|VM_WRITE)))) {
+ trace_spf_vma_access(_RET_IP_, vma, address);
ret = VM_FAULT_SIGSEGV;
goto out_put;
}
@@ -3900,8 +3930,11 @@
pol = __get_vma_policy(vma, address);
if (!pol)
pol = get_task_policy(current);
- if (pol && pol->mode == MPOL_INTERLEAVE)
+
+ if (pol && pol->mode == MPOL_INTERLEAVE) {
+ trace_spf_vma_notsup(_RET_IP_, vma, address);
goto out_put;
+ }
#endif
/*
@@ -3962,8 +3995,10 @@
* We need to re-validate the VMA after checking the bounds, otherwise
* we might have a false positive on the bounds.
*/
- if (read_seqcount_retry(&vma->vm_sequence, seq))
+ if (read_seqcount_retry(&vma->vm_sequence, seq)) {
+ trace_spf_vma_changed(_RET_IP_, vma, address);
goto out_put;
+ }
mem_cgroup_oom_enable();
ret = handle_pte_fault(&fe);
@@ -3982,6 +4017,7 @@
return ret;
out_walk:
+ trace_spf_vma_notsup(_RET_IP_, vma, address);
local_irq_enable();
out_put:
put_vma(vma);