mm/memblock: fix a race between search and remove
no-map-fixup feature for dma-removed reserve region does a
late memblock remove. This late removal is a slight
deviation from conventional memblock flow. That leads to a
race between pfn_valid->memblock_search, and
memblock_remove.
To fix this, use a read seqlock in memblock_search, which
would ensure minimum overhead in search path. And export
two APIs to let code doing late memblock remove apply
write seqlock. write seqlock would ensure that search
retries if the list is updated concurrently.
The two exported APIs which should be called before
and after modifying memblock regions, late in boot, are
- memblock_region_resize_late_begin
- memblock_region_resize_late_end
The code to alter memblock regions should be guarded by
these APIs.
CRs-fixed: 967728
Change-Id: I6a10c3e980002048aafeaf829a16119848c6a099
Signed-off-by: Shiraz Hashim <shashim@codeaurora.org>
Signed-off-by: Patrick Daly <pdaly@codeaurora.org>
diff --git a/mm/memblock.c b/mm/memblock.c
index 5d2a991..7949841 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -19,6 +19,8 @@
#include <linux/debugfs.h>
#include <linux/seq_file.h>
#include <linux/memblock.h>
+#include <linux/preempt.h>
+#include <linux/seqlock.h>
#include <asm/sections.h>
#include <linux/io.h>
@@ -31,6 +33,7 @@
static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS] __initdata_memblock;
#endif
+static seqcount_t memblock_seq;
struct memblock memblock __initdata_memblock = {
.memory.regions = memblock_memory_init_regions,
.memory.cnt = 1, /* empty dummy entry */
@@ -1541,7 +1544,8 @@
(phys_addr_t)ULLONG_MAX);
}
-static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr)
+static int __init_memblock __memblock_search(struct memblock_type *type,
+ phys_addr_t addr)
{
unsigned int left = 0, right = type->cnt;
@@ -1559,6 +1563,20 @@
return -1;
}
+static int __init_memblock memblock_search(struct memblock_type *type,
+ phys_addr_t addr)
+{
+ int ret;
+ unsigned long seq;
+
+ do {
+ seq = raw_read_seqcount_begin(&memblock_seq);
+ ret = __memblock_search(type, addr);
+ } while (unlikely(read_seqcount_retry(&memblock_seq, seq)));
+
+ return ret;
+}
+
bool __init memblock_is_reserved(phys_addr_t addr)
{
return memblock_search(&memblock.reserved, addr) != -1;
@@ -1716,6 +1734,32 @@
memblock_can_resize = 1;
}
+static void __init_memblock memblock_resize_late(int begin)
+{
+ static int memblock_can_resize_old;
+
+ if (begin) {
+ preempt_disable();
+ memblock_can_resize_old = memblock_can_resize;
+ memblock_can_resize = 0;
+ raw_write_seqcount_begin(&memblock_seq);
+ } else {
+ raw_write_seqcount_end(&memblock_seq);
+ memblock_can_resize = memblock_can_resize_old;
+ preempt_enable();
+ }
+}
+
+void __init_memblock memblock_region_resize_late_begin(void)
+{
+ memblock_resize_late(1);
+}
+
+void __init_memblock memblock_region_resize_late_end(void)
+{
+ memblock_resize_late(0);
+}
+
static int __init early_memblock(char *p)
{
if (p && strstr(p, "debug"))