cma: redirect page allocation to CMA
CMA pages are designed to be used as fallback for movable allocations
and cannot be used for non-movable allocations. If CMA pages are
utilized poorly, non-movable allocations may end up getting starved if
all regular movable pages are allocated and the only pages left are
CMA. Always using CMA pages first creates unacceptable performance
problems. As a midway alternative, use CMA pages for certain
userspace allocations. The userspace pages can be migrated or dropped
quickly which giving decent utilization.
Change-Id: I6165dda01b705309eebabc6dfa67146b7a95c174
CRs-Fixed: 452508
[lauraa@codeaurora.org: Missing CONFIG_CMA guards, add commit text]
Signed-off-by: Laura Abbott <lauraa@codeaurora.org>
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 1de24ac..adf29ab 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1121,6 +1121,37 @@
return page;
}
+static struct page *__rmqueue_cma(struct zone *zone, unsigned int order,
+ int migratetype)
+{
+ struct page *page = 0;
+#ifdef CONFIG_CMA
+ if (migratetype == MIGRATE_MOVABLE && !zone->cma_alloc)
+ page = __rmqueue_smallest(zone, order, MIGRATE_CMA);
+ else
+#endif
+retry_reserve :
+ page = __rmqueue_smallest(zone, order, migratetype);
+
+
+ if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
+ page = __rmqueue_fallback(zone, order, migratetype);
+
+ /*
+ * Use MIGRATE_RESERVE rather than fail an allocation. goto
+ * is used because __rmqueue_smallest is an inline function
+ * and we want just one call site
+ */
+ if (!page) {
+ migratetype = MIGRATE_RESERVE;
+ goto retry_reserve;
+ }
+ }
+
+ trace_mm_page_alloc_zone_locked(page, order, migratetype);
+ return page;
+}
+
/*
* Obtain a specified number of elements from the buddy allocator, all under
* a single hold of the lock, for efficiency. Add them to the supplied list.
@@ -1128,13 +1159,17 @@
*/
static int rmqueue_bulk(struct zone *zone, unsigned int order,
unsigned long count, struct list_head *list,
- int migratetype, int cold)
+ int migratetype, int cold, int cma)
{
int mt = migratetype, i;
spin_lock(&zone->lock);
for (i = 0; i < count; ++i) {
- struct page *page = __rmqueue(zone, order, migratetype);
+ struct page *page;
+ if (cma)
+ page = __rmqueue_cma(zone, order, migratetype);
+ else
+ page = __rmqueue(zone, order, migratetype);
if (unlikely(page == NULL))
break;
@@ -1480,7 +1515,8 @@
if (list_empty(list)) {
pcp->count += rmqueue_bulk(zone, 0,
pcp->batch, list,
- migratetype, cold);
+ migratetype, cold,
+ gfp_flags & __GFP_CMA);
if (unlikely(list_empty(list)))
goto failed;
}
@@ -1507,7 +1543,10 @@
WARN_ON_ONCE(order > 1);
}
spin_lock_irqsave(&zone->lock, flags);
- page = __rmqueue(zone, order, migratetype);
+ if (gfp_flags & __GFP_CMA)
+ page = __rmqueue_cma(zone, order, migratetype);
+ else
+ page = __rmqueue(zone, order, migratetype);
spin_unlock(&zone->lock);
if (!page)
goto failed;
@@ -5884,6 +5923,8 @@
if (ret)
goto done;
+ zone->cma_alloc = 1;
+
ret = __alloc_contig_migrate_range(start, end);
if (ret)
goto done;
@@ -5948,6 +5989,7 @@
done:
undo_isolate_page_range(pfn_max_align_down(start),
pfn_max_align_up(end), migratetype);
+ zone->cma_alloc = 0;
return ret;
}