mm: add cma pcp list
Add a cma pcp list in order to increase cma memory utilization.
Increased cma memory utilization will improve overall memory
utilization because free cma pages are ignored when memory reclaim
is done with gfp mask GFP_KERNEL.
Since most memory reclaim is done by kswapd, which uses a gfp mask
of GFP_KERNEL, by increasing cma memory utilization we are therefore
ensuring that less aggressive memory reclaim takes place.
Increased cma memory utilization will improve performance,
for example it will increase app concurrency.
Change-Id: I809589a25c6abca51f1c963f118adfc78e955cf9
Signed-off-by: Liam Mark <lmark@codeaurora.org>
[vinmenon@codeaurora.org: fix !CONFIG_CMA compile time issues]
Signed-off-by: Vinayak Menon <vinmenon@codeaurora.org>
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 7eb13a2..8a0490a 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -233,10 +233,10 @@
"Unmovable",
"Movable",
"Reclaimable",
- "HighAtomic",
#ifdef CONFIG_CMA
"CMA",
#endif
+ "HighAtomic",
#ifdef CONFIG_MEMORY_ISOLATION
"Isolate",
#endif
@@ -2216,24 +2216,22 @@
return page;
}
-static struct page *__rmqueue_cma(struct zone *zone, unsigned int order,
- int migratetype)
+#ifdef CONFIG_CMA
+static struct page *__rmqueue_cma(struct zone *zone, unsigned int order)
{
struct page *page = 0;
-#ifdef CONFIG_CMA
- if (migratetype == MIGRATE_MOVABLE && !zone->cma_alloc)
- page = __rmqueue_cma_fallback(zone, order);
- else
-#endif
- page = __rmqueue_smallest(zone, order, migratetype);
-
- if (unlikely(!page)) {
- page = __rmqueue_fallback(zone, order, migratetype);
- }
-
- trace_mm_page_alloc_zone_locked(page, order, migratetype);
+ if (IS_ENABLED(CONFIG_CMA))
+ if (!zone->cma_alloc)
+ page = __rmqueue_cma_fallback(zone, order);
+ trace_mm_page_alloc_zone_locked(page, order, MIGRATE_CMA);
return page;
}
+#else
+static inline struct page *__rmqueue_cma(struct zone *zone, unsigned int order)
+{
+ return NULL;
+}
+#endif
/*
* Obtain a specified number of elements from the buddy allocator, all under
@@ -2242,15 +2240,21 @@
*/
static int rmqueue_bulk(struct zone *zone, unsigned int order,
unsigned long count, struct list_head *list,
- int migratetype, bool cold, int cma)
+ int migratetype, bool cold)
{
int i, alloced = 0;
spin_lock(&zone->lock);
for (i = 0; i < count; ++i) {
struct page *page;
- if (cma)
- page = __rmqueue_cma(zone, order, migratetype);
+
+ /*
+ * If migrate type CMA is being requested only try to
+ * satisfy the request with CMA pages to try and increase
+ * CMA utlization.
+ */
+ if (is_migrate_cma(migratetype))
+ page = __rmqueue_cma(zone, order);
else
page = __rmqueue(zone, order, migratetype);
if (unlikely(page == NULL))
@@ -2290,6 +2294,28 @@
return alloced;
}
+/*
+ * Return the pcp list that corresponds to the migrate type if that list isn't
+ * empty.
+ * If the list is empty return NULL.
+ */
+static struct list_head *get_populated_pcp_list(struct zone *zone,
+ unsigned int order, struct per_cpu_pages *pcp,
+ int migratetype, int cold)
+{
+ struct list_head *list = &pcp->lists[migratetype];
+
+ if (list_empty(list)) {
+ pcp->count += rmqueue_bulk(zone, order,
+ pcp->batch, list,
+ migratetype, cold);
+
+ if (list_empty(list))
+ list = NULL;
+ }
+ return list;
+}
+
#ifdef CONFIG_NUMA
/*
* Called from the vmstat counter updater to drain pagesets of this
@@ -2650,23 +2676,33 @@
int migratetype)
{
unsigned long flags;
- struct page *page;
+ struct page *page = NULL;
bool cold = ((gfp_flags & __GFP_COLD) != 0);
if (likely(order == 0)) {
struct per_cpu_pages *pcp;
- struct list_head *list;
+ struct list_head *list = NULL;
local_irq_save(flags);
do {
pcp = &this_cpu_ptr(zone->pageset)->pcp;
- list = &pcp->lists[migratetype];
- if (list_empty(list)) {
- pcp->count += rmqueue_bulk(zone, 0,
- pcp->batch, list,
- migratetype, cold,
- gfp_flags & __GFP_CMA);
- if (unlikely(list_empty(list)))
+
+ /* First try to get CMA pages */
+ if (migratetype == MIGRATE_MOVABLE &&
+ gfp_flags & __GFP_CMA) {
+ list = get_populated_pcp_list(zone, 0, pcp,
+ get_cma_migrate_type(), cold);
+ }
+
+ if (list == NULL) {
+ /*
+ * Either CMA is not suitable or there are no
+ * free CMA pages.
+ */
+ list = get_populated_pcp_list(zone, 0, pcp,
+ migratetype, cold);
+ if (unlikely(list == NULL) ||
+ unlikely(list_empty(list)))
goto failed;
}
@@ -2694,13 +2730,14 @@
if (page)
trace_mm_page_alloc_zone_locked(page, order, migratetype);
}
- if (!page) {
- if (gfp_flags & __GFP_CMA)
- page = __rmqueue_cma(zone, order, migratetype);
- else
- page = __rmqueue(zone, order, migratetype);
- }
+ if (!page && migratetype == MIGRATE_MOVABLE &&
+ gfp_flags & __GFP_CMA)
+ page = __rmqueue_cma(zone, order);
+
+ if (!page)
+ page = __rmqueue(zone, order, migratetype);
} while (page && check_new_pages(page, order));
+
spin_unlock(&zone->lock);
if (!page)
goto failed;