percpu: improve first chunk initial area map handling
Impact: no functional change
When the first chunk is created, its initial area map is not allocated
because kmalloc isn't online yet. The map is allocated and
initialized on the first allocation request on the chunk. This works
fine but the scattering of initialization logic between the init
function and allocation path is a bit confusing.
This patch makes the first chunk initialize and use minimal statically
allocated map from pcpu_setpu_first_chunk(). The map resizing path
still needs to handle this specially but it's more straight-forward
and gives more latitude to the init path. This will ease future
changes.
Signed-off-by: Tejun Heo <tj@kernel.org>
diff --git a/mm/percpu.c b/mm/percpu.c
index 9531590..503ccad 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -93,9 +93,6 @@
void *pcpu_base_addr __read_mostly;
EXPORT_SYMBOL_GPL(pcpu_base_addr);
-/* the size of kernel static area */
-static int pcpu_static_size __read_mostly;
-
/*
* One mutex to rule them all.
*
@@ -316,15 +313,28 @@
/* reallocation required? */
if (chunk->map_alloc < target) {
- int new_alloc = chunk->map_alloc;
+ int new_alloc;
int *new;
+ new_alloc = PCPU_DFL_MAP_ALLOC;
while (new_alloc < target)
new_alloc *= 2;
- new = pcpu_realloc(chunk->map,
- chunk->map_alloc * sizeof(new[0]),
- new_alloc * sizeof(new[0]));
+ if (chunk->map_alloc < PCPU_DFL_MAP_ALLOC) {
+ /*
+ * map_alloc smaller than the default size
+ * indicates that the chunk is one of the
+ * first chunks and still using static map.
+ * Allocate a dynamic one and copy.
+ */
+ new = pcpu_realloc(NULL, 0, new_alloc * sizeof(new[0]));
+ if (new)
+ memcpy(new, chunk->map,
+ chunk->map_alloc * sizeof(new[0]));
+ } else
+ new = pcpu_realloc(chunk->map,
+ chunk->map_alloc * sizeof(new[0]),
+ new_alloc * sizeof(new[0]));
if (!new)
return -ENOMEM;
@@ -367,22 +377,6 @@
int max_contig = 0;
int i, off;
- /*
- * The static chunk initially doesn't have map attached
- * because kmalloc wasn't available during init. Give it one.
- */
- if (unlikely(!chunk->map)) {
- chunk->map = pcpu_realloc(NULL, 0,
- PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0]));
- if (!chunk->map)
- return -ENOMEM;
-
- chunk->map_alloc = PCPU_DFL_MAP_ALLOC;
- chunk->map[chunk->map_used++] = -pcpu_static_size;
- if (chunk->free_size)
- chunk->map[chunk->map_used++] = chunk->free_size;
- }
-
for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++])) {
bool is_last = i + 1 == chunk->map_used;
int head, tail;
@@ -874,12 +868,14 @@
pcpu_populate_pte_fn_t populate_pte_fn)
{
static struct vm_struct first_vm;
+ static int smap[2];
struct pcpu_chunk *schunk;
unsigned int cpu;
int nr_pages;
int err, i;
/* santiy checks */
+ BUILD_BUG_ON(ARRAY_SIZE(smap) >= PCPU_DFL_MAP_ALLOC);
BUG_ON(!static_size);
BUG_ON(!unit_size && dyn_size);
BUG_ON(unit_size && unit_size < static_size + dyn_size);
@@ -893,7 +889,6 @@
pcpu_unit_pages = max_t(int, PCPU_MIN_UNIT_SIZE >> PAGE_SHIFT,
PFN_UP(static_size));
- pcpu_static_size = static_size;
pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT;
pcpu_chunk_size = num_possible_cpus() * pcpu_unit_size;
pcpu_chunk_struct_size = sizeof(struct pcpu_chunk)
@@ -912,14 +907,20 @@
schunk = alloc_bootmem(pcpu_chunk_struct_size);
INIT_LIST_HEAD(&schunk->list);
schunk->vm = &first_vm;
+ schunk->map = smap;
+ schunk->map_alloc = ARRAY_SIZE(smap);
if (dyn_size)
schunk->free_size = dyn_size;
else
- schunk->free_size = pcpu_unit_size - pcpu_static_size;
+ schunk->free_size = pcpu_unit_size - static_size;
schunk->contig_hint = schunk->free_size;
+ schunk->map[schunk->map_used++] = -static_size;
+ if (schunk->free_size)
+ schunk->map[schunk->map_used++] = schunk->free_size;
+
/* allocate vm address */
first_vm.flags = VM_ALLOC;
first_vm.size = pcpu_chunk_size;
@@ -948,7 +949,7 @@
*pcpu_chunk_pagep(schunk, cpu, i) = page;
}
- BUG_ON(i < PFN_UP(pcpu_static_size));
+ BUG_ON(i < PFN_UP(static_size));
if (nr_pages < 0)
nr_pages = i;