arch/x86/mm/srat_64.c - kernel/msm-5.4 - Gitiles

 /*
  * ACPI 3.0 based NUMA setup
  * Copyright 2004 Andi Kleen, SuSE Labs.
  *
  * Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
  *
  * Called from acpi_numa_init while reading the SRAT and SLIT tables.
  * Assumes all memory regions belonging to a single proximity domain
  * are in one chunk. Holes between them will be included in the node.
  */

 #include <linux/kernel.h>
 #include <linux/acpi.h>
 #include <linux/mmzone.h>
 #include <linux/bitmap.h>
 #include <linux/module.h>
 #include <linux/topology.h>
 #include <linux/bootmem.h>
 #include <linux/memblock.h>
 #include <linux/mm.h>
 #include <asm/proto.h>
 #include <asm/numa.h>
 #include <asm/e820.h>
 #include <asm/apic.h>
 #include <asm/uv/uv.h>

 int acpi_numa __initdata;

 static struct bootnode nodes_add[MAX_NUMNODES];

 static __init int setup_node(int pxm)
 {
 	return acpi_map_pxm_to_node(pxm);
 }

 static __init void bad_srat(void)
 {
 	printk(KERN_ERR "SRAT: SRAT not used.\n");
 	acpi_numa = -1;
 	memset(nodes_add, 0, sizeof(nodes_add));
 }

 static __init inline int srat_disabled(void)
 {
 	return acpi_numa < 0;
 }

 /* Callback for SLIT parsing */
 void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
 {
 	int i, j;

 	for (i = 0; i < slit->locality_count; i++)
 		for (j = 0; j < slit->locality_count; j++)
 			numa_set_distance(pxm_to_node(i), pxm_to_node(j),
 				slit->entry[slit->locality_count * i + j]);
 }

 /* Callback for Proximity Domain -> x2APIC mapping */
 void __init
 acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
 {
 	int pxm, node;
 	int apic_id;

 	if (srat_disabled())
 		return;
 	if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) {
 		bad_srat();
 		return;
 	}
 	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
 		return;
 	pxm = pa->proximity_domain;
 	node = setup_node(pxm);
 	if (node < 0) {
 		printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
 		bad_srat();
 		return;
 	}

 	apic_id = pa->apic_id;
 	if (apic_id >= MAX_LOCAL_APIC) {
 		printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
 		return;
 	}
 	set_apicid_to_node(apic_id, node);
 	node_set(node, numa_nodes_parsed);
 	acpi_numa = 1;
 	printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u\n",
 	       pxm, apic_id, node);
 }

 /* Callback for Proximity Domain -> LAPIC mapping */
 void __init
 acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
 {
 	int pxm, node;
 	int apic_id;

 	if (srat_disabled())
 		return;
 	if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
 		bad_srat();
 		return;
 	}
 	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
 		return;
 	pxm = pa->proximity_domain_lo;
 	node = setup_node(pxm);
 	if (node < 0) {
 		printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
 		bad_srat();
 		return;
 	}

 	if (get_uv_system_type() >= UV_X2APIC)
 		apic_id = (pa->apic_id << 8) | pa->local_sapic_eid;
 	else
 		apic_id = pa->apic_id;

 	if (apic_id >= MAX_LOCAL_APIC) {
 		printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
 		return;
 	}

 	set_apicid_to_node(apic_id, node);
 	node_set(node, numa_nodes_parsed);
 	acpi_numa = 1;
 	printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u\n",
 	       pxm, apic_id, node);
 }

 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
 static inline int save_add_info(void) {return 1;}
 #else
 static inline int save_add_info(void) {return 0;}
 #endif
 /*
  * Update nodes_add[]
  * This code supports one contiguous hot add area per node
  */
 static void __init
 update_nodes_add(int node, unsigned long start, unsigned long end)
 {
 	unsigned long s_pfn = start >> PAGE_SHIFT;
 	unsigned long e_pfn = end >> PAGE_SHIFT;
 	int changed = 0;
 	struct bootnode *nd = &nodes_add[node];

 	/* I had some trouble with strange memory hotadd regions breaking
 	   the boot. Be very strict here and reject anything unexpected.
 	   If you want working memory hotadd write correct SRATs.

 	   The node size check is a basic sanity check to guard against
 	   mistakes */
 	if ((signed long)(end - start) < NODE_MIN_SIZE) {
 		printk(KERN_ERR "SRAT: Hotplug area too small\n");
 		return;
 	}

 	/* This check might be a bit too strict, but I'm keeping it for now. */
 	if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) {
 		printk(KERN_ERR
 			"SRAT: Hotplug area %lu -> %lu has existing memory\n",
 			s_pfn, e_pfn);
 		return;
 	}

 	/* Looks good */

 	if (nd->start == nd->end) {
 		nd->start = start;
 		nd->end = end;
 		changed = 1;
 	} else {
 		if (nd->start == end) {
 			nd->start = start;
 			changed = 1;
 		}
 		if (nd->end == start) {
 			nd->end = end;
 			changed = 1;
 		}
 		if (!changed)
 			printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
 	}

 	if (changed) {
 		node_set(node, numa_nodes_parsed);
 		printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n",
 				 nd->start, nd->end);
 	}
 }

 /* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
 void __init
 acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
 {
 	unsigned long start, end;
 	int node, pxm;

 	if (srat_disabled())
 		return;
 	if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
 		bad_srat();
 		return;
 	}
 	if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
 		return;

 	if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
 		return;
 	start = ma->base_address;
 	end = start + ma->length;
 	pxm = ma->proximity_domain;
 	node = setup_node(pxm);
 	if (node < 0) {
 		printk(KERN_ERR "SRAT: Too many proximity domains.\n");
 		bad_srat();
 		return;
 	}

 	if (numa_add_memblk(node, start, end) < 0) {
 		bad_srat();
 		return;
 	}

 	printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm,
 	       start, end);

 	if (ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE)
 		update_nodes_add(node, start, end);
 }

 void __init acpi_numa_arch_fixup(void) {}

 int __init x86_acpi_numa_init(void)
 {
 	int ret;

 	ret = acpi_numa_init();
 	if (ret < 0)
 		return ret;
 	return srat_disabled() ? -EINVAL : 0;
 }

 #if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) || defined(CONFIG_ACPI_HOTPLUG_MEMORY)
 int memory_add_physaddr_to_nid(u64 start)
 {
 	int i, ret = 0;

 	for_each_node(i)
 		if (nodes_add[i].start <= start && nodes_add[i].end > start)
 			ret = i;

 	return ret;
 }
 EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
 #endif
	/*
	* ACPI 3.0 based NUMA setup
	* Copyright 2004 Andi Kleen, SuSE Labs.
	*
	* Reads the ACPI SRAT table to figure out what memory belongs to which CPUs.
	*
	* Called from acpi_numa_init while reading the SRAT and SLIT tables.
	* Assumes all memory regions belonging to a single proximity domain
	* are in one chunk. Holes between them will be included in the node.
	*/

	#include <linux/kernel.h>
	#include <linux/acpi.h>
	#include <linux/mmzone.h>
	#include <linux/bitmap.h>
	#include <linux/module.h>
	#include <linux/topology.h>
	#include <linux/bootmem.h>
	#include <linux/memblock.h>
	#include <linux/mm.h>
	#include <asm/proto.h>
	#include <asm/numa.h>
	#include <asm/e820.h>
	#include <asm/apic.h>
	#include <asm/uv/uv.h>

	int acpi_numa __initdata;

	static struct bootnode nodes_add[MAX_NUMNODES];

	static __init int setup_node(int pxm)
	{
	return acpi_map_pxm_to_node(pxm);
	}

	static __init void bad_srat(void)
	{
	printk(KERN_ERR "SRAT: SRAT not used.\n");
	acpi_numa = -1;
	memset(nodes_add, 0, sizeof(nodes_add));
	}

	static __init inline int srat_disabled(void)
	{
	return acpi_numa < 0;
	}

	/* Callback for SLIT parsing */
	void __init acpi_numa_slit_init(struct acpi_table_slit *slit)
	{
	int i, j;

	for (i = 0; i < slit->locality_count; i++)
	for (j = 0; j < slit->locality_count; j++)
	numa_set_distance(pxm_to_node(i), pxm_to_node(j),
	slit->entry[slit->locality_count * i + j]);
	}

	/* Callback for Proximity Domain -> x2APIC mapping */
	void __init
	acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa)
	{
	int pxm, node;
	int apic_id;

	if (srat_disabled())
	return;
	if (pa->header.length < sizeof(struct acpi_srat_x2apic_cpu_affinity)) {
	bad_srat();
	return;
	}
	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
	return;
	pxm = pa->proximity_domain;
	node = setup_node(pxm);
	if (node < 0) {
	printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
	bad_srat();
	return;
	}

	apic_id = pa->apic_id;
	if (apic_id >= MAX_LOCAL_APIC) {
	printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
	return;
	}
	set_apicid_to_node(apic_id, node);
	node_set(node, numa_nodes_parsed);
	acpi_numa = 1;
	printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%04x -> Node %u\n",
	pxm, apic_id, node);
	}

	/* Callback for Proximity Domain -> LAPIC mapping */
	void __init
	acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa)
	{
	int pxm, node;
	int apic_id;

	if (srat_disabled())
	return;
	if (pa->header.length != sizeof(struct acpi_srat_cpu_affinity)) {
	bad_srat();
	return;
	}
	if ((pa->flags & ACPI_SRAT_CPU_ENABLED) == 0)
	return;
	pxm = pa->proximity_domain_lo;
	node = setup_node(pxm);
	if (node < 0) {
	printk(KERN_ERR "SRAT: Too many proximity domains %x\n", pxm);
	bad_srat();
	return;
	}

	if (get_uv_system_type() >= UV_X2APIC)
	apic_id = (pa->apic_id << 8) \| pa->local_sapic_eid;
	else
	apic_id = pa->apic_id;

	if (apic_id >= MAX_LOCAL_APIC) {
	printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u skipped apicid that is too big\n", pxm, apic_id, node);
	return;
	}

	set_apicid_to_node(apic_id, node);
	node_set(node, numa_nodes_parsed);
	acpi_numa = 1;
	printk(KERN_INFO "SRAT: PXM %u -> APIC 0x%02x -> Node %u\n",
	pxm, apic_id, node);
	}

	#ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
	static inline int save_add_info(void) {return 1;}
	#else
	static inline int save_add_info(void) {return 0;}
	#endif
	/*
	* Update nodes_add[]
	* This code supports one contiguous hot add area per node
	*/
	static void __init
	update_nodes_add(int node, unsigned long start, unsigned long end)
	{
	unsigned long s_pfn = start >> PAGE_SHIFT;
	unsigned long e_pfn = end >> PAGE_SHIFT;
	int changed = 0;
	struct bootnode *nd = &nodes_add[node];

	/* I had some trouble with strange memory hotadd regions breaking
	the boot. Be very strict here and reject anything unexpected.
	If you want working memory hotadd write correct SRATs.

	The node size check is a basic sanity check to guard against
	mistakes */
	if ((signed long)(end - start) < NODE_MIN_SIZE) {
	printk(KERN_ERR "SRAT: Hotplug area too small\n");
	return;
	}

	/* This check might be a bit too strict, but I'm keeping it for now. */
	if (absent_pages_in_range(s_pfn, e_pfn) != e_pfn - s_pfn) {
	printk(KERN_ERR
	"SRAT: Hotplug area %lu -> %lu has existing memory\n",
	s_pfn, e_pfn);
	return;
	}

	/* Looks good */

	if (nd->start == nd->end) {
	nd->start = start;
	nd->end = end;
	changed = 1;
	} else {
	if (nd->start == end) {
	nd->start = start;
	changed = 1;
	}
	if (nd->end == start) {
	nd->end = end;
	changed = 1;
	}
	if (!changed)
	printk(KERN_ERR "SRAT: Hotplug zone not continuous. Partly ignored\n");
	}

	if (changed) {
	node_set(node, numa_nodes_parsed);
	printk(KERN_INFO "SRAT: hot plug zone found %Lx - %Lx\n",
	nd->start, nd->end);
	}
	}

	/* Callback for parsing of the Proximity Domain <-> Memory Area mappings */
	void __init
	acpi_numa_memory_affinity_init(struct acpi_srat_mem_affinity *ma)
	{
	unsigned long start, end;
	int node, pxm;

	if (srat_disabled())
	return;
	if (ma->header.length != sizeof(struct acpi_srat_mem_affinity)) {
	bad_srat();
	return;
	}
	if ((ma->flags & ACPI_SRAT_MEM_ENABLED) == 0)
	return;

	if ((ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && !save_add_info())
	return;
	start = ma->base_address;
	end = start + ma->length;
	pxm = ma->proximity_domain;
	node = setup_node(pxm);
	if (node < 0) {
	printk(KERN_ERR "SRAT: Too many proximity domains.\n");
	bad_srat();
	return;
	}

	if (numa_add_memblk(node, start, end) < 0) {
	bad_srat();
	return;
	}

	printk(KERN_INFO "SRAT: Node %u PXM %u %lx-%lx\n", node, pxm,
	start, end);

	if (ma->flags & ACPI_SRAT_MEM_HOT_PLUGGABLE)
	update_nodes_add(node, start, end);
	}

	void __init acpi_numa_arch_fixup(void) {}

	int __init x86_acpi_numa_init(void)
	{
	int ret;

	ret = acpi_numa_init();
	if (ret < 0)
	return ret;
	return srat_disabled() ? -EINVAL : 0;
	}

	#if defined(CONFIG_MEMORY_HOTPLUG_SPARSE) \|\| defined(CONFIG_ACPI_HOTPLUG_MEMORY)
	int memory_add_physaddr_to_nid(u64 start)
	{
	int i, ret = 0;

	for_each_node(i)
	if (nodes_add[i].start <= start && nodes_add[i].end > start)
	ret = i;

	return ret;
	}
	EXPORT_SYMBOL_GPL(memory_add_physaddr_to_nid);
	#endif