blob: 6d18b70ed5a9bb808f2c976450671e7034e3169c [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#include <linux/highmem.h>
Paul Gortmaker4b599fe2016-07-13 20:18:55 -04002#include <linux/export.h>
Pekka Enberg867c5b522009-03-03 14:10:12 +02003#include <linux/swap.h> /* for totalram_pages */
Jiang Liu7b4b2a02013-07-03 15:03:11 -07004#include <linux/bootmem.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07005
6void *kmap(struct page *page)
7{
8 might_sleep();
9 if (!PageHighMem(page))
10 return page_address(page);
11 return kmap_high(page);
12}
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070013EXPORT_SYMBOL(kmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -070014
15void kunmap(struct page *page)
16{
17 if (in_interrupt())
18 BUG();
19 if (!PageHighMem(page))
20 return;
21 kunmap_high(page);
22}
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070023EXPORT_SYMBOL(kunmap);
Linus Torvalds1da177e2005-04-16 15:20:36 -070024
25/*
26 * kmap_atomic/kunmap_atomic is significantly faster than kmap/kunmap because
27 * no global lock is needed and because the kmap code must perform a global TLB
28 * invalidation when the kmap pool wraps.
29 *
Figo.zhang565b0c12009-06-29 12:02:55 +080030 * However when holding an atomic kmap it is not legal to sleep, so atomic
Linus Torvalds1da177e2005-04-16 15:20:36 -070031 * kmaps are appropriate for short, tight code paths only.
32 */
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070033void *kmap_atomic_prot(struct page *page, pgprot_t prot)
Linus Torvalds1da177e2005-04-16 15:20:36 -070034{
Linus Torvalds1da177e2005-04-16 15:20:36 -070035 unsigned long vaddr;
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070036 int idx, type;
Andrew Morton9c312052008-03-28 11:47:34 -070037
David Hildenbrand2cb7c9c2015-05-11 17:52:09 +020038 preempt_disable();
Peter Zijlstraa8663742006-12-06 20:32:20 -080039 pagefault_disable();
Ingo Molnar656dad32007-02-10 01:46:36 -080040
Linus Torvalds1da177e2005-04-16 15:20:36 -070041 if (!PageHighMem(page))
42 return page_address(page);
43
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070044 type = kmap_atomic_idx_push();
Andrew Morton4150d3f2007-09-11 15:24:10 -070045 idx = type + KM_TYPE_NR*smp_processor_id();
Linus Torvalds1da177e2005-04-16 15:20:36 -070046 vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
Andrew Morton4150d3f2007-09-11 15:24:10 -070047 BUG_ON(!pte_none(*(kmap_pte-idx)));
Jeremy Fitzhardingece6234b2007-05-02 19:27:15 +020048 set_pte(kmap_pte-idx, mk_pte(page, prot));
Konrad Rzeszutek Wilk2cd1c8d2011-11-15 14:49:09 -080049 arch_flush_lazy_mmu_mode();
Linus Torvalds1da177e2005-04-16 15:20:36 -070050
Andrew Morton4150d3f2007-09-11 15:24:10 -070051 return (void *)vaddr;
Linus Torvalds1da177e2005-04-16 15:20:36 -070052}
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070053EXPORT_SYMBOL(kmap_atomic_prot);
Linus Torvalds1da177e2005-04-16 15:20:36 -070054
Cong Wanga24401b2011-11-26 10:53:39 +080055void *kmap_atomic(struct page *page)
Jeremy Fitzhardingece6234b2007-05-02 19:27:15 +020056{
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070057 return kmap_atomic_prot(page, kmap_prot);
Jeremy Fitzhardingece6234b2007-05-02 19:27:15 +020058}
Cong Wanga24401b2011-11-26 10:53:39 +080059EXPORT_SYMBOL(kmap_atomic);
Linus Torvalds1da177e2005-04-16 15:20:36 -070060
Ingo Molnardd63fdc2009-03-13 03:20:49 +010061/*
62 * This is the same as kmap_atomic() but can map memory that doesn't
Vivek Goyal60e64d42005-06-25 14:58:19 -070063 * have a struct page associated with it.
64 */
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070065void *kmap_atomic_pfn(unsigned long pfn)
Vivek Goyal60e64d42005-06-25 14:58:19 -070066{
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070067 return kmap_atomic_prot_pfn(pfn, kmap_prot);
Vivek Goyal60e64d42005-06-25 14:58:19 -070068}
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070069EXPORT_SYMBOL_GPL(kmap_atomic_pfn);
70
71void __kunmap_atomic(void *kvaddr)
72{
73 unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK;
74
75 if (vaddr >= __fix_to_virt(FIX_KMAP_END) &&
76 vaddr <= __fix_to_virt(FIX_KMAP_BEGIN)) {
77 int idx, type;
78
Peter Zijlstra20273942010-10-27 15:32:58 -070079 type = kmap_atomic_idx();
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070080 idx = type + KM_TYPE_NR * smp_processor_id();
81
82#ifdef CONFIG_DEBUG_HIGHMEM
83 WARN_ON_ONCE(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx));
84#endif
85 /*
86 * Force other mappings to Oops if they'll try to access this
87 * pte without first remap it. Keeping stale mappings around
88 * is a bad idea also, in case the page changes cacheability
89 * attributes or becomes a protected page in a hypervisor.
90 */
91 kpte_clear_flush(kmap_pte-idx, vaddr);
Peter Zijlstra20273942010-10-27 15:32:58 -070092 kmap_atomic_idx_pop();
Konrad Rzeszutek Wilk2cd1c8d2011-11-15 14:49:09 -080093 arch_flush_lazy_mmu_mode();
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -070094 }
95#ifdef CONFIG_DEBUG_HIGHMEM
96 else {
97 BUG_ON(vaddr < PAGE_OFFSET);
98 BUG_ON(vaddr >= (unsigned long)high_memory);
99 }
100#endif
101
102 pagefault_enable();
David Hildenbrand2cb7c9c2015-05-11 17:52:09 +0200103 preempt_enable();
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -0700104}
105EXPORT_SYMBOL(__kunmap_atomic);
Vivek Goyal60e64d42005-06-25 14:58:19 -0700106
Pekka Enberg867c5b522009-03-03 14:10:12 +0200107void __init set_highmem_pages_init(void)
108{
109 struct zone *zone;
110 int nid;
111
Jiang Liu7b4b2a02013-07-03 15:03:11 -0700112 /*
113 * Explicitly reset zone->managed_pages because set_highmem_pages_init()
114 * is invoked before free_all_bootmem()
115 */
116 reset_all_zones_managed_pages();
Pekka Enberg867c5b522009-03-03 14:10:12 +0200117 for_each_zone(zone) {
118 unsigned long zone_start_pfn, zone_end_pfn;
119
120 if (!is_highmem(zone))
121 continue;
122
123 zone_start_pfn = zone->zone_start_pfn;
124 zone_end_pfn = zone_start_pfn + zone->spanned_pages;
125
126 nid = zone_to_nid(zone);
127 printk(KERN_INFO "Initializing %s for node %d (%08lx:%08lx)\n",
128 zone->name, nid, zone_start_pfn, zone_end_pfn);
129
130 add_highpages_with_active_regions(nid, zone_start_pfn,
131 zone_end_pfn);
132 }
Pekka Enberg867c5b522009-03-03 14:10:12 +0200133}