blob: d52aaa7dc08839de97674ab2ca7d7c7436d2de3e [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Huang, Ying5b836832008-01-30 13:31:19 +01002/*
3 * x86_64 specific EFI support functions
4 * Based on Extensible Firmware Interface Specification version 1.0
5 *
6 * Copyright (C) 2005-2008 Intel Co.
7 * Fenghua Yu <fenghua.yu@intel.com>
8 * Bibo Mao <bibo.mao@intel.com>
9 * Chandramouli Narayanan <mouli@linux.intel.com>
10 * Huang Ying <ying.huang@intel.com>
11 *
12 * Code to convert EFI to E820 map has been implemented in elilo bootloader
13 * based on a EFI patch by Edgar Hucek. Based on the E820 map, the page table
14 * is setup appropriately for EFI runtime code.
15 * - mouli 06/14/2007.
16 *
17 */
18
Matt Fleming26d7f652015-10-25 10:26:35 +000019#define pr_fmt(fmt) "efi: " fmt
20
Huang, Ying5b836832008-01-30 13:31:19 +010021#include <linux/kernel.h>
22#include <linux/init.h>
23#include <linux/mm.h>
24#include <linux/types.h>
25#include <linux/spinlock.h>
26#include <linux/bootmem.h>
27#include <linux/ioport.h>
Arnd Bergmann5ab788d2016-05-30 20:57:50 +020028#include <linux/mc146818rtc.h>
Huang, Ying5b836832008-01-30 13:31:19 +010029#include <linux/efi.h>
Andy Lutomirski116fef62018-01-31 07:56:22 -080030#include <linux/export.h>
Huang, Ying5b836832008-01-30 13:31:19 +010031#include <linux/uaccess.h>
32#include <linux/io.h>
33#include <linux/reboot.h>
David Howells0d01ff22013-04-11 23:51:01 +010034#include <linux/slab.h>
Matt Flemingf6697df2016-11-12 21:04:24 +000035#include <linux/ucs2_string.h>
Tom Lendacky1379edd2017-10-20 09:30:49 -050036#include <linux/mem_encrypt.h>
Huang, Ying5b836832008-01-30 13:31:19 +010037
38#include <asm/setup.h>
39#include <asm/page.h>
Ingo Molnar66441bd2017-01-27 10:27:10 +010040#include <asm/e820/api.h>
Huang, Ying5b836832008-01-30 13:31:19 +010041#include <asm/pgtable.h>
42#include <asm/tlbflush.h>
Huang, Ying5b836832008-01-30 13:31:19 +010043#include <asm/proto.h>
44#include <asm/efi.h>
Huang, Ying4de0d4a2008-02-13 17:22:41 +080045#include <asm/cacheflush.h>
Brian Gerst3819cd42009-01-23 11:03:29 +090046#include <asm/fixmap.h>
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +010047#include <asm/realmode.h>
Matt Fleming4f9dbcf2014-01-10 18:48:30 +000048#include <asm/time.h>
Matt Fleming67a91082015-11-27 21:09:34 +000049#include <asm/pgalloc.h>
Huang, Ying5b836832008-01-30 13:31:19 +010050
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +010051/*
Baoquan Heb1d17762017-04-04 17:02:43 +010052 * We allocate runtime services regions top-down, starting from -4G, i.e.
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +010053 * 0xffff_ffff_0000_0000 and limit EFI VA mapping space to 64G.
54 */
Mathias Krause8266e312014-09-21 17:26:54 +020055static u64 efi_va = EFI_VA_START;
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +010056
Matt Flemingc9f2a9a2015-11-27 21:09:33 +000057struct efi_scratch efi_scratch;
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +010058
Matthew Garrett9cd2b072011-05-05 15:19:43 -040059static void __init early_code_mapping_set_exec(int executable)
Huang, Ying5b836832008-01-30 13:31:19 +010060{
61 efi_memory_desc_t *md;
Huang, Ying5b836832008-01-30 13:31:19 +010062
Huang, Yinga2172e22008-01-30 13:33:55 +010063 if (!(__supported_pte_mask & _PAGE_NX))
64 return;
65
Matthew Garrett916f6762011-05-25 09:53:13 -040066 /* Make EFI service code area executable */
Matt Fleming78ce2482016-04-25 21:06:38 +010067 for_each_efi_memory_desc(md) {
Matthew Garrett916f6762011-05-25 09:53:13 -040068 if (md->type == EFI_RUNTIME_SERVICES_CODE ||
69 md->type == EFI_BOOT_SERVICES_CODE)
Matthew Garrett9cd2b072011-05-05 15:19:43 -040070 efi_set_executable(md, executable);
Huang, Ying5b836832008-01-30 13:31:19 +010071 }
72}
73
Ingo Molnar744937b2015-03-03 07:48:50 +010074pgd_t * __init efi_call_phys_prolog(void)
Huang, Ying5b836832008-01-30 13:31:19 +010075{
Baoquan He94133e42017-05-26 12:36:50 +010076 unsigned long vaddr, addr_pgd, addr_p4d, addr_pud;
77 pgd_t *save_pgd, *pgd_k, *pgd_efi;
78 p4d_t *p4d, *p4d_k, *p4d_efi;
79 pud_t *pud;
Ingo Molnar744937b2015-03-03 07:48:50 +010080
Nathan Zimmerb8f2c212013-01-08 09:02:43 -060081 int pgd;
Baoquan He94133e42017-05-26 12:36:50 +010082 int n_pgds, i, j;
Huang, Ying5b836832008-01-30 13:31:19 +010083
Matt Flemingc9f2a9a2015-11-27 21:09:33 +000084 if (!efi_enabled(EFI_OLD_MEMMAP)) {
Andy Lutomirski6c690ee2017-06-12 10:26:14 -070085 save_pgd = (pgd_t *)__read_cr3();
Matt Flemingc9f2a9a2015-11-27 21:09:33 +000086 write_cr3((unsigned long)efi_scratch.efi_pgt);
87 goto out;
88 }
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +010089
Matthew Garrett9cd2b072011-05-05 15:19:43 -040090 early_code_mapping_set_exec(1);
Nathan Zimmerb8f2c212013-01-08 09:02:43 -060091
92 n_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT), PGDIR_SIZE);
Markus Elfring20ebc152016-08-25 11:34:03 +020093 save_pgd = kmalloc_array(n_pgds, sizeof(*save_pgd), GFP_KERNEL);
Nathan Zimmerb8f2c212013-01-08 09:02:43 -060094
Baoquan He94133e42017-05-26 12:36:50 +010095 /*
96 * Build 1:1 identity mapping for efi=old_map usage. Note that
97 * PAGE_OFFSET is PGDIR_SIZE aligned when KASLR is disabled, while
98 * it is PUD_SIZE ALIGNED with KASLR enabled. So for a given physical
99 * address X, the pud_index(X) != pud_index(__va(X)), we can only copy
100 * PUD entry of __va(X) to fill in pud entry of X to build 1:1 mapping.
101 * This means here we can only reuse the PMD tables of the direct mapping.
102 */
Nathan Zimmerb8f2c212013-01-08 09:02:43 -0600103 for (pgd = 0; pgd < n_pgds; pgd++) {
Baoquan He94133e42017-05-26 12:36:50 +0100104 addr_pgd = (unsigned long)(pgd * PGDIR_SIZE);
105 vaddr = (unsigned long)__va(pgd * PGDIR_SIZE);
106 pgd_efi = pgd_offset_k(addr_pgd);
107 save_pgd[pgd] = *pgd_efi;
108
109 p4d = p4d_alloc(&init_mm, pgd_efi, addr_pgd);
110 if (!p4d) {
111 pr_err("Failed to allocate p4d table!\n");
112 goto out;
113 }
114
115 for (i = 0; i < PTRS_PER_P4D; i++) {
116 addr_p4d = addr_pgd + i * P4D_SIZE;
117 p4d_efi = p4d + p4d_index(addr_p4d);
118
119 pud = pud_alloc(&init_mm, p4d_efi, addr_p4d);
120 if (!pud) {
121 pr_err("Failed to allocate pud table!\n");
122 goto out;
123 }
124
125 for (j = 0; j < PTRS_PER_PUD; j++) {
126 addr_pud = addr_p4d + j * PUD_SIZE;
127
128 if (addr_pud > (max_pfn << PAGE_SHIFT))
129 break;
130
131 vaddr = (unsigned long)__va(addr_pud);
132
133 pgd_k = pgd_offset_k(vaddr);
134 p4d_k = p4d_offset(pgd_k, vaddr);
135 pud[j] = *pud_offset(p4d_k, vaddr);
136 }
137 }
Jiri Kosinade53c372018-01-05 22:35:41 +0100138 pgd_offset_k(pgd * PGDIR_SIZE)->pgd &= ~_PAGE_NX;
Nathan Zimmerb8f2c212013-01-08 09:02:43 -0600139 }
Jiri Kosinade53c372018-01-05 22:35:41 +0100140
Matt Flemingc9f2a9a2015-11-27 21:09:33 +0000141out:
Huang, Ying5b836832008-01-30 13:31:19 +0100142 __flush_tlb_all();
Ingo Molnar744937b2015-03-03 07:48:50 +0100143
144 return save_pgd;
Huang, Ying5b836832008-01-30 13:31:19 +0100145}
146
Ingo Molnar744937b2015-03-03 07:48:50 +0100147void __init efi_call_phys_epilog(pgd_t *save_pgd)
Huang, Ying5b836832008-01-30 13:31:19 +0100148{
149 /*
150 * After the lock is released, the original page table is restored.
151 */
Baoquan He94133e42017-05-26 12:36:50 +0100152 int pgd_idx, i;
Ingo Molnar744937b2015-03-03 07:48:50 +0100153 int nr_pgds;
Baoquan He94133e42017-05-26 12:36:50 +0100154 pgd_t *pgd;
155 p4d_t *p4d;
156 pud_t *pud;
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +0100157
Matt Flemingc9f2a9a2015-11-27 21:09:33 +0000158 if (!efi_enabled(EFI_OLD_MEMMAP)) {
159 write_cr3((unsigned long)save_pgd);
160 __flush_tlb_all();
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +0100161 return;
Matt Flemingc9f2a9a2015-11-27 21:09:33 +0000162 }
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +0100163
Ingo Molnar744937b2015-03-03 07:48:50 +0100164 nr_pgds = DIV_ROUND_UP((max_pfn << PAGE_SHIFT) , PGDIR_SIZE);
165
Baoquan He94133e42017-05-26 12:36:50 +0100166 for (pgd_idx = 0; pgd_idx < nr_pgds; pgd_idx++) {
167 pgd = pgd_offset_k(pgd_idx * PGDIR_SIZE);
Ingo Molnar744937b2015-03-03 07:48:50 +0100168 set_pgd(pgd_offset_k(pgd_idx * PGDIR_SIZE), save_pgd[pgd_idx]);
169
Baoquan He94133e42017-05-26 12:36:50 +0100170 if (!(pgd_val(*pgd) & _PAGE_PRESENT))
171 continue;
172
173 for (i = 0; i < PTRS_PER_P4D; i++) {
174 p4d = p4d_offset(pgd,
175 pgd_idx * PGDIR_SIZE + i * P4D_SIZE);
176
177 if (!(p4d_val(*p4d) & _PAGE_PRESENT))
178 continue;
179
180 pud = (pud_t *)p4d_page_vaddr(*p4d);
181 pud_free(&init_mm, pud);
182 }
183
184 p4d = (p4d_t *)pgd_page_vaddr(*pgd);
185 p4d_free(&init_mm, p4d);
186 }
187
Nathan Zimmerb8f2c212013-01-08 09:02:43 -0600188 kfree(save_pgd);
Ingo Molnar744937b2015-03-03 07:48:50 +0100189
Huang, Ying5b836832008-01-30 13:31:19 +0100190 __flush_tlb_all();
Matthew Garrett9cd2b072011-05-05 15:19:43 -0400191 early_code_mapping_set_exec(0);
Huang, Ying5b836832008-01-30 13:31:19 +0100192}
Keith Packarde1ad7832011-12-11 16:12:42 -0800193
Andy Lutomirski116fef62018-01-31 07:56:22 -0800194pgd_t *efi_pgd;
195EXPORT_SYMBOL_GPL(efi_pgd);
Matt Fleming67a91082015-11-27 21:09:34 +0000196
197/*
198 * We need our own copy of the higher levels of the page tables
199 * because we want to avoid inserting EFI region mappings (EFI_VA_END
200 * to EFI_VA_START) into the standard kernel page tables. Everything
201 * else can be shared, see efi_sync_low_kernel_mappings().
Dave Hansend9e9a642017-12-04 15:07:39 +0100202 *
203 * We don't want the pgd on the pgd_list and cannot use pgd_alloc() for the
204 * allocation.
Matt Fleming67a91082015-11-27 21:09:34 +0000205 */
206int __init efi_alloc_page_tables(void)
207{
208 pgd_t *pgd;
Kirill A. Shutemove9813162017-03-17 21:55:11 +0300209 p4d_t *p4d;
Matt Fleming67a91082015-11-27 21:09:34 +0000210 pud_t *pud;
211 gfp_t gfp_mask;
212
213 if (efi_enabled(EFI_OLD_MEMMAP))
214 return 0;
215
Levin, Alexander (Sasha Levin)75f296d2017-11-15 17:35:54 -0800216 gfp_mask = GFP_KERNEL | __GFP_ZERO;
Dave Hansend9e9a642017-12-04 15:07:39 +0100217 efi_pgd = (pgd_t *)__get_free_pages(gfp_mask, PGD_ALLOCATION_ORDER);
Matt Fleming67a91082015-11-27 21:09:34 +0000218 if (!efi_pgd)
219 return -ENOMEM;
220
221 pgd = efi_pgd + pgd_index(EFI_VA_END);
Kirill A. Shutemove9813162017-03-17 21:55:11 +0300222 p4d = p4d_alloc(&init_mm, pgd, EFI_VA_END);
223 if (!p4d) {
Matt Fleming67a91082015-11-27 21:09:34 +0000224 free_page((unsigned long)efi_pgd);
225 return -ENOMEM;
226 }
227
Kirill A. Shutemove9813162017-03-17 21:55:11 +0300228 pud = pud_alloc(&init_mm, p4d, EFI_VA_END);
229 if (!pud) {
230 if (CONFIG_PGTABLE_LEVELS > 4)
231 free_page((unsigned long) pgd_page_vaddr(*pgd));
232 free_page((unsigned long)efi_pgd);
233 return -ENOMEM;
234 }
Matt Fleming67a91082015-11-27 21:09:34 +0000235
236 return 0;
237}
238
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +0100239/*
240 * Add low kernel mappings for passing arguments to EFI functions.
241 */
242void efi_sync_low_kernel_mappings(void)
243{
Matt Fleming67a91082015-11-27 21:09:34 +0000244 unsigned num_entries;
245 pgd_t *pgd_k, *pgd_efi;
Kirill A. Shutemove0c4f672017-03-13 17:33:05 +0300246 p4d_t *p4d_k, *p4d_efi;
Matt Fleming67a91082015-11-27 21:09:34 +0000247 pud_t *pud_k, *pud_efi;
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +0100248
249 if (efi_enabled(EFI_OLD_MEMMAP))
250 return;
251
Matt Fleming67a91082015-11-27 21:09:34 +0000252 /*
253 * We can share all PGD entries apart from the one entry that
254 * covers the EFI runtime mapping space.
255 *
256 * Make sure the EFI runtime region mappings are guaranteed to
257 * only span a single PGD entry and that the entry also maps
258 * other important kernel regions.
259 */
Kirill A. Shutemovc65e7742018-02-14 14:16:53 +0300260 MAYBE_BUILD_BUG_ON(pgd_index(EFI_VA_END) != pgd_index(MODULES_END));
261 MAYBE_BUILD_BUG_ON((EFI_VA_START & PGDIR_MASK) !=
Matt Fleming67a91082015-11-27 21:09:34 +0000262 (EFI_VA_END & PGDIR_MASK));
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +0100263
Matt Fleming67a91082015-11-27 21:09:34 +0000264 pgd_efi = efi_pgd + pgd_index(PAGE_OFFSET);
265 pgd_k = pgd_offset_k(PAGE_OFFSET);
266
267 num_entries = pgd_index(EFI_VA_END) - pgd_index(PAGE_OFFSET);
268 memcpy(pgd_efi, pgd_k, sizeof(pgd_t) * num_entries);
269
270 /*
Kirill A. Shutemove9813162017-03-17 21:55:11 +0300271 * As with PGDs, we share all P4D entries apart from the one entry
272 * that covers the EFI runtime mapping space.
273 */
274 BUILD_BUG_ON(p4d_index(EFI_VA_END) != p4d_index(MODULES_END));
275 BUILD_BUG_ON((EFI_VA_START & P4D_MASK) != (EFI_VA_END & P4D_MASK));
276
277 pgd_efi = efi_pgd + pgd_index(EFI_VA_END);
278 pgd_k = pgd_offset_k(EFI_VA_END);
279 p4d_efi = p4d_offset(pgd_efi, 0);
280 p4d_k = p4d_offset(pgd_k, 0);
281
282 num_entries = p4d_index(EFI_VA_END);
283 memcpy(p4d_efi, p4d_k, sizeof(p4d_t) * num_entries);
284
285 /*
Matt Fleming67a91082015-11-27 21:09:34 +0000286 * We share all the PUD entries apart from those that map the
287 * EFI regions. Copy around them.
288 */
289 BUILD_BUG_ON((EFI_VA_START & ~PUD_MASK) != 0);
290 BUILD_BUG_ON((EFI_VA_END & ~PUD_MASK) != 0);
291
Kirill A. Shutemove9813162017-03-17 21:55:11 +0300292 p4d_efi = p4d_offset(pgd_efi, EFI_VA_END);
293 p4d_k = p4d_offset(pgd_k, EFI_VA_END);
Kirill A. Shutemove0c4f672017-03-13 17:33:05 +0300294 pud_efi = pud_offset(p4d_efi, 0);
Kirill A. Shutemove0c4f672017-03-13 17:33:05 +0300295 pud_k = pud_offset(p4d_k, 0);
Matt Fleming67a91082015-11-27 21:09:34 +0000296
297 num_entries = pud_index(EFI_VA_END);
298 memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries);
299
Kirill A. Shutemove0c4f672017-03-13 17:33:05 +0300300 pud_efi = pud_offset(p4d_efi, EFI_VA_START);
Kirill A. Shutemove0c4f672017-03-13 17:33:05 +0300301 pud_k = pud_offset(p4d_k, EFI_VA_START);
Matt Fleming67a91082015-11-27 21:09:34 +0000302
303 num_entries = PTRS_PER_PUD - pud_index(EFI_VA_START);
304 memcpy(pud_efi, pud_k, sizeof(pud_t) * num_entries);
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +0100305}
306
Matt Flemingf6697df2016-11-12 21:04:24 +0000307/*
308 * Wrapper for slow_virt_to_phys() that handles NULL addresses.
309 */
310static inline phys_addr_t
311virt_to_phys_or_null_size(void *va, unsigned long size)
312{
313 bool bad_size;
314
315 if (!va)
316 return 0;
317
318 if (virt_addr_valid(va))
319 return virt_to_phys(va);
320
321 /*
322 * A fully aligned variable on the stack is guaranteed not to
323 * cross a page bounary. Try to catch strings on the stack by
324 * checking that 'size' is a power of two.
325 */
326 bad_size = size > PAGE_SIZE || !is_power_of_2(size);
327
328 WARN_ON(!IS_ALIGNED((unsigned long)va, size) || bad_size);
329
330 return slow_virt_to_phys(va);
331}
332
333#define virt_to_phys_or_null(addr) \
334 virt_to_phys_or_null_size((addr), sizeof(*(addr)))
335
Mathias Krause4e78eb052014-09-07 19:42:17 +0200336int __init efi_setup_page_tables(unsigned long pa_memmap, unsigned num_pages)
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +0100337{
Tom Lendacky38eeccc2017-07-17 16:10:15 -0500338 unsigned long pfn, text, pf;
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000339 struct page *page;
Matt Fleming994448f2014-03-05 18:15:37 +0000340 unsigned npages;
Borislav Petkovb7b898a2014-01-18 12:48:17 +0100341 pgd_t *pgd;
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +0100342
Borislav Petkovb7b898a2014-01-18 12:48:17 +0100343 if (efi_enabled(EFI_OLD_MEMMAP))
344 return 0;
345
Tom Lendacky38eeccc2017-07-17 16:10:15 -0500346 /*
347 * Since the PGD is encrypted, set the encryption mask so that when
348 * this value is loaded into cr3 the PGD will be decrypted during
349 * the pagetable walk.
350 */
351 efi_scratch.efi_pgt = (pgd_t *)__sme_pa(efi_pgd);
Matt Fleming67a91082015-11-27 21:09:34 +0000352 pgd = efi_pgd;
Borislav Petkovb7b898a2014-01-18 12:48:17 +0100353
354 /*
355 * It can happen that the physical address of new_memmap lands in memory
356 * which is not mapped in the EFI page table. Therefore we need to go
357 * and ident-map those pages containing the map before calling
358 * phys_efi_set_virtual_address_map().
359 */
Matt Flemingedc3b912015-11-27 21:09:31 +0000360 pfn = pa_memmap >> PAGE_SHIFT;
Tom Lendacky38eeccc2017-07-17 16:10:15 -0500361 pf = _PAGE_NX | _PAGE_RW | _PAGE_ENC;
362 if (kernel_map_pages_in_pgd(pgd, pfn, pa_memmap, num_pages, pf)) {
Borislav Petkovb7b898a2014-01-18 12:48:17 +0100363 pr_err("Error ident-mapping new memmap (0x%lx)!\n", pa_memmap);
364 return 1;
365 }
366
367 efi_scratch.use_pgd = true;
368
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000369 /*
Jiri Kosinabf29bdd2017-01-27 22:25:52 +0000370 * Certain firmware versions are way too sentimential and still believe
371 * they are exclusive and unquestionable owners of the first physical page,
372 * even though they explicitly mark it as EFI_CONVENTIONAL_MEMORY
373 * (but then write-access it later during SetVirtualAddressMap()).
374 *
375 * Create a 1:1 mapping for this page, to avoid triple faults during early
376 * boot with such firmware. We are free to hand this page to the BIOS,
377 * as trim_bios_range() will reserve the first page and isolate it away
378 * from memory allocators anyway.
379 */
Tom Lendacky1379edd2017-10-20 09:30:49 -0500380 pf = _PAGE_RW;
381 if (sev_active())
382 pf |= _PAGE_ENC;
383
384 if (kernel_map_pages_in_pgd(pgd, 0x0, 0x0, 1, pf)) {
Jiri Kosinabf29bdd2017-01-27 22:25:52 +0000385 pr_err("Failed to create 1:1 mapping for the first page!\n");
386 return 1;
387 }
388
389 /*
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000390 * When making calls to the firmware everything needs to be 1:1
391 * mapped and addressable with 32-bit pointers. Map the kernel
392 * text and allocate a new stack because we can't rely on the
393 * stack pointer being < 4GB.
394 */
Matt Fleming12976672016-09-19 13:09:09 +0100395 if (!IS_ENABLED(CONFIG_EFI_MIXED) || efi_is_native())
Matt Fleming994448f2014-03-05 18:15:37 +0000396 return 0;
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000397
398 page = alloc_page(GFP_KERNEL|__GFP_DMA32);
399 if (!page)
400 panic("Unable to allocate EFI runtime stack < 4GB\n");
401
402 efi_scratch.phys_stack = virt_to_phys(page_address(page));
403 efi_scratch.phys_stack += PAGE_SIZE; /* stack grows down */
404
Sai Praneeth2ad510d2016-02-17 12:36:06 +0000405 npages = (_etext - _text) >> PAGE_SHIFT;
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000406 text = __pa(_text);
Matt Flemingedc3b912015-11-27 21:09:31 +0000407 pfn = text >> PAGE_SHIFT;
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000408
Tom Lendacky38eeccc2017-07-17 16:10:15 -0500409 pf = _PAGE_RW | _PAGE_ENC;
410 if (kernel_map_pages_in_pgd(pgd, pfn, text, npages, pf)) {
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000411 pr_err("Failed to map kernel text 1:1\n");
Matt Fleming994448f2014-03-05 18:15:37 +0000412 return 1;
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000413 }
Borislav Petkovb7b898a2014-01-18 12:48:17 +0100414
415 return 0;
416}
417
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +0100418static void __init __map_region(efi_memory_desc_t *md, u64 va)
419{
Sai Praneeth15f003d2016-02-17 12:36:04 +0000420 unsigned long flags = _PAGE_RW;
Matt Flemingedc3b912015-11-27 21:09:31 +0000421 unsigned long pfn;
Matt Fleming67a91082015-11-27 21:09:34 +0000422 pgd_t *pgd = efi_pgd;
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +0100423
424 if (!(md->attribute & EFI_MEMORY_WB))
Matt Flemingedc3b912015-11-27 21:09:31 +0000425 flags |= _PAGE_PCD;
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +0100426
Tom Lendacky1379edd2017-10-20 09:30:49 -0500427 if (sev_active())
428 flags |= _PAGE_ENC;
429
Matt Flemingedc3b912015-11-27 21:09:31 +0000430 pfn = md->phys_addr >> PAGE_SHIFT;
431 if (kernel_map_pages_in_pgd(pgd, pfn, va, md->num_pages, flags))
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +0100432 pr_warn("Error mapping PA 0x%llx -> VA 0x%llx!\n",
433 md->phys_addr, va);
434}
435
436void __init efi_map_region(efi_memory_desc_t *md)
437{
438 unsigned long size = md->num_pages << PAGE_SHIFT;
439 u64 pa = md->phys_addr;
440
441 if (efi_enabled(EFI_OLD_MEMMAP))
442 return old_map_region(md);
443
444 /*
445 * Make sure the 1:1 mappings are present as a catch-all for b0rked
446 * firmware which doesn't update all internal pointers after switching
447 * to virtual mode and would otherwise crap on us.
448 */
449 __map_region(md, md->phys_addr);
450
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000451 /*
452 * Enforce the 1:1 mapping as the default virtual address when
453 * booting in EFI mixed mode, because even though we may be
454 * running a 64-bit kernel, the firmware may only be 32-bit.
455 */
456 if (!efi_is_native () && IS_ENABLED(CONFIG_EFI_MIXED)) {
457 md->virt_addr = md->phys_addr;
458 return;
459 }
460
Borislav Petkovd2f7cbe2013-10-31 17:25:08 +0100461 efi_va -= size;
462
463 /* Is PA 2M-aligned? */
464 if (!(pa & (PMD_SIZE - 1))) {
465 efi_va &= PMD_MASK;
466 } else {
467 u64 pa_offset = pa & (PMD_SIZE - 1);
468 u64 prev_va = efi_va;
469
470 /* get us the same offset within this 2M page */
471 efi_va = (efi_va & PMD_MASK) + pa_offset;
472
473 if (efi_va > prev_va)
474 efi_va -= PMD_SIZE;
475 }
476
477 if (efi_va < EFI_VA_END) {
478 pr_warn(FW_WARN "VA address range overflow!\n");
479 return;
480 }
481
482 /* Do the VA map */
483 __map_region(md, efi_va);
484 md->virt_addr = efi_va;
485}
486
Dave Young3b266492013-12-20 18:02:14 +0800487/*
488 * kexec kernel will use efi_map_region_fixed to map efi runtime memory ranges.
489 * md->virt_addr is the original virtual address which had been mapped in kexec
490 * 1st kernel.
491 */
492void __init efi_map_region_fixed(efi_memory_desc_t *md)
493{
Alex Thorlton0513fe12016-08-05 18:59:35 -0500494 __map_region(md, md->phys_addr);
Dave Young3b266492013-12-20 18:02:14 +0800495 __map_region(md, md->virt_addr);
496}
497
Keith Packarde1ad7832011-12-11 16:12:42 -0800498void __iomem *__init efi_ioremap(unsigned long phys_addr, unsigned long size,
Matt Fleming3e8fa262012-10-19 13:25:46 +0100499 u32 type, u64 attribute)
Keith Packarde1ad7832011-12-11 16:12:42 -0800500{
501 unsigned long last_map_pfn;
502
503 if (type == EFI_MEMORY_MAPPED_IO)
504 return ioremap(phys_addr, size);
505
506 last_map_pfn = init_memory_mapping(phys_addr, phys_addr + size);
507 if ((last_map_pfn << PAGE_SHIFT) < phys_addr + size) {
508 unsigned long top = last_map_pfn << PAGE_SHIFT;
Matt Fleming3e8fa262012-10-19 13:25:46 +0100509 efi_ioremap(top, size - (top - phys_addr), type, attribute);
Keith Packarde1ad7832011-12-11 16:12:42 -0800510 }
511
Matt Fleming3e8fa262012-10-19 13:25:46 +0100512 if (!(attribute & EFI_MEMORY_WB))
513 efi_memory_uc((u64)(unsigned long)__va(phys_addr), size);
514
Keith Packarde1ad7832011-12-11 16:12:42 -0800515 return (void __iomem *)__va(phys_addr);
516}
Dave Young1fec0532013-12-20 18:02:19 +0800517
518void __init parse_efi_setup(u64 phys_addr, u32 data_len)
519{
520 efi_setup = phys_addr + sizeof(struct setup_data);
Dave Young1fec0532013-12-20 18:02:19 +0800521}
Borislav Petkovc55d0162014-02-14 08:24:24 +0100522
Sai Praneeth18141e82017-01-31 13:21:37 +0000523static int __init efi_update_mappings(efi_memory_desc_t *md, unsigned long pf)
Borislav Petkovc55d0162014-02-14 08:24:24 +0100524{
Sai Praneeth6d0cc882016-02-17 12:36:05 +0000525 unsigned long pfn;
526 pgd_t *pgd = efi_pgd;
Sai Praneeth18141e82017-01-31 13:21:37 +0000527 int err1, err2;
528
529 /* Update the 1:1 mapping */
530 pfn = md->phys_addr >> PAGE_SHIFT;
531 err1 = kernel_map_pages_in_pgd(pgd, pfn, md->phys_addr, md->num_pages, pf);
532 if (err1) {
533 pr_err("Error while updating 1:1 mapping PA 0x%llx -> VA 0x%llx!\n",
534 md->phys_addr, md->virt_addr);
535 }
536
537 err2 = kernel_map_pages_in_pgd(pgd, pfn, md->virt_addr, md->num_pages, pf);
538 if (err2) {
539 pr_err("Error while updating VA mapping PA 0x%llx -> VA 0x%llx!\n",
540 md->phys_addr, md->virt_addr);
541 }
542
543 return err1 || err2;
544}
545
546static int __init efi_update_mem_attr(struct mm_struct *mm, efi_memory_desc_t *md)
547{
548 unsigned long pf = 0;
549
550 if (md->attribute & EFI_MEMORY_XP)
551 pf |= _PAGE_NX;
552
553 if (!(md->attribute & EFI_MEMORY_RO))
554 pf |= _PAGE_RW;
555
Tom Lendacky1379edd2017-10-20 09:30:49 -0500556 if (sev_active())
557 pf |= _PAGE_ENC;
558
Sai Praneeth18141e82017-01-31 13:21:37 +0000559 return efi_update_mappings(md, pf);
560}
561
562void __init efi_runtime_update_mappings(void)
563{
Sai Praneeth6d0cc882016-02-17 12:36:05 +0000564 efi_memory_desc_t *md;
Sai Praneeth6d0cc882016-02-17 12:36:05 +0000565
566 if (efi_enabled(EFI_OLD_MEMMAP)) {
567 if (__supported_pte_mask & _PAGE_NX)
568 runtime_code_page_mkexec();
569 return;
570 }
571
Sai Praneeth18141e82017-01-31 13:21:37 +0000572 /*
573 * Use the EFI Memory Attribute Table for mapping permissions if it
574 * exists, since it is intended to supersede EFI_PROPERTIES_TABLE.
575 */
576 if (efi_enabled(EFI_MEM_ATTR)) {
577 efi_memattr_apply_permissions(NULL, efi_update_mem_attr);
578 return;
579 }
580
581 /*
582 * EFI_MEMORY_ATTRIBUTES_TABLE is intended to replace
583 * EFI_PROPERTIES_TABLE. So, use EFI_PROPERTIES_TABLE to update
584 * permissions only if EFI_MEMORY_ATTRIBUTES_TABLE is not
585 * published by the firmware. Even if we find a buggy implementation of
586 * EFI_MEMORY_ATTRIBUTES_TABLE, don't fall back to
587 * EFI_PROPERTIES_TABLE, because of the same reason.
588 */
589
Sai Praneeth6d0cc882016-02-17 12:36:05 +0000590 if (!efi_enabled(EFI_NX_PE_DATA))
Borislav Petkovc55d0162014-02-14 08:24:24 +0100591 return;
592
Matt Fleming78ce2482016-04-25 21:06:38 +0100593 for_each_efi_memory_desc(md) {
Sai Praneeth6d0cc882016-02-17 12:36:05 +0000594 unsigned long pf = 0;
Sai Praneeth6d0cc882016-02-17 12:36:05 +0000595
596 if (!(md->attribute & EFI_MEMORY_RUNTIME))
597 continue;
598
599 if (!(md->attribute & EFI_MEMORY_WB))
600 pf |= _PAGE_PCD;
601
602 if ((md->attribute & EFI_MEMORY_XP) ||
603 (md->type == EFI_RUNTIME_SERVICES_DATA))
604 pf |= _PAGE_NX;
605
606 if (!(md->attribute & EFI_MEMORY_RO) &&
607 (md->type != EFI_RUNTIME_SERVICES_CODE))
608 pf |= _PAGE_RW;
609
Tom Lendacky1379edd2017-10-20 09:30:49 -0500610 if (sev_active())
611 pf |= _PAGE_ENC;
612
Sai Praneeth18141e82017-01-31 13:21:37 +0000613 efi_update_mappings(md, pf);
Sai Praneeth6d0cc882016-02-17 12:36:05 +0000614 }
Borislav Petkovc55d0162014-02-14 08:24:24 +0100615}
Borislav Petkov11cc8512014-01-18 12:48:15 +0100616
617void __init efi_dump_pagetable(void)
618{
619#ifdef CONFIG_EFI_PGT_DUMP
Sai Praneethac81d3d2017-06-02 13:52:06 +0000620 if (efi_enabled(EFI_OLD_MEMMAP))
621 ptdump_walk_pgd_level(NULL, swapper_pg_dir);
622 else
623 ptdump_walk_pgd_level(NULL, efi_pgd);
Borislav Petkov11cc8512014-01-18 12:48:15 +0100624#endif
625}
Matt Fleming994448f2014-03-05 18:15:37 +0000626
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000627#ifdef CONFIG_EFI_MIXED
628extern efi_status_t efi64_thunk(u32, ...);
629
630#define runtime_service32(func) \
631({ \
632 u32 table = (u32)(unsigned long)efi.systab; \
633 u32 *rt, *___f; \
634 \
635 rt = (u32 *)(table + offsetof(efi_system_table_32_t, runtime)); \
636 ___f = (u32 *)(*rt + offsetof(efi_runtime_services_32_t, func)); \
637 *___f; \
638})
639
640/*
641 * Switch to the EFI page tables early so that we can access the 1:1
642 * runtime services mappings which are not mapped in any other page
643 * tables. This function must be called before runtime_service32().
644 *
645 * Also, disable interrupts because the IDT points to 64-bit handlers,
646 * which aren't going to function correctly when we switch to 32-bit.
647 */
648#define efi_thunk(f, ...) \
649({ \
650 efi_status_t __s; \
Alex Thorlton21f86622016-06-25 08:20:29 +0100651 unsigned long __flags; \
652 u32 __func; \
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000653 \
Alex Thorlton21f86622016-06-25 08:20:29 +0100654 local_irq_save(__flags); \
655 arch_efi_call_virt_setup(); \
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000656 \
Alex Thorlton21f86622016-06-25 08:20:29 +0100657 __func = runtime_service32(f); \
658 __s = efi64_thunk(__func, __VA_ARGS__); \
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000659 \
Alex Thorlton21f86622016-06-25 08:20:29 +0100660 arch_efi_call_virt_teardown(); \
661 local_irq_restore(__flags); \
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000662 \
663 __s; \
664})
665
666efi_status_t efi_thunk_set_virtual_address_map(
667 void *phys_set_virtual_address_map,
668 unsigned long memory_map_size,
669 unsigned long descriptor_size,
670 u32 descriptor_version,
671 efi_memory_desc_t *virtual_map)
672{
673 efi_status_t status;
674 unsigned long flags;
675 u32 func;
676
677 efi_sync_low_kernel_mappings();
678 local_irq_save(flags);
679
Andy Lutomirski6c690ee2017-06-12 10:26:14 -0700680 efi_scratch.prev_cr3 = __read_cr3();
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000681 write_cr3((unsigned long)efi_scratch.efi_pgt);
682 __flush_tlb_all();
683
684 func = (u32)(unsigned long)phys_set_virtual_address_map;
685 status = efi64_thunk(func, memory_map_size, descriptor_size,
686 descriptor_version, virtual_map);
687
688 write_cr3(efi_scratch.prev_cr3);
689 __flush_tlb_all();
690 local_irq_restore(flags);
691
692 return status;
693}
694
695static efi_status_t efi_thunk_get_time(efi_time_t *tm, efi_time_cap_t *tc)
696{
697 efi_status_t status;
698 u32 phys_tm, phys_tc;
699
700 spin_lock(&rtc_lock);
701
Matt Flemingf6697df2016-11-12 21:04:24 +0000702 phys_tm = virt_to_phys_or_null(tm);
703 phys_tc = virt_to_phys_or_null(tc);
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000704
705 status = efi_thunk(get_time, phys_tm, phys_tc);
706
707 spin_unlock(&rtc_lock);
708
709 return status;
710}
711
712static efi_status_t efi_thunk_set_time(efi_time_t *tm)
713{
714 efi_status_t status;
715 u32 phys_tm;
716
717 spin_lock(&rtc_lock);
718
Matt Flemingf6697df2016-11-12 21:04:24 +0000719 phys_tm = virt_to_phys_or_null(tm);
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000720
721 status = efi_thunk(set_time, phys_tm);
722
723 spin_unlock(&rtc_lock);
724
725 return status;
726}
727
728static efi_status_t
729efi_thunk_get_wakeup_time(efi_bool_t *enabled, efi_bool_t *pending,
730 efi_time_t *tm)
731{
732 efi_status_t status;
733 u32 phys_enabled, phys_pending, phys_tm;
734
735 spin_lock(&rtc_lock);
736
Matt Flemingf6697df2016-11-12 21:04:24 +0000737 phys_enabled = virt_to_phys_or_null(enabled);
738 phys_pending = virt_to_phys_or_null(pending);
739 phys_tm = virt_to_phys_or_null(tm);
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000740
741 status = efi_thunk(get_wakeup_time, phys_enabled,
742 phys_pending, phys_tm);
743
744 spin_unlock(&rtc_lock);
745
746 return status;
747}
748
749static efi_status_t
750efi_thunk_set_wakeup_time(efi_bool_t enabled, efi_time_t *tm)
751{
752 efi_status_t status;
753 u32 phys_tm;
754
755 spin_lock(&rtc_lock);
756
Matt Flemingf6697df2016-11-12 21:04:24 +0000757 phys_tm = virt_to_phys_or_null(tm);
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000758
759 status = efi_thunk(set_wakeup_time, enabled, phys_tm);
760
761 spin_unlock(&rtc_lock);
762
763 return status;
764}
765
Matt Flemingf6697df2016-11-12 21:04:24 +0000766static unsigned long efi_name_size(efi_char16_t *name)
767{
768 return ucs2_strsize(name, EFI_VAR_NAME_LEN) + 1;
769}
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000770
771static efi_status_t
772efi_thunk_get_variable(efi_char16_t *name, efi_guid_t *vendor,
773 u32 *attr, unsigned long *data_size, void *data)
774{
775 efi_status_t status;
776 u32 phys_name, phys_vendor, phys_attr;
777 u32 phys_data_size, phys_data;
778
Matt Flemingf6697df2016-11-12 21:04:24 +0000779 phys_data_size = virt_to_phys_or_null(data_size);
780 phys_vendor = virt_to_phys_or_null(vendor);
781 phys_name = virt_to_phys_or_null_size(name, efi_name_size(name));
782 phys_attr = virt_to_phys_or_null(attr);
783 phys_data = virt_to_phys_or_null_size(data, *data_size);
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000784
785 status = efi_thunk(get_variable, phys_name, phys_vendor,
786 phys_attr, phys_data_size, phys_data);
787
788 return status;
789}
790
791static efi_status_t
792efi_thunk_set_variable(efi_char16_t *name, efi_guid_t *vendor,
793 u32 attr, unsigned long data_size, void *data)
794{
795 u32 phys_name, phys_vendor, phys_data;
796 efi_status_t status;
797
Matt Flemingf6697df2016-11-12 21:04:24 +0000798 phys_name = virt_to_phys_or_null_size(name, efi_name_size(name));
799 phys_vendor = virt_to_phys_or_null(vendor);
800 phys_data = virt_to_phys_or_null_size(data, data_size);
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000801
802 /* If data_size is > sizeof(u32) we've got problems */
803 status = efi_thunk(set_variable, phys_name, phys_vendor,
804 attr, data_size, phys_data);
805
806 return status;
807}
808
809static efi_status_t
810efi_thunk_get_next_variable(unsigned long *name_size,
811 efi_char16_t *name,
812 efi_guid_t *vendor)
813{
814 efi_status_t status;
815 u32 phys_name_size, phys_name, phys_vendor;
816
Matt Flemingf6697df2016-11-12 21:04:24 +0000817 phys_name_size = virt_to_phys_or_null(name_size);
818 phys_vendor = virt_to_phys_or_null(vendor);
819 phys_name = virt_to_phys_or_null_size(name, *name_size);
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000820
821 status = efi_thunk(get_next_variable, phys_name_size,
822 phys_name, phys_vendor);
823
824 return status;
825}
826
827static efi_status_t
828efi_thunk_get_next_high_mono_count(u32 *count)
829{
830 efi_status_t status;
831 u32 phys_count;
832
Matt Flemingf6697df2016-11-12 21:04:24 +0000833 phys_count = virt_to_phys_or_null(count);
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000834 status = efi_thunk(get_next_high_mono_count, phys_count);
835
836 return status;
837}
838
839static void
840efi_thunk_reset_system(int reset_type, efi_status_t status,
841 unsigned long data_size, efi_char16_t *data)
842{
843 u32 phys_data;
844
Matt Flemingf6697df2016-11-12 21:04:24 +0000845 phys_data = virt_to_phys_or_null_size(data, data_size);
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000846
847 efi_thunk(reset_system, reset_type, status, data_size, phys_data);
848}
849
850static efi_status_t
851efi_thunk_update_capsule(efi_capsule_header_t **capsules,
852 unsigned long count, unsigned long sg_list)
853{
854 /*
855 * To properly support this function we would need to repackage
856 * 'capsules' because the firmware doesn't understand 64-bit
857 * pointers.
858 */
859 return EFI_UNSUPPORTED;
860}
861
862static efi_status_t
863efi_thunk_query_variable_info(u32 attr, u64 *storage_space,
864 u64 *remaining_space,
865 u64 *max_variable_size)
866{
867 efi_status_t status;
868 u32 phys_storage, phys_remaining, phys_max;
869
870 if (efi.runtime_version < EFI_2_00_SYSTEM_TABLE_REVISION)
871 return EFI_UNSUPPORTED;
872
Matt Flemingf6697df2016-11-12 21:04:24 +0000873 phys_storage = virt_to_phys_or_null(storage_space);
874 phys_remaining = virt_to_phys_or_null(remaining_space);
875 phys_max = virt_to_phys_or_null(max_variable_size);
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000876
Matt Fleming9a110402014-03-16 17:46:46 +0000877 status = efi_thunk(query_variable_info, attr, phys_storage,
Matt Fleming4f9dbcf2014-01-10 18:48:30 +0000878 phys_remaining, phys_max);
879
880 return status;
881}
882
883static efi_status_t
884efi_thunk_query_capsule_caps(efi_capsule_header_t **capsules,
885 unsigned long count, u64 *max_size,
886 int *reset_type)
887{
888 /*
889 * To properly support this function we would need to repackage
890 * 'capsules' because the firmware doesn't understand 64-bit
891 * pointers.
892 */
893 return EFI_UNSUPPORTED;
894}
895
896void efi_thunk_runtime_setup(void)
897{
898 efi.get_time = efi_thunk_get_time;
899 efi.set_time = efi_thunk_set_time;
900 efi.get_wakeup_time = efi_thunk_get_wakeup_time;
901 efi.set_wakeup_time = efi_thunk_set_wakeup_time;
902 efi.get_variable = efi_thunk_get_variable;
903 efi.get_next_variable = efi_thunk_get_next_variable;
904 efi.set_variable = efi_thunk_set_variable;
905 efi.get_next_high_mono_count = efi_thunk_get_next_high_mono_count;
906 efi.reset_system = efi_thunk_reset_system;
907 efi.query_variable_info = efi_thunk_query_variable_info;
908 efi.update_capsule = efi_thunk_update_capsule;
909 efi.query_capsule_caps = efi_thunk_query_capsule_caps;
910}
911#endif /* CONFIG_EFI_MIXED */