1 /*
2 * Hibernation support for x86-64
3 *
4 * Distribute under GPLv2
5 *
6 * Copyright (c) 2007 Rafael J. Wysocki <rjw@sisk.pl>
7 * Copyright (c) 2002 Pavel Machek <pavel@ucw.cz>
8 * Copyright (c) 2001 Patrick Mochel <mochel@osdl.org>
9 */
10
11 #include <linux/gfp.h>
12 #include <linux/smp.h>
13 #include <linux/suspend.h>
14 #include <linux/scatterlist.h>
15 #include <linux/kdebug.h>
16
17 #include <crypto/hash.h>
18
19 #include <asm/e820/api.h>
20 #include <asm/init.h>
21 #include <asm/proto.h>
22 #include <asm/page.h>
23 #include <asm/pgtable.h>
24 #include <asm/mtrr.h>
25 #include <asm/sections.h>
26 #include <asm/suspend.h>
27 #include <asm/tlbflush.h>
28
29 /* Defined in hibernate_asm_64.S */
30 extern asmlinkage __visible int restore_image(void);
31
32 /*
33 * Address to jump to in the last phase of restore in order to get to the image
34 * kernel's text (this value is passed in the image header).
35 */
36 unsigned long restore_jump_address __visible;
37 unsigned long jump_address_phys;
38
39 /*
40 * Value of the cr3 register from before the hibernation (this value is passed
41 * in the image header).
42 */
43 unsigned long restore_cr3 __visible;
44
45 unsigned long temp_level4_pgt __visible;
46
47 unsigned long relocated_restore_code __visible;
48
set_up_temporary_text_mapping(pgd_t * pgd)49 static int set_up_temporary_text_mapping(pgd_t *pgd)
50 {
51 pmd_t *pmd;
52 pud_t *pud;
53 p4d_t *p4d = NULL;
54 pgprot_t pgtable_prot = __pgprot(_KERNPG_TABLE);
55 pgprot_t pmd_text_prot = __pgprot(__PAGE_KERNEL_LARGE_EXEC);
56
57 /* Filter out unsupported __PAGE_KERNEL* bits: */
58 pgprot_val(pmd_text_prot) &= __default_kernel_pte_mask;
59 pgprot_val(pgtable_prot) &= __default_kernel_pte_mask;
60
61 /*
62 * The new mapping only has to cover the page containing the image
63 * kernel's entry point (jump_address_phys), because the switch over to
64 * it is carried out by relocated code running from a page allocated
65 * specifically for this purpose and covered by the identity mapping, so
66 * the temporary kernel text mapping is only needed for the final jump.
67 * Moreover, in that mapping the virtual address of the image kernel's
68 * entry point must be the same as its virtual address in the image
69 * kernel (restore_jump_address), so the image kernel's
70 * restore_registers() code doesn't find itself in a different area of
71 * the virtual address space after switching over to the original page
72 * tables used by the image kernel.
73 */
74
75 if (pgtable_l5_enabled()) {
76 p4d = (p4d_t *)get_safe_page(GFP_ATOMIC);
77 if (!p4d)
78 return -ENOMEM;
79 }
80
81 pud = (pud_t *)get_safe_page(GFP_ATOMIC);
82 if (!pud)
83 return -ENOMEM;
84
85 pmd = (pmd_t *)get_safe_page(GFP_ATOMIC);
86 if (!pmd)
87 return -ENOMEM;
88
89 set_pmd(pmd + pmd_index(restore_jump_address),
90 __pmd((jump_address_phys & PMD_MASK) | pgprot_val(pmd_text_prot)));
91 set_pud(pud + pud_index(restore_jump_address),
92 __pud(__pa(pmd) | pgprot_val(pgtable_prot)));
93 if (p4d) {
94 p4d_t new_p4d = __p4d(__pa(pud) | pgprot_val(pgtable_prot));
95 pgd_t new_pgd = __pgd(__pa(p4d) | pgprot_val(pgtable_prot));
96
97 set_p4d(p4d + p4d_index(restore_jump_address), new_p4d);
98 set_pgd(pgd + pgd_index(restore_jump_address), new_pgd);
99 } else {
100 /* No p4d for 4-level paging: point the pgd to the pud page table */
101 pgd_t new_pgd = __pgd(__pa(pud) | pgprot_val(pgtable_prot));
102 set_pgd(pgd + pgd_index(restore_jump_address), new_pgd);
103 }
104
105 return 0;
106 }
107
alloc_pgt_page(void * context)108 static void *alloc_pgt_page(void *context)
109 {
110 return (void *)get_safe_page(GFP_ATOMIC);
111 }
112
set_up_temporary_mappings(void)113 static int set_up_temporary_mappings(void)
114 {
115 struct x86_mapping_info info = {
116 .alloc_pgt_page = alloc_pgt_page,
117 .page_flag = __PAGE_KERNEL_LARGE_EXEC,
118 .offset = __PAGE_OFFSET,
119 };
120 unsigned long mstart, mend;
121 pgd_t *pgd;
122 int result;
123 int i;
124
125 pgd = (pgd_t *)get_safe_page(GFP_ATOMIC);
126 if (!pgd)
127 return -ENOMEM;
128
129 /* Prepare a temporary mapping for the kernel text */
130 result = set_up_temporary_text_mapping(pgd);
131 if (result)
132 return result;
133
134 /* Set up the direct mapping from scratch */
135 for (i = 0; i < nr_pfn_mapped; i++) {
136 mstart = pfn_mapped[i].start << PAGE_SHIFT;
137 mend = pfn_mapped[i].end << PAGE_SHIFT;
138
139 result = kernel_ident_mapping_init(&info, pgd, mstart, mend);
140 if (result)
141 return result;
142 }
143
144 temp_level4_pgt = __pa(pgd);
145 return 0;
146 }
147
relocate_restore_code(void)148 static int relocate_restore_code(void)
149 {
150 pgd_t *pgd;
151 p4d_t *p4d;
152 pud_t *pud;
153 pmd_t *pmd;
154 pte_t *pte;
155
156 relocated_restore_code = get_safe_page(GFP_ATOMIC);
157 if (!relocated_restore_code)
158 return -ENOMEM;
159
160 memcpy((void *)relocated_restore_code, core_restore_code, PAGE_SIZE);
161
162 /* Make the page containing the relocated code executable */
163 pgd = (pgd_t *)__va(read_cr3_pa()) +
164 pgd_index(relocated_restore_code);
165 p4d = p4d_offset(pgd, relocated_restore_code);
166 if (p4d_large(*p4d)) {
167 set_p4d(p4d, __p4d(p4d_val(*p4d) & ~_PAGE_NX));
168 goto out;
169 }
170 pud = pud_offset(p4d, relocated_restore_code);
171 if (pud_large(*pud)) {
172 set_pud(pud, __pud(pud_val(*pud) & ~_PAGE_NX));
173 goto out;
174 }
175 pmd = pmd_offset(pud, relocated_restore_code);
176 if (pmd_large(*pmd)) {
177 set_pmd(pmd, __pmd(pmd_val(*pmd) & ~_PAGE_NX));
178 goto out;
179 }
180 pte = pte_offset_kernel(pmd, relocated_restore_code);
181 set_pte(pte, __pte(pte_val(*pte) & ~_PAGE_NX));
182 out:
183 __flush_tlb_all();
184 return 0;
185 }
186
swsusp_arch_resume(void)187 asmlinkage int swsusp_arch_resume(void)
188 {
189 int error;
190
191 /* We have got enough memory and from now on we cannot recover */
192 error = set_up_temporary_mappings();
193 if (error)
194 return error;
195
196 error = relocate_restore_code();
197 if (error)
198 return error;
199
200 restore_image();
201 return 0;
202 }
203
204 /*
205 * pfn_is_nosave - check if given pfn is in the 'nosave' section
206 */
207
pfn_is_nosave(unsigned long pfn)208 int pfn_is_nosave(unsigned long pfn)
209 {
210 unsigned long nosave_begin_pfn = __pa_symbol(&__nosave_begin) >> PAGE_SHIFT;
211 unsigned long nosave_end_pfn = PAGE_ALIGN(__pa_symbol(&__nosave_end)) >> PAGE_SHIFT;
212 return (pfn >= nosave_begin_pfn) && (pfn < nosave_end_pfn);
213 }
214
215 #define MD5_DIGEST_SIZE 16
216
217 struct restore_data_record {
218 unsigned long jump_address;
219 unsigned long jump_address_phys;
220 unsigned long cr3;
221 unsigned long magic;
222 u8 e820_digest[MD5_DIGEST_SIZE];
223 };
224
225 #define RESTORE_MAGIC 0x23456789ABCDEF01UL
226
227 #if IS_BUILTIN(CONFIG_CRYPTO_MD5)
228 /**
229 * get_e820_md5 - calculate md5 according to given e820 table
230 *
231 * @table: the e820 table to be calculated
232 * @buf: the md5 result to be stored to
233 */
get_e820_md5(struct e820_table * table,void * buf)234 static int get_e820_md5(struct e820_table *table, void *buf)
235 {
236 struct crypto_shash *tfm;
237 struct shash_desc *desc;
238 int size;
239 int ret = 0;
240
241 tfm = crypto_alloc_shash("md5", 0, 0);
242 if (IS_ERR(tfm))
243 return -ENOMEM;
244
245 desc = kmalloc(sizeof(struct shash_desc) + crypto_shash_descsize(tfm),
246 GFP_KERNEL);
247 if (!desc) {
248 ret = -ENOMEM;
249 goto free_tfm;
250 }
251
252 desc->tfm = tfm;
253 desc->flags = 0;
254
255 size = offsetof(struct e820_table, entries) +
256 sizeof(struct e820_entry) * table->nr_entries;
257
258 if (crypto_shash_digest(desc, (u8 *)table, size, buf))
259 ret = -EINVAL;
260
261 kzfree(desc);
262
263 free_tfm:
264 crypto_free_shash(tfm);
265 return ret;
266 }
267
hibernation_e820_save(void * buf)268 static void hibernation_e820_save(void *buf)
269 {
270 get_e820_md5(e820_table_firmware, buf);
271 }
272
hibernation_e820_mismatch(void * buf)273 static bool hibernation_e820_mismatch(void *buf)
274 {
275 int ret;
276 u8 result[MD5_DIGEST_SIZE];
277
278 memset(result, 0, MD5_DIGEST_SIZE);
279 /* If there is no digest in suspend kernel, let it go. */
280 if (!memcmp(result, buf, MD5_DIGEST_SIZE))
281 return false;
282
283 ret = get_e820_md5(e820_table_firmware, result);
284 if (ret)
285 return true;
286
287 return memcmp(result, buf, MD5_DIGEST_SIZE) ? true : false;
288 }
289 #else
hibernation_e820_save(void * buf)290 static void hibernation_e820_save(void *buf)
291 {
292 }
293
hibernation_e820_mismatch(void * buf)294 static bool hibernation_e820_mismatch(void *buf)
295 {
296 /* If md5 is not builtin for restore kernel, let it go. */
297 return false;
298 }
299 #endif
300
301 /**
302 * arch_hibernation_header_save - populate the architecture specific part
303 * of a hibernation image header
304 * @addr: address to save the data at
305 */
arch_hibernation_header_save(void * addr,unsigned int max_size)306 int arch_hibernation_header_save(void *addr, unsigned int max_size)
307 {
308 struct restore_data_record *rdr = addr;
309
310 if (max_size < sizeof(struct restore_data_record))
311 return -EOVERFLOW;
312 rdr->jump_address = (unsigned long)restore_registers;
313 rdr->jump_address_phys = __pa_symbol(restore_registers);
314
315 /*
316 * The restore code fixes up CR3 and CR4 in the following sequence:
317 *
318 * [in hibernation asm]
319 * 1. CR3 <= temporary page tables
320 * 2. CR4 <= mmu_cr4_features (from the kernel that restores us)
321 * 3. CR3 <= rdr->cr3
322 * 4. CR4 <= mmu_cr4_features (from us, i.e. the image kernel)
323 * [in restore_processor_state()]
324 * 5. CR4 <= saved CR4
325 * 6. CR3 <= saved CR3
326 *
327 * Our mmu_cr4_features has CR4.PCIDE=0, and toggling
328 * CR4.PCIDE while CR3's PCID bits are nonzero is illegal, so
329 * rdr->cr3 needs to point to valid page tables but must not
330 * have any of the PCID bits set.
331 */
332 rdr->cr3 = restore_cr3 & ~CR3_PCID_MASK;
333
334 rdr->magic = RESTORE_MAGIC;
335
336 hibernation_e820_save(rdr->e820_digest);
337
338 return 0;
339 }
340
341 /**
342 * arch_hibernation_header_restore - read the architecture specific data
343 * from the hibernation image header
344 * @addr: address to read the data from
345 */
arch_hibernation_header_restore(void * addr)346 int arch_hibernation_header_restore(void *addr)
347 {
348 struct restore_data_record *rdr = addr;
349
350 restore_jump_address = rdr->jump_address;
351 jump_address_phys = rdr->jump_address_phys;
352 restore_cr3 = rdr->cr3;
353
354 if (rdr->magic != RESTORE_MAGIC) {
355 pr_crit("Unrecognized hibernate image header format!\n");
356 return -EINVAL;
357 }
358
359 if (hibernation_e820_mismatch(rdr->e820_digest)) {
360 pr_crit("Hibernate inconsistent memory map detected!\n");
361 return -ENODEV;
362 }
363
364 return 0;
365 }
366
