1 /*
2  * Page table handling routines for radix page table.
3  *
4  * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version
9  * 2 of the License, or (at your option) any later version.
10  */
11 
12 #define pr_fmt(fmt) "radix-mmu: " fmt
13 
14 #include <linux/kernel.h>
15 #include <linux/sched/mm.h>
16 #include <linux/memblock.h>
17 #include <linux/of_fdt.h>
18 #include <linux/mm.h>
19 #include <linux/string_helpers.h>
20 #include <linux/stop_machine.h>
21 
22 #include <asm/pgtable.h>
23 #include <asm/pgalloc.h>
24 #include <asm/mmu_context.h>
25 #include <asm/dma.h>
26 #include <asm/machdep.h>
27 #include <asm/mmu.h>
28 #include <asm/firmware.h>
29 #include <asm/powernv.h>
30 #include <asm/sections.h>
31 #include <asm/trace.h>
32 
33 #include <trace/events/thp.h>
34 
35 unsigned int mmu_pid_bits;
36 unsigned int mmu_base_pid;
37 
native_register_process_table(unsigned long base,unsigned long pg_sz,unsigned long table_size)38 static int native_register_process_table(unsigned long base, unsigned long pg_sz,
39 					 unsigned long table_size)
40 {
41 	unsigned long patb0, patb1;
42 
43 	patb0 = be64_to_cpu(partition_tb[0].patb0);
44 	patb1 = base | table_size | PATB_GR;
45 
46 	mmu_partition_table_set_entry(0, patb0, patb1);
47 
48 	return 0;
49 }
50 
early_alloc_pgtable(unsigned long size,int nid,unsigned long region_start,unsigned long region_end)51 static __ref void *early_alloc_pgtable(unsigned long size, int nid,
52 			unsigned long region_start, unsigned long region_end)
53 {
54 	unsigned long pa = 0;
55 	void *pt;
56 
57 	if (region_start || region_end) /* has region hint */
58 		pa = memblock_alloc_range(size, size, region_start, region_end,
59 						MEMBLOCK_NONE);
60 	else if (nid != -1) /* has node hint */
61 		pa = memblock_alloc_base_nid(size, size,
62 						MEMBLOCK_ALLOC_ANYWHERE,
63 						nid, MEMBLOCK_NONE);
64 
65 	if (!pa)
66 		pa = memblock_alloc_base(size, size, MEMBLOCK_ALLOC_ANYWHERE);
67 
68 	BUG_ON(!pa);
69 
70 	pt = __va(pa);
71 	memset(pt, 0, size);
72 
73 	return pt;
74 }
75 
early_map_kernel_page(unsigned long ea,unsigned long pa,pgprot_t flags,unsigned int map_page_size,int nid,unsigned long region_start,unsigned long region_end)76 static int early_map_kernel_page(unsigned long ea, unsigned long pa,
77 			  pgprot_t flags,
78 			  unsigned int map_page_size,
79 			  int nid,
80 			  unsigned long region_start, unsigned long region_end)
81 {
82 	unsigned long pfn = pa >> PAGE_SHIFT;
83 	pgd_t *pgdp;
84 	pud_t *pudp;
85 	pmd_t *pmdp;
86 	pte_t *ptep;
87 
88 	pgdp = pgd_offset_k(ea);
89 	if (pgd_none(*pgdp)) {
90 		pudp = early_alloc_pgtable(PUD_TABLE_SIZE, nid,
91 						region_start, region_end);
92 		pgd_populate(&init_mm, pgdp, pudp);
93 	}
94 	pudp = pud_offset(pgdp, ea);
95 	if (map_page_size == PUD_SIZE) {
96 		ptep = (pte_t *)pudp;
97 		goto set_the_pte;
98 	}
99 	if (pud_none(*pudp)) {
100 		pmdp = early_alloc_pgtable(PMD_TABLE_SIZE, nid,
101 						region_start, region_end);
102 		pud_populate(&init_mm, pudp, pmdp);
103 	}
104 	pmdp = pmd_offset(pudp, ea);
105 	if (map_page_size == PMD_SIZE) {
106 		ptep = pmdp_ptep(pmdp);
107 		goto set_the_pte;
108 	}
109 	if (!pmd_present(*pmdp)) {
110 		ptep = early_alloc_pgtable(PAGE_SIZE, nid,
111 						region_start, region_end);
112 		pmd_populate_kernel(&init_mm, pmdp, ptep);
113 	}
114 	ptep = pte_offset_kernel(pmdp, ea);
115 
116 set_the_pte:
117 	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
118 	smp_wmb();
119 	return 0;
120 }
121 
122 /*
123  * nid, region_start, and region_end are hints to try to place the page
124  * table memory in the same node or region.
125  */
__map_kernel_page(unsigned long ea,unsigned long pa,pgprot_t flags,unsigned int map_page_size,int nid,unsigned long region_start,unsigned long region_end)126 static int __map_kernel_page(unsigned long ea, unsigned long pa,
127 			  pgprot_t flags,
128 			  unsigned int map_page_size,
129 			  int nid,
130 			  unsigned long region_start, unsigned long region_end)
131 {
132 	unsigned long pfn = pa >> PAGE_SHIFT;
133 	pgd_t *pgdp;
134 	pud_t *pudp;
135 	pmd_t *pmdp;
136 	pte_t *ptep;
137 	/*
138 	 * Make sure task size is correct as per the max adddr
139 	 */
140 	BUILD_BUG_ON(TASK_SIZE_USER64 > RADIX_PGTABLE_RANGE);
141 
142 	if (unlikely(!slab_is_available()))
143 		return early_map_kernel_page(ea, pa, flags, map_page_size,
144 						nid, region_start, region_end);
145 
146 	/*
147 	 * Should make page table allocation functions be able to take a
148 	 * node, so we can place kernel page tables on the right nodes after
149 	 * boot.
150 	 */
151 	pgdp = pgd_offset_k(ea);
152 	pudp = pud_alloc(&init_mm, pgdp, ea);
153 	if (!pudp)
154 		return -ENOMEM;
155 	if (map_page_size == PUD_SIZE) {
156 		ptep = (pte_t *)pudp;
157 		goto set_the_pte;
158 	}
159 	pmdp = pmd_alloc(&init_mm, pudp, ea);
160 	if (!pmdp)
161 		return -ENOMEM;
162 	if (map_page_size == PMD_SIZE) {
163 		ptep = pmdp_ptep(pmdp);
164 		goto set_the_pte;
165 	}
166 	ptep = pte_alloc_kernel(pmdp, ea);
167 	if (!ptep)
168 		return -ENOMEM;
169 
170 set_the_pte:
171 	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
172 	smp_wmb();
173 	return 0;
174 }
175 
radix__map_kernel_page(unsigned long ea,unsigned long pa,pgprot_t flags,unsigned int map_page_size)176 int radix__map_kernel_page(unsigned long ea, unsigned long pa,
177 			  pgprot_t flags,
178 			  unsigned int map_page_size)
179 {
180 	return __map_kernel_page(ea, pa, flags, map_page_size, -1, 0, 0);
181 }
182 
183 #ifdef CONFIG_STRICT_KERNEL_RWX
radix__change_memory_range(unsigned long start,unsigned long end,unsigned long clear)184 void radix__change_memory_range(unsigned long start, unsigned long end,
185 				unsigned long clear)
186 {
187 	unsigned long idx;
188 	pgd_t *pgdp;
189 	pud_t *pudp;
190 	pmd_t *pmdp;
191 	pte_t *ptep;
192 
193 	start = ALIGN_DOWN(start, PAGE_SIZE);
194 	end = PAGE_ALIGN(end); // aligns up
195 
196 	pr_debug("Changing flags on range %lx-%lx removing 0x%lx\n",
197 		 start, end, clear);
198 
199 	for (idx = start; idx < end; idx += PAGE_SIZE) {
200 		pgdp = pgd_offset_k(idx);
201 		pudp = pud_alloc(&init_mm, pgdp, idx);
202 		if (!pudp)
203 			continue;
204 		if (pud_huge(*pudp)) {
205 			ptep = (pte_t *)pudp;
206 			goto update_the_pte;
207 		}
208 		pmdp = pmd_alloc(&init_mm, pudp, idx);
209 		if (!pmdp)
210 			continue;
211 		if (pmd_huge(*pmdp)) {
212 			ptep = pmdp_ptep(pmdp);
213 			goto update_the_pte;
214 		}
215 		ptep = pte_alloc_kernel(pmdp, idx);
216 		if (!ptep)
217 			continue;
218 update_the_pte:
219 		radix__pte_update(&init_mm, idx, ptep, clear, 0, 0);
220 	}
221 
222 	radix__flush_tlb_kernel_range(start, end);
223 }
224 
radix__mark_rodata_ro(void)225 void radix__mark_rodata_ro(void)
226 {
227 	unsigned long start, end;
228 
229 	start = (unsigned long)_stext;
230 	end = (unsigned long)__init_begin;
231 
232 	radix__change_memory_range(start, end, _PAGE_WRITE);
233 }
234 
radix__mark_initmem_nx(void)235 void radix__mark_initmem_nx(void)
236 {
237 	unsigned long start = (unsigned long)__init_begin;
238 	unsigned long end = (unsigned long)__init_end;
239 
240 	radix__change_memory_range(start, end, _PAGE_EXEC);
241 }
242 #endif /* CONFIG_STRICT_KERNEL_RWX */
243 
print_mapping(unsigned long start,unsigned long end,unsigned long size)244 static inline void __meminit print_mapping(unsigned long start,
245 					   unsigned long end,
246 					   unsigned long size)
247 {
248 	char buf[10];
249 
250 	if (end <= start)
251 		return;
252 
253 	string_get_size(size, 1, STRING_UNITS_2, buf, sizeof(buf));
254 
255 	pr_info("Mapped 0x%016lx-0x%016lx with %s pages\n", start, end, buf);
256 }
257 
create_physical_mapping(unsigned long start,unsigned long end,int nid)258 static int __meminit create_physical_mapping(unsigned long start,
259 					     unsigned long end,
260 					     int nid)
261 {
262 	unsigned long vaddr, addr, mapping_size = 0;
263 	pgprot_t prot;
264 	unsigned long max_mapping_size;
265 #ifdef CONFIG_STRICT_KERNEL_RWX
266 	int split_text_mapping = 1;
267 #else
268 	int split_text_mapping = 0;
269 #endif
270 	int psize;
271 
272 	start = _ALIGN_UP(start, PAGE_SIZE);
273 	for (addr = start; addr < end; addr += mapping_size) {
274 		unsigned long gap, previous_size;
275 		int rc;
276 
277 		gap = end - addr;
278 		previous_size = mapping_size;
279 		max_mapping_size = PUD_SIZE;
280 
281 retry:
282 		if (IS_ALIGNED(addr, PUD_SIZE) && gap >= PUD_SIZE &&
283 		    mmu_psize_defs[MMU_PAGE_1G].shift &&
284 		    PUD_SIZE <= max_mapping_size) {
285 			mapping_size = PUD_SIZE;
286 			psize = MMU_PAGE_1G;
287 		} else if (IS_ALIGNED(addr, PMD_SIZE) && gap >= PMD_SIZE &&
288 			   mmu_psize_defs[MMU_PAGE_2M].shift) {
289 			mapping_size = PMD_SIZE;
290 			psize = MMU_PAGE_2M;
291 		} else {
292 			mapping_size = PAGE_SIZE;
293 			psize = mmu_virtual_psize;
294 		}
295 
296 		if (split_text_mapping && (mapping_size == PUD_SIZE) &&
297 			(addr <= __pa_symbol(__init_begin)) &&
298 			(addr + mapping_size) >= __pa_symbol(_stext)) {
299 			max_mapping_size = PMD_SIZE;
300 			goto retry;
301 		}
302 
303 		if (split_text_mapping && (mapping_size == PMD_SIZE) &&
304 		    (addr <= __pa_symbol(__init_begin)) &&
305 		    (addr + mapping_size) >= __pa_symbol(_stext)) {
306 			mapping_size = PAGE_SIZE;
307 			psize = mmu_virtual_psize;
308 		}
309 
310 		if (mapping_size != previous_size) {
311 			print_mapping(start, addr, previous_size);
312 			start = addr;
313 		}
314 
315 		vaddr = (unsigned long)__va(addr);
316 
317 		if (overlaps_kernel_text(vaddr, vaddr + mapping_size) ||
318 		    overlaps_interrupt_vector_text(vaddr, vaddr + mapping_size))
319 			prot = PAGE_KERNEL_X;
320 		else
321 			prot = PAGE_KERNEL;
322 
323 		rc = __map_kernel_page(vaddr, addr, prot, mapping_size, nid, start, end);
324 		if (rc)
325 			return rc;
326 
327 		update_page_count(psize, 1);
328 	}
329 
330 	print_mapping(start, addr, mapping_size);
331 	return 0;
332 }
333 
radix_init_pgtable(void)334 void __init radix_init_pgtable(void)
335 {
336 	unsigned long rts_field;
337 	struct memblock_region *reg;
338 
339 	/* We don't support slb for radix */
340 	mmu_slb_size = 0;
341 	/*
342 	 * Create the linear mapping, using standard page size for now
343 	 */
344 	for_each_memblock(memory, reg) {
345 		/*
346 		 * The memblock allocator  is up at this point, so the
347 		 * page tables will be allocated within the range. No
348 		 * need or a node (which we don't have yet).
349 		 */
350 		WARN_ON(create_physical_mapping(reg->base,
351 						reg->base + reg->size,
352 						-1));
353 	}
354 
355 	/* Find out how many PID bits are supported */
356 	if (cpu_has_feature(CPU_FTR_HVMODE)) {
357 		if (!mmu_pid_bits)
358 			mmu_pid_bits = 20;
359 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
360 		/*
361 		 * When KVM is possible, we only use the top half of the
362 		 * PID space to avoid collisions between host and guest PIDs
363 		 * which can cause problems due to prefetch when exiting the
364 		 * guest with AIL=3
365 		 */
366 		mmu_base_pid = 1 << (mmu_pid_bits - 1);
367 #else
368 		mmu_base_pid = 1;
369 #endif
370 	} else {
371 		/* The guest uses the bottom half of the PID space */
372 		if (!mmu_pid_bits)
373 			mmu_pid_bits = 19;
374 		mmu_base_pid = 1;
375 	}
376 
377 	/*
378 	 * Allocate Partition table and process table for the
379 	 * host.
380 	 */
381 	BUG_ON(PRTB_SIZE_SHIFT > 36);
382 	process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT, -1, 0, 0);
383 	/*
384 	 * Fill in the process table.
385 	 */
386 	rts_field = radix__get_tree_size();
387 	process_tb->prtb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE);
388 	/*
389 	 * Fill in the partition table. We are suppose to use effective address
390 	 * of process table here. But our linear mapping also enable us to use
391 	 * physical address here.
392 	 */
393 	register_process_table(__pa(process_tb), 0, PRTB_SIZE_SHIFT - 12);
394 	pr_info("Process table %p and radix root for kernel: %p\n", process_tb, init_mm.pgd);
395 	asm volatile("ptesync" : : : "memory");
396 	asm volatile(PPC_TLBIE_5(%0,%1,2,1,1) : :
397 		     "r" (TLBIEL_INVAL_SET_LPID), "r" (0));
398 	asm volatile("eieio; tlbsync; ptesync" : : : "memory");
399 	trace_tlbie(0, 0, TLBIEL_INVAL_SET_LPID, 0, 2, 1, 1);
400 
401 	/*
402 	 * The init_mm context is given the first available (non-zero) PID,
403 	 * which is the "guard PID" and contains no page table. PIDR should
404 	 * never be set to zero because that duplicates the kernel address
405 	 * space at the 0x0... offset (quadrant 0)!
406 	 *
407 	 * An arbitrary PID that may later be allocated by the PID allocator
408 	 * for userspace processes must not be used either, because that
409 	 * would cause stale user mappings for that PID on CPUs outside of
410 	 * the TLB invalidation scheme (because it won't be in mm_cpumask).
411 	 *
412 	 * So permanently carve out one PID for the purpose of a guard PID.
413 	 */
414 	init_mm.context.id = mmu_base_pid;
415 	mmu_base_pid++;
416 }
417 
radix_init_partition_table(void)418 static void __init radix_init_partition_table(void)
419 {
420 	unsigned long rts_field, dw0;
421 
422 	mmu_partition_table_init();
423 	rts_field = radix__get_tree_size();
424 	dw0 = rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE | PATB_HR;
425 	mmu_partition_table_set_entry(0, dw0, 0);
426 
427 	pr_info("Initializing Radix MMU\n");
428 	pr_info("Partition table %p\n", partition_tb);
429 }
430 
radix_init_native(void)431 void __init radix_init_native(void)
432 {
433 	register_process_table = native_register_process_table;
434 }
435 
get_idx_from_shift(unsigned int shift)436 static int __init get_idx_from_shift(unsigned int shift)
437 {
438 	int idx = -1;
439 
440 	switch (shift) {
441 	case 0xc:
442 		idx = MMU_PAGE_4K;
443 		break;
444 	case 0x10:
445 		idx = MMU_PAGE_64K;
446 		break;
447 	case 0x15:
448 		idx = MMU_PAGE_2M;
449 		break;
450 	case 0x1e:
451 		idx = MMU_PAGE_1G;
452 		break;
453 	}
454 	return idx;
455 }
456 
radix_dt_scan_page_sizes(unsigned long node,const char * uname,int depth,void * data)457 static int __init radix_dt_scan_page_sizes(unsigned long node,
458 					   const char *uname, int depth,
459 					   void *data)
460 {
461 	int size = 0;
462 	int shift, idx;
463 	unsigned int ap;
464 	const __be32 *prop;
465 	const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
466 
467 	/* We are scanning "cpu" nodes only */
468 	if (type == NULL || strcmp(type, "cpu") != 0)
469 		return 0;
470 
471 	/* Find MMU PID size */
472 	prop = of_get_flat_dt_prop(node, "ibm,mmu-pid-bits", &size);
473 	if (prop && size == 4)
474 		mmu_pid_bits = be32_to_cpup(prop);
475 
476 	/* Grab page size encodings */
477 	prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size);
478 	if (!prop)
479 		return 0;
480 
481 	pr_info("Page sizes from device-tree:\n");
482 	for (; size >= 4; size -= 4, ++prop) {
483 
484 		struct mmu_psize_def *def;
485 
486 		/* top 3 bit is AP encoding */
487 		shift = be32_to_cpu(prop[0]) & ~(0xe << 28);
488 		ap = be32_to_cpu(prop[0]) >> 29;
489 		pr_info("Page size shift = %d AP=0x%x\n", shift, ap);
490 
491 		idx = get_idx_from_shift(shift);
492 		if (idx < 0)
493 			continue;
494 
495 		def = &mmu_psize_defs[idx];
496 		def->shift = shift;
497 		def->ap  = ap;
498 	}
499 
500 	/* needed ? */
501 	cur_cpu_spec->mmu_features &= ~MMU_FTR_NO_SLBIE_B;
502 	return 1;
503 }
504 
radix__early_init_devtree(void)505 void __init radix__early_init_devtree(void)
506 {
507 	int rc;
508 
509 	/*
510 	 * Try to find the available page sizes in the device-tree
511 	 */
512 	rc = of_scan_flat_dt(radix_dt_scan_page_sizes, NULL);
513 	if (rc != 0)  /* Found */
514 		goto found;
515 	/*
516 	 * let's assume we have page 4k and 64k support
517 	 */
518 	mmu_psize_defs[MMU_PAGE_4K].shift = 12;
519 	mmu_psize_defs[MMU_PAGE_4K].ap = 0x0;
520 
521 	mmu_psize_defs[MMU_PAGE_64K].shift = 16;
522 	mmu_psize_defs[MMU_PAGE_64K].ap = 0x5;
523 found:
524 #ifdef CONFIG_SPARSEMEM_VMEMMAP
525 	if (mmu_psize_defs[MMU_PAGE_2M].shift) {
526 		/*
527 		 * map vmemmap using 2M if available
528 		 */
529 		mmu_vmemmap_psize = MMU_PAGE_2M;
530 	}
531 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
532 	return;
533 }
534 
radix_init_amor(void)535 static void radix_init_amor(void)
536 {
537 	/*
538 	* In HV mode, we init AMOR (Authority Mask Override Register) so that
539 	* the hypervisor and guest can setup IAMR (Instruction Authority Mask
540 	* Register), enable key 0 and set it to 1.
541 	*
542 	* AMOR = 0b1100 .... 0000 (Mask for key 0 is 11)
543 	*/
544 	mtspr(SPRN_AMOR, (3ul << 62));
545 }
546 
radix_init_iamr(void)547 static void radix_init_iamr(void)
548 {
549 	/*
550 	 * Radix always uses key0 of the IAMR to determine if an access is
551 	 * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction
552 	 * fetch.
553 	 */
554 	mtspr(SPRN_IAMR, (1ul << 62));
555 }
556 
radix__early_init_mmu(void)557 void __init radix__early_init_mmu(void)
558 {
559 	unsigned long lpcr;
560 
561 #ifdef CONFIG_PPC_64K_PAGES
562 	/* PAGE_SIZE mappings */
563 	mmu_virtual_psize = MMU_PAGE_64K;
564 #else
565 	mmu_virtual_psize = MMU_PAGE_4K;
566 #endif
567 
568 #ifdef CONFIG_SPARSEMEM_VMEMMAP
569 	/* vmemmap mapping */
570 	mmu_vmemmap_psize = mmu_virtual_psize;
571 #endif
572 	/*
573 	 * initialize page table size
574 	 */
575 	__pte_index_size = RADIX_PTE_INDEX_SIZE;
576 	__pmd_index_size = RADIX_PMD_INDEX_SIZE;
577 	__pud_index_size = RADIX_PUD_INDEX_SIZE;
578 	__pgd_index_size = RADIX_PGD_INDEX_SIZE;
579 	__pud_cache_index = RADIX_PUD_INDEX_SIZE;
580 	__pte_table_size = RADIX_PTE_TABLE_SIZE;
581 	__pmd_table_size = RADIX_PMD_TABLE_SIZE;
582 	__pud_table_size = RADIX_PUD_TABLE_SIZE;
583 	__pgd_table_size = RADIX_PGD_TABLE_SIZE;
584 
585 	__pmd_val_bits = RADIX_PMD_VAL_BITS;
586 	__pud_val_bits = RADIX_PUD_VAL_BITS;
587 	__pgd_val_bits = RADIX_PGD_VAL_BITS;
588 
589 	__kernel_virt_start = RADIX_KERN_VIRT_START;
590 	__kernel_virt_size = RADIX_KERN_VIRT_SIZE;
591 	__vmalloc_start = RADIX_VMALLOC_START;
592 	__vmalloc_end = RADIX_VMALLOC_END;
593 	__kernel_io_start = RADIX_KERN_IO_START;
594 	vmemmap = (struct page *)RADIX_VMEMMAP_BASE;
595 	ioremap_bot = IOREMAP_BASE;
596 
597 #ifdef CONFIG_PCI
598 	pci_io_base = ISA_IO_BASE;
599 #endif
600 	__pte_frag_nr = RADIX_PTE_FRAG_NR;
601 	__pte_frag_size_shift = RADIX_PTE_FRAG_SIZE_SHIFT;
602 	__pmd_frag_nr = RADIX_PMD_FRAG_NR;
603 	__pmd_frag_size_shift = RADIX_PMD_FRAG_SIZE_SHIFT;
604 
605 	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
606 		radix_init_native();
607 		lpcr = mfspr(SPRN_LPCR);
608 		mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
609 		radix_init_partition_table();
610 		radix_init_amor();
611 	} else {
612 		radix_init_pseries();
613 	}
614 
615 	memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
616 
617 	radix_init_iamr();
618 	radix_init_pgtable();
619 	/* Switch to the guard PID before turning on MMU */
620 	radix__switch_mmu_context(NULL, &init_mm);
621 	if (cpu_has_feature(CPU_FTR_HVMODE))
622 		tlbiel_all();
623 }
624 
radix__early_init_mmu_secondary(void)625 void radix__early_init_mmu_secondary(void)
626 {
627 	unsigned long lpcr;
628 	/*
629 	 * update partition table control register and UPRT
630 	 */
631 	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
632 		lpcr = mfspr(SPRN_LPCR);
633 		mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
634 
635 		mtspr(SPRN_PTCR,
636 		      __pa(partition_tb) | (PATB_SIZE_SHIFT - 12));
637 		radix_init_amor();
638 	}
639 	radix_init_iamr();
640 
641 	radix__switch_mmu_context(NULL, &init_mm);
642 	if (cpu_has_feature(CPU_FTR_HVMODE))
643 		tlbiel_all();
644 }
645 
radix__mmu_cleanup_all(void)646 void radix__mmu_cleanup_all(void)
647 {
648 	unsigned long lpcr;
649 
650 	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
651 		lpcr = mfspr(SPRN_LPCR);
652 		mtspr(SPRN_LPCR, lpcr & ~LPCR_UPRT);
653 		mtspr(SPRN_PTCR, 0);
654 		powernv_set_nmmu_ptcr(0);
655 		radix__flush_tlb_all();
656 	}
657 }
658 
radix__setup_initial_memory_limit(phys_addr_t first_memblock_base,phys_addr_t first_memblock_size)659 void radix__setup_initial_memory_limit(phys_addr_t first_memblock_base,
660 				phys_addr_t first_memblock_size)
661 {
662 	/* We don't currently support the first MEMBLOCK not mapping 0
663 	 * physical on those processors
664 	 */
665 	BUG_ON(first_memblock_base != 0);
666 
667 	/*
668 	 * Radix mode is not limited by RMA / VRMA addressing.
669 	 */
670 	ppc64_rma_size = ULONG_MAX;
671 }
672 
673 #ifdef CONFIG_MEMORY_HOTPLUG
free_pte_table(pte_t * pte_start,pmd_t * pmd)674 static void free_pte_table(pte_t *pte_start, pmd_t *pmd)
675 {
676 	pte_t *pte;
677 	int i;
678 
679 	for (i = 0; i < PTRS_PER_PTE; i++) {
680 		pte = pte_start + i;
681 		if (!pte_none(*pte))
682 			return;
683 	}
684 
685 	pte_free_kernel(&init_mm, pte_start);
686 	pmd_clear(pmd);
687 }
688 
free_pmd_table(pmd_t * pmd_start,pud_t * pud)689 static void free_pmd_table(pmd_t *pmd_start, pud_t *pud)
690 {
691 	pmd_t *pmd;
692 	int i;
693 
694 	for (i = 0; i < PTRS_PER_PMD; i++) {
695 		pmd = pmd_start + i;
696 		if (!pmd_none(*pmd))
697 			return;
698 	}
699 
700 	pmd_free(&init_mm, pmd_start);
701 	pud_clear(pud);
702 }
703 
704 struct change_mapping_params {
705 	pte_t *pte;
706 	unsigned long start;
707 	unsigned long end;
708 	unsigned long aligned_start;
709 	unsigned long aligned_end;
710 };
711 
stop_machine_change_mapping(void * data)712 static int __meminit stop_machine_change_mapping(void *data)
713 {
714 	struct change_mapping_params *params =
715 			(struct change_mapping_params *)data;
716 
717 	if (!data)
718 		return -1;
719 
720 	spin_unlock(&init_mm.page_table_lock);
721 	pte_clear(&init_mm, params->aligned_start, params->pte);
722 	create_physical_mapping(params->aligned_start, params->start, -1);
723 	create_physical_mapping(params->end, params->aligned_end, -1);
724 	spin_lock(&init_mm.page_table_lock);
725 	return 0;
726 }
727 
remove_pte_table(pte_t * pte_start,unsigned long addr,unsigned long end)728 static void remove_pte_table(pte_t *pte_start, unsigned long addr,
729 			     unsigned long end)
730 {
731 	unsigned long next;
732 	pte_t *pte;
733 
734 	pte = pte_start + pte_index(addr);
735 	for (; addr < end; addr = next, pte++) {
736 		next = (addr + PAGE_SIZE) & PAGE_MASK;
737 		if (next > end)
738 			next = end;
739 
740 		if (!pte_present(*pte))
741 			continue;
742 
743 		if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(next)) {
744 			/*
745 			 * The vmemmap_free() and remove_section_mapping()
746 			 * codepaths call us with aligned addresses.
747 			 */
748 			WARN_ONCE(1, "%s: unaligned range\n", __func__);
749 			continue;
750 		}
751 
752 		pte_clear(&init_mm, addr, pte);
753 	}
754 }
755 
756 /*
757  * clear the pte and potentially split the mapping helper
758  */
split_kernel_mapping(unsigned long addr,unsigned long end,unsigned long size,pte_t * pte)759 static void __meminit split_kernel_mapping(unsigned long addr, unsigned long end,
760 				unsigned long size, pte_t *pte)
761 {
762 	unsigned long mask = ~(size - 1);
763 	unsigned long aligned_start = addr & mask;
764 	unsigned long aligned_end = addr + size;
765 	struct change_mapping_params params;
766 	bool split_region = false;
767 
768 	if ((end - addr) < size) {
769 		/*
770 		 * We're going to clear the PTE, but not flushed
771 		 * the mapping, time to remap and flush. The
772 		 * effects if visible outside the processor or
773 		 * if we are running in code close to the
774 		 * mapping we cleared, we are in trouble.
775 		 */
776 		if (overlaps_kernel_text(aligned_start, addr) ||
777 			overlaps_kernel_text(end, aligned_end)) {
778 			/*
779 			 * Hack, just return, don't pte_clear
780 			 */
781 			WARN_ONCE(1, "Linear mapping %lx->%lx overlaps kernel "
782 				  "text, not splitting\n", addr, end);
783 			return;
784 		}
785 		split_region = true;
786 	}
787 
788 	if (split_region) {
789 		params.pte = pte;
790 		params.start = addr;
791 		params.end = end;
792 		params.aligned_start = addr & ~(size - 1);
793 		params.aligned_end = min_t(unsigned long, aligned_end,
794 				(unsigned long)__va(memblock_end_of_DRAM()));
795 		stop_machine(stop_machine_change_mapping, &params, NULL);
796 		return;
797 	}
798 
799 	pte_clear(&init_mm, addr, pte);
800 }
801 
remove_pmd_table(pmd_t * pmd_start,unsigned long addr,unsigned long end)802 static void remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
803 			     unsigned long end)
804 {
805 	unsigned long next;
806 	pte_t *pte_base;
807 	pmd_t *pmd;
808 
809 	pmd = pmd_start + pmd_index(addr);
810 	for (; addr < end; addr = next, pmd++) {
811 		next = pmd_addr_end(addr, end);
812 
813 		if (!pmd_present(*pmd))
814 			continue;
815 
816 		if (pmd_huge(*pmd)) {
817 			split_kernel_mapping(addr, end, PMD_SIZE, (pte_t *)pmd);
818 			continue;
819 		}
820 
821 		pte_base = (pte_t *)pmd_page_vaddr(*pmd);
822 		remove_pte_table(pte_base, addr, next);
823 		free_pte_table(pte_base, pmd);
824 	}
825 }
826 
remove_pud_table(pud_t * pud_start,unsigned long addr,unsigned long end)827 static void remove_pud_table(pud_t *pud_start, unsigned long addr,
828 			     unsigned long end)
829 {
830 	unsigned long next;
831 	pmd_t *pmd_base;
832 	pud_t *pud;
833 
834 	pud = pud_start + pud_index(addr);
835 	for (; addr < end; addr = next, pud++) {
836 		next = pud_addr_end(addr, end);
837 
838 		if (!pud_present(*pud))
839 			continue;
840 
841 		if (pud_huge(*pud)) {
842 			split_kernel_mapping(addr, end, PUD_SIZE, (pte_t *)pud);
843 			continue;
844 		}
845 
846 		pmd_base = (pmd_t *)pud_page_vaddr(*pud);
847 		remove_pmd_table(pmd_base, addr, next);
848 		free_pmd_table(pmd_base, pud);
849 	}
850 }
851 
remove_pagetable(unsigned long start,unsigned long end)852 static void __meminit remove_pagetable(unsigned long start, unsigned long end)
853 {
854 	unsigned long addr, next;
855 	pud_t *pud_base;
856 	pgd_t *pgd;
857 
858 	spin_lock(&init_mm.page_table_lock);
859 
860 	for (addr = start; addr < end; addr = next) {
861 		next = pgd_addr_end(addr, end);
862 
863 		pgd = pgd_offset_k(addr);
864 		if (!pgd_present(*pgd))
865 			continue;
866 
867 		if (pgd_huge(*pgd)) {
868 			split_kernel_mapping(addr, end, PGDIR_SIZE, (pte_t *)pgd);
869 			continue;
870 		}
871 
872 		pud_base = (pud_t *)pgd_page_vaddr(*pgd);
873 		remove_pud_table(pud_base, addr, next);
874 	}
875 
876 	spin_unlock(&init_mm.page_table_lock);
877 	radix__flush_tlb_kernel_range(start, end);
878 }
879 
radix__create_section_mapping(unsigned long start,unsigned long end,int nid)880 int __meminit radix__create_section_mapping(unsigned long start, unsigned long end, int nid)
881 {
882 	return create_physical_mapping(start, end, nid);
883 }
884 
radix__remove_section_mapping(unsigned long start,unsigned long end)885 int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end)
886 {
887 	remove_pagetable(start, end);
888 	return 0;
889 }
890 #endif /* CONFIG_MEMORY_HOTPLUG */
891 
892 #ifdef CONFIG_SPARSEMEM_VMEMMAP
__map_kernel_page_nid(unsigned long ea,unsigned long pa,pgprot_t flags,unsigned int map_page_size,int nid)893 static int __map_kernel_page_nid(unsigned long ea, unsigned long pa,
894 				 pgprot_t flags, unsigned int map_page_size,
895 				 int nid)
896 {
897 	return __map_kernel_page(ea, pa, flags, map_page_size, nid, 0, 0);
898 }
899 
radix__vmemmap_create_mapping(unsigned long start,unsigned long page_size,unsigned long phys)900 int __meminit radix__vmemmap_create_mapping(unsigned long start,
901 				      unsigned long page_size,
902 				      unsigned long phys)
903 {
904 	/* Create a PTE encoding */
905 	unsigned long flags = _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_KERNEL_RW;
906 	int nid = early_pfn_to_nid(phys >> PAGE_SHIFT);
907 	int ret;
908 
909 	ret = __map_kernel_page_nid(start, phys, __pgprot(flags), page_size, nid);
910 	BUG_ON(ret);
911 
912 	return 0;
913 }
914 
915 #ifdef CONFIG_MEMORY_HOTPLUG
radix__vmemmap_remove_mapping(unsigned long start,unsigned long page_size)916 void __meminit radix__vmemmap_remove_mapping(unsigned long start, unsigned long page_size)
917 {
918 	remove_pagetable(start, start + page_size);
919 }
920 #endif
921 #endif
922 
923 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
924 
radix__pmd_hugepage_update(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,unsigned long clr,unsigned long set)925 unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
926 				  pmd_t *pmdp, unsigned long clr,
927 				  unsigned long set)
928 {
929 	unsigned long old;
930 
931 #ifdef CONFIG_DEBUG_VM
932 	WARN_ON(!radix__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
933 	assert_spin_locked(pmd_lockptr(mm, pmdp));
934 #endif
935 
936 	old = radix__pte_update(mm, addr, (pte_t *)pmdp, clr, set, 1);
937 	trace_hugepage_update(addr, old, clr, set);
938 
939 	return old;
940 }
941 
radix__pmdp_collapse_flush(struct vm_area_struct * vma,unsigned long address,pmd_t * pmdp)942 pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
943 			pmd_t *pmdp)
944 
945 {
946 	pmd_t pmd;
947 
948 	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
949 	VM_BUG_ON(radix__pmd_trans_huge(*pmdp));
950 	VM_BUG_ON(pmd_devmap(*pmdp));
951 	/*
952 	 * khugepaged calls this for normal pmd
953 	 */
954 	pmd = *pmdp;
955 	pmd_clear(pmdp);
956 
957 	/*FIXME!!  Verify whether we need this kick below */
958 	serialize_against_pte_lookup(vma->vm_mm);
959 
960 	radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
961 
962 	return pmd;
963 }
964 
965 /*
966  * For us pgtable_t is pte_t *. Inorder to save the deposisted
967  * page table, we consider the allocated page table as a list
968  * head. On withdraw we need to make sure we zero out the used
969  * list_head memory area.
970  */
radix__pgtable_trans_huge_deposit(struct mm_struct * mm,pmd_t * pmdp,pgtable_t pgtable)971 void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
972 				 pgtable_t pgtable)
973 {
974         struct list_head *lh = (struct list_head *) pgtable;
975 
976         assert_spin_locked(pmd_lockptr(mm, pmdp));
977 
978         /* FIFO */
979         if (!pmd_huge_pte(mm, pmdp))
980                 INIT_LIST_HEAD(lh);
981         else
982                 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
983         pmd_huge_pte(mm, pmdp) = pgtable;
984 }
985 
radix__pgtable_trans_huge_withdraw(struct mm_struct * mm,pmd_t * pmdp)986 pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
987 {
988         pte_t *ptep;
989         pgtable_t pgtable;
990         struct list_head *lh;
991 
992         assert_spin_locked(pmd_lockptr(mm, pmdp));
993 
994         /* FIFO */
995         pgtable = pmd_huge_pte(mm, pmdp);
996         lh = (struct list_head *) pgtable;
997         if (list_empty(lh))
998                 pmd_huge_pte(mm, pmdp) = NULL;
999         else {
1000                 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
1001                 list_del(lh);
1002         }
1003         ptep = (pte_t *) pgtable;
1004         *ptep = __pte(0);
1005         ptep++;
1006         *ptep = __pte(0);
1007         return pgtable;
1008 }
1009 
1010 
radix__pmdp_huge_get_and_clear(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp)1011 pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,
1012 			       unsigned long addr, pmd_t *pmdp)
1013 {
1014 	pmd_t old_pmd;
1015 	unsigned long old;
1016 
1017 	old = radix__pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
1018 	old_pmd = __pmd(old);
1019 	/*
1020 	 * Serialize against find_current_mm_pte which does lock-less
1021 	 * lookup in page tables with local interrupts disabled. For huge pages
1022 	 * it casts pmd_t to pte_t. Since format of pte_t is different from
1023 	 * pmd_t we want to prevent transit from pmd pointing to page table
1024 	 * to pmd pointing to huge page (and back) while interrupts are disabled.
1025 	 * We clear pmd to possibly replace it with page table pointer in
1026 	 * different code paths. So make sure we wait for the parallel
1027 	 * find_current_mm_pte to finish.
1028 	 */
1029 	serialize_against_pte_lookup(mm);
1030 	return old_pmd;
1031 }
1032 
radix__has_transparent_hugepage(void)1033 int radix__has_transparent_hugepage(void)
1034 {
1035 	/* For radix 2M at PMD level means thp */
1036 	if (mmu_psize_defs[MMU_PAGE_2M].shift == PMD_SHIFT)
1037 		return 1;
1038 	return 0;
1039 }
1040 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1041 
radix__ptep_set_access_flags(struct vm_area_struct * vma,pte_t * ptep,pte_t entry,unsigned long address,int psize)1042 void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
1043 				  pte_t entry, unsigned long address, int psize)
1044 {
1045 	struct mm_struct *mm = vma->vm_mm;
1046 	unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED |
1047 					      _PAGE_RW | _PAGE_EXEC);
1048 
1049 	unsigned long change = pte_val(entry) ^ pte_val(*ptep);
1050 	/*
1051 	 * To avoid NMMU hang while relaxing access, we need mark
1052 	 * the pte invalid in between.
1053 	 */
1054 	if ((change & _PAGE_RW) && atomic_read(&mm->context.copros) > 0) {
1055 		unsigned long old_pte, new_pte;
1056 
1057 		old_pte = __radix_pte_update(ptep, _PAGE_PRESENT, _PAGE_INVALID);
1058 		/*
1059 		 * new value of pte
1060 		 */
1061 		new_pte = old_pte | set;
1062 		radix__flush_tlb_page_psize(mm, address, psize);
1063 		__radix_pte_update(ptep, _PAGE_INVALID, new_pte);
1064 	} else {
1065 		__radix_pte_update(ptep, 0, set);
1066 		/*
1067 		 * Book3S does not require a TLB flush when relaxing access
1068 		 * restrictions when the address space is not attached to a
1069 		 * NMMU, because the core MMU will reload the pte after taking
1070 		 * an access fault, which is defined by the architectue.
1071 		 */
1072 	}
1073 	/* See ptesync comment in radix__set_pte_at */
1074 }
1075