55 #define NODEDATA_ALIGN(addr, node)						\  argument
57 	     (((node)*PERCPU_PAGE_SIZE) & (MAX_NODE_ALIGN_OFFSET - 1)))
72 				  int node)  in build_node_maps()  argument
79 	if (!mem_data[node].min_pfn) {  in build_node_maps()
80 		mem_data[node].min_pfn = spfn;  in build_node_maps()
81 		mem_data[node].max_pfn = epfn;  in build_node_maps()
83 		mem_data[node].min_pfn = min(spfn, mem_data[node].min_pfn);  in build_node_maps()
84 		mem_data[node].max_pfn = max(epfn, mem_data[node].max_pfn);  in build_node_maps()
98 static int __meminit early_nr_cpus_node(int node)  in early_nr_cpus_node()  argument
103 		if (node == node_cpuid[cpu].nid)  in early_nr_cpus_node()
113 static unsigned long __meminit compute_pernodesize(int node)  in compute_pernodesize()  argument
117 	cpus = early_nr_cpus_node(node);  in compute_pernodesize()
119 	pernodesize += node * L1_CACHE_BYTES;  in compute_pernodesize()
136 static void *per_cpu_node_setup(void *cpu_data, int node)  in per_cpu_node_setup()  argument
144 		if (node != node_cpuid[cpu].nid)  in per_cpu_node_setup()
189 	int node, prev_node, unit, nr_units;  in setup_per_cpu_areas()  local
205 	for_each_node(node)  in setup_per_cpu_areas()
207 			if (node == node_cpuid[cpu].nid)  in setup_per_cpu_areas()
234 		node = node_cpuid[cpu].nid;  in setup_per_cpu_areas()
236 		if (node == prev_node) {  in setup_per_cpu_areas()
240 		prev_node = node;  in setup_per_cpu_areas()
259 static void __init fill_pernode(int node, unsigned long pernode,  in fill_pernode()  argument
263 	int cpus = early_nr_cpus_node(node);  in fill_pernode()
265 	mem_data[node].pernode_addr = pernode;  in fill_pernode()
266 	mem_data[node].pernode_size = pernodesize;  in fill_pernode()
271 	pernode += node * L1_CACHE_BYTES;  in fill_pernode()
273 	pgdat_list[node] = __va(pernode);  in fill_pernode()
276 	mem_data[node].node_data = __va(pernode);  in fill_pernode()
280 	cpu_data = per_cpu_node_setup(cpu_data, node);  in fill_pernode()
314 				     int node)  in find_pernode_space()  argument
326 	if (spfn < mem_data[node].min_pfn || epfn > mem_data[node].max_pfn)  in find_pernode_space()
330 	if (mem_data[node].pernode_addr)  in find_pernode_space()
337 	pernodesize = compute_pernodesize(node);  in find_pernode_space()
338 	pernode = NODEDATA_ALIGN(start, node);  in find_pernode_space()
342 		fill_pernode(node, pernode, pernodesize);  in find_pernode_space()
357 	int node;  in reserve_pernode_space()  local
359 	for_each_online_node(node) {  in reserve_pernode_space()
360 		if (node_isset(node, memory_less_mask))  in reserve_pernode_space()
364 		size = mem_data[node].pernode_size;  in reserve_pernode_space()
365 		base = __pa(mem_data[node].pernode_addr);  in reserve_pernode_space()
373 	int node;  in scatter_node_data()  local
383 	for_each_node(node) {  in scatter_node_data()
384 		if (pgdat_list[node]) {  in scatter_node_data()
385 			dst = LOCAL_DATA_ADDR(pgdat_list[node])->pg_data_ptrs;  in scatter_node_data()
400 	int cpu, node;  in initialize_pernode_data()  local
407 		node = node_cpuid[cpu].nid;  in initialize_pernode_data()
409 			mem_data[node].node_data;  in initialize_pernode_data()
415 		node = node_cpuid[cpu].nid;  in initialize_pernode_data()
418 		cpu0_cpu_info->node_data = mem_data[node].node_data;  in initialize_pernode_data()
434 	int bestnode = NUMA_NO_NODE, node, anynode = 0;  in memory_less_node_alloc()  local
436 	for_each_online_node(node) {  in memory_less_node_alloc()
437 		if (node_isset(node, memory_less_mask))  in memory_less_node_alloc()
439 		else if (node_distance(nid, node) < best) {  in memory_less_node_alloc()
440 			best = node_distance(nid, node);  in memory_less_node_alloc()
441 			bestnode = node;  in memory_less_node_alloc()
443 		anynode = node;  in memory_less_node_alloc()
469 	int node;  in memory_less_nodes()  local
471 	for_each_node_mask(node, memory_less_mask) {  in memory_less_nodes()
472 		pernodesize = compute_pernodesize(node);  in memory_less_nodes()
473 		pernode = memory_less_node_alloc(node, pernodesize);  in memory_less_nodes()
474 		fill_pernode(node, __pa(pernode), pernodesize);  in memory_less_nodes()
488 	int node;  in find_memory()  local
507 	for_each_online_node(node)  in find_memory()
508 		if (mem_data[node].min_pfn)  in find_memory()
509 			node_clear(node, memory_less_mask);  in find_memory()
599 	int node;  in paging_init()  local
615 	for_each_online_node(node) {  in paging_init()
616 		pfn_offset = mem_data[node].min_pfn;  in paging_init()
619 		NODE_DATA(node)->node_mem_map = vmem_map + pfn_offset;  in paging_init()
621 		if (mem_data[node].max_pfn > max_pfn)  in paging_init()
622 			max_pfn = mem_data[node].max_pfn;  in paging_init()
656 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,  in vmemmap_populate()  argument
659 	return vmemmap_populate_basepages(start, end, node);  in vmemmap_populate()