73 	.bdata = &bootmem_node_data[0]
125 static void __init link_bootmem(bootmem_data_t *bdata)  in link_bootmem()  argument
130 		if (bdata->node_min_pfn < ent->node_min_pfn) {  in link_bootmem()
131 			list_add_tail(&bdata->list, &ent->list);  in link_bootmem()
136 	list_add_tail(&bdata->list, &bdata_list);  in link_bootmem()
142 static unsigned long __init init_bootmem_core(bootmem_data_t *bdata,  in init_bootmem_core()  argument
148 	bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));  in init_bootmem_core()
149 	bdata->node_min_pfn = start;  in init_bootmem_core()
150 	bdata->node_low_pfn = end;  in init_bootmem_core()
151 	link_bootmem(bdata);  in init_bootmem_core()
158 	memset(bdata->node_bootmem_map, 0xff, mapsize);  in init_bootmem_core()
161 		bdata - bootmem_node_data, start, mapstart, end, mapsize);  in init_bootmem_core()
178 	return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn);  in init_bootmem_node()
192 	return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);  in init_bootmem()
210 static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)  in free_all_bootmem_core()  argument
215 	if (!bdata->node_bootmem_map)  in free_all_bootmem_core()
218 	map = bdata->node_bootmem_map;  in free_all_bootmem_core()
219 	start = bdata->node_min_pfn;  in free_all_bootmem_core()
220 	end = bdata->node_low_pfn;  in free_all_bootmem_core()
223 		bdata - bootmem_node_data, start, end);  in free_all_bootmem_core()
229 		idx = start - bdata->node_min_pfn;  in free_all_bootmem_core()
270 	cur = bdata->node_min_pfn;  in free_all_bootmem_core()
271 	page = virt_to_page(bdata->node_bootmem_map);  in free_all_bootmem_core()
272 	pages = bdata->node_low_pfn - bdata->node_min_pfn;  in free_all_bootmem_core()
277 	bdata->node_bootmem_map = NULL;  in free_all_bootmem_core()
279 	bdebug("nid=%td released=%lx\n", bdata - bootmem_node_data, count);  in free_all_bootmem_core()
310 	bootmem_data_t *bdata;  in free_all_bootmem()  local
314 	list_for_each_entry(bdata, &bdata_list, list)  in free_all_bootmem()
315 		total_pages += free_all_bootmem_core(bdata);  in free_all_bootmem()
322 static void __init __free(bootmem_data_t *bdata,  in __free()  argument
327 	bdebug("nid=%td start=%lx end=%lx\n", bdata - bootmem_node_data,  in __free()
328 		sidx + bdata->node_min_pfn,  in __free()
329 		eidx + bdata->node_min_pfn);  in __free()
331 	if (WARN_ON(bdata->node_bootmem_map == NULL))  in __free()
334 	if (bdata->hint_idx > sidx)  in __free()
335 		bdata->hint_idx = sidx;  in __free()
338 		if (!test_and_clear_bit(idx, bdata->node_bootmem_map))  in __free()
342 static int __init __reserve(bootmem_data_t *bdata, unsigned long sidx,  in __reserve()  argument
349 		bdata - bootmem_node_data,  in __reserve()
350 		sidx + bdata->node_min_pfn,  in __reserve()
351 		eidx + bdata->node_min_pfn,  in __reserve()
354 	if (WARN_ON(bdata->node_bootmem_map == NULL))  in __reserve()
358 		if (test_and_set_bit(idx, bdata->node_bootmem_map)) {  in __reserve()
360 				__free(bdata, sidx, idx);  in __reserve()
364 				idx + bdata->node_min_pfn);  in __reserve()
369 static int __init mark_bootmem_node(bootmem_data_t *bdata,  in mark_bootmem_node()  argument
376 		bdata - bootmem_node_data, start, end, reserve, flags);  in mark_bootmem_node()
378 	BUG_ON(start < bdata->node_min_pfn);  in mark_bootmem_node()
379 	BUG_ON(end > bdata->node_low_pfn);  in mark_bootmem_node()
381 	sidx = start - bdata->node_min_pfn;  in mark_bootmem_node()
382 	eidx = end - bdata->node_min_pfn;  in mark_bootmem_node()
385 		return __reserve(bdata, sidx, eidx, flags);  in mark_bootmem_node()
387 		__free(bdata, sidx, eidx);  in mark_bootmem_node()
395 	bootmem_data_t *bdata;  in mark_bootmem()  local
398 	list_for_each_entry(bdata, &bdata_list, list) {  in mark_bootmem()
402 		if (pos < bdata->node_min_pfn ||  in mark_bootmem()
403 		    pos >= bdata->node_low_pfn) {  in mark_bootmem()
408 		max = min(bdata->node_low_pfn, end);  in mark_bootmem()
410 		err = mark_bootmem_node(bdata, pos, max, reserve, flags);  in mark_bootmem()
418 		pos = bdata->node_low_pfn;  in mark_bootmem()
433 	mark_bootmem_node(pgdat->bdata, start, end, 0, 0);  in free_bootmem_node()
469 	return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);  in reserve_bootmem_node()
495 static unsigned long __init align_idx(struct bootmem_data *bdata,  in align_idx()  argument
498 	unsigned long base = bdata->node_min_pfn;  in align_idx()
508 static unsigned long __init align_off(struct bootmem_data *bdata,  in align_off()  argument
511 	unsigned long base = PFN_PHYS(bdata->node_min_pfn);  in align_off()
518 static void * __init alloc_bootmem_bdata(struct bootmem_data *bdata,  in alloc_bootmem_bdata()  argument
526 		bdata - bootmem_node_data, size, PAGE_ALIGN(size) >> PAGE_SHIFT,  in alloc_bootmem_bdata()
533 	if (!bdata->node_bootmem_map)  in alloc_bootmem_bdata()
536 	min = bdata->node_min_pfn;  in alloc_bootmem_bdata()
537 	max = bdata->node_low_pfn;  in alloc_bootmem_bdata()
554 	sidx = start - bdata->node_min_pfn;  in alloc_bootmem_bdata()
555 	midx = max - bdata->node_min_pfn;  in alloc_bootmem_bdata()
557 	if (bdata->hint_idx > sidx) {  in alloc_bootmem_bdata()
563 		sidx = align_idx(bdata, bdata->hint_idx, step);  in alloc_bootmem_bdata()
571 		sidx = find_next_zero_bit(bdata->node_bootmem_map, midx, sidx);  in alloc_bootmem_bdata()
572 		sidx = align_idx(bdata, sidx, step);  in alloc_bootmem_bdata()
579 			if (test_bit(i, bdata->node_bootmem_map)) {  in alloc_bootmem_bdata()
580 				sidx = align_idx(bdata, i, step);  in alloc_bootmem_bdata()
586 		if (bdata->last_end_off & (PAGE_SIZE - 1) &&  in alloc_bootmem_bdata()
587 				PFN_DOWN(bdata->last_end_off) + 1 == sidx)  in alloc_bootmem_bdata()
588 			start_off = align_off(bdata, bdata->last_end_off, align);  in alloc_bootmem_bdata()
595 		bdata->last_end_off = end_off;  in alloc_bootmem_bdata()
596 		bdata->hint_idx = PFN_UP(end_off);  in alloc_bootmem_bdata()
601 		if (__reserve(bdata, PFN_DOWN(start_off) + merge,  in alloc_bootmem_bdata()
605 		region = phys_to_virt(PFN_PHYS(bdata->node_min_pfn) +  in alloc_bootmem_bdata()
617 		sidx = align_idx(bdata, fallback - 1, step);  in alloc_bootmem_bdata()
630 	bootmem_data_t *bdata;  in alloc_bootmem_core()  local
636 	list_for_each_entry(bdata, &bdata_list, list) {  in alloc_bootmem_core()
637 		if (goal && bdata->node_low_pfn <= PFN_DOWN(goal))  in alloc_bootmem_core()
639 		if (limit && bdata->node_min_pfn >= PFN_DOWN(limit))  in alloc_bootmem_core()
642 		region = alloc_bootmem_bdata(bdata, size, align, goal, limit);  in alloc_bootmem_core()
714 	ptr = alloc_bootmem_bdata(pgdat->bdata, size, align, goal, limit);  in ___alloc_bootmem_node_nopanic()
778 		ptr = alloc_bootmem_bdata(pgdat->bdata, size, align,  in __alloc_bootmem_node_high()