38 struct cma cma_areas[MAX_CMA_AREAS];
41 phys_addr_t cma_get_base(const struct cma *cma)  in cma_get_base()  argument
43 	return PFN_PHYS(cma->base_pfn);  in cma_get_base()
46 unsigned long cma_get_size(const struct cma *cma)  in cma_get_size()  argument
48 	return cma->count << PAGE_SHIFT;  in cma_get_size()
51 const char *cma_get_name(const struct cma *cma)  in cma_get_name()  argument
53 	return cma->name;  in cma_get_name()
56 static unsigned long cma_bitmap_aligned_mask(const struct cma *cma,  in cma_bitmap_aligned_mask()  argument
59 	if (align_order <= cma->order_per_bit)  in cma_bitmap_aligned_mask()
61 	return (1UL << (align_order - cma->order_per_bit)) - 1;  in cma_bitmap_aligned_mask()
68 static unsigned long cma_bitmap_aligned_offset(const struct cma *cma,  in cma_bitmap_aligned_offset()  argument
71 	return (cma->base_pfn & ((1UL << align_order) - 1))  in cma_bitmap_aligned_offset()
72 		>> cma->order_per_bit;  in cma_bitmap_aligned_offset()
75 static unsigned long cma_bitmap_pages_to_bits(const struct cma *cma,  in cma_bitmap_pages_to_bits()  argument
78 	return ALIGN(pages, 1UL << cma->order_per_bit) >> cma->order_per_bit;  in cma_bitmap_pages_to_bits()
81 static void cma_clear_bitmap(struct cma *cma, unsigned long pfn,  in cma_clear_bitmap()  argument
87 	bitmap_no = (pfn - cma->base_pfn) >> cma->order_per_bit;  in cma_clear_bitmap()
88 	bitmap_count = cma_bitmap_pages_to_bits(cma, count);  in cma_clear_bitmap()
90 	spin_lock_irqsave(&cma->lock, flags);  in cma_clear_bitmap()
91 	bitmap_clear(cma->bitmap, bitmap_no, bitmap_count);  in cma_clear_bitmap()
92 	spin_unlock_irqrestore(&cma->lock, flags);  in cma_clear_bitmap()
95 static void __init cma_activate_area(struct cma *cma)  in cma_activate_area()  argument
97 	unsigned long base_pfn = cma->base_pfn, pfn;  in cma_activate_area()
100 	cma->bitmap = bitmap_zalloc(cma_bitmap_maxno(cma), GFP_KERNEL);  in cma_activate_area()
101 	if (!cma->bitmap)  in cma_activate_area()
111 	for (pfn = base_pfn + 1; pfn < base_pfn + cma->count; pfn++) {  in cma_activate_area()
117 	for (pfn = base_pfn; pfn < base_pfn + cma->count;  in cma_activate_area()
121 	spin_lock_init(&cma->lock);  in cma_activate_area()
124 	INIT_HLIST_HEAD(&cma->mem_head);  in cma_activate_area()
125 	spin_lock_init(&cma->mem_head_lock);  in cma_activate_area()
131 	bitmap_free(cma->bitmap);  in cma_activate_area()
134 	for (pfn = base_pfn; pfn < base_pfn + cma->count; pfn++)  in cma_activate_area()
136 	totalcma_pages -= cma->count;  in cma_activate_area()
137 	cma->count = 0;  in cma_activate_area()
138 	pr_err("CMA area %s could not be activated\n", cma->name);  in cma_activate_area()
168 				 struct cma **res_cma)  in cma_init_reserved_mem()
170 	struct cma *cma;  in cma_init_reserved_mem()  local
197 	cma = &cma_areas[cma_area_count];  in cma_init_reserved_mem()
200 		snprintf(cma->name, CMA_MAX_NAME, name);  in cma_init_reserved_mem()
202 		snprintf(cma->name, CMA_MAX_NAME,  "cma%d\n", cma_area_count);  in cma_init_reserved_mem()
204 	cma->base_pfn = PFN_DOWN(base);  in cma_init_reserved_mem()
205 	cma->count = size >> PAGE_SHIFT;  in cma_init_reserved_mem()
206 	cma->order_per_bit = order_per_bit;  in cma_init_reserved_mem()
207 	*res_cma = cma;  in cma_init_reserved_mem()
237 			bool fixed, const char *name, struct cma **res_cma,  in cma_declare_contiguous_nid()
388 static void cma_debug_show_areas(struct cma *cma)  in cma_debug_show_areas()  argument
393 	unsigned long nbits = cma_bitmap_maxno(cma);  in cma_debug_show_areas()
395 	spin_lock_irq(&cma->lock);  in cma_debug_show_areas()
398 		next_zero_bit = find_next_zero_bit(cma->bitmap, nbits, start);  in cma_debug_show_areas()
401 		next_set_bit = find_next_bit(cma->bitmap, nbits, next_zero_bit);  in cma_debug_show_areas()
403 		nr_part = nr_zero << cma->order_per_bit;  in cma_debug_show_areas()
409 	pr_cont("=> %lu free of %lu total pages\n", nr_total, cma->count);  in cma_debug_show_areas()
410 	spin_unlock_irq(&cma->lock);  in cma_debug_show_areas()
413 static inline void cma_debug_show_areas(struct cma *cma) { }  in cma_debug_show_areas()  argument
426 struct page *cma_alloc(struct cma *cma, unsigned long count,  in cma_alloc()  argument
437 	if (!cma || !cma->count || !cma->bitmap)  in cma_alloc()
440 	pr_debug("%s(cma %p, count %lu, align %d)\n", __func__, (void *)cma,  in cma_alloc()
446 	trace_cma_alloc_start(cma->name, count, align);  in cma_alloc()
448 	mask = cma_bitmap_aligned_mask(cma, align);  in cma_alloc()
449 	offset = cma_bitmap_aligned_offset(cma, align);  in cma_alloc()
450 	bitmap_maxno = cma_bitmap_maxno(cma);  in cma_alloc()
451 	bitmap_count = cma_bitmap_pages_to_bits(cma, count);  in cma_alloc()
457 		spin_lock_irq(&cma->lock);  in cma_alloc()
458 		bitmap_no = bitmap_find_next_zero_area_off(cma->bitmap,  in cma_alloc()
462 			spin_unlock_irq(&cma->lock);  in cma_alloc()
465 		bitmap_set(cma->bitmap, bitmap_no, bitmap_count);  in cma_alloc()
471 		spin_unlock_irq(&cma->lock);  in cma_alloc()
473 		pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit);  in cma_alloc()
482 		cma_clear_bitmap(cma, pfn, count);  in cma_alloc()
489 		trace_cma_alloc_busy_retry(cma->name, pfn, pfn_to_page(pfn),  in cma_alloc()
495 	trace_cma_alloc_finish(cma->name, pfn, page, count, align);  in cma_alloc()
509 				   __func__, cma->name, count, ret);  in cma_alloc()
510 		cma_debug_show_areas(cma);  in cma_alloc()
517 		cma_sysfs_account_success_pages(cma, count);  in cma_alloc()
520 		if (cma)  in cma_alloc()
521 			cma_sysfs_account_fail_pages(cma, count);  in cma_alloc()
537 bool cma_release(struct cma *cma, const struct page *pages,  in cma_release()  argument
542 	if (!cma || !pages)  in cma_release()
549 	if (pfn < cma->base_pfn || pfn >= cma->base_pfn + cma->count)  in cma_release()
552 	VM_BUG_ON(pfn + count > cma->base_pfn + cma->count);  in cma_release()
555 	cma_clear_bitmap(cma, pfn, count);  in cma_release()
556 	trace_cma_release(cma->name, pfn, pages, count);  in cma_release()
561 int cma_for_each_area(int (*it)(struct cma *cma, void *data), void *data)  in cma_for_each_area()  argument