60 static size_t binder_alloc_buffer_size(struct binder_alloc *alloc,  in binder_alloc_buffer_size()  argument
63 	if (list_is_last(&buffer->entry, &alloc->buffers))  in binder_alloc_buffer_size()
64 		return alloc->buffer + alloc->buffer_size - buffer->user_data;  in binder_alloc_buffer_size()
68 static void binder_insert_free_buffer(struct binder_alloc *alloc,  in binder_insert_free_buffer()  argument
71 	struct rb_node **p = &alloc->free_buffers.rb_node;  in binder_insert_free_buffer()
79 	new_buffer_size = binder_alloc_buffer_size(alloc, new_buffer);  in binder_insert_free_buffer()
83 		      alloc->pid, new_buffer_size, new_buffer);  in binder_insert_free_buffer()
90 		buffer_size = binder_alloc_buffer_size(alloc, buffer);  in binder_insert_free_buffer()
98 	rb_insert_color(&new_buffer->rb_node, &alloc->free_buffers);  in binder_insert_free_buffer()
102 		struct binder_alloc *alloc, struct binder_buffer *new_buffer)  in binder_insert_allocated_buffer_locked()  argument
104 	struct rb_node **p = &alloc->allocated_buffers.rb_node;  in binder_insert_allocated_buffer_locked()
123 	rb_insert_color(&new_buffer->rb_node, &alloc->allocated_buffers);  in binder_insert_allocated_buffer_locked()
127 		struct binder_alloc *alloc,  in binder_alloc_prepare_to_free_locked()  argument
130 	struct rb_node *n = alloc->allocated_buffers.rb_node;  in binder_alloc_prepare_to_free_locked()
170 struct binder_buffer *binder_alloc_prepare_to_free(struct binder_alloc *alloc,  in binder_alloc_prepare_to_free()  argument
175 	mutex_lock(&alloc->mutex);  in binder_alloc_prepare_to_free()
176 	buffer = binder_alloc_prepare_to_free_locked(alloc, user_ptr);  in binder_alloc_prepare_to_free()
177 	mutex_unlock(&alloc->mutex);  in binder_alloc_prepare_to_free()
181 static int binder_update_page_range(struct binder_alloc *alloc, int allocate,  in binder_update_page_range()  argument
192 		     "%d: %s pages %pK-%pK\n", alloc->pid,  in binder_update_page_range()
198 	trace_binder_update_page_range(alloc, allocate, start, end);  in binder_update_page_range()
204 		page = &alloc->pages[(page_addr - alloc->buffer) / PAGE_SIZE];  in binder_update_page_range()
211 	if (need_mm && mmget_not_zero(alloc->vma_vm_mm))  in binder_update_page_range()
212 		mm = alloc->vma_vm_mm;  in binder_update_page_range()
216 		vma = alloc->vma;  in binder_update_page_range()
222 				   alloc->pid);  in binder_update_page_range()
231 		index = (page_addr - alloc->buffer) / PAGE_SIZE;  in binder_update_page_range()
232 		page = &alloc->pages[index];  in binder_update_page_range()
235 			trace_binder_alloc_lru_start(alloc, index);  in binder_update_page_range()
240 			trace_binder_alloc_lru_end(alloc, index);  in binder_update_page_range()
247 		trace_binder_alloc_page_start(alloc, index);  in binder_update_page_range()
253 				alloc->pid, page_addr);  in binder_update_page_range()
256 		page->alloc = alloc;  in binder_update_page_range()
263 			       alloc->pid, user_page_addr);  in binder_update_page_range()
267 		if (index + 1 > alloc->pages_high)  in binder_update_page_range()
268 			alloc->pages_high = index + 1;  in binder_update_page_range()
270 		trace_binder_alloc_page_end(alloc, index);  in binder_update_page_range()
285 		index = (page_addr - alloc->buffer) / PAGE_SIZE;  in binder_update_page_range()
286 		page = &alloc->pages[index];  in binder_update_page_range()
288 		trace_binder_free_lru_start(alloc, index);  in binder_update_page_range()
293 		trace_binder_free_lru_end(alloc, index);  in binder_update_page_range()
312 static inline void binder_alloc_set_vma(struct binder_alloc *alloc,  in binder_alloc_set_vma()  argument
316 		alloc->vma_vm_mm = vma->vm_mm;  in binder_alloc_set_vma()
324 	alloc->vma = vma;  in binder_alloc_set_vma()
328 		struct binder_alloc *alloc)  in binder_alloc_get_vma()  argument
332 	if (alloc->vma) {  in binder_alloc_get_vma()
335 		vma = alloc->vma;  in binder_alloc_get_vma()
341 				struct binder_alloc *alloc,  in binder_alloc_new_buf_locked()  argument
347 	struct rb_node *n = alloc->free_buffers.rb_node;  in binder_alloc_new_buf_locked()
356 	if (!binder_alloc_get_vma(alloc)) {  in binder_alloc_new_buf_locked()
359 				   alloc->pid);  in binder_alloc_new_buf_locked()
369 				alloc->pid, data_size, offsets_size);  in binder_alloc_new_buf_locked()
376 				alloc->pid, extra_buffers_size);  in binder_alloc_new_buf_locked()
380 	    alloc->free_async_space < size + sizeof(struct binder_buffer)) {  in binder_alloc_new_buf_locked()
383 			      alloc->pid, size);  in binder_alloc_new_buf_locked()
393 		buffer_size = binder_alloc_buffer_size(alloc, buffer);  in binder_alloc_new_buf_locked()
413 		for (n = rb_first(&alloc->allocated_buffers); n != NULL;  in binder_alloc_new_buf_locked()
416 			buffer_size = binder_alloc_buffer_size(alloc, buffer);  in binder_alloc_new_buf_locked()
422 		for (n = rb_first(&alloc->free_buffers); n != NULL;  in binder_alloc_new_buf_locked()
425 			buffer_size = binder_alloc_buffer_size(alloc, buffer);  in binder_alloc_new_buf_locked()
433 				   alloc->pid, size);  in binder_alloc_new_buf_locked()
443 		buffer_size = binder_alloc_buffer_size(alloc, buffer);  in binder_alloc_new_buf_locked()
448 		      alloc->pid, size, buffer, buffer_size);  in binder_alloc_new_buf_locked()
457 	ret = binder_update_page_range(alloc, 1, (void __user *)  in binder_alloc_new_buf_locked()
468 			       __func__, alloc->pid);  in binder_alloc_new_buf_locked()
474 		binder_insert_free_buffer(alloc, new_buffer);  in binder_alloc_new_buf_locked()
477 	rb_erase(best_fit, &alloc->free_buffers);  in binder_alloc_new_buf_locked()
480 	binder_insert_allocated_buffer_locked(alloc, buffer);  in binder_alloc_new_buf_locked()
483 		      alloc->pid, size, buffer);  in binder_alloc_new_buf_locked()
489 		alloc->free_async_space -= size + sizeof(struct binder_buffer);  in binder_alloc_new_buf_locked()
492 			      alloc->pid, size, alloc->free_async_space);  in binder_alloc_new_buf_locked()
497 	binder_update_page_range(alloc, 0, (void __user *)  in binder_alloc_new_buf_locked()
518 struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc,  in binder_alloc_new_buf()  argument
526 	mutex_lock(&alloc->mutex);  in binder_alloc_new_buf()
527 	buffer = binder_alloc_new_buf_locked(alloc, data_size, offsets_size,  in binder_alloc_new_buf()
529 	mutex_unlock(&alloc->mutex);  in binder_alloc_new_buf()
544 static void binder_delete_free_buffer(struct binder_alloc *alloc,  in binder_delete_free_buffer()  argument
549 	BUG_ON(alloc->buffers.next == &buffer->entry);  in binder_delete_free_buffer()
556 				   alloc->pid, buffer->user_data,  in binder_delete_free_buffer()
560 	if (!list_is_last(&buffer->entry, &alloc->buffers)) {  in binder_delete_free_buffer()
566 					   alloc->pid,  in binder_delete_free_buffer()
575 				   alloc->pid, buffer->user_data);  in binder_delete_free_buffer()
582 				   alloc->pid, buffer->user_data,  in binder_delete_free_buffer()
585 		binder_update_page_range(alloc, 0, buffer_start_page(buffer),  in binder_delete_free_buffer()
592 static void binder_free_buf_locked(struct binder_alloc *alloc,  in binder_free_buf_locked()  argument
597 	buffer_size = binder_alloc_buffer_size(alloc, buffer);  in binder_free_buf_locked()
605 		      alloc->pid, buffer, size, buffer_size);  in binder_free_buf_locked()
610 	BUG_ON(buffer->user_data < alloc->buffer);  in binder_free_buf_locked()
611 	BUG_ON(buffer->user_data > alloc->buffer + alloc->buffer_size);  in binder_free_buf_locked()
614 		alloc->free_async_space += size + sizeof(struct binder_buffer);  in binder_free_buf_locked()
618 			      alloc->pid, size, alloc->free_async_space);  in binder_free_buf_locked()
621 	binder_update_page_range(alloc, 0,  in binder_free_buf_locked()
626 	rb_erase(&buffer->rb_node, &alloc->allocated_buffers);  in binder_free_buf_locked()
628 	if (!list_is_last(&buffer->entry, &alloc->buffers)) {  in binder_free_buf_locked()
632 			rb_erase(&next->rb_node, &alloc->free_buffers);  in binder_free_buf_locked()
633 			binder_delete_free_buffer(alloc, next);  in binder_free_buf_locked()
636 	if (alloc->buffers.next != &buffer->entry) {  in binder_free_buf_locked()
640 			binder_delete_free_buffer(alloc, buffer);  in binder_free_buf_locked()
641 			rb_erase(&prev->rb_node, &alloc->free_buffers);  in binder_free_buf_locked()
645 	binder_insert_free_buffer(alloc, buffer);  in binder_free_buf_locked()
655 void binder_alloc_free_buf(struct binder_alloc *alloc,  in binder_alloc_free_buf()  argument
658 	mutex_lock(&alloc->mutex);  in binder_alloc_free_buf()
659 	binder_free_buf_locked(alloc, buffer);  in binder_alloc_free_buf()
660 	mutex_unlock(&alloc->mutex);  in binder_alloc_free_buf()
676 int binder_alloc_mmap_handler(struct binder_alloc *alloc,  in binder_alloc_mmap_handler()  argument
684 	if (alloc->buffer) {  in binder_alloc_mmap_handler()
690 	alloc->buffer = (void __user *)vma->vm_start;  in binder_alloc_mmap_handler()
693 	alloc->buffer_size = min_t(unsigned long, vma->vm_end - vma->vm_start,  in binder_alloc_mmap_handler()
695 	alloc->pages = kcalloc(alloc->buffer_size / PAGE_SIZE,  in binder_alloc_mmap_handler()
696 			       sizeof(alloc->pages[0]),  in binder_alloc_mmap_handler()
698 	if (alloc->pages == NULL) {  in binder_alloc_mmap_handler()
711 	buffer->user_data = alloc->buffer;  in binder_alloc_mmap_handler()
712 	list_add(&buffer->entry, &alloc->buffers);  in binder_alloc_mmap_handler()
714 	binder_insert_free_buffer(alloc, buffer);  in binder_alloc_mmap_handler()
715 	alloc->free_async_space = alloc->buffer_size / 2;  in binder_alloc_mmap_handler()
716 	binder_alloc_set_vma(alloc, vma);  in binder_alloc_mmap_handler()
717 	mmgrab(alloc->vma_vm_mm);  in binder_alloc_mmap_handler()
722 	kfree(alloc->pages);  in binder_alloc_mmap_handler()
723 	alloc->pages = NULL;  in binder_alloc_mmap_handler()
726 	alloc->buffer = NULL;  in binder_alloc_mmap_handler()
731 			   alloc->pid, vma->vm_start, vma->vm_end,  in binder_alloc_mmap_handler()
737 void binder_alloc_deferred_release(struct binder_alloc *alloc)  in binder_alloc_deferred_release()  argument
744 	mutex_lock(&alloc->mutex);  in binder_alloc_deferred_release()
745 	BUG_ON(alloc->vma);  in binder_alloc_deferred_release()
747 	while ((n = rb_first(&alloc->allocated_buffers))) {  in binder_alloc_deferred_release()
753 		binder_free_buf_locked(alloc, buffer);  in binder_alloc_deferred_release()
757 	while (!list_empty(&alloc->buffers)) {  in binder_alloc_deferred_release()
758 		buffer = list_first_entry(&alloc->buffers,  in binder_alloc_deferred_release()
763 		WARN_ON_ONCE(!list_empty(&alloc->buffers));  in binder_alloc_deferred_release()
768 	if (alloc->pages) {  in binder_alloc_deferred_release()
771 		for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) {  in binder_alloc_deferred_release()
775 			if (!alloc->pages[i].page_ptr)  in binder_alloc_deferred_release()
779 					      &alloc->pages[i].lru);  in binder_alloc_deferred_release()
780 			page_addr = alloc->buffer + i * PAGE_SIZE;  in binder_alloc_deferred_release()
783 				     __func__, alloc->pid, i, page_addr,  in binder_alloc_deferred_release()
785 			__free_page(alloc->pages[i].page_ptr);  in binder_alloc_deferred_release()
788 		kfree(alloc->pages);  in binder_alloc_deferred_release()
790 	mutex_unlock(&alloc->mutex);  in binder_alloc_deferred_release()
791 	if (alloc->vma_vm_mm)  in binder_alloc_deferred_release()
792 		mmdrop(alloc->vma_vm_mm);  in binder_alloc_deferred_release()
796 		     __func__, alloc->pid, buffers, page_count);  in binder_alloc_deferred_release()
818 				  struct binder_alloc *alloc)  in binder_alloc_print_allocated()  argument
822 	mutex_lock(&alloc->mutex);  in binder_alloc_print_allocated()
823 	for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n))  in binder_alloc_print_allocated()
826 	mutex_unlock(&alloc->mutex);  in binder_alloc_print_allocated()
835 			      struct binder_alloc *alloc)  in binder_alloc_print_pages()  argument
843 	mutex_lock(&alloc->mutex);  in binder_alloc_print_pages()
844 	for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) {  in binder_alloc_print_pages()
845 		page = &alloc->pages[i];  in binder_alloc_print_pages()
853 	mutex_unlock(&alloc->mutex);  in binder_alloc_print_pages()
855 	seq_printf(m, "  pages high watermark: %zu\n", alloc->pages_high);  in binder_alloc_print_pages()
864 int binder_alloc_get_allocated_count(struct binder_alloc *alloc)  in binder_alloc_get_allocated_count()  argument
869 	mutex_lock(&alloc->mutex);  in binder_alloc_get_allocated_count()
870 	for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n))  in binder_alloc_get_allocated_count()
872 	mutex_unlock(&alloc->mutex);  in binder_alloc_get_allocated_count()
885 void binder_alloc_vma_close(struct binder_alloc *alloc)  in binder_alloc_vma_close()  argument
887 	binder_alloc_set_vma(alloc, NULL);  in binder_alloc_vma_close()
909 	struct binder_alloc *alloc;  in binder_alloc_free_page()  local
914 	alloc = page->alloc;  in binder_alloc_free_page()
915 	if (!mutex_trylock(&alloc->mutex))  in binder_alloc_free_page()
921 	index = page - alloc->pages;  in binder_alloc_free_page()
922 	page_addr = (uintptr_t)alloc->buffer + index * PAGE_SIZE;  in binder_alloc_free_page()
924 	mm = alloc->vma_vm_mm;  in binder_alloc_free_page()
929 	vma = binder_alloc_get_vma(alloc);  in binder_alloc_free_page()
935 		trace_binder_unmap_user_start(alloc, index);  in binder_alloc_free_page()
939 		trace_binder_unmap_user_end(alloc, index);  in binder_alloc_free_page()
944 	trace_binder_unmap_kernel_start(alloc, index);  in binder_alloc_free_page()
949 	trace_binder_unmap_kernel_end(alloc, index);  in binder_alloc_free_page()
952 	mutex_unlock(&alloc->mutex);  in binder_alloc_free_page()
959 	mutex_unlock(&alloc->mutex);  in binder_alloc_free_page()
994 void binder_alloc_init(struct binder_alloc *alloc)  in binder_alloc_init()  argument
996 	alloc->pid = current->group_leader->pid;  in binder_alloc_init()
997 	mutex_init(&alloc->mutex);  in binder_alloc_init()
998 	INIT_LIST_HEAD(&alloc->buffers);  in binder_alloc_init()
1032 static inline bool check_buffer(struct binder_alloc *alloc,  in check_buffer()  argument
1036 	size_t buffer_size = binder_alloc_buffer_size(alloc, buffer);  in check_buffer()
1064 static struct page *binder_alloc_get_page(struct binder_alloc *alloc,  in binder_alloc_get_page()  argument
1070 		(buffer->user_data - alloc->buffer);  in binder_alloc_get_page()
1075 	lru_page = &alloc->pages[index];  in binder_alloc_get_page()
1093 binder_alloc_copy_user_to_buffer(struct binder_alloc *alloc,  in binder_alloc_copy_user_to_buffer()  argument
1099 	if (!check_buffer(alloc, buffer, buffer_offset, bytes))  in binder_alloc_copy_user_to_buffer()
1109 		page = binder_alloc_get_page(alloc, buffer,  in binder_alloc_copy_user_to_buffer()
1124 static int binder_alloc_do_buffer_copy(struct binder_alloc *alloc,  in binder_alloc_do_buffer_copy()  argument
1132 	if (!check_buffer(alloc, buffer, buffer_offset, bytes))  in binder_alloc_do_buffer_copy()
1142 		page = binder_alloc_get_page(alloc, buffer,  in binder_alloc_do_buffer_copy()
1164 int binder_alloc_copy_to_buffer(struct binder_alloc *alloc,  in binder_alloc_copy_to_buffer()  argument
1170 	return binder_alloc_do_buffer_copy(alloc, true, buffer, buffer_offset,  in binder_alloc_copy_to_buffer()
1174 int binder_alloc_copy_from_buffer(struct binder_alloc *alloc,  in binder_alloc_copy_from_buffer()  argument
1180 	return binder_alloc_do_buffer_copy(alloc, false, buffer, buffer_offset,  in binder_alloc_copy_from_buffer()