60 static size_t binder_alloc_buffer_size(struct binder_alloc *alloc,  in binder_alloc_buffer_size()  argument
63 	if (list_is_last(&buffer->entry, &alloc->buffers))  in binder_alloc_buffer_size()
64 		return alloc->buffer + alloc->buffer_size - buffer->user_data;  in binder_alloc_buffer_size()
68 static void binder_insert_free_buffer(struct binder_alloc *alloc,  in binder_insert_free_buffer()  argument
71 	struct rb_node **p = &alloc->free_buffers.rb_node;  in binder_insert_free_buffer()
79 	new_buffer_size = binder_alloc_buffer_size(alloc, new_buffer);  in binder_insert_free_buffer()
83 		      alloc->pid, new_buffer_size, new_buffer);  in binder_insert_free_buffer()
90 		buffer_size = binder_alloc_buffer_size(alloc, buffer);  in binder_insert_free_buffer()
98 	rb_insert_color(&new_buffer->rb_node, &alloc->free_buffers);  in binder_insert_free_buffer()
102 		struct binder_alloc *alloc, struct binder_buffer *new_buffer)  in binder_insert_allocated_buffer_locked()  argument
104 	struct rb_node **p = &alloc->allocated_buffers.rb_node;  in binder_insert_allocated_buffer_locked()
123 	rb_insert_color(&new_buffer->rb_node, &alloc->allocated_buffers);  in binder_insert_allocated_buffer_locked()
127 		struct binder_alloc *alloc,  in binder_alloc_prepare_to_free_locked()  argument
130 	struct rb_node *n = alloc->allocated_buffers.rb_node;  in binder_alloc_prepare_to_free_locked()
170 struct binder_buffer *binder_alloc_prepare_to_free(struct binder_alloc *alloc,  in binder_alloc_prepare_to_free()  argument
175 	mutex_lock(&alloc->mutex);  in binder_alloc_prepare_to_free()
176 	buffer = binder_alloc_prepare_to_free_locked(alloc, user_ptr);  in binder_alloc_prepare_to_free()
177 	mutex_unlock(&alloc->mutex);  in binder_alloc_prepare_to_free()
181 static int binder_update_page_range(struct binder_alloc *alloc, int allocate,  in binder_update_page_range()  argument
192 		     "%d: %s pages %pK-%pK\n", alloc->pid,  in binder_update_page_range()
198 	trace_binder_update_page_range(alloc, allocate, start, end);  in binder_update_page_range()
204 		page = &alloc->pages[(page_addr - alloc->buffer) / PAGE_SIZE];  in binder_update_page_range()
211 	if (need_mm && mmget_not_zero(alloc->mm))  in binder_update_page_range()
212 		mm = alloc->mm;  in binder_update_page_range()
216 		vma = vma_lookup(mm, alloc->vma_addr);  in binder_update_page_range()
222 				   alloc->pid);  in binder_update_page_range()
231 		index = (page_addr - alloc->buffer) / PAGE_SIZE;  in binder_update_page_range()
232 		page = &alloc->pages[index];  in binder_update_page_range()
235 			trace_binder_alloc_lru_start(alloc, index);  in binder_update_page_range()
240 			trace_binder_alloc_lru_end(alloc, index);  in binder_update_page_range()
247 		trace_binder_alloc_page_start(alloc, index);  in binder_update_page_range()
253 				alloc->pid, page_addr);  in binder_update_page_range()
256 		page->alloc = alloc;  in binder_update_page_range()
263 			       alloc->pid, user_page_addr);  in binder_update_page_range()
267 		if (index + 1 > alloc->pages_high)  in binder_update_page_range()
268 			alloc->pages_high = index + 1;  in binder_update_page_range()
270 		trace_binder_alloc_page_end(alloc, index);  in binder_update_page_range()
283 		index = (page_addr - alloc->buffer) / PAGE_SIZE;  in binder_update_page_range()
284 		page = &alloc->pages[index];  in binder_update_page_range()
286 		trace_binder_free_lru_start(alloc, index);  in binder_update_page_range()
291 		trace_binder_free_lru_end(alloc, index);  in binder_update_page_range()
313 		struct binder_alloc *alloc)  in binder_alloc_get_vma()  argument
317 	if (alloc->vma_addr)  in binder_alloc_get_vma()
318 		vma = vma_lookup(alloc->mm, alloc->vma_addr);  in binder_alloc_get_vma()
323 static bool debug_low_async_space_locked(struct binder_alloc *alloc, int pid)  in debug_low_async_space_locked()  argument
337 	for (n = rb_first(&alloc->allocated_buffers); n != NULL;  in debug_low_async_space_locked()
344 		total_alloc_size += binder_alloc_buffer_size(alloc, buffer)  in debug_low_async_space_locked()
354 	if (num_buffers > 50 || total_alloc_size > alloc->buffer_size / 4) {  in debug_low_async_space_locked()
357 			      alloc->pid, pid, num_buffers, total_alloc_size);  in debug_low_async_space_locked()
358 		if (!alloc->oneway_spam_detected) {  in debug_low_async_space_locked()
359 			alloc->oneway_spam_detected = true;  in debug_low_async_space_locked()
367 				struct binder_alloc *alloc,  in binder_alloc_new_buf_locked()  argument
374 	struct rb_node *n = alloc->free_buffers.rb_node;  in binder_alloc_new_buf_locked()
383 	mmap_read_lock(alloc->mm);  in binder_alloc_new_buf_locked()
384 	if (!binder_alloc_get_vma(alloc)) {  in binder_alloc_new_buf_locked()
385 		mmap_read_unlock(alloc->mm);  in binder_alloc_new_buf_locked()
388 				   alloc->pid);  in binder_alloc_new_buf_locked()
391 	mmap_read_unlock(alloc->mm);  in binder_alloc_new_buf_locked()
399 				alloc->pid, data_size, offsets_size);  in binder_alloc_new_buf_locked()
406 				alloc->pid, extra_buffers_size);  in binder_alloc_new_buf_locked()
410 	    alloc->free_async_space < size + sizeof(struct binder_buffer)) {  in binder_alloc_new_buf_locked()
413 			      alloc->pid, size);  in binder_alloc_new_buf_locked()
423 		buffer_size = binder_alloc_buffer_size(alloc, buffer);  in binder_alloc_new_buf_locked()
443 		for (n = rb_first(&alloc->allocated_buffers); n != NULL;  in binder_alloc_new_buf_locked()
446 			buffer_size = binder_alloc_buffer_size(alloc, buffer);  in binder_alloc_new_buf_locked()
452 		for (n = rb_first(&alloc->free_buffers); n != NULL;  in binder_alloc_new_buf_locked()
455 			buffer_size = binder_alloc_buffer_size(alloc, buffer);  in binder_alloc_new_buf_locked()
463 				   alloc->pid, size);  in binder_alloc_new_buf_locked()
473 		buffer_size = binder_alloc_buffer_size(alloc, buffer);  in binder_alloc_new_buf_locked()
478 		      alloc->pid, size, buffer, buffer_size);  in binder_alloc_new_buf_locked()
487 	ret = binder_update_page_range(alloc, 1, (void __user *)  in binder_alloc_new_buf_locked()
498 			       __func__, alloc->pid);  in binder_alloc_new_buf_locked()
504 		binder_insert_free_buffer(alloc, new_buffer);  in binder_alloc_new_buf_locked()
507 	rb_erase(best_fit, &alloc->free_buffers);  in binder_alloc_new_buf_locked()
510 	binder_insert_allocated_buffer_locked(alloc, buffer);  in binder_alloc_new_buf_locked()
513 		      alloc->pid, size, buffer);  in binder_alloc_new_buf_locked()
521 		alloc->free_async_space -= size + sizeof(struct binder_buffer);  in binder_alloc_new_buf_locked()
524 			      alloc->pid, size, alloc->free_async_space);  in binder_alloc_new_buf_locked()
525 		if (alloc->free_async_space < alloc->buffer_size / 10) {  in binder_alloc_new_buf_locked()
531 			buffer->oneway_spam_suspect = debug_low_async_space_locked(alloc, pid);  in binder_alloc_new_buf_locked()
533 			alloc->oneway_spam_detected = false;  in binder_alloc_new_buf_locked()
539 	binder_update_page_range(alloc, 0, (void __user *)  in binder_alloc_new_buf_locked()
561 struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc,  in binder_alloc_new_buf()  argument
570 	mutex_lock(&alloc->mutex);  in binder_alloc_new_buf()
571 	buffer = binder_alloc_new_buf_locked(alloc, data_size, offsets_size,  in binder_alloc_new_buf()
573 	mutex_unlock(&alloc->mutex);  in binder_alloc_new_buf()
588 static void binder_delete_free_buffer(struct binder_alloc *alloc,  in binder_delete_free_buffer()  argument
594 	BUG_ON(alloc->buffers.next == &buffer->entry);  in binder_delete_free_buffer()
601 				   alloc->pid, buffer->user_data,  in binder_delete_free_buffer()
605 	if (!list_is_last(&buffer->entry, &alloc->buffers)) {  in binder_delete_free_buffer()
611 					   alloc->pid,  in binder_delete_free_buffer()
620 				   alloc->pid, buffer->user_data);  in binder_delete_free_buffer()
627 				   alloc->pid, buffer->user_data,  in binder_delete_free_buffer()
630 		binder_update_page_range(alloc, 0, buffer_start_page(buffer),  in binder_delete_free_buffer()
637 static void binder_free_buf_locked(struct binder_alloc *alloc,  in binder_free_buf_locked()  argument
642 	buffer_size = binder_alloc_buffer_size(alloc, buffer);  in binder_free_buf_locked()
650 		      alloc->pid, buffer, size, buffer_size);  in binder_free_buf_locked()
655 	BUG_ON(buffer->user_data < alloc->buffer);  in binder_free_buf_locked()
656 	BUG_ON(buffer->user_data > alloc->buffer + alloc->buffer_size);  in binder_free_buf_locked()
659 		alloc->free_async_space += buffer_size + sizeof(struct binder_buffer);  in binder_free_buf_locked()
663 			      alloc->pid, size, alloc->free_async_space);  in binder_free_buf_locked()
666 	binder_update_page_range(alloc, 0,  in binder_free_buf_locked()
671 	rb_erase(&buffer->rb_node, &alloc->allocated_buffers);  in binder_free_buf_locked()
673 	if (!list_is_last(&buffer->entry, &alloc->buffers)) {  in binder_free_buf_locked()
677 			rb_erase(&next->rb_node, &alloc->free_buffers);  in binder_free_buf_locked()
678 			binder_delete_free_buffer(alloc, next);  in binder_free_buf_locked()
681 	if (alloc->buffers.next != &buffer->entry) {  in binder_free_buf_locked()
685 			binder_delete_free_buffer(alloc, buffer);  in binder_free_buf_locked()
686 			rb_erase(&prev->rb_node, &alloc->free_buffers);  in binder_free_buf_locked()
690 	binder_insert_free_buffer(alloc, buffer);  in binder_free_buf_locked()
693 static void binder_alloc_clear_buf(struct binder_alloc *alloc,
702 void binder_alloc_free_buf(struct binder_alloc *alloc,  in binder_alloc_free_buf()  argument
714 		binder_alloc_clear_buf(alloc, buffer);  in binder_alloc_free_buf()
717 	mutex_lock(&alloc->mutex);  in binder_alloc_free_buf()
718 	binder_free_buf_locked(alloc, buffer);  in binder_alloc_free_buf()
719 	mutex_unlock(&alloc->mutex);  in binder_alloc_free_buf()
735 int binder_alloc_mmap_handler(struct binder_alloc *alloc,  in binder_alloc_mmap_handler()  argument
742 	if (unlikely(vma->vm_mm != alloc->mm)) {  in binder_alloc_mmap_handler()
749 	if (alloc->buffer_size) {  in binder_alloc_mmap_handler()
754 	alloc->buffer_size = min_t(unsigned long, vma->vm_end - vma->vm_start,  in binder_alloc_mmap_handler()
758 	alloc->buffer = (void __user *)vma->vm_start;  in binder_alloc_mmap_handler()
760 	alloc->pages = kcalloc(alloc->buffer_size / PAGE_SIZE,  in binder_alloc_mmap_handler()
761 			       sizeof(alloc->pages[0]),  in binder_alloc_mmap_handler()
763 	if (alloc->pages == NULL) {  in binder_alloc_mmap_handler()
776 	buffer->user_data = alloc->buffer;  in binder_alloc_mmap_handler()
777 	list_add(&buffer->entry, &alloc->buffers);  in binder_alloc_mmap_handler()
779 	binder_insert_free_buffer(alloc, buffer);  in binder_alloc_mmap_handler()
780 	alloc->free_async_space = alloc->buffer_size / 2;  in binder_alloc_mmap_handler()
781 	alloc->vma_addr = vma->vm_start;  in binder_alloc_mmap_handler()
786 	kfree(alloc->pages);  in binder_alloc_mmap_handler()
787 	alloc->pages = NULL;  in binder_alloc_mmap_handler()
789 	alloc->buffer = NULL;  in binder_alloc_mmap_handler()
791 	alloc->buffer_size = 0;  in binder_alloc_mmap_handler()
797 			   alloc->pid, vma->vm_start, vma->vm_end,  in binder_alloc_mmap_handler()
803 void binder_alloc_deferred_release(struct binder_alloc *alloc)  in binder_alloc_deferred_release()  argument
810 	mutex_lock(&alloc->mutex);  in binder_alloc_deferred_release()
811 	BUG_ON(alloc->vma_addr &&  in binder_alloc_deferred_release()
812 	       vma_lookup(alloc->mm, alloc->vma_addr));  in binder_alloc_deferred_release()
814 	while ((n = rb_first(&alloc->allocated_buffers))) {  in binder_alloc_deferred_release()
821 			binder_alloc_clear_buf(alloc, buffer);  in binder_alloc_deferred_release()
824 		binder_free_buf_locked(alloc, buffer);  in binder_alloc_deferred_release()
828 	while (!list_empty(&alloc->buffers)) {  in binder_alloc_deferred_release()
829 		buffer = list_first_entry(&alloc->buffers,  in binder_alloc_deferred_release()
834 		WARN_ON_ONCE(!list_empty(&alloc->buffers));  in binder_alloc_deferred_release()
839 	if (alloc->pages) {  in binder_alloc_deferred_release()
842 		for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) {  in binder_alloc_deferred_release()
846 			if (!alloc->pages[i].page_ptr)  in binder_alloc_deferred_release()
850 					      &alloc->pages[i].lru);  in binder_alloc_deferred_release()
851 			page_addr = alloc->buffer + i * PAGE_SIZE;  in binder_alloc_deferred_release()
854 				     __func__, alloc->pid, i, page_addr,  in binder_alloc_deferred_release()
856 			__free_page(alloc->pages[i].page_ptr);  in binder_alloc_deferred_release()
859 		kfree(alloc->pages);  in binder_alloc_deferred_release()
861 	mutex_unlock(&alloc->mutex);  in binder_alloc_deferred_release()
862 	if (alloc->mm)  in binder_alloc_deferred_release()
863 		mmdrop(alloc->mm);  in binder_alloc_deferred_release()
867 		     __func__, alloc->pid, buffers, page_count);  in binder_alloc_deferred_release()
889 				  struct binder_alloc *alloc)  in binder_alloc_print_allocated()  argument
893 	mutex_lock(&alloc->mutex);  in binder_alloc_print_allocated()
894 	for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n))  in binder_alloc_print_allocated()
897 	mutex_unlock(&alloc->mutex);  in binder_alloc_print_allocated()
906 			      struct binder_alloc *alloc)  in binder_alloc_print_pages()  argument
914 	mutex_lock(&alloc->mutex);  in binder_alloc_print_pages()
920 	mmap_read_lock(alloc->mm);  in binder_alloc_print_pages()
921 	if (binder_alloc_get_vma(alloc) == NULL) {  in binder_alloc_print_pages()
922 		mmap_read_unlock(alloc->mm);  in binder_alloc_print_pages()
926 	mmap_read_unlock(alloc->mm);  in binder_alloc_print_pages()
927 	for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) {  in binder_alloc_print_pages()
928 		page = &alloc->pages[i];  in binder_alloc_print_pages()
938 	mutex_unlock(&alloc->mutex);  in binder_alloc_print_pages()
940 	seq_printf(m, "  pages high watermark: %zu\n", alloc->pages_high);  in binder_alloc_print_pages()
949 int binder_alloc_get_allocated_count(struct binder_alloc *alloc)  in binder_alloc_get_allocated_count()  argument
954 	mutex_lock(&alloc->mutex);  in binder_alloc_get_allocated_count()
955 	for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n))  in binder_alloc_get_allocated_count()
957 	mutex_unlock(&alloc->mutex);  in binder_alloc_get_allocated_count()
970 void binder_alloc_vma_close(struct binder_alloc *alloc)  in binder_alloc_vma_close()  argument
972 	alloc->vma_addr = 0;  in binder_alloc_vma_close()
994 	struct binder_alloc *alloc;  in binder_alloc_free_page()  local
999 	alloc = page->alloc;  in binder_alloc_free_page()
1000 	if (!mutex_trylock(&alloc->mutex))  in binder_alloc_free_page()
1006 	index = page - alloc->pages;  in binder_alloc_free_page()
1007 	page_addr = (uintptr_t)alloc->buffer + index * PAGE_SIZE;  in binder_alloc_free_page()
1009 	mm = alloc->mm;  in binder_alloc_free_page()
1014 	vma = binder_alloc_get_vma(alloc);  in binder_alloc_free_page()
1020 		trace_binder_unmap_user_start(alloc, index);  in binder_alloc_free_page()
1024 		trace_binder_unmap_user_end(alloc, index);  in binder_alloc_free_page()
1029 	trace_binder_unmap_kernel_start(alloc, index);  in binder_alloc_free_page()
1034 	trace_binder_unmap_kernel_end(alloc, index);  in binder_alloc_free_page()
1037 	mutex_unlock(&alloc->mutex);  in binder_alloc_free_page()
1044 	mutex_unlock(&alloc->mutex);  in binder_alloc_free_page()
1075 void binder_alloc_init(struct binder_alloc *alloc)  in binder_alloc_init()  argument
1077 	alloc->pid = current->group_leader->pid;  in binder_alloc_init()
1078 	alloc->mm = current->mm;  in binder_alloc_init()
1079 	mmgrab(alloc->mm);  in binder_alloc_init()
1080 	mutex_init(&alloc->mutex);  in binder_alloc_init()
1081 	INIT_LIST_HEAD(&alloc->buffers);  in binder_alloc_init()
1115 static inline bool check_buffer(struct binder_alloc *alloc,  in check_buffer()  argument
1119 	size_t buffer_size = binder_alloc_buffer_size(alloc, buffer);  in check_buffer()
1147 static struct page *binder_alloc_get_page(struct binder_alloc *alloc,  in binder_alloc_get_page()  argument
1153 		(buffer->user_data - alloc->buffer);  in binder_alloc_get_page()
1158 	lru_page = &alloc->pages[index];  in binder_alloc_get_page()
1170 static void binder_alloc_clear_buf(struct binder_alloc *alloc,  in binder_alloc_clear_buf()  argument
1173 	size_t bytes = binder_alloc_buffer_size(alloc, buffer);  in binder_alloc_clear_buf()
1181 		page = binder_alloc_get_page(alloc, buffer,  in binder_alloc_clear_buf()
1203 binder_alloc_copy_user_to_buffer(struct binder_alloc *alloc,  in binder_alloc_copy_user_to_buffer()  argument
1209 	if (!check_buffer(alloc, buffer, buffer_offset, bytes))  in binder_alloc_copy_user_to_buffer()
1219 		page = binder_alloc_get_page(alloc, buffer,  in binder_alloc_copy_user_to_buffer()
1234 static int binder_alloc_do_buffer_copy(struct binder_alloc *alloc,  in binder_alloc_do_buffer_copy()  argument
1242 	if (!check_buffer(alloc, buffer, buffer_offset, bytes))  in binder_alloc_do_buffer_copy()
1250 		page = binder_alloc_get_page(alloc, buffer,  in binder_alloc_do_buffer_copy()
1265 int binder_alloc_copy_to_buffer(struct binder_alloc *alloc,  in binder_alloc_copy_to_buffer()  argument
1271 	return binder_alloc_do_buffer_copy(alloc, true, buffer, buffer_offset,  in binder_alloc_copy_to_buffer()
1275 int binder_alloc_copy_from_buffer(struct binder_alloc *alloc,  in binder_alloc_copy_from_buffer()  argument
1281 	return binder_alloc_do_buffer_copy(alloc, false, buffer, buffer_offset,  in binder_alloc_copy_from_buffer()