1 // SPDX-License-Identifier: GPL-2.0-only
13  * 1) one per-CPU thread takes a per-page pthread_mutex in a random
15  *    area_src), and increments a per-page counter in the same page,
18  * 2) another per-CPU thread handles the userfaults generated by
22  * 3) one last per-CPU thread transfers the memory in the background
32  * per-CPU threads 1 by triggering userfaults inside
82 /* Whether to test uffd write-protection */
92 static int uffd = -1;
111  * to avoid non alignment faults on non-x86 archs.
117 				  sizeof(unsigned long long) - 1) &	\
119 						  -  1)))
130     "# 10MiB-~6GiB 999 bounces anonymous test, "
214 			   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);  in anon_allocate_area()
269 	 * MADV_DONTEED won't work. So exercise -EEXIST on a alias  in hugetlb_alias_mapping()
271 	 * this way we'll exercise the -EEXEC at the fs level.  in hugetlb_alias_mapping()
383 			if (close(pipefd[i]))  in uffd_test_ctx_clear()
384 				err("close pipefd");  in uffd_test_ctx_clear()
395 	if (uffd != -1) {  in uffd_test_ctx_clear()
396 		if (close(uffd))  in uffd_test_ctx_clear()
397 			err("close uffd");  in uffd_test_ctx_clear()
398 		uffd = -1;  in uffd_test_ctx_clear()
414 	uffd_test_ops->allocate_area((void **)&area_src);  in uffd_test_ctx_init_ext()
415 	uffd_test_ops->allocate_area((void **)&area_dst);  in uffd_test_ctx_init_ext()
430 		 * zero, so leave a placeholder below always non-zero  in uffd_test_ctx_init_ext()
455 	uffd_test_ops->release_pages(area_dst);  in uffd_test_ctx_init_ext()
484 	prms.range.start = start;  in wp_range()
485 	prms.range.len = len;  in wp_range()
486 	/* Undo write-protect, do wakeup after that */  in wp_range()
498 	req.range.start = start;  in continue_range()
499 	req.range.len = len;  in continue_range()
509 	 * error (-EEXIST) on purpose, to verify doing so doesn't cause a BUG.  in continue_range()
513 	if (ret >= 0 || req.mapped != -EEXIST)  in continue_range()
529 		seed = (unsigned int) time(NULL) - bounces;  in locking_thread()
537 		page_nr = -bounces;  in locking_thread()
572 	uffd_test_ops->alias_mapping(&uffdio_copy->dst,  in retry_copy_page()
573 				     uffdio_copy->len,  in retry_copy_page()
577 		if (uffdio_copy->copy != -EEXIST)  in retry_copy_page()
579 			    (int64_t)uffdio_copy->copy);  in retry_copy_page()
582 		    (int64_t)uffdio_copy->copy);  in retry_copy_page()
614 		if (uffdio_copy.copy != -EEXIST)  in __copy_page()
662 	if (msg->event != UFFD_EVENT_PAGEFAULT)  in uffd_handle_page_fault()
663 		err("unexpected msg event %u", msg->event);  in uffd_handle_page_fault()
665 	if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) {  in uffd_handle_page_fault()
667 		wp_range(uffd, msg->arg.pagefault.address, page_size, false);  in uffd_handle_page_fault()
668 		stats->wp_faults++;  in uffd_handle_page_fault()
669 	} else if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_MINOR) {  in uffd_handle_page_fault()
676 		 * To prove we can modify the original range for testing  in uffd_handle_page_fault()
677 		 * purposes, we're going to bit flip this range before  in uffd_handle_page_fault()
681 		 * area_dst (non-UFFD-registered) and area_dst_alias  in uffd_handle_page_fault()
682 		 * (UFFD-registered).  in uffd_handle_page_fault()
686 				   ((char *)msg->arg.pagefault.address -  in uffd_handle_page_fault()
690 		continue_range(uffd, msg->arg.pagefault.address, page_size);  in uffd_handle_page_fault()
691 		stats->minor_faults++;  in uffd_handle_page_fault()
694 		if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)  in uffd_handle_page_fault()
697 		offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;  in uffd_handle_page_fault()
698 		offset &= ~(page_size-1);  in uffd_handle_page_fault()
701 			stats->missing_faults++;  in uffd_handle_page_fault()
708 	unsigned long cpu = stats->cpu;  in uffd_poll_thread()
721 		ret = poll(pollfd, 2, -1);  in uffd_poll_thread()
741 			close(uffd);  in uffd_poll_thread()
746 			uffd_reg.range.start = msg.arg.remove.start;  in uffd_poll_thread()
747 			uffd_reg.range.len = msg.arg.remove.end -  in uffd_poll_thread()
749 			if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range))  in uffd_poll_thread()
794 	 * If we need to test uffd-wp, set it up now.  Then we'll have  in background_thread()
796 	 * can be write-protected for testing  in background_thread()
849 	 * area_src (but they're guaranteed to get -EEXIST from  in stress()
853 	uffd_test_ops->release_pages(area_src);  in stress()
891  * For non-cooperative userfaultfd test we fork() a process that will
906  * test robustness use case - we release monitored area, fork a process
932 		lastnr = (unsigned long)-1;  in faulting_process()
1003 	uffd_test_ops->release_pages(area_dst);  in faulting_process()
1016 	uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,  in retry_uffdio_zeropage()
1017 				     uffdio_zeropage->range.len,  in retry_uffdio_zeropage()
1020 		if (uffdio_zeropage->zeropage != -EEXIST)  in retry_uffdio_zeropage()
1022 			    (int64_t)uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
1025 		    (int64_t)uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
1036 	has_zeropage = uffd_test_ops->expected_ioctls & (1 << _UFFDIO_ZEROPAGE);  in __uffdio_zeropage()
1040 	uffdio_zeropage.range.start = (unsigned long) area_dst + offset;  in __uffdio_zeropage()
1041 	uffdio_zeropage.range.len = page_size;  in __uffdio_zeropage()
1049 		else if (res != -EINVAL)  in __uffdio_zeropage()
1050 			err("UFFDIO_ZEROPAGE not -EINVAL");  in __uffdio_zeropage()
1084 	uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_zeropage_test()
1085 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_zeropage_test()
1092 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_zeropage_test()
1123 	uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_events_test()
1124 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_events_test()
1131 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_events_test()
1177 	uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_sig_test()
1178 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_sig_test()
1185 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_sig_test()
1192 	uffd_test_ops->release_pages(area_dst);  in userfaultfd_sig_test()
1253 	uffdio_register.range.start = (unsigned long)area_dst_alias;  in userfaultfd_minor_test()
1254 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_minor_test()
1259 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_minor_test()
1265 	 * After registering with UFFD, populate the non-UFFD-registered side of  in userfaultfd_minor_test()
1269 		memset(area_dst + (p * page_size), p % ((uint8_t)-1),  in userfaultfd_minor_test()
1277 	 * Read each of the pages back using the UFFD-registered mapping. We  in userfaultfd_minor_test()
1279 	 * fault. uffd_poll_thread will resolve the fault by bit-flipping the  in userfaultfd_minor_test()
1287 		expected_byte = ~((uint8_t)(p % ((uint8_t)-1)));  in userfaultfd_minor_test()
1338 			err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
1352 		 * After fork() uffd-wp bit should be gone as long as we're  in pagemap_test_fork()
1369 	/* Pagemap tests uffd-wp only */  in userfaultfd_pagemap_test()
1377 	printf("testing uffd-wp with pagemap (pgsize=%u): ", test_pgsize);  in userfaultfd_pagemap_test()
1393 	uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_pagemap_test()
1394 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_pagemap_test()
1406 	/* Make sure uffd-wp bit dropped when fork */  in userfaultfd_pagemap_test()
1408 		err("Detected stall uffd-wp bit in child");  in userfaultfd_pagemap_test()
1417 	/* Uffd-wp should persist even swapped out */  in userfaultfd_pagemap_test()
1420 	/* Make sure uffd-wp bit dropped when fork */  in userfaultfd_pagemap_test()
1422 		err("Detected stall uffd-wp bit in child");  in userfaultfd_pagemap_test()
1434 	close(pagemap_fd);  in userfaultfd_pagemap_test()
1458 	while (bounces--) {  in userfaultfd_stress()
1481 		uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_stress()
1482 		uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_stress()
1488 		expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_stress()
1494 			uffdio_register.range.start = (unsigned long)  in userfaultfd_stress()
1508 		 * return -EEXIST). The problem comes at the next  in userfaultfd_stress()
1513 		 * area_src would lead to -EEXIST failure during the  in userfaultfd_stress()
1524 		uffd_test_ops->release_pages(area_dst);  in userfaultfd_stress()
1538 		if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range))  in userfaultfd_stress()
1541 			uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_stress()
1543 				  &uffdio_register.range))  in userfaultfd_stress()
1570 		 * behavior on fork() (file-backed memory normally drops ptes  in userfaultfd_stress()
1576 		 * Hard-code for x86_64 for now for 2M THP, as x86_64 is  in userfaultfd_stress()
1577 		 * currently the only one that supports uffd-wp  in userfaultfd_stress()
1587  * Copied from mlock2-tests.c
1615 		/* Only enable write-protect test for anonymous test */  in set_test_type()