1 // SPDX-License-Identifier: GPL-2.0-only
13  * 1) one per-CPU thread takes a per-page pthread_mutex in a random
15  *    area_src), and increments a per-page counter in the same page,
18  * 2) another per-CPU thread handles the userfaults generated by
22  * 3) one last per-CPU thread transfers the memory in the background
32  * per-CPU threads 1 by triggering userfaults inside
92 /* Whether to test uffd write-protection */
101 static int uffd = -1;
121  * to avoid non alignment faults on non-x86 archs.
127 				  sizeof(unsigned long long) - 1) &	\
129 						  -  1)))
134 #define factor_of_2(x) ((x) ^ ((x) & ((x) - 1)))
147     "# 10MiB-~6GiB 999 bounces anonymous test, "
159 	fprintf(stderr, "\tsyscall - Use userfaultfd(2) (default)\n");  in usage()
160 	fprintf(stderr, "\tdev - Use /dev/userfaultfd instead of userfaultfd(2)\n");  in usage()
161 	fprintf(stderr, "\tcollapse - Test MADV_COLLAPSE of UFFDIO_REGISTER_MODE_MINOR\n"  in usage()
243 			   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);  in anon_allocate_area()
272 			-1,  in hugetlb_allocate_area()
447 	close(fd);  in __userfaultfd_open_dev()
491 			if (close(pipefd[i]))  in uffd_test_ctx_clear()
492 				err("close pipefd");  in uffd_test_ctx_clear()
503 	if (uffd != -1) {  in uffd_test_ctx_clear()
504 		if (close(uffd))  in uffd_test_ctx_clear()
505 			err("close uffd");  in uffd_test_ctx_clear()
506 		uffd = -1;  in uffd_test_ctx_clear()
522 	uffd_test_ops->allocate_area((void **)&area_src, true);  in uffd_test_ctx_init()
523 	uffd_test_ops->allocate_area((void **)&area_dst, false);  in uffd_test_ctx_init()
538 		 * zero, so leave a placeholder below always non-zero  in uffd_test_ctx_init()
563 	uffd_test_ops->release_pages(area_dst);  in uffd_test_ctx_init()
587 	prms.range.start = start;  in wp_range()
588 	prms.range.len = len;  in wp_range()
589 	/* Undo write-protect, do wakeup after that */  in wp_range()
601 	req.range.start = start;  in continue_range()
602 	req.range.len = len;  in continue_range()
612 	 * error (-EEXIST) on purpose, to verify doing so doesn't cause a BUG.  in continue_range()
616 	if (ret >= 0 || req.mapped != -EEXIST)  in continue_range()
628 		page_nr = -bounces;  in locking_thread()
656 	uffd_test_ops->alias_mapping(&uffdio_copy->dst,  in retry_copy_page()
657 				     uffdio_copy->len,  in retry_copy_page()
661 		if (uffdio_copy->copy != -EEXIST)  in retry_copy_page()
663 			    (int64_t)uffdio_copy->copy);  in retry_copy_page()
666 		    (int64_t)uffdio_copy->copy);  in retry_copy_page()
698 		if (uffdio_copy.copy != -EEXIST)  in __copy_page()
746 	if (msg->event != UFFD_EVENT_PAGEFAULT)  in uffd_handle_page_fault()
747 		err("unexpected msg event %u", msg->event);  in uffd_handle_page_fault()
749 	if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) {  in uffd_handle_page_fault()
751 		wp_range(uffd, msg->arg.pagefault.address, page_size, false);  in uffd_handle_page_fault()
752 		stats->wp_faults++;  in uffd_handle_page_fault()
753 	} else if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_MINOR) {  in uffd_handle_page_fault()
760 		 * To prove we can modify the original range for testing  in uffd_handle_page_fault()
761 		 * purposes, we're going to bit flip this range before  in uffd_handle_page_fault()
765 		 * area_dst (non-UFFD-registered) and area_dst_alias  in uffd_handle_page_fault()
766 		 * (UFFD-registered).  in uffd_handle_page_fault()
770 				   ((char *)msg->arg.pagefault.address -  in uffd_handle_page_fault()
774 		continue_range(uffd, msg->arg.pagefault.address, page_size);  in uffd_handle_page_fault()
775 		stats->minor_faults++;  in uffd_handle_page_fault()
791 		 * logically a pthread-compatible lib can implement the  in uffd_handle_page_fault()
798 		if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)  in uffd_handle_page_fault()
801 		offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;  in uffd_handle_page_fault()
802 		offset &= ~(page_size-1);  in uffd_handle_page_fault()
805 			stats->missing_faults++;  in uffd_handle_page_fault()
812 	unsigned long cpu = stats->cpu;  in uffd_poll_thread()
825 		ret = poll(pollfd, 2, -1);  in uffd_poll_thread()
848 			close(uffd);  in uffd_poll_thread()
853 			uffd_reg.range.start = msg.arg.remove.start;  in uffd_poll_thread()
854 			uffd_reg.range.len = msg.arg.remove.end -  in uffd_poll_thread()
856 			if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range))  in uffd_poll_thread()
902 	 * If we need to test uffd-wp, set it up now.  Then we'll have  in background_thread()
904 	 * can be write-protected for testing  in background_thread()
957 	 * area_src (but they're guaranteed to get -EEXIST from  in stress()
961 	uffd_test_ops->release_pages(area_src);  in stress()
999  * For non-cooperative userfaultfd test we fork() a process that will
1014  * test robustness use case - we release monitored area, fork a process
1037 		lastnr = (unsigned long)-1;  in faulting_process()
1105 	uffd_test_ops->release_pages(area_dst);  in faulting_process()
1118 	uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,  in retry_uffdio_zeropage()
1119 				     uffdio_zeropage->range.len,  in retry_uffdio_zeropage()
1122 		if (uffdio_zeropage->zeropage != -EEXIST)  in retry_uffdio_zeropage()
1124 			    (int64_t)uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
1127 		    (int64_t)uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
1140 	uffdio_zeropage.range.start = (unsigned long) area_dst + offset;  in __uffdio_zeropage()
1141 	uffdio_zeropage.range.len = page_size;  in __uffdio_zeropage()
1149 		else if (res != -EINVAL)  in __uffdio_zeropage()
1150 			err("UFFDIO_ZEROPAGE not -EINVAL");  in __uffdio_zeropage()
1183 	uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_zeropage_test()
1184 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_zeropage_test()
1220 	uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_events_test()
1221 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_events_test()
1272 	uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_sig_test()
1273 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_sig_test()
1286 	uffd_test_ops->release_pages(area_dst);  in userfaultfd_sig_test()
1323 		expected_byte = ~((uint8_t)(i % ((uint8_t)-1)));  in check_memory_contents()
1348 	uffdio_register.range.start = (unsigned long)area_dst_alias;  in userfaultfd_minor_test()
1349 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_minor_test()
1358 	 * After registering with UFFD, populate the non-UFFD-registered side of  in userfaultfd_minor_test()
1362 		memset(area_dst + (p * page_size), p % ((uint8_t)-1),  in userfaultfd_minor_test()
1370 	 * Read each of the pages back using the UFFD-registered mapping. We  in userfaultfd_minor_test()
1372 	 * fault. uffd_poll_thread will resolve the fault by bit-flipping the  in userfaultfd_minor_test()
1385 		printf("testing collapse of uffd memory into PMD-mapped THPs:");  in userfaultfd_minor_test()
1390 		uffd_test_ops->check_pmd_mapping(area_dst,  in userfaultfd_minor_test()
1394 		 * This won't cause uffd-fault - it purely just makes sure there  in userfaultfd_minor_test()
1438 			err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \
1452 		 * After fork() uffd-wp bit should be gone as long as we're  in pagemap_test_fork()
1469 	/* Pagemap tests uffd-wp only */  in userfaultfd_pagemap_test()
1477 	printf("testing uffd-wp with pagemap (pgsize=%u): ", test_pgsize);  in userfaultfd_pagemap_test()
1493 	uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_pagemap_test()
1494 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_pagemap_test()
1506 	/* Make sure uffd-wp bit dropped when fork */  in userfaultfd_pagemap_test()
1508 		err("Detected stall uffd-wp bit in child");  in userfaultfd_pagemap_test()
1517 	/* Uffd-wp should persist even swapped out */  in userfaultfd_pagemap_test()
1520 	/* Make sure uffd-wp bit dropped when fork */  in userfaultfd_pagemap_test()
1522 		err("Detected stall uffd-wp bit in child");  in userfaultfd_pagemap_test()
1534 	close(pagemap_fd);  in userfaultfd_pagemap_test()
1557 	while (bounces--) {  in userfaultfd_stress()
1578 		uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_stress()
1579 		uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_stress()
1589 			uffdio_register.range.start = (unsigned long)  in userfaultfd_stress()
1603 		 * return -EEXIST). The problem comes at the next  in userfaultfd_stress()
1608 		 * area_src would lead to -EEXIST failure during the  in userfaultfd_stress()
1619 		uffd_test_ops->release_pages(area_dst);  in userfaultfd_stress()
1633 		if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range))  in userfaultfd_stress()
1636 			uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_stress()
1638 				  &uffdio_register.range))  in userfaultfd_stress()
1661 		 * behavior on fork() (file-backed memory normally drops ptes  in userfaultfd_stress()
1667 		 * Hard-code for x86_64 for now for 2M THP, as x86_64 is  in userfaultfd_stress()
1668 		 * currently the only one that supports uffd-wp  in userfaultfd_stress()
1678  * Copied from mlock2-tests.c
1773 	close(uffd);  in parse_test_type_arg()
1774 	uffd = -1;  in parse_test_type_arg()
1801 	if (test_collapse && bytes & (hpage_size - 1))  in main()
1813 		 * This means that nr_cpus must divide (N * (2 << (H-P))  in main()