Lines Matching +full:data +full:- +full:mirror
1 // SPDX-License-Identifier: GPL-2.0
4 * the linux kernel to help device drivers mirror a process address space in
6 * makes communication and data exchange a lot easier.
42 void *mirror; member
54 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1)))
86 self->page_size = sysconf(_SC_PAGE_SIZE); in FIXTURE_SETUP()
87 self->page_shift = ffs(self->page_size) - 1; in FIXTURE_SETUP()
89 self->fd = hmm_open(0); in FIXTURE_SETUP()
90 ASSERT_GE(self->fd, 0); in FIXTURE_SETUP()
95 self->page_size = sysconf(_SC_PAGE_SIZE); in FIXTURE_SETUP()
96 self->page_shift = ffs(self->page_size) - 1; in FIXTURE_SETUP()
98 self->fd0 = hmm_open(0); in FIXTURE_SETUP()
99 ASSERT_GE(self->fd0, 0); in FIXTURE_SETUP()
100 self->fd1 = hmm_open(1); in FIXTURE_SETUP()
101 ASSERT_GE(self->fd1, 0); in FIXTURE_SETUP()
106 int ret = close(self->fd); in FIXTURE_TEARDOWN()
109 self->fd = -1; in FIXTURE_TEARDOWN()
114 int ret = close(self->fd0); in FIXTURE_TEARDOWN()
117 self->fd0 = -1; in FIXTURE_TEARDOWN()
119 ret = close(self->fd1); in FIXTURE_TEARDOWN()
121 self->fd1 = -1; in FIXTURE_TEARDOWN()
133 cmd.addr = (__u64)buffer->ptr; in hmm_dmirror_cmd()
134 cmd.ptr = (__u64)buffer->mirror; in hmm_dmirror_cmd()
143 return -errno; in hmm_dmirror_cmd()
145 buffer->cpages = cmd.cpages; in hmm_dmirror_cmd()
146 buffer->faults = cmd.faults; in hmm_dmirror_cmd()
156 if (buffer->ptr) in hmm_buffer_free()
157 munmap(buffer->ptr, buffer->size); in hmm_buffer_free()
158 free(buffer->mirror); in hmm_buffer_free()
177 } while (r == -1 && errno == EINTR); in hmm_create_file()
182 return -1; in hmm_create_file()
190 static int fd = -1; in hmm_random()
234 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
236 size = npages << self->page_shift; in TEST_F()
241 buffer->fd = -1; in TEST_F()
242 buffer->size = size; in TEST_F()
243 buffer->mirror = malloc(size); in TEST_F()
244 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
246 buffer->ptr = mmap(NULL, size, in TEST_F()
249 buffer->fd, 0); in TEST_F()
250 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
256 i = 2 * self->page_size / sizeof(*ptr); in TEST_F()
257 for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
260 /* Set buffer permission to read-only. */ in TEST_F()
261 ret = mprotect(buffer->ptr, size, PROT_READ); in TEST_F()
265 val = *(int *)(buffer->ptr + self->page_size); in TEST_F()
269 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
271 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
272 ASSERT_EQ(buffer->faults, 1); in TEST_F()
275 ptr = buffer->mirror; in TEST_F()
276 for (i = 0; i < 2 * self->page_size / sizeof(*ptr); ++i) in TEST_F()
297 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
299 size = npages << self->page_shift; in TEST_F()
304 buffer->fd = -1; in TEST_F()
305 buffer->size = size; in TEST_F()
306 buffer->mirror = malloc(size); in TEST_F()
307 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
309 buffer->ptr = mmap(NULL, size, in TEST_F()
312 buffer->fd, 0); in TEST_F()
313 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
316 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
319 /* Initialize mirror buffer so we can verify it isn't written. */ in TEST_F()
320 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
321 ptr[i] = -i; in TEST_F()
324 ret = mprotect(buffer->ptr, size, PROT_NONE); in TEST_F()
328 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
329 ASSERT_EQ(ret, -EFAULT); in TEST_F()
332 ret = mprotect(buffer->ptr, size, PROT_READ); in TEST_F()
334 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
338 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
339 ASSERT_EQ(ptr[i], -i); in TEST_F()
356 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
358 size = npages << self->page_shift; in TEST_F()
363 buffer->fd = -1; in TEST_F()
364 buffer->size = size; in TEST_F()
365 buffer->mirror = malloc(size); in TEST_F()
366 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
368 buffer->ptr = mmap(NULL, size, in TEST_F()
371 buffer->fd, 0); in TEST_F()
372 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
374 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
375 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
379 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
381 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
382 ASSERT_EQ(buffer->faults, 1); in TEST_F()
385 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
404 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
406 size = npages << self->page_shift; in TEST_F()
411 buffer->fd = -1; in TEST_F()
412 buffer->size = size; in TEST_F()
413 buffer->mirror = malloc(size); in TEST_F()
414 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
416 buffer->ptr = mmap(NULL, size, in TEST_F()
419 buffer->fd, 0); in TEST_F()
420 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
423 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1); in TEST_F()
425 ASSERT_EQ(buffer->cpages, 1); in TEST_F()
426 ASSERT_EQ(buffer->faults, 1); in TEST_F()
428 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
429 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
433 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
434 ASSERT_EQ(ret, -EPERM); in TEST_F()
437 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
441 ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ); in TEST_F()
445 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
447 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
448 ASSERT_EQ(buffer->faults, 1); in TEST_F()
451 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
459 * will copy-on-write if a child process inherits the mapping.
472 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
474 size = npages << self->page_shift; in TEST_F()
479 buffer->fd = -1; in TEST_F()
480 buffer->size = size; in TEST_F()
481 buffer->mirror = malloc(size); in TEST_F()
482 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
484 buffer->ptr = mmap(NULL, size, in TEST_F()
487 buffer->fd, 0); in TEST_F()
488 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
490 /* Initialize buffer->ptr so we can tell if it is written. */ in TEST_F()
491 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
494 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
495 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
496 ptr[i] = -i; in TEST_F()
499 if (pid == -1) in TEST_F()
506 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
512 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
514 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
515 ASSERT_EQ(ptr[i], -i); in TEST_F()
517 /* The child process needs its own mirror to its own mm. */ in TEST_F()
524 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
525 ASSERT_EQ(buffer->faults, 1); in TEST_F()
528 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
529 ASSERT_EQ(ptr[i], -i); in TEST_F()
537 * will not copy-on-write if a child process inherits the mapping.
550 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
552 size = npages << self->page_shift; in TEST_F()
557 buffer->fd = -1; in TEST_F()
558 buffer->size = size; in TEST_F()
559 buffer->mirror = malloc(size); in TEST_F()
560 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
562 buffer->ptr = mmap(NULL, size, in TEST_F()
565 buffer->fd, 0); in TEST_F()
566 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
568 /* Initialize buffer->ptr so we can tell if it is written. */ in TEST_F()
569 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
572 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
573 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
574 ptr[i] = -i; in TEST_F()
577 if (pid == -1) in TEST_F()
584 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
585 ASSERT_EQ(ptr[i], -i); in TEST_F()
590 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
592 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
593 ASSERT_EQ(ptr[i], -i); in TEST_F()
595 /* The child process needs its own mirror to its own mm. */ in TEST_F()
602 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
603 ASSERT_EQ(buffer->faults, 1); in TEST_F()
606 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
607 ASSERT_EQ(ptr[i], -i); in TEST_F()
632 buffer->fd = -1; in TEST_F()
633 buffer->size = size; in TEST_F()
634 buffer->mirror = malloc(size); in TEST_F()
635 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
637 buffer->ptr = mmap(NULL, size, in TEST_F()
640 buffer->fd, 0); in TEST_F()
641 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
644 npages = size >> self->page_shift; in TEST_F()
645 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); in TEST_F()
648 old_ptr = buffer->ptr; in TEST_F()
649 buffer->ptr = map; in TEST_F()
651 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
652 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
656 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
658 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
659 ASSERT_EQ(buffer->faults, 1); in TEST_F()
662 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
665 buffer->ptr = old_ptr; in TEST_F()
689 for (idx = 0; --n > 0; ) { in TEST_F()
694 npages = size >> self->page_shift; in TEST_F()
699 buffer->ptr = get_hugepage_region(size, GHR_STRICT); in TEST_F()
700 if (buffer->ptr == NULL) { in TEST_F()
705 buffer->fd = -1; in TEST_F()
706 buffer->size = size; in TEST_F()
707 buffer->mirror = malloc(size); in TEST_F()
708 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
710 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
711 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
715 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
717 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
718 ASSERT_EQ(buffer->faults, 1); in TEST_F()
721 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
724 free_hugepage_region(buffer->ptr); in TEST_F()
725 buffer->ptr = NULL; in TEST_F()
744 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
746 size = npages << self->page_shift; in TEST_F()
754 buffer->fd = fd; in TEST_F()
755 buffer->size = size; in TEST_F()
756 buffer->mirror = malloc(size); in TEST_F()
757 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
760 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
762 len = pwrite(fd, buffer->mirror, size, 0); in TEST_F()
764 memset(buffer->mirror, 0, size); in TEST_F()
766 buffer->ptr = mmap(NULL, size, in TEST_F()
769 buffer->fd, 0); in TEST_F()
770 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
773 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
775 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
776 ASSERT_EQ(buffer->faults, 1); in TEST_F()
779 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
799 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
801 size = npages << self->page_shift; in TEST_F()
809 buffer->fd = fd; in TEST_F()
810 buffer->size = size; in TEST_F()
811 buffer->mirror = malloc(size); in TEST_F()
812 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
814 buffer->ptr = mmap(NULL, size, in TEST_F()
817 buffer->fd, 0); in TEST_F()
818 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
820 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
821 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
825 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
827 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
828 ASSERT_EQ(buffer->faults, 1); in TEST_F()
831 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
835 len = pread(fd, buffer->mirror, size, 0); in TEST_F()
837 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
855 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
857 size = npages << self->page_shift; in TEST_F()
862 buffer->fd = -1; in TEST_F()
863 buffer->size = size; in TEST_F()
864 buffer->mirror = malloc(size); in TEST_F()
865 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
867 buffer->ptr = mmap(NULL, size, in TEST_F()
870 buffer->fd, 0); in TEST_F()
871 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
874 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
878 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, npages); in TEST_F()
880 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
883 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
903 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
905 size = npages << self->page_shift; in TEST_F()
910 buffer->fd = -1; in TEST_F()
911 buffer->size = size; in TEST_F()
912 buffer->mirror = malloc(size); in TEST_F()
913 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
915 buffer->ptr = mmap(NULL, size, in TEST_F()
918 buffer->fd, 0); in TEST_F()
919 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
922 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
926 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, npages); in TEST_F()
928 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
931 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
935 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) in TEST_F()
939 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, npages); in TEST_F()
941 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
944 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
960 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
962 size = npages << self->page_shift; in TEST_F()
967 buffer->fd = -1; in TEST_F()
968 buffer->size = size; in TEST_F()
969 buffer->mirror = malloc(size); in TEST_F()
970 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
972 buffer->ptr = mmap(NULL, size, in TEST_F()
975 buffer->fd, 0); in TEST_F()
976 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
979 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, npages); in TEST_F()
980 ASSERT_EQ(ret, -ENOENT); in TEST_F()
999 size = npages << self->page_shift; in TEST_F()
1004 buffer->fd = -1; in TEST_F()
1005 buffer->size = size; in TEST_F()
1006 buffer->mirror = malloc(size); in TEST_F()
1007 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1010 buffer->ptr = mmap(NULL, size, in TEST_F()
1013 buffer->fd, 0); in TEST_F()
1014 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1015 p = buffer->ptr; in TEST_F()
1018 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_MIGRATE, buffer, npages); in TEST_F()
1019 ASSERT_EQ(ret, -EINVAL); in TEST_F()
1022 ret = munmap(buffer->ptr + self->page_size, self->page_size); in TEST_F()
1026 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_MIGRATE, buffer, 3); in TEST_F()
1027 ASSERT_EQ(ret, -EINVAL); in TEST_F()
1029 /* Page 2 will be a read-only zero page. */ in TEST_F()
1030 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, in TEST_F()
1033 ptr = (int *)(buffer->ptr + 2 * self->page_size); in TEST_F()
1037 /* Page 3 will be read-only. */ in TEST_F()
1038 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, in TEST_F()
1041 ptr = (int *)(buffer->ptr + 3 * self->page_size); in TEST_F()
1043 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, in TEST_F()
1047 /* Page 4-5 will be read-write. */ in TEST_F()
1048 ret = mprotect(buffer->ptr + 4 * self->page_size, 2 * self->page_size, in TEST_F()
1051 ptr = (int *)(buffer->ptr + 4 * self->page_size); in TEST_F()
1053 ptr = (int *)(buffer->ptr + 5 * self->page_size); in TEST_F()
1056 /* Now try to migrate pages 2-5 to device 1. */ in TEST_F()
1057 buffer->ptr = p + 2 * self->page_size; in TEST_F()
1058 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_MIGRATE, buffer, 4); in TEST_F()
1060 ASSERT_EQ(buffer->cpages, 4); in TEST_F()
1063 buffer->ptr = p + 5 * self->page_size; in TEST_F()
1064 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_MIGRATE, buffer, 1); in TEST_F()
1065 ASSERT_EQ(ret, -ENOENT); in TEST_F()
1066 buffer->ptr = p; in TEST_F()
1068 buffer->ptr = p; in TEST_F()
1086 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1088 size = npages << self->page_shift; in TEST_F()
1094 buffer->fd = -1; in TEST_F()
1095 buffer->size = size; in TEST_F()
1096 buffer->mirror = malloc(size); in TEST_F()
1097 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1099 buffer->ptr = mmap(NULL, size, in TEST_F()
1102 buffer->fd, 0); in TEST_F()
1103 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1106 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1110 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, in TEST_F()
1113 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1116 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1120 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1140 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1142 size = npages << self->page_shift; in TEST_F()
1148 buffer->fd = -1; in TEST_F()
1149 buffer->size = size; in TEST_F()
1150 buffer->mirror = malloc(size); in TEST_F()
1151 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1153 buffer->ptr = mmap(NULL, size, in TEST_F()
1156 buffer->fd, 0); in TEST_F()
1157 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1160 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1164 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, in TEST_F()
1167 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1168 ASSERT_EQ(buffer->faults, 1); in TEST_F()
1171 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1184 munmap(buffer->ptr + buffer->size / 2, buffer->size / 2); in unmap_buffer()
1185 buffer->ptr = NULL; in unmap_buffer()
1200 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1202 size = npages << self->page_shift; in TEST_F()
1214 buffer->fd = -1; in TEST_F()
1215 buffer->size = size; in TEST_F()
1216 buffer->mirror = malloc(size); in TEST_F()
1217 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1219 buffer->ptr = mmap(NULL, size, in TEST_F()
1222 buffer->fd, 0); in TEST_F()
1223 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1226 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1233 rc = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, in TEST_F()
1236 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1237 ASSERT_EQ(buffer->faults, 1); in TEST_F()
1240 for (i = 0, ptr = buffer->mirror; in TEST_F()
1266 size = npages << self->page_shift; in TEST_F()
1271 buffer->fd = -1; in TEST_F()
1272 buffer->size = size; in TEST_F()
1273 buffer->mirror = malloc(npages); in TEST_F()
1274 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1277 buffer->ptr = mmap(NULL, size, in TEST_F()
1280 buffer->fd, 0); in TEST_F()
1281 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1282 p = buffer->ptr; in TEST_F()
1285 ret = munmap(buffer->ptr + self->page_size, self->page_size); in TEST_F()
1288 /* Page 2 will be read-only zero page. */ in TEST_F()
1289 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, in TEST_F()
1292 ptr = (int *)(buffer->ptr + 2 * self->page_size); in TEST_F()
1296 /* Page 3 will be read-only. */ in TEST_F()
1297 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, in TEST_F()
1300 ptr = (int *)(buffer->ptr + 3 * self->page_size); in TEST_F()
1302 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, in TEST_F()
1306 /* Page 4-6 will be read-write. */ in TEST_F()
1307 ret = mprotect(buffer->ptr + 4 * self->page_size, 3 * self->page_size, in TEST_F()
1310 ptr = (int *)(buffer->ptr + 4 * self->page_size); in TEST_F()
1314 buffer->ptr = p + 5 * self->page_size; in TEST_F()
1315 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_MIGRATE, buffer, 1); in TEST_F()
1317 ASSERT_EQ(buffer->cpages, 1); in TEST_F()
1320 buffer->ptr = p + 6 * self->page_size; in TEST_F()
1321 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_MIGRATE, buffer, 1); in TEST_F()
1323 ASSERT_EQ(buffer->cpages, 1); in TEST_F()
1326 buffer->ptr = p; in TEST_F()
1327 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_SNAPSHOT, buffer, npages); in TEST_F()
1329 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1332 m = buffer->mirror; in TEST_F()
1367 for (idx = 0; --n > 0; ) { in TEST_F()
1372 npages = size >> self->page_shift; in TEST_F()
1377 buffer->ptr = get_hugepage_region(size, GHR_STRICT); in TEST_F()
1378 if (buffer->ptr == NULL) { in TEST_F()
1383 buffer->size = size; in TEST_F()
1384 buffer->mirror = malloc(npages); in TEST_F()
1385 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1387 /* Initialize the pages the device will snapshot in buffer->ptr. */ in TEST_F()
1388 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1392 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); in TEST_F()
1394 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1397 m = buffer->mirror; in TEST_F()
1402 /* Make the region read-only. */ in TEST_F()
1403 ret = mprotect(buffer->ptr, size, PROT_READ); in TEST_F()
1407 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); in TEST_F()
1409 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1412 m = buffer->mirror; in TEST_F()
1417 free_hugepage_region(buffer->ptr); in TEST_F()
1418 buffer->ptr = NULL; in TEST_F()
1436 size = npages << self->page_shift; in TEST_F()
1441 buffer->fd = -1; in TEST_F()
1442 buffer->size = size; in TEST_F()
1443 buffer->mirror = malloc(npages); in TEST_F()
1444 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1447 buffer->ptr = mmap(NULL, size, in TEST_F()
1450 buffer->fd, 0); in TEST_F()
1451 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1454 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1457 /* Make region read-only. */ in TEST_F()
1458 ret = mprotect(buffer->ptr, size, PROT_READ); in TEST_F()
1462 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
1464 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1465 ASSERT_EQ(buffer->faults, 1); in TEST_F()
1468 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1472 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
1474 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1475 ASSERT_EQ(buffer->faults, 1); in TEST_F()
1478 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1482 ret = munmap(buffer->ptr + self->page_size, self->page_size); in TEST_F()
1500 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1502 size = npages << self->page_shift; in TEST_F()
1507 buffer->fd = -1; in TEST_F()
1508 buffer->size = size; in TEST_F()
1509 buffer->mirror = malloc(size); in TEST_F()
1510 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1512 buffer->ptr = mmap(NULL, size, in TEST_F()
1515 buffer->fd, 0); in TEST_F()
1516 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1519 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1523 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); in TEST_F()
1525 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1528 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1532 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1535 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1539 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_CHECK_EXCLUSIVE, buffer, npages); in TEST_F()
1554 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1556 size = npages << self->page_shift; in TEST_F()
1561 buffer->fd = -1; in TEST_F()
1562 buffer->size = size; in TEST_F()
1563 buffer->mirror = malloc(size); in TEST_F()
1564 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1566 buffer->ptr = mmap(NULL, size, in TEST_F()
1569 buffer->fd, 0); in TEST_F()
1570 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1573 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1577 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); in TEST_F()
1579 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1582 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1585 ret = mprotect(buffer->ptr, size, PROT_READ); in TEST_F()
1589 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
1590 ASSERT_EQ(ret, -EPERM); in TEST_F()
1596 * Check copy-on-write works.
1607 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1609 size = npages << self->page_shift; in TEST_F()
1614 buffer->fd = -1; in TEST_F()
1615 buffer->size = size; in TEST_F()
1616 buffer->mirror = malloc(size); in TEST_F()
1617 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1619 buffer->ptr = mmap(NULL, size, in TEST_F()
1622 buffer->fd, 0); in TEST_F()
1623 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1626 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1630 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages); in TEST_F()
1632 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1637 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1640 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()