1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright 2020 Google LLC
4 */
5 #define _GNU_SOURCE
6
7 #include <errno.h>
8 #include <stdlib.h>
9 #include <string.h>
10 #include <sys/mman.h>
11 #include <time.h>
12
13 #include "../kselftest.h"
14
15 #define EXPECT_SUCCESS 0
16 #define EXPECT_FAILURE 1
17 #define NON_OVERLAPPING 0
18 #define OVERLAPPING 1
19 #define NS_PER_SEC 1000000000ULL
20 #define VALIDATION_DEFAULT_THRESHOLD 4 /* 4MB */
21 #define VALIDATION_NO_THRESHOLD 0 /* Verify the entire region */
22
23 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
24 #define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
25
26 struct config {
27 unsigned long long src_alignment;
28 unsigned long long dest_alignment;
29 unsigned long long region_size;
30 int overlapping;
31 };
32
33 struct test {
34 const char *name;
35 struct config config;
36 int expect_failure;
37 };
38
39 enum {
40 _1KB = 1ULL << 10, /* 1KB -> not page aligned */
41 _4KB = 4ULL << 10,
42 _8KB = 8ULL << 10,
43 _1MB = 1ULL << 20,
44 _2MB = 2ULL << 20,
45 _4MB = 4ULL << 20,
46 _1GB = 1ULL << 30,
47 _2GB = 2ULL << 30,
48 PMD = _2MB,
49 PUD = _1GB,
50 };
51
52 #define PTE page_size
53
54 #define MAKE_TEST(source_align, destination_align, size, \
55 overlaps, should_fail, test_name) \
56 (struct test){ \
57 .name = test_name, \
58 .config = { \
59 .src_alignment = source_align, \
60 .dest_alignment = destination_align, \
61 .region_size = size, \
62 .overlapping = overlaps, \
63 }, \
64 .expect_failure = should_fail \
65 }
66
67 /*
68 * Returns the start address of the mapping on success, else returns
69 * NULL on failure.
70 */
get_source_mapping(struct config c)71 static void *get_source_mapping(struct config c)
72 {
73 unsigned long long addr = 0ULL;
74 void *src_addr = NULL;
75 retry:
76 addr += c.src_alignment;
77 src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE,
78 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
79 -1, 0);
80 if (src_addr == MAP_FAILED) {
81 if (errno == EPERM || errno == EEXIST)
82 goto retry;
83 goto error;
84 }
85 /*
86 * Check that the address is aligned to the specified alignment.
87 * Addresses which have alignments that are multiples of that
88 * specified are not considered valid. For instance, 1GB address is
89 * 2MB-aligned, however it will not be considered valid for a
90 * requested alignment of 2MB. This is done to reduce coincidental
91 * alignment in the tests.
92 */
93 if (((unsigned long long) src_addr & (c.src_alignment - 1)) ||
94 !((unsigned long long) src_addr & c.src_alignment))
95 goto retry;
96
97 if (!src_addr)
98 goto error;
99
100 return src_addr;
101 error:
102 ksft_print_msg("Failed to map source region: %s\n",
103 strerror(errno));
104 return NULL;
105 }
106
107 /* Returns the time taken for the remap on success else returns -1. */
remap_region(struct config c,unsigned int threshold_mb,char pattern_seed)108 static long long remap_region(struct config c, unsigned int threshold_mb,
109 char pattern_seed)
110 {
111 void *addr, *src_addr, *dest_addr;
112 unsigned long long i;
113 struct timespec t_start = {0, 0}, t_end = {0, 0};
114 long long start_ns, end_ns, align_mask, ret, offset;
115 unsigned long long threshold;
116
117 if (threshold_mb == VALIDATION_NO_THRESHOLD)
118 threshold = c.region_size;
119 else
120 threshold = MIN(threshold_mb * _1MB, c.region_size);
121
122 src_addr = get_source_mapping(c);
123 if (!src_addr) {
124 ret = -1;
125 goto out;
126 }
127
128 /* Set byte pattern */
129 srand(pattern_seed);
130 for (i = 0; i < threshold; i++)
131 memset((char *) src_addr + i, (char) rand(), 1);
132
133 /* Mask to zero out lower bits of address for alignment */
134 align_mask = ~(c.dest_alignment - 1);
135 /* Offset of destination address from the end of the source region */
136 offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment;
137 addr = (void *) (((unsigned long long) src_addr + c.region_size
138 + offset) & align_mask);
139
140 /* See comment in get_source_mapping() */
141 if (!((unsigned long long) addr & c.dest_alignment))
142 addr = (void *) ((unsigned long long) addr | c.dest_alignment);
143
144 clock_gettime(CLOCK_MONOTONIC, &t_start);
145 dest_addr = mremap(src_addr, c.region_size, c.region_size,
146 MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr);
147 clock_gettime(CLOCK_MONOTONIC, &t_end);
148
149 if (dest_addr == MAP_FAILED) {
150 ksft_print_msg("mremap failed: %s\n", strerror(errno));
151 ret = -1;
152 goto clean_up_src;
153 }
154
155 /* Verify byte pattern after remapping */
156 srand(pattern_seed);
157 for (i = 0; i < threshold; i++) {
158 char c = (char) rand();
159
160 if (((char *) dest_addr)[i] != c) {
161 ksft_print_msg("Data after remap doesn't match at offset %d\n",
162 i);
163 ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
164 ((char *) dest_addr)[i] & 0xff);
165 ret = -1;
166 goto clean_up_dest;
167 }
168 }
169
170 start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec;
171 end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec;
172 ret = end_ns - start_ns;
173
174 /*
175 * Since the destination address is specified using MREMAP_FIXED, subsequent
176 * mremap will unmap any previous mapping at the address range specified by
177 * dest_addr and region_size. This significantly affects the remap time of
178 * subsequent tests. So we clean up mappings after each test.
179 */
180 clean_up_dest:
181 munmap(dest_addr, c.region_size);
182 clean_up_src:
183 munmap(src_addr, c.region_size);
184 out:
185 return ret;
186 }
187
run_mremap_test_case(struct test test_case,int * failures,unsigned int threshold_mb,unsigned int pattern_seed)188 static void run_mremap_test_case(struct test test_case, int *failures,
189 unsigned int threshold_mb,
190 unsigned int pattern_seed)
191 {
192 long long remap_time = remap_region(test_case.config, threshold_mb,
193 pattern_seed);
194
195 if (remap_time < 0) {
196 if (test_case.expect_failure)
197 ksft_test_result_pass("%s\n\tExpected mremap failure\n",
198 test_case.name);
199 else {
200 ksft_test_result_fail("%s\n", test_case.name);
201 *failures += 1;
202 }
203 } else {
204 /*
205 * Comparing mremap time is only applicable if entire region
206 * was faulted in.
207 */
208 if (threshold_mb == VALIDATION_NO_THRESHOLD ||
209 test_case.config.region_size <= threshold_mb * _1MB)
210 ksft_test_result_pass("%s\n\tmremap time: %12lldns\n",
211 test_case.name, remap_time);
212 else
213 ksft_test_result_pass("%s\n", test_case.name);
214 }
215 }
216
usage(const char * cmd)217 static void usage(const char *cmd)
218 {
219 fprintf(stderr,
220 "Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n"
221 "-t\t only validate threshold_mb of the remapped region\n"
222 " \t if 0 is supplied no threshold is used; all tests\n"
223 " \t are run and remapped regions validated fully.\n"
224 " \t The default threshold used is 4MB.\n"
225 "-p\t provide a seed to generate the random pattern for\n"
226 " \t validating the remapped region.\n", cmd);
227 }
228
parse_args(int argc,char ** argv,unsigned int * threshold_mb,unsigned int * pattern_seed)229 static int parse_args(int argc, char **argv, unsigned int *threshold_mb,
230 unsigned int *pattern_seed)
231 {
232 const char *optstr = "t:p:";
233 int opt;
234
235 while ((opt = getopt(argc, argv, optstr)) != -1) {
236 switch (opt) {
237 case 't':
238 *threshold_mb = atoi(optarg);
239 break;
240 case 'p':
241 *pattern_seed = atoi(optarg);
242 break;
243 default:
244 usage(argv[0]);
245 return -1;
246 }
247 }
248
249 if (optind < argc) {
250 usage(argv[0]);
251 return -1;
252 }
253
254 return 0;
255 }
256
257 #define MAX_TEST 13
258 #define MAX_PERF_TEST 3
main(int argc,char ** argv)259 int main(int argc, char **argv)
260 {
261 int failures = 0;
262 int i, run_perf_tests;
263 unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD;
264 unsigned int pattern_seed;
265 struct test test_cases[MAX_TEST];
266 struct test perf_test_cases[MAX_PERF_TEST];
267 int page_size;
268 time_t t;
269
270 pattern_seed = (unsigned int) time(&t);
271
272 if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0)
273 exit(EXIT_FAILURE);
274
275 ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n",
276 threshold_mb, pattern_seed);
277
278 page_size = sysconf(_SC_PAGESIZE);
279
280 /* Expected mremap failures */
281 test_cases[0] = MAKE_TEST(page_size, page_size, page_size,
282 OVERLAPPING, EXPECT_FAILURE,
283 "mremap - Source and Destination Regions Overlapping");
284
285 test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size,
286 NON_OVERLAPPING, EXPECT_FAILURE,
287 "mremap - Destination Address Misaligned (1KB-aligned)");
288 test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size,
289 NON_OVERLAPPING, EXPECT_FAILURE,
290 "mremap - Source Address Misaligned (1KB-aligned)");
291
292 /* Src addr PTE aligned */
293 test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2,
294 NON_OVERLAPPING, EXPECT_SUCCESS,
295 "8KB mremap - Source PTE-aligned, Destination PTE-aligned");
296
297 /* Src addr 1MB aligned */
298 test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
299 "2MB mremap - Source 1MB-aligned, Destination PTE-aligned");
300 test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
301 "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned");
302
303 /* Src addr PMD aligned */
304 test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
305 "4MB mremap - Source PMD-aligned, Destination PTE-aligned");
306 test_cases[7] = MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
307 "4MB mremap - Source PMD-aligned, Destination 1MB-aligned");
308 test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
309 "4MB mremap - Source PMD-aligned, Destination PMD-aligned");
310
311 /* Src addr PUD aligned */
312 test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
313 "2GB mremap - Source PUD-aligned, Destination PTE-aligned");
314 test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
315 "2GB mremap - Source PUD-aligned, Destination 1MB-aligned");
316 test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
317 "2GB mremap - Source PUD-aligned, Destination PMD-aligned");
318 test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
319 "2GB mremap - Source PUD-aligned, Destination PUD-aligned");
320
321 perf_test_cases[0] = MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
322 "1GB mremap - Source PTE-aligned, Destination PTE-aligned");
323 /*
324 * mremap 1GB region - Page table level aligned time
325 * comparison.
326 */
327 perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
328 "1GB mremap - Source PMD-aligned, Destination PMD-aligned");
329 perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
330 "1GB mremap - Source PUD-aligned, Destination PUD-aligned");
331
332 run_perf_tests = (threshold_mb == VALIDATION_NO_THRESHOLD) ||
333 (threshold_mb * _1MB >= _1GB);
334
335 ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ?
336 ARRAY_SIZE(perf_test_cases) : 0));
337
338 for (i = 0; i < ARRAY_SIZE(test_cases); i++)
339 run_mremap_test_case(test_cases[i], &failures, threshold_mb,
340 pattern_seed);
341
342 if (run_perf_tests) {
343 ksft_print_msg("\n%s\n",
344 "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:");
345 for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++)
346 run_mremap_test_case(perf_test_cases[i], &failures,
347 threshold_mb, pattern_seed);
348 }
349
350 if (failures > 0)
351 ksft_exit_fail();
352 else
353 ksft_exit_pass();
354 }
355
