Lines Matching refs:mas
65 static int get_alloc_node_count(struct ma_state *mas) in get_alloc_node_count() argument
68 struct maple_alloc *node = mas->alloc; in get_alloc_node_count()
79 static void check_mas_alloc_node_count(struct ma_state *mas) in check_mas_alloc_node_count() argument
81 mas_node_count_gfp(mas, MAPLE_ALLOC_SLOTS + 1, GFP_KERNEL); in check_mas_alloc_node_count()
82 mas_node_count_gfp(mas, MAPLE_ALLOC_SLOTS + 3, GFP_KERNEL); in check_mas_alloc_node_count()
83 MT_BUG_ON(mas->tree, get_alloc_node_count(mas) != mas->alloc->total); in check_mas_alloc_node_count()
84 mas_destroy(mas); in check_mas_alloc_node_count()
99 MA_STATE(mas, mt, 0, 0); in check_new_node()
101 check_mas_alloc_node_count(&mas); in check_new_node()
107 mas_node_count(&mas, 3); in check_new_node()
109 MT_BUG_ON(mt, mas_alloc_req(&mas) != 3); in check_new_node()
111 MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); in check_new_node()
112 MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); in check_new_node()
114 MT_BUG_ON(mt, mas_allocated(&mas) != 3); in check_new_node()
115 mn = mas_pop_node(&mas); in check_new_node()
118 MT_BUG_ON(mt, mas.alloc == NULL); in check_new_node()
119 MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); in check_new_node()
120 mas_push_node(&mas, mn); in check_new_node()
121 mas_nomem(&mas, GFP_KERNEL); /* free */ in check_new_node()
128 mas_set_alloc_req(&mas, 1); in check_new_node()
130 MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); in check_new_node()
131 mas_set_err(&mas, -ENOMEM); in check_new_node()
133 MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); in check_new_node()
135 mn = mas_pop_node(&mas); in check_new_node()
140 MT_BUG_ON(mt, mas_allocated(&mas) != 0); in check_new_node()
144 mas.node = MAS_START; in check_new_node()
145 mas_nomem(&mas, GFP_KERNEL); in check_new_node()
147 mas_node_count(&mas, 3); in check_new_node()
149 mas_nomem(&mas, GFP_KERNEL); in check_new_node()
151 MT_BUG_ON(mt, mas_allocated(&mas) != 3); in check_new_node()
153 MT_BUG_ON(mt, mas_alloc_req(&mas) != 0); in check_new_node()
155 MT_BUG_ON(mt, mas.alloc == NULL); in check_new_node()
156 MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); in check_new_node()
157 MT_BUG_ON(mt, mas.alloc->slot[1] == NULL); in check_new_node()
159 MT_BUG_ON(mt, mas_allocated(&mas) != 3); in check_new_node()
161 mas_nomem(&mas, GFP_KERNEL); in check_new_node()
164 mas_set_alloc_req(&mas, 1); in check_new_node()
165 MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); in check_new_node()
166 mas_set_err(&mas, -ENOMEM); in check_new_node()
168 MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); in check_new_node()
169 MT_BUG_ON(mt, mas_allocated(&mas) != 1); in check_new_node()
171 mn = mas_pop_node(&mas); in check_new_node()
173 MT_BUG_ON(mt, mas_allocated(&mas) != 0); in check_new_node()
177 MT_BUG_ON(mt, mas_allocated(&mas) != 0); in check_new_node()
178 mas_push_node(&mas, mn); in check_new_node()
179 MT_BUG_ON(mt, mas_allocated(&mas) != 1); in check_new_node()
180 MT_BUG_ON(mt, mas.alloc->node_count); in check_new_node()
182 mas_set_alloc_req(&mas, 2); /* request 2 more. */ in check_new_node()
183 MT_BUG_ON(mt, mas_alloc_req(&mas) != 2); in check_new_node()
184 mas_set_err(&mas, -ENOMEM); in check_new_node()
185 MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); in check_new_node()
186 MT_BUG_ON(mt, mas_allocated(&mas) != 3); in check_new_node()
187 MT_BUG_ON(mt, mas.alloc == NULL); in check_new_node()
188 MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); in check_new_node()
189 MT_BUG_ON(mt, mas.alloc->slot[1] == NULL); in check_new_node()
191 mn = mas_pop_node(&mas); in check_new_node()
192 MT_BUG_ON(mt, mas_allocated(&mas) != i); in check_new_node()
200 mas_set_alloc_req(&mas, total); /* request 2 more. */ in check_new_node()
201 MT_BUG_ON(mt, mas_alloc_req(&mas) != total); in check_new_node()
202 mas_set_err(&mas, -ENOMEM); in check_new_node()
203 MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); in check_new_node()
223 MT_BUG_ON(mt, mas.alloc->node_count != e); in check_new_node()
224 mn = mas_pop_node(&mas); in check_new_node()
226 MT_BUG_ON(mt, mas_allocated(&mas) != i - 1); in check_new_node()
234 mas_set_alloc_req(&mas, i); in check_new_node()
235 mas_set_err(&mas, -ENOMEM); in check_new_node()
236 MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); in check_new_node()
238 mn = mas_pop_node(&mas); in check_new_node()
239 MT_BUG_ON(mt, mas_allocated(&mas) != j - 1); in check_new_node()
242 mas_push_node(&mas, mn); in check_new_node()
243 MT_BUG_ON(mt, mas_allocated(&mas) != j); in check_new_node()
244 mn = mas_pop_node(&mas); in check_new_node()
246 MT_BUG_ON(mt, mas_allocated(&mas) != j - 1); in check_new_node()
250 MT_BUG_ON(mt, mas_allocated(&mas) != 0); in check_new_node()
252 mas_set_alloc_req(&mas, i); in check_new_node()
253 mas_set_err(&mas, -ENOMEM); in check_new_node()
254 MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); in check_new_node()
256 MT_BUG_ON(mt, mas_allocated(&mas) != i - j); in check_new_node()
257 nodes[j] = mas_pop_node(&mas); in check_new_node()
258 MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1); in check_new_node()
263 mas_push_node(&mas, nodes[j]); in check_new_node()
264 MT_BUG_ON(mt, mas_allocated(&mas) != i - j); in check_new_node()
266 MT_BUG_ON(mt, mas_allocated(&mas) != i); in check_new_node()
268 MT_BUG_ON(mt, mas_allocated(&mas) != i - j); in check_new_node()
269 mn = mas_pop_node(&mas); in check_new_node()
273 MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1); in check_new_node()
275 MT_BUG_ON(mt, mas_nomem(&mas, GFP_KERNEL)); in check_new_node()
281 mas_node_count(&mas, total); in check_new_node()
283 mas_nomem(&mas, GFP_KERNEL); in check_new_node()
284 MT_BUG_ON(mt, !mas.alloc); in check_new_node()
286 smn = mas.alloc; in check_new_node()
296 MT_BUG_ON(mt, mas_allocated(&mas) != total); in check_new_node()
297 mas_nomem(&mas, GFP_KERNEL); /* Free. */ in check_new_node()
299 MT_BUG_ON(mt, mas_allocated(&mas) != 0); in check_new_node()
301 mas_node_count(&mas, i); /* Request */ in check_new_node()
302 mas_nomem(&mas, GFP_KERNEL); /* Fill request */ in check_new_node()
303 MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */ in check_new_node()
305 mn = mas_pop_node(&mas); /* get the next node. */ in check_new_node()
311 MT_BUG_ON(mt, mas_allocated(&mas) != 0); in check_new_node()
316 mas_node_count(&mas, i); /* Request */ in check_new_node()
317 mas_nomem(&mas, GFP_KERNEL); /* Fill request */ in check_new_node()
318 MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */ in check_new_node()
320 mn = mas_pop_node(&mas); /* get the next node. */ in check_new_node()
326 MT_BUG_ON(mt, mas_allocated(&mas) != 0); in check_new_node()
341 MT_BUG_ON(mt, mas_allocated(&mas) != 0); in check_new_node()
342 mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 1); /* Request */ in check_new_node()
343 MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); in check_new_node()
344 MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); in check_new_node()
345 MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); in check_new_node()
346 MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS); in check_new_node()
348 mn = mas_pop_node(&mas); /* get the next node. */ in check_new_node()
351 MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS); in check_new_node()
352 MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); in check_new_node()
354 mas_push_node(&mas, mn); in check_new_node()
355 MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); in check_new_node()
356 MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS); in check_new_node()
359 mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 2); /* Request */ in check_new_node()
360 MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); in check_new_node()
361 MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); in check_new_node()
362 MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); in check_new_node()
363 MT_BUG_ON(mt, mas_alloc_req(&mas)); in check_new_node()
364 MT_BUG_ON(mt, mas.alloc->node_count != 1); in check_new_node()
365 MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2); in check_new_node()
366 mn = mas_pop_node(&mas); in check_new_node()
368 MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); in check_new_node()
369 MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS); in check_new_node()
370 mas_push_node(&mas, mn); in check_new_node()
371 MT_BUG_ON(mt, mas.alloc->node_count != 1); in check_new_node()
372 MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2); in check_new_node()
373 mn = mas_pop_node(&mas); in check_new_node()
378 mn = mas_pop_node(&mas); in check_new_node()
383 MT_BUG_ON(mt, mas_allocated(&mas) != 0); in check_new_node()
387 mas.node = MA_ERROR(-ENOMEM); in check_new_node()
388 mas_node_count(&mas, i); /* Request */ in check_new_node()
389 mas_nomem(&mas, GFP_KERNEL); /* Fill request */ in check_new_node()
390 mn = mas_pop_node(&mas); /* get the next node. */ in check_new_node()
391 mas_push_node(&mas, mn); /* put it back */ in check_new_node()
392 mas_destroy(&mas); in check_new_node()
394 mas.node = MA_ERROR(-ENOMEM); in check_new_node()
395 mas_node_count(&mas, i); /* Request */ in check_new_node()
396 mas_nomem(&mas, GFP_KERNEL); /* Fill request */ in check_new_node()
397 mn = mas_pop_node(&mas); /* get the next node. */ in check_new_node()
398 mn2 = mas_pop_node(&mas); /* get the next node. */ in check_new_node()
399 mas_push_node(&mas, mn); /* put them back */ in check_new_node()
400 mas_push_node(&mas, mn2); in check_new_node()
401 mas_destroy(&mas); in check_new_node()
403 mas.node = MA_ERROR(-ENOMEM); in check_new_node()
404 mas_node_count(&mas, i); /* Request */ in check_new_node()
405 mas_nomem(&mas, GFP_KERNEL); /* Fill request */ in check_new_node()
406 mn = mas_pop_node(&mas); /* get the next node. */ in check_new_node()
407 mn2 = mas_pop_node(&mas); /* get the next node. */ in check_new_node()
408 mn3 = mas_pop_node(&mas); /* get the next node. */ in check_new_node()
409 mas_push_node(&mas, mn); /* put them back */ in check_new_node()
410 mas_push_node(&mas, mn2); in check_new_node()
411 mas_push_node(&mas, mn3); in check_new_node()
412 mas_destroy(&mas); in check_new_node()
414 mas.node = MA_ERROR(-ENOMEM); in check_new_node()
415 mas_node_count(&mas, i); /* Request */ in check_new_node()
416 mas_nomem(&mas, GFP_KERNEL); /* Fill request */ in check_new_node()
417 mn = mas_pop_node(&mas); /* get the next node. */ in check_new_node()
420 mas_destroy(&mas); in check_new_node()
422 mas.node = MA_ERROR(-ENOMEM); in check_new_node()
423 mas_node_count(&mas, i); /* Request */ in check_new_node()
424 mas_nomem(&mas, GFP_KERNEL); /* Fill request */ in check_new_node()
425 mn = mas_pop_node(&mas); /* get the next node. */ in check_new_node()
428 mn = mas_pop_node(&mas); /* get the next node. */ in check_new_node()
431 mn = mas_pop_node(&mas); /* get the next node. */ in check_new_node()
434 mas_destroy(&mas); in check_new_node()
437 mas.node = MA_ERROR(-ENOMEM); in check_new_node()
438 mas_node_count(&mas, 5); /* Request */ in check_new_node()
439 mas_nomem(&mas, GFP_KERNEL); /* Fill request */ in check_new_node()
440 MT_BUG_ON(mt, mas_allocated(&mas) != 5); in check_new_node()
441 mas.node = MA_ERROR(-ENOMEM); in check_new_node()
442 mas_node_count(&mas, 10); /* Request */ in check_new_node()
443 mas_nomem(&mas, GFP_KERNEL); /* Fill request */ in check_new_node()
444 mas.node = MAS_START; in check_new_node()
445 MT_BUG_ON(mt, mas_allocated(&mas) != 10); in check_new_node()
446 mas_destroy(&mas); in check_new_node()
448 mas.node = MA_ERROR(-ENOMEM); in check_new_node()
449 mas_node_count(&mas, MAPLE_ALLOC_SLOTS - 1); /* Request */ in check_new_node()
450 mas_nomem(&mas, GFP_KERNEL); /* Fill request */ in check_new_node()
451 MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS - 1); in check_new_node()
452 mas.node = MA_ERROR(-ENOMEM); in check_new_node()
453 mas_node_count(&mas, 10 + MAPLE_ALLOC_SLOTS - 1); /* Request */ in check_new_node()
454 mas_nomem(&mas, GFP_KERNEL); /* Fill request */ in check_new_node()
455 mas.node = MAS_START; in check_new_node()
456 MT_BUG_ON(mt, mas_allocated(&mas) != 10 + MAPLE_ALLOC_SLOTS - 1); in check_new_node()
457 mas_destroy(&mas); in check_new_node()
816 static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node, in mas_node_walk() argument
828 (*range_max) = (*range_min) = mas->index; in mas_node_walk()
832 mas->offset = mas->index = mas->min; in mas_node_walk()
842 prev = mas->min; in mas_node_walk()
843 index = mas->index; in mas_node_walk()
862 max = mas->max; in mas_node_walk()
866 mas->offset = offset; in mas_node_walk()
871 mas->max = max; in mas_node_walk()
872 mas->min = prev; in mas_node_walk()
888 static inline bool mas_descend_walk(struct ma_state *mas, in mas_descend_walk() argument
895 next = mas->node; in mas_descend_walk()
899 mas_node_walk(mas, node, type, range_min, range_max); in mas_descend_walk()
900 next = mas_slot(mas, ma_slots(node, type), mas->offset); in mas_descend_walk()
908 mas->node = next; in mas_descend_walk()
923 static inline bool mas_tree_walk(struct ma_state *mas, unsigned long *range_min, in mas_tree_walk() argument
930 mas_start(mas); in mas_tree_walk()
931 if (mas_is_none(mas)) in mas_tree_walk()
934 if (mas_is_ptr(mas)) { in mas_tree_walk()
936 if (!mas->index) in mas_tree_walk()
942 ret = mas_descend_walk(mas, range_min, range_max); in mas_tree_walk()
943 if (unlikely(mte_dead_node(mas->node))) { in mas_tree_walk()
944 mas->node = MAS_START; in mas_tree_walk()
951 mas->offset = MAPLE_NODE_SLOTS; in mas_tree_walk()
955 static inline void *mas_range_load(struct ma_state *mas, in mas_range_load() argument
960 unsigned long index = mas->index; in mas_range_load()
962 if (mas_is_none(mas) || mas_is_paused(mas)) in mas_range_load()
963 mas->node = MAS_START; in mas_range_load()
965 if (mas_tree_walk(mas, range_min, range_max)) in mas_range_load()
966 if (unlikely(mas->node == MAS_ROOT)) in mas_range_load()
967 return mas_root(mas); in mas_range_load()
969 if (likely(mas->offset != MAPLE_NODE_SLOTS)) in mas_range_load()
970 entry = mas_get_slot(mas, mas->offset); in mas_range_load()
972 if (mas_dead_node(mas, index)) in mas_range_load()
988 MA_STATE(mas, mt, 0, 0); in check_erase2_testset()
1089 mas_reset(&mas); in check_erase2_testset()
1090 mas.index = 0; in check_erase2_testset()
1092 mas_for_each(&mas, foo, ULONG_MAX) { in check_erase2_testset()
1094 if (addr == mas.index) { in check_erase2_testset()
1095 mt_dump(mas.tree, mt_dump_hex); in check_erase2_testset()
1097 mas.index, mas.last); in check_erase2_testset()
1100 addr = mas.index; in check_erase2_testset()
1104 pr_err("mas: %lu -> %p\n", mas.index, foo); in check_erase2_testset()
1118 MT_BUG_ON(mt, mtree_load(mas.tree, 0) != NULL); in check_erase2_testset()
33896 MA_STATE(mas, mt, 0, 0); in check_erase2_sets()
33918 mas_for_each(&mas, entry, ULONG_MAX) { in check_erase2_sets()
33958 mas_reset(&mas); in check_erase2_sets()
33962 mas_empty_area_rev(&mas, 12288, 140014592737280, 0x2000); in check_erase2_sets()
33963 MT_BUG_ON(mt, mas.last != 140014592573439); in check_erase2_sets()
33966 mas_reset(&mas); in check_erase2_sets()
33967 mas.tree = mt; in check_erase2_sets()
33969 mas.index = 0; in check_erase2_sets()
33973 mas_for_each(&mas, entry, ULONG_MAX) { in check_erase2_sets()
33981 mas_reset(&mas); in check_erase2_sets()
33986 mas_empty_area_rev(&mas, 0, 140373518663680, 4096); in check_erase2_sets()
34001 mas_reset(&mas); in check_erase2_sets()
34005 mas_empty_area_rev(&mas, 4096, 139921865637888, 0x6000); in check_erase2_sets()
34006 MT_BUG_ON(mt, mas.last != 139921865547775); in check_erase2_sets()
34016 mas_reset(&mas); in check_erase2_sets()
34020 mas_empty_area_rev(&mas, 4096, 139953197334528, 0x1000); in check_erase2_sets()
34021 MT_BUG_ON(mt, mas.last != 139953197322239); in check_erase2_sets()
34032 mas_reset(&mas); in check_erase2_sets()
34036 mas_empty_area_rev(&mas, 4096, 140222972858368, 2215936); in check_erase2_sets()
34037 MT_BUG_ON(mt, mas.last != 140222968475647); in check_erase2_sets()
34050 mas_reset(&mas); in check_erase2_sets()
34054 mas.index = 140656779083776; in check_erase2_sets()
34055 entry = mas_find(&mas, ULONG_MAX); in check_erase2_sets()
34057 entry = mas_prev(&mas, 0); in check_erase2_sets()
34119 mas_reset(&mas); in check_erase2_sets()
34123 mas_empty_area_rev(&mas, 4096, 140109042671616, 409600); in check_erase2_sets()
34124 MT_BUG_ON(mt, mas.last != 140109040959487); in check_erase2_sets()
34131 mas_reset(&mas); in check_erase2_sets()
34141 mas_reset(&mas); in check_erase2_sets()
34145 mas_empty_area_rev(&mas, 4096, 139918413357056, 2097152); in check_erase2_sets()
34147 mas.index = (mas.last + 1 - 2097152 - 0) & (~2093056); in check_erase2_sets()
34148 MT_BUG_ON(mt, mas.index != 139918401601536); in check_erase2_sets()
34157 mas_reset(&mas); in check_erase2_sets()
34170 mas_reset(&mas); in check_erase2_sets()
34183 mas_reset(&mas); in check_erase2_sets()
34192 mas_reset(&mas); in check_erase2_sets()
34212 mas_reset(&mas); in check_erase2_sets()
34216 mas_empty_area_rev(&mas, 4096, 140583656296448, 134217728); in check_erase2_sets()
34217 MT_BUG_ON(mt, mas.last != 140583003750399); in check_erase2_sets()
34227 mas_reset(&mas); in check_erase2_sets()
34237 mas_reset(&mas); in check_erase2_sets()
34247 mas_reset(&mas); in check_erase2_sets()
34255 mas_reset(&mas); in check_erase2_sets()
34263 mas_reset(&mas); in check_erase2_sets()
34271 mas_reset(&mas); in check_erase2_sets()
34278 mas_reset(&mas); in check_erase2_sets()
34285 mas_reset(&mas); in check_erase2_sets()
34294 mas_reset(&mas); in check_erase2_sets()
34298 mas_empty_area_rev(&mas, 4096, 4052029440, 28672); in check_erase2_sets()
34299 MT_BUG_ON(mt, mas.last != 4041211903); in check_erase2_sets()
34306 mas_reset(&mas); in check_erase2_sets()
34351 MA_STATE(mas, test->mt, 0, 0); in rcu_reader_fwd()
34360 mas_set(&mas, test->index[index]); in rcu_reader_fwd()
34361 mas_for_each(&mas, entry, test->last[index + 9]) { in rcu_reader_fwd()
34373 if (mas.index == alt_start) { in rcu_reader_fwd()
34388 if (mas.index == r_start) { in rcu_reader_fwd()
34391 } else if (mas.index == alt_start) { in rcu_reader_fwd()
34399 RCU_MT_BUG_ON(test, mas.index != r_start); in rcu_reader_fwd()
34400 RCU_MT_BUG_ON(test, mas.last != r_end); in rcu_reader_fwd()
34416 printk("!!%lu-%lu -> %p not %p or %p\n", mas.index, mas.last, entry, expected, alt); in rcu_reader_fwd()
34439 printk("!!%lu-%lu -> %p not %p\n", mas.index, mas.last, entry, expected); in rcu_reader_fwd()
34461 MA_STATE(mas, test->mt, 0, 0); in rcu_reader_rev()
34471 mas_set(&mas, test->index[index + i]); in rcu_reader_rev()
34479 entry = mas_prev(&mas, test->index[index]); in rcu_reader_rev()
34486 if (mas.index == alt_start) { in rcu_reader_rev()
34502 if (mas.index == r_start) { in rcu_reader_rev()
34506 } else if (mas.index == alt_start) { in rcu_reader_rev()
34520 if (mas.index != r_start) { in rcu_reader_rev()
34525 mas.index, mas.last, entry, in rcu_reader_rev()
34529 RCU_MT_BUG_ON(test, mas.index != r_start); in rcu_reader_rev()
34530 RCU_MT_BUG_ON(test, mas.last != r_end); in rcu_reader_rev()
34565 mas.index, mas.last, entry, in rcu_reader_rev()
34573 printk("%lu-%lu %p != %p\n", mas.index, in rcu_reader_rev()
34574 mas.last, entry, expected); in rcu_reader_rev()
34890 MA_STATE(mas, test->mt, test->range_start, test->range_start); in rcu_loop()
34901 mas_for_each(&mas, entry, test->range_end) { in rcu_loop()
34903 expected = xa_mk_value(mas.index ? mas.index / 10 : 0); in rcu_loop()
34906 if (mas.index < test->index || mas.index > test->last) { in rcu_loop()
34909 mas.index, mas.last, entry, expected); in rcu_loop()
34924 mas_set(&mas, test->range_start); in rcu_loop()
34967 MA_STATE(mas, test->mt, test->index, test->index); in rcu_slot_store_reader()
34973 mas_walk(&mas); in rcu_slot_store_reader()
34975 RCU_MT_BUG_ON(test, (test->index - mas.index) != in rcu_slot_store_reader()
34976 (mas.last - test->last)); in rcu_slot_store_reader()
35332 static void mas_dfs_preorder(struct ma_state *mas) in mas_dfs_preorder() argument
35339 if (mas->node == MAS_START) { in mas_dfs_preorder()
35340 mas_start(mas); in mas_dfs_preorder()
35344 if (mte_is_leaf(mas->node) && mte_is_root(mas->node)) in mas_dfs_preorder()
35348 end = mas_data_end(mas); in mas_dfs_preorder()
35349 if (mte_is_leaf(mas->node) || in mas_dfs_preorder()
35351 if (mte_is_root(mas->node)) in mas_dfs_preorder()
35354 slot = mte_parent_slot(mas->node) + 1; in mas_dfs_preorder()
35355 mas_ascend(mas); in mas_dfs_preorder()
35359 prev = mas->node; in mas_dfs_preorder()
35360 mas->node = mas_get_slot(mas, slot); in mas_dfs_preorder()
35361 if (!mas->node || slot > end) { in mas_dfs_preorder()
35365 mas->node = prev; in mas_dfs_preorder()
35366 slot = mte_parent_slot(mas->node) + 1; in mas_dfs_preorder()
35367 mas_ascend(mas); in mas_dfs_preorder()
35371 mas->max = mas_safe_pivot(mas, pivots, slot, mte_node_type(prev)); in mas_dfs_preorder()
35372 mas->min = mas_safe_min(mas, pivots, slot); in mas_dfs_preorder()
35376 mas->node = MAS_NONE; in mas_dfs_preorder()
35384 MA_STATE(mas, mt, 0, 0); in check_dfs_preorder()
35394 mas_dfs_preorder(&mas); in check_dfs_preorder()
35395 } while (!mas_is_none(&mas)); in check_dfs_preorder()
35400 mas_reset(&mas); in check_dfs_preorder()
35408 mas_dfs_preorder(&mas); in check_dfs_preorder()
35409 } while (!mas_is_none(&mas)); in check_dfs_preorder()
35415 mas_reset(&mas); in check_dfs_preorder()
35420 mas_dfs_preorder(&mas); in check_dfs_preorder()
35421 } while (!mas_is_none(&mas)); in check_dfs_preorder()
35427 mas_reset(&mas); in check_dfs_preorder()
35430 mas_expected_entries(&mas, max); in check_dfs_preorder()
35432 mas.index = mas.last = count; in check_dfs_preorder()
35433 mas_store(&mas, xa_mk_value(count)); in check_dfs_preorder()
35434 MT_BUG_ON(mt, mas_is_err(&mas)); in check_dfs_preorder()
35436 mas_destroy(&mas); in check_dfs_preorder()
35455 MA_STATE(mas, mt, 10, 20); in check_prealloc()
35462 mas_set_range(&mas, 470, 500); in check_prealloc()
35463 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); in check_prealloc()
35464 allocated = mas_allocated(&mas); in check_prealloc()
35465 height = mas_mt_height(&mas); in check_prealloc()
35468 mas_destroy(&mas); in check_prealloc()
35469 allocated = mas_allocated(&mas); in check_prealloc()
35472 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); in check_prealloc()
35473 allocated = mas_allocated(&mas); in check_prealloc()
35474 height = mas_mt_height(&mas); in check_prealloc()
35477 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); in check_prealloc()
35478 mas_destroy(&mas); in check_prealloc()
35479 allocated = mas_allocated(&mas); in check_prealloc()
35483 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); in check_prealloc()
35484 allocated = mas_allocated(&mas); in check_prealloc()
35485 height = mas_mt_height(&mas); in check_prealloc()
35487 mn = mas_pop_node(&mas); in check_prealloc()
35488 MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); in check_prealloc()
35491 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); in check_prealloc()
35492 mas_destroy(&mas); in check_prealloc()
35493 allocated = mas_allocated(&mas); in check_prealloc()
35496 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); in check_prealloc()
35497 allocated = mas_allocated(&mas); in check_prealloc()
35498 height = mas_mt_height(&mas); in check_prealloc()
35500 mn = mas_pop_node(&mas); in check_prealloc()
35501 MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); in check_prealloc()
35502 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); in check_prealloc()
35503 mas_destroy(&mas); in check_prealloc()
35504 allocated = mas_allocated(&mas); in check_prealloc()
35509 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); in check_prealloc()
35510 allocated = mas_allocated(&mas); in check_prealloc()
35511 height = mas_mt_height(&mas); in check_prealloc()
35513 mn = mas_pop_node(&mas); in check_prealloc()
35514 MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); in check_prealloc()
35515 mas_push_node(&mas, mn); in check_prealloc()
35516 MT_BUG_ON(mt, mas_allocated(&mas) != allocated); in check_prealloc()
35517 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); in check_prealloc()
35518 mas_destroy(&mas); in check_prealloc()
35519 allocated = mas_allocated(&mas); in check_prealloc()
35522 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); in check_prealloc()
35523 allocated = mas_allocated(&mas); in check_prealloc()
35524 height = mas_mt_height(&mas); in check_prealloc()
35526 mas_store_prealloc(&mas, ptr); in check_prealloc()
35527 MT_BUG_ON(mt, mas_allocated(&mas) != 0); in check_prealloc()
35530 mas_set_range(&mas, 6, 9); in check_prealloc()
35531 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); in check_prealloc()
35532 allocated = mas_allocated(&mas); in check_prealloc()
35534 mas_store_prealloc(&mas, ptr); in check_prealloc()
35535 MT_BUG_ON(mt, mas_allocated(&mas) != 0); in check_prealloc()
35537 mas_set_range(&mas, 6, 10); in check_prealloc()
35538 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); in check_prealloc()
35539 allocated = mas_allocated(&mas); in check_prealloc()
35540 height = mas_mt_height(&mas); in check_prealloc()
35542 mas_store_prealloc(&mas, ptr); in check_prealloc()
35543 MT_BUG_ON(mt, mas_allocated(&mas) != 0); in check_prealloc()
35546 mas_set_range(&mas, 54, 54); in check_prealloc()
35547 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); in check_prealloc()
35548 allocated = mas_allocated(&mas); in check_prealloc()
35549 height = mas_mt_height(&mas); in check_prealloc()
35551 mas_store_prealloc(&mas, ptr); in check_prealloc()
35552 MT_BUG_ON(mt, mas_allocated(&mas) != 0); in check_prealloc()
35555 mas_set_range(&mas, 1, 100); in check_prealloc()
35556 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); in check_prealloc()
35557 allocated = mas_allocated(&mas); in check_prealloc()
35558 height = mas_mt_height(&mas); in check_prealloc()
35560 mas_destroy(&mas); in check_prealloc()
35564 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); in check_prealloc()
35565 allocated = mas_allocated(&mas); in check_prealloc()
35566 height = mas_mt_height(&mas); in check_prealloc()
35569 mas_store_prealloc(&mas, ptr); in check_prealloc()
35570 MT_BUG_ON(mt, mas_allocated(&mas) != 0); in check_prealloc()
35571 mas_set_range(&mas, 0, 200); in check_prealloc()
35573 MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); in check_prealloc()
35574 allocated = mas_allocated(&mas); in check_prealloc()
35575 height = mas_mt_height(&mas); in check_prealloc()
35584 MA_STATE(mas, mt, 1200, 2380); in check_spanning_write()
35590 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_spanning_write()
35591 mas_set(&mas, 1205); in check_spanning_write()
35592 MT_BUG_ON(mt, mas_walk(&mas) != NULL); in check_spanning_write()
35600 mas_set_range(&mas, 9, 50006); /* Will expand to 0 - ULONG_MAX */ in check_spanning_write()
35601 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_spanning_write()
35602 mas_set(&mas, 1205); in check_spanning_write()
35603 MT_BUG_ON(mt, mas_walk(&mas) != NULL); in check_spanning_write()
35613 mas_set_range(&mas, 0, 12900); /* Spans more than 2 levels */ in check_spanning_write()
35615 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_spanning_write()
35616 mas_set(&mas, 1205); in check_spanning_write()
35617 MT_BUG_ON(mt, mas_walk(&mas) != NULL); in check_spanning_write()
35626 mas_set_range(&mas, 0, 300); in check_spanning_write()
35628 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_spanning_write()
35629 mas_set(&mas, 15); in check_spanning_write()
35630 MT_BUG_ON(mt, mas_walk(&mas) != NULL); in check_spanning_write()
35639 mas_set_range(&mas, 0, 12865); in check_spanning_write()
35641 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_spanning_write()
35642 mas_set(&mas, 15); in check_spanning_write()
35643 MT_BUG_ON(mt, mas_walk(&mas) != NULL); in check_spanning_write()
35652 mas_set_range(&mas, 90, 13665); in check_spanning_write()
35654 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_spanning_write()
35655 mas_set(&mas, 95); in check_spanning_write()
35656 MT_BUG_ON(mt, mas_walk(&mas) != NULL); in check_spanning_write()
35665 mas_set_range(&mas, 46805, 49995); in check_spanning_write()
35667 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_spanning_write()
35668 mas_set(&mas, 46815); in check_spanning_write()
35669 MT_BUG_ON(mt, mas_walk(&mas) != NULL); in check_spanning_write()
35681 mas_set_range(&mas, 32395, 49995); in check_spanning_write()
35683 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_spanning_write()
35684 mas_set(&mas, 46815); in check_spanning_write()
35685 MT_BUG_ON(mt, mas_walk(&mas) != NULL); in check_spanning_write()
35696 mas_set_range(&mas, 38875, 43190); in check_spanning_write()
35698 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_spanning_write()
35699 mas_set(&mas, 38900); in check_spanning_write()
35700 MT_BUG_ON(mt, mas_walk(&mas) != NULL); in check_spanning_write()
35709 mas_set(&mas, 47606); in check_spanning_write()
35710 mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); in check_spanning_write()
35711 mas_set(&mas, 47607); in check_spanning_write()
35712 mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); in check_spanning_write()
35713 mas_set(&mas, 47608); in check_spanning_write()
35714 mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); in check_spanning_write()
35715 mas_set(&mas, 47609); in check_spanning_write()
35716 mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); in check_spanning_write()
35718 mas_ascend(&mas); in check_spanning_write()
35719 MT_BUG_ON(mt, (mas_data_end(&mas)) != mt_slot_count(mas.node) - 1); in check_spanning_write()
35720 mas_set_range(&mas, 11516, 48940); in check_spanning_write()
35721 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_spanning_write()
35729 mas_set_range(&mas, 43200, 49999); in check_spanning_write()
35731 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_spanning_write()
35732 mas_set(&mas, 43200); in check_spanning_write()
35733 MT_BUG_ON(mt, mas_walk(&mas) != NULL); in check_spanning_write()
35742 mas_set_range(&mas, 76, 875); in check_spanning_write()
35743 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_spanning_write()
35753 MA_STATE(mas, mt, 959, 959); in check_null_expand()
35758 mas_lock(&mas); in check_null_expand()
35759 mas_walk(&mas); in check_null_expand()
35760 data_end = mas_data_end(&mas); in check_null_expand()
35761 mas_set_range(&mas, 959, 963); in check_null_expand()
35762 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_null_expand()
35764 MT_BUG_ON(mt, data_end != mas_data_end(&mas)); in check_null_expand()
35767 mas_set(&mas, 880); in check_null_expand()
35768 mas_walk(&mas); in check_null_expand()
35769 data_end = mas_data_end(&mas); in check_null_expand()
35770 mas_set_range(&mas, 884, 887); in check_null_expand()
35771 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_null_expand()
35775 MT_BUG_ON(mt, data_end != mas_data_end(&mas)); in check_null_expand()
35779 mas_set(&mas, 890); in check_null_expand()
35780 mas_walk(&mas); in check_null_expand()
35781 data_end = mas_data_end(&mas); in check_null_expand()
35782 mas_set_range(&mas, 900, 905); in check_null_expand()
35783 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_null_expand()
35789 MT_BUG_ON(mt, data_end - 2 != mas_data_end(&mas)); in check_null_expand()
35793 mas_set(&mas, 800); in check_null_expand()
35794 mas_walk(&mas); in check_null_expand()
35795 data_end = mas_data_end(&mas); in check_null_expand()
35796 mas_set_range(&mas, 810, 825); in check_null_expand()
35797 mas_store_gfp(&mas, NULL, GFP_KERNEL); in check_null_expand()
35803 MT_BUG_ON(mt, data_end - 4 != mas_data_end(&mas)); in check_null_expand()
35805 mas_unlock(&mas); in check_null_expand()