47 #define tprintf(x...) do { if (g && g->p.show_details >= 0) printf(x); } while (0)
53 #define dprintf(x...) do { if (g && g->p.show_details >= 1) printf(x); } while (0)
162 static struct global_info	*g = NULL;  variable
173 	OPT_STRING('G', "mb_global"	, &p0.mb_global_str,	"MB", "global  memory (MBs)"),
230 	for (i = 0; i < g->p.nr_nodes; i++) {  in nr_numa_nodes()
282 		for (cpu = 0; cpu < g->p.nr_cpus; cpu++)  in bind_to_cpu()
285 		BUG_ON(target_cpu < 0 || target_cpu >= g->p.nr_cpus);  in bind_to_cpu()
307 		for (cpu = 0; cpu < g->p.nr_cpus; cpu++)  in bind_to_node()
340 	ret = set_mempolicy(MPOL_DEFAULT, NULL, g->p.nr_nodes-1);  in mempol_restore()
353 	BUG_ON(g->p.nr_nodes > (int)sizeof(nodemask)*8);  in bind_to_memnode()
399 			if (ret && !g->print_once) {  in alloc_data()
400 				g->print_once = 1;  in alloc_data()
406 			if (ret && !g->print_once) {  in alloc_data()
407 				g->print_once = 1;  in alloc_data()
455 	return alloc_data(bytes, MAP_SHARED, 1, g->p.init_cpu0,  g->p.thp, g->p.init_random);  in zalloc_shared_data()
463 	return alloc_data(bytes, MAP_SHARED, 0, g->p.init_cpu0,  g->p.thp, g->p.init_random);  in setup_shared_data()
472 	return alloc_data(bytes, MAP_PRIVATE, 0, g->p.init_cpu0,  g->p.thp, g->p.init_random);  in setup_private_data()
514 	if (!g->p.cpu_list_str)  in parse_setup_cpu_list()
517 	dprintf("g->p.nr_tasks: %d\n", g->p.nr_tasks);  in parse_setup_cpu_list()
519 	str0 = str = strdup(g->p.cpu_list_str);  in parse_setup_cpu_list()
554 			BUG_ON(step <= 0 || step >= g->p.nr_cpus);  in parse_setup_cpu_list()
566 			BUG_ON(bind_len <= 0 || bind_len > g->p.nr_cpus);  in parse_setup_cpu_list()
579 		if (bind_cpu_0 >= g->p.nr_cpus || bind_cpu_1 >= g->p.nr_cpus) {  in parse_setup_cpu_list()
580 			printf("\nTest not applicable, system has only %d CPUs.\n", g->p.nr_cpus);  in parse_setup_cpu_list()
593 				if (t >= g->p.nr_tasks) {  in parse_setup_cpu_list()
597 				td = g->threads + t;  in parse_setup_cpu_list()
609 					BUG_ON(cpu < 0 || cpu >= g->p.nr_cpus);  in parse_setup_cpu_list()
620 	if (t < g->p.nr_tasks)  in parse_setup_cpu_list()
621 		printf("# NOTE: %d tasks bound, %d tasks unbound\n", t, g->p.nr_tasks - t);  in parse_setup_cpu_list()
651 	if (!g->p.node_list_str)  in parse_setup_node_list()
654 	dprintf("g->p.nr_tasks: %d\n", g->p.nr_tasks);  in parse_setup_node_list()
656 	str0 = str = strdup(g->p.node_list_str);  in parse_setup_node_list()
690 			BUG_ON(step <= 0 || step >= g->p.nr_nodes);  in parse_setup_node_list()
703 		if (bind_node_0 >= g->p.nr_nodes || bind_node_1 >= g->p.nr_nodes) {  in parse_setup_node_list()
704 			printf("\nTest not applicable, system has only %d nodes.\n", g->p.nr_nodes);  in parse_setup_node_list()
715 				if (t >= g->p.nr_tasks || !node_has_cpus(bind_node)) {  in parse_setup_node_list()
719 				td = g->threads + t;  in parse_setup_node_list()
735 	if (t < g->p.nr_tasks)  in parse_setup_node_list()
736 		printf("# NOTE: %d tasks mem-bound, %d tasks unbound\n", t, g->p.nr_tasks - t);  in parse_setup_node_list()
767 	if (g->p.data_reads)  in access_data()
769 	if (g->p.data_writes)  in access_data()
798 	if (g->p.data_zero_memset && !g->p.data_rand_walk) {  in do_work()
805 	chunk_1 = words/g->p.nr_loops;  in do_work()
811 	if (g->p.data_rand_walk) {  in do_work()
823 			if (g->p.data_zero_memset) {  in do_work()
830 	} else if (!g->p.data_backwards || (nr + loop) & 1) {  in do_work()
875 	g->threads[task_nr].curr_cpu = cpu;  in update_curr_cpu()
886  * to a single node. A count of g->p.nr_nodes means it's
895 	for (t = 0; t < g->p.nr_threads; t++) {  in count_process_nodes()
900 		task_nr = process_nr*g->p.nr_threads + t;  in count_process_nodes()
901 		td = g->threads + task_nr;  in count_process_nodes()
930 	for (p = 0; p < g->p.nr_proc; p++) {  in count_node_processes()
931 		for (t = 0; t < g->p.nr_threads; t++) {  in count_node_processes()
936 			task_nr = p*g->p.nr_threads + t;  in count_node_processes()
937 			td = g->threads + task_nr;  in count_node_processes()
958 	for (p = 0; p < g->p.nr_proc; p++) {  in calc_convergence_compression()
994 	if (!g->p.show_convergence && !g->p.measure_convergence)  in calc_convergence()
997 	for (node = 0; node < g->p.nr_nodes; node++)  in calc_convergence()
1003 	for (t = 0; t < g->p.nr_tasks; t++) {  in calc_convergence()
1004 		struct thread_data *td = g->threads + t;  in calc_convergence()
1023 	nr_min = g->p.nr_tasks;  in calc_convergence()
1026 	for (node = 0; node < g->p.nr_nodes; node++) {  in calc_convergence()
1036 	BUG_ON(sum > g->p.nr_tasks);  in calc_convergence()
1038 	if (0 && (sum < g->p.nr_tasks))  in calc_convergence()
1044 	 * to g->p.nr_proc:  in calc_convergence()
1048 	for (node = 0; node < g->p.nr_nodes; node++) {  in calc_convergence()
1075 	if (strong && process_groups == g->p.nr_proc) {  in calc_convergence()
1079 			if (g->p.measure_convergence) {  in calc_convergence()
1080 				g->all_converged = true;  in calc_convergence()
1081 				g->stop_work = true;  in calc_convergence()
1096 		(double)(l+1)/g->p.nr_loops*100.0, runtime_ns_max / NSEC_PER_SEC / 60.0);  in show_summary()
1100 	if (g->p.show_details >= 0)  in show_summary()
1112 	int details = g->p.show_details;  in worker_thread()
1130 	global_data = g->data;  in worker_thread()
1132 	thread_data = setup_private_data(g->p.bytes_thread);  in worker_thread()
1137 	if (process_nr == g->p.nr_proc-1 && thread_nr == g->p.nr_threads-1)  in worker_thread()
1149 	if (g->p.serialize_startup) {  in worker_thread()
1150 		pthread_mutex_lock(&g->startup_mutex);  in worker_thread()
1151 		g->nr_tasks_started++;  in worker_thread()
1153 		if (g->nr_tasks_started == g->p.nr_tasks)  in worker_thread()
1154 			pthread_cond_signal(&g->startup_cond);  in worker_thread()
1156 		pthread_mutex_unlock(&g->startup_mutex);  in worker_thread()
1159 		pthread_mutex_lock(&g->start_work_mutex);  in worker_thread()
1160 		g->start_work = false;  in worker_thread()
1161 		g->nr_tasks_working++;  in worker_thread()
1162 		while (!g->start_work)  in worker_thread()
1163 			pthread_cond_wait(&g->start_work_cond, &g->start_work_mutex);  in worker_thread()
1165 		pthread_mutex_unlock(&g->start_work_mutex);  in worker_thread()
1173 	for (l = 0; l < g->p.nr_loops; l++) {  in worker_thread()
1176 		if (g->stop_work)  in worker_thread()
1179 		val += do_work(global_data,  g->p.bytes_global,  process_nr, g->p.nr_proc,	l, val);  in worker_thread()
1180 		val += do_work(process_data, g->p.bytes_process, thread_nr,  g->p.nr_threads,	l, val);  in worker_thread()
1181 		val += do_work(thread_data,  g->p.bytes_thread,  0,          1,		l, val);  in worker_thread()
1183 		if (g->p.sleep_usecs) {  in worker_thread()
1185 			usleep(g->p.sleep_usecs);  in worker_thread()
1191 		if (g->p.bytes_process_locked) {  in worker_thread()
1193 			val += do_work(process_data, g->p.bytes_process_locked, thread_nr,  g->p.nr_threads,	l, val);  in worker_thread()
1197 		work_done = g->p.bytes_global + g->p.bytes_process +  in worker_thread()
1198 			    g->p.bytes_process_locked + g->p.bytes_thread;  in worker_thread()
1203 		if (details < 0 && !g->p.perturb_secs && !g->p.measure_convergence && !g->p.nr_secs)  in worker_thread()
1211 		if (g->p.nr_secs) {  in worker_thread()
1213 			if ((u32)diff.tv_sec >= g->p.nr_secs) {  in worker_thread()
1214 				g->stop_work = true;  in worker_thread()
1224 		 * Perturb the first task's equilibrium every g->p.perturb_secs seconds,  in worker_thread()
1227 …if (first_task && g->p.perturb_secs && (int)(stop.tv_sec - last_perturbance) >= g->p.perturb_secs)…  in worker_thread()
1239 			this_cpu = g->threads[task_nr].curr_cpu;  in worker_thread()
1240 			if (this_cpu < g->p.nr_cpus/2)  in worker_thread()
1241 				target_cpu = g->p.nr_cpus-1;  in worker_thread()
1288 	free_data(thread_data, g->p.bytes_thread);  in worker_thread()
1290 	pthread_mutex_lock(&g->stop_work_mutex);  in worker_thread()
1291 	g->bytes_done += bytes_done;  in worker_thread()
1292 	pthread_mutex_unlock(&g->stop_work_mutex);  in worker_thread()
1317 	task_nr = process_nr*g->p.nr_threads;  in worker_process()
1318 	td = g->threads + task_nr;  in worker_process()
1323 	pthreads = zalloc(g->p.nr_threads * sizeof(pthread_t));  in worker_process()
1324 	process_data = setup_private_data(g->p.bytes_process);  in worker_process()
1326 	if (g->p.show_details >= 3) {  in worker_process()
1328 			process_nr, g->data, process_data);  in worker_process()
1331 	for (t = 0; t < g->p.nr_threads; t++) {  in worker_process()
1332 		task_nr = process_nr*g->p.nr_threads + t;  in worker_process()
1333 		td = g->threads + task_nr;  in worker_process()
1347 	for (t = 0; t < g->p.nr_threads; t++) {  in worker_process()
1352 	free_data(process_data, g->p.bytes_process);  in worker_process()
1358 	if (g->p.show_details < 0)  in print_summary()
1363 		g->p.nr_tasks, g->p.nr_tasks == 1 ? "task" : "tasks", nr_numa_nodes(), g->p.nr_cpus);  in print_summary()
1365 			g->p.nr_loops, g->p.bytes_global/1024/1024);  in print_summary()
1367 			g->p.nr_loops, g->p.bytes_process/1024/1024);  in print_summary()
1369 			g->p.nr_loops, g->p.bytes_thread/1024/1024);  in print_summary()
1378 	ssize_t size = sizeof(*g->threads)*g->p.nr_tasks;  in init_thread_data()
1381 	g->threads = zalloc_shared_data(size);  in init_thread_data()
1383 	for (t = 0; t < g->p.nr_tasks; t++) {  in init_thread_data()
1384 		struct thread_data *td = g->threads + t;  in init_thread_data()
1392 		for (cpu = 0; cpu < g->p.nr_cpus; cpu++)  in init_thread_data()
1399 	ssize_t size = sizeof(*g->threads)*g->p.nr_tasks;  in deinit_thread_data()
1401 	free_data(g->threads, size);  in deinit_thread_data()
1406 	g = (void *)alloc_data(sizeof(*g), MAP_SHARED, 1, 0, 0 /* THP */, 0);  in init()
1409 	g->p = p0;  in init()
1411 	g->p.nr_cpus = numa_num_configured_cpus();  in init()
1413 	g->p.nr_nodes = numa_max_node() + 1;  in init()
1416 	BUG_ON(g->p.nr_nodes > MAX_NR_NODES || g->p.nr_nodes < 0);  in init()
1418 	if (g->p.show_quiet && !g->p.show_details)  in init()
1419 		g->p.show_details = -1;  in init()
1422 	if (!g->p.mb_global_str && !g->p.mb_proc_str && !g->p.mb_thread_str)  in init()
1425 	if (g->p.mb_global_str) {  in init()
1426 		g->p.mb_global = atof(g->p.mb_global_str);  in init()
1427 		BUG_ON(g->p.mb_global < 0);  in init()
1430 	if (g->p.mb_proc_str) {  in init()
1431 		g->p.mb_proc = atof(g->p.mb_proc_str);  in init()
1432 		BUG_ON(g->p.mb_proc < 0);  in init()
1435 	if (g->p.mb_proc_locked_str) {  in init()
1436 		g->p.mb_proc_locked = atof(g->p.mb_proc_locked_str);  in init()
1437 		BUG_ON(g->p.mb_proc_locked < 0);  in init()
1438 		BUG_ON(g->p.mb_proc_locked > g->p.mb_proc);  in init()
1441 	if (g->p.mb_thread_str) {  in init()
1442 		g->p.mb_thread = atof(g->p.mb_thread_str);  in init()
1443 		BUG_ON(g->p.mb_thread < 0);  in init()
1446 	BUG_ON(g->p.nr_threads <= 0);  in init()
1447 	BUG_ON(g->p.nr_proc <= 0);  in init()
1449 	g->p.nr_tasks = g->p.nr_proc*g->p.nr_threads;  in init()
1451 	g->p.bytes_global		= g->p.mb_global	*1024L*1024L;  in init()
1452 	g->p.bytes_process		= g->p.mb_proc		*1024L*1024L;  in init()
1453 	g->p.bytes_process_locked	= g->p.mb_proc_locked	*1024L*1024L;  in init()
1454 	g->p.bytes_thread		= g->p.mb_thread	*1024L*1024L;  in init()
1456 	g->data = setup_shared_data(g->p.bytes_global);  in init()
1459 	init_global_mutex(&g->start_work_mutex);  in init()
1460 	init_global_cond(&g->start_work_cond);  in init()
1461 	init_global_mutex(&g->startup_mutex);  in init()
1462 	init_global_cond(&g->startup_cond);  in init()
1463 	init_global_mutex(&g->stop_work_mutex);  in init()
1479 	free_data(g->data, g->p.bytes_global);  in deinit()
1480 	g->data = NULL;  in deinit()
1484 	free_data(g, sizeof(*g));  in deinit()
1485 	g = NULL;  in deinit()
1497 	if (!g->p.show_quiet)  in print_res()
1519 	pids = zalloc(g->p.nr_proc * sizeof(*pids));  in __bench_numa()
1522 	if (g->p.serialize_startup) {  in __bench_numa()
1529 	for (i = 0; i < g->p.nr_proc; i++) {  in __bench_numa()
1544 	if (g->p.serialize_startup) {  in __bench_numa()
1552 		pthread_mutex_lock(&g->startup_mutex);  in __bench_numa()
1553 		while (g->nr_tasks_started != g->p.nr_tasks)  in __bench_numa()
1554 			pthread_cond_wait(&g->startup_cond, &g->startup_mutex);  in __bench_numa()
1556 		pthread_mutex_unlock(&g->startup_mutex);  in __bench_numa()
1560 			pthread_mutex_lock(&g->start_work_mutex);  in __bench_numa()
1561 			threads_ready = (g->nr_tasks_working == g->p.nr_tasks);  in __bench_numa()
1562 			pthread_mutex_unlock(&g->start_work_mutex);  in __bench_numa()
1580 		pthread_mutex_lock(&g->start_work_mutex);  in __bench_numa()
1581 		g->start_work = true;  in __bench_numa()
1582 		pthread_mutex_unlock(&g->start_work_mutex);  in __bench_numa()
1583 		pthread_cond_broadcast(&g->start_work_cond);  in __bench_numa()
1591 	for (i = 0; i < g->p.nr_proc; i++) {  in __bench_numa()
1601 	for (t = 0; t < g->p.nr_tasks; t++) {  in __bench_numa()
1602 		u64 thread_runtime_ns = g->threads[t].runtime_ns;  in __bench_numa()
1622 	bytes = g->bytes_done;  in __bench_numa()
1623 	runtime_avg = (double)runtime_ns_sum / g->p.nr_tasks / NSEC_PER_SEC;  in __bench_numa()
1625 	if (g->p.measure_convergence) {  in __bench_numa()
1643 	print_res(name, bytes / g->p.nr_tasks / 1e9,  in __bench_numa()
1649 	print_res(name, runtime_sec_max * NSEC_PER_SEC / (bytes / g->p.nr_tasks),  in __bench_numa()
1652 	print_res(name, bytes / g->p.nr_tasks / 1e9 / runtime_sec_max,  in __bench_numa()
1658 	if (g->p.show_details >= 2) {  in __bench_numa()
1661 		for (p = 0; p < g->p.nr_proc; p++) {  in __bench_numa()
1662 			for (t = 0; t < g->p.nr_threads; t++) {  in __bench_numa()
1664 				td = g->threads + p*g->p.nr_threads + t;  in __bench_numa()