47 #define tprintf(x...) do { if (g && g->p.show_details >= 0) printf(x); } while (0)
53 #define dprintf(x...) do { if (g && g->p.show_details >= 1) printf(x); } while (0)
162 static struct global_info	*g = NULL;  variable
173 	OPT_STRING('G', "mb_global"	, &p0.mb_global_str,	"MB", "global  memory (MBs)"),
230 	for (i = 0; i < g->p.nr_nodes; i++) {  in nr_numa_nodes()
282 		for (cpu = 0; cpu < g->p.nr_cpus; cpu++)  in bind_to_cpu()
285 		BUG_ON(target_cpu < 0 || target_cpu >= g->p.nr_cpus);  in bind_to_cpu()
307 		for (cpu = 0; cpu < g->p.nr_cpus; cpu++)  in bind_to_node()
340 	ret = set_mempolicy(MPOL_DEFAULT, NULL, g->p.nr_nodes-1);  in mempol_restore()
403 			if (ret && !g->print_once) {  in alloc_data()
404 				g->print_once = 1;  in alloc_data()
410 			if (ret && !g->print_once) {  in alloc_data()
411 				g->print_once = 1;  in alloc_data()
459 	return alloc_data(bytes, MAP_SHARED, 1, g->p.init_cpu0,  g->p.thp, g->p.init_random);  in zalloc_shared_data()
467 	return alloc_data(bytes, MAP_SHARED, 0, g->p.init_cpu0,  g->p.thp, g->p.init_random);  in setup_shared_data()
476 	return alloc_data(bytes, MAP_PRIVATE, 0, g->p.init_cpu0,  g->p.thp, g->p.init_random);  in setup_private_data()
518 	if (!g->p.cpu_list_str)  in parse_setup_cpu_list()
521 	dprintf("g->p.nr_tasks: %d\n", g->p.nr_tasks);  in parse_setup_cpu_list()
523 	str0 = str = strdup(g->p.cpu_list_str);  in parse_setup_cpu_list()
558 			BUG_ON(step <= 0 || step >= g->p.nr_cpus);  in parse_setup_cpu_list()
570 			BUG_ON(bind_len <= 0 || bind_len > g->p.nr_cpus);  in parse_setup_cpu_list()
583 		if (bind_cpu_0 >= g->p.nr_cpus || bind_cpu_1 >= g->p.nr_cpus) {  in parse_setup_cpu_list()
584 			printf("\nTest not applicable, system has only %d CPUs.\n", g->p.nr_cpus);  in parse_setup_cpu_list()
597 				if (t >= g->p.nr_tasks) {  in parse_setup_cpu_list()
601 				td = g->threads + t;  in parse_setup_cpu_list()
613 					BUG_ON(cpu < 0 || cpu >= g->p.nr_cpus);  in parse_setup_cpu_list()
624 	if (t < g->p.nr_tasks)  in parse_setup_cpu_list()
625 		printf("# NOTE: %d tasks bound, %d tasks unbound\n", t, g->p.nr_tasks - t);  in parse_setup_cpu_list()
655 	if (!g->p.node_list_str)  in parse_setup_node_list()
658 	dprintf("g->p.nr_tasks: %d\n", g->p.nr_tasks);  in parse_setup_node_list()
660 	str0 = str = strdup(g->p.node_list_str);  in parse_setup_node_list()
694 			BUG_ON(step <= 0 || step >= g->p.nr_nodes);  in parse_setup_node_list()
707 		if (bind_node_0 >= g->p.nr_nodes || bind_node_1 >= g->p.nr_nodes) {  in parse_setup_node_list()
708 			printf("\nTest not applicable, system has only %d nodes.\n", g->p.nr_nodes);  in parse_setup_node_list()
719 				if (t >= g->p.nr_tasks || !node_has_cpus(bind_node)) {  in parse_setup_node_list()
723 				td = g->threads + t;  in parse_setup_node_list()
739 	if (t < g->p.nr_tasks)  in parse_setup_node_list()
740 		printf("# NOTE: %d tasks mem-bound, %d tasks unbound\n", t, g->p.nr_tasks - t);  in parse_setup_node_list()
771 	if (g->p.data_reads)  in access_data()
773 	if (g->p.data_writes)  in access_data()
802 	if (g->p.data_zero_memset && !g->p.data_rand_walk) {  in do_work()
809 	chunk_1 = words/g->p.nr_loops;  in do_work()
815 	if (g->p.data_rand_walk) {  in do_work()
827 			if (g->p.data_zero_memset) {  in do_work()
834 	} else if (!g->p.data_backwards || (nr + loop) & 1) {  in do_work()
879 	g->threads[task_nr].curr_cpu = cpu;  in update_curr_cpu()
888  * to a single node. A count of g->p.nr_nodes means it's
897 	node_present = (char *)malloc(g->p.nr_nodes * sizeof(char));  in count_process_nodes()
899 	for (nodes = 0; nodes < g->p.nr_nodes; nodes++)  in count_process_nodes()
902 	for (t = 0; t < g->p.nr_threads; t++) {  in count_process_nodes()
907 		task_nr = process_nr*g->p.nr_threads + t;  in count_process_nodes()
908 		td = g->threads + task_nr;  in count_process_nodes()
921 	for (n = 0; n < g->p.nr_nodes; n++)  in count_process_nodes()
940 	for (p = 0; p < g->p.nr_proc; p++) {  in count_node_processes()
941 		for (t = 0; t < g->p.nr_threads; t++) {  in count_node_processes()
946 			task_nr = p*g->p.nr_threads + t;  in count_node_processes()
947 			td = g->threads + task_nr;  in count_node_processes()
968 	for (p = 0; p < g->p.nr_proc; p++) {  in calc_convergence_compression()
1004 	if (!g->p.show_convergence && !g->p.measure_convergence)  in calc_convergence()
1007 	nodes = (int *)malloc(g->p.nr_nodes * sizeof(int));  in calc_convergence()
1009 	for (node = 0; node < g->p.nr_nodes; node++)  in calc_convergence()
1015 	for (t = 0; t < g->p.nr_tasks; t++) {  in calc_convergence()
1016 		struct thread_data *td = g->threads + t;  in calc_convergence()
1035 	nr_min = g->p.nr_tasks;  in calc_convergence()
1038 	for (node = 0; node < g->p.nr_nodes; node++) {  in calc_convergence()
1048 	BUG_ON(sum > g->p.nr_tasks);  in calc_convergence()
1050 	if (0 && (sum < g->p.nr_tasks)) {  in calc_convergence()
1058 	 * to g->p.nr_proc:  in calc_convergence()
1062 	for (node = 0; node < g->p.nr_nodes; node++) {  in calc_convergence()
1089 	if (strong && process_groups == g->p.nr_proc) {  in calc_convergence()
1093 			if (g->p.measure_convergence) {  in calc_convergence()
1094 				g->all_converged = true;  in calc_convergence()
1095 				g->stop_work = true;  in calc_convergence()
1112 		(double)(l+1)/g->p.nr_loops*100.0, runtime_ns_max / NSEC_PER_SEC / 60.0);  in show_summary()
1116 	if (g->p.show_details >= 0)  in show_summary()
1128 	int details = g->p.show_details;  in worker_thread()
1146 	global_data = g->data;  in worker_thread()
1148 	thread_data = setup_private_data(g->p.bytes_thread);  in worker_thread()
1153 	if (process_nr == g->p.nr_proc-1 && thread_nr == g->p.nr_threads-1)  in worker_thread()
1165 	if (g->p.serialize_startup) {  in worker_thread()
1166 		pthread_mutex_lock(&g->startup_mutex);  in worker_thread()
1167 		g->nr_tasks_started++;  in worker_thread()
1169 		if (g->nr_tasks_started == g->p.nr_tasks)  in worker_thread()
1170 			pthread_cond_signal(&g->startup_cond);  in worker_thread()
1172 		pthread_mutex_unlock(&g->startup_mutex);  in worker_thread()
1175 		pthread_mutex_lock(&g->start_work_mutex);  in worker_thread()
1176 		g->start_work = false;  in worker_thread()
1177 		g->nr_tasks_working++;  in worker_thread()
1178 		while (!g->start_work)  in worker_thread()
1179 			pthread_cond_wait(&g->start_work_cond, &g->start_work_mutex);  in worker_thread()
1181 		pthread_mutex_unlock(&g->start_work_mutex);  in worker_thread()
1189 	for (l = 0; l < g->p.nr_loops; l++) {  in worker_thread()
1192 		if (g->stop_work)  in worker_thread()
1195 		val += do_work(global_data,  g->p.bytes_global,  process_nr, g->p.nr_proc,	l, val);  in worker_thread()
1196 		val += do_work(process_data, g->p.bytes_process, thread_nr,  g->p.nr_threads,	l, val);  in worker_thread()
1197 		val += do_work(thread_data,  g->p.bytes_thread,  0,          1,		l, val);  in worker_thread()
1199 		if (g->p.sleep_usecs) {  in worker_thread()
1201 			usleep(g->p.sleep_usecs);  in worker_thread()
1207 		if (g->p.bytes_process_locked) {  in worker_thread()
1209 			val += do_work(process_data, g->p.bytes_process_locked, thread_nr,  g->p.nr_threads,	l, val);  in worker_thread()
1213 		work_done = g->p.bytes_global + g->p.bytes_process +  in worker_thread()
1214 			    g->p.bytes_process_locked + g->p.bytes_thread;  in worker_thread()
1219 		if (details < 0 && !g->p.perturb_secs && !g->p.measure_convergence && !g->p.nr_secs)  in worker_thread()
1227 		if (g->p.nr_secs) {  in worker_thread()
1229 			if ((u32)diff.tv_sec >= g->p.nr_secs) {  in worker_thread()
1230 				g->stop_work = true;  in worker_thread()
1240 		 * Perturb the first task's equilibrium every g->p.perturb_secs seconds,  in worker_thread()
1243 …if (first_task && g->p.perturb_secs && (int)(stop.tv_sec - last_perturbance) >= g->p.perturb_secs)…  in worker_thread()
1255 			this_cpu = g->threads[task_nr].curr_cpu;  in worker_thread()
1256 			if (this_cpu < g->p.nr_cpus/2)  in worker_thread()
1257 				target_cpu = g->p.nr_cpus-1;  in worker_thread()
1304 	free_data(thread_data, g->p.bytes_thread);  in worker_thread()
1306 	pthread_mutex_lock(&g->stop_work_mutex);  in worker_thread()
1307 	g->bytes_done += bytes_done;  in worker_thread()
1308 	pthread_mutex_unlock(&g->stop_work_mutex);  in worker_thread()
1333 	task_nr = process_nr*g->p.nr_threads;  in worker_process()
1334 	td = g->threads + task_nr;  in worker_process()
1339 	pthreads = zalloc(g->p.nr_threads * sizeof(pthread_t));  in worker_process()
1340 	process_data = setup_private_data(g->p.bytes_process);  in worker_process()
1342 	if (g->p.show_details >= 3) {  in worker_process()
1344 			process_nr, g->data, process_data);  in worker_process()
1347 	for (t = 0; t < g->p.nr_threads; t++) {  in worker_process()
1348 		task_nr = process_nr*g->p.nr_threads + t;  in worker_process()
1349 		td = g->threads + task_nr;  in worker_process()
1363 	for (t = 0; t < g->p.nr_threads; t++) {  in worker_process()
1368 	free_data(process_data, g->p.bytes_process);  in worker_process()
1374 	if (g->p.show_details < 0)  in print_summary()
1379 		g->p.nr_tasks, g->p.nr_tasks == 1 ? "task" : "tasks", nr_numa_nodes(), g->p.nr_cpus);  in print_summary()
1381 			g->p.nr_loops, g->p.bytes_global/1024/1024);  in print_summary()
1383 			g->p.nr_loops, g->p.bytes_process/1024/1024);  in print_summary()
1385 			g->p.nr_loops, g->p.bytes_thread/1024/1024);  in print_summary()
1394 	ssize_t size = sizeof(*g->threads)*g->p.nr_tasks;  in init_thread_data()
1397 	g->threads = zalloc_shared_data(size);  in init_thread_data()
1399 	for (t = 0; t < g->p.nr_tasks; t++) {  in init_thread_data()
1400 		struct thread_data *td = g->threads + t;  in init_thread_data()
1408 		for (cpu = 0; cpu < g->p.nr_cpus; cpu++)  in init_thread_data()
1415 	ssize_t size = sizeof(*g->threads)*g->p.nr_tasks;  in deinit_thread_data()
1417 	free_data(g->threads, size);  in deinit_thread_data()
1422 	g = (void *)alloc_data(sizeof(*g), MAP_SHARED, 1, 0, 0 /* THP */, 0);  in init()
1425 	g->p = p0;  in init()
1427 	g->p.nr_cpus = numa_num_configured_cpus();  in init()
1429 	g->p.nr_nodes = numa_max_node() + 1;  in init()
1432 	BUG_ON(g->p.nr_nodes < 0);  in init()
1434 	if (g->p.show_quiet && !g->p.show_details)  in init()
1435 		g->p.show_details = -1;  in init()
1438 	if (!g->p.mb_global_str && !g->p.mb_proc_str && !g->p.mb_thread_str)  in init()
1441 	if (g->p.mb_global_str) {  in init()
1442 		g->p.mb_global = atof(g->p.mb_global_str);  in init()
1443 		BUG_ON(g->p.mb_global < 0);  in init()
1446 	if (g->p.mb_proc_str) {  in init()
1447 		g->p.mb_proc = atof(g->p.mb_proc_str);  in init()
1448 		BUG_ON(g->p.mb_proc < 0);  in init()
1451 	if (g->p.mb_proc_locked_str) {  in init()
1452 		g->p.mb_proc_locked = atof(g->p.mb_proc_locked_str);  in init()
1453 		BUG_ON(g->p.mb_proc_locked < 0);  in init()
1454 		BUG_ON(g->p.mb_proc_locked > g->p.mb_proc);  in init()
1457 	if (g->p.mb_thread_str) {  in init()
1458 		g->p.mb_thread = atof(g->p.mb_thread_str);  in init()
1459 		BUG_ON(g->p.mb_thread < 0);  in init()
1462 	BUG_ON(g->p.nr_threads <= 0);  in init()
1463 	BUG_ON(g->p.nr_proc <= 0);  in init()
1465 	g->p.nr_tasks = g->p.nr_proc*g->p.nr_threads;  in init()
1467 	g->p.bytes_global		= g->p.mb_global	*1024L*1024L;  in init()
1468 	g->p.bytes_process		= g->p.mb_proc		*1024L*1024L;  in init()
1469 	g->p.bytes_process_locked	= g->p.mb_proc_locked	*1024L*1024L;  in init()
1470 	g->p.bytes_thread		= g->p.mb_thread	*1024L*1024L;  in init()
1472 	g->data = setup_shared_data(g->p.bytes_global);  in init()
1475 	init_global_mutex(&g->start_work_mutex);  in init()
1476 	init_global_cond(&g->start_work_cond);  in init()
1477 	init_global_mutex(&g->startup_mutex);  in init()
1478 	init_global_cond(&g->startup_cond);  in init()
1479 	init_global_mutex(&g->stop_work_mutex);  in init()
1495 	free_data(g->data, g->p.bytes_global);  in deinit()
1496 	g->data = NULL;  in deinit()
1500 	free_data(g, sizeof(*g));  in deinit()
1501 	g = NULL;  in deinit()
1513 	if (!g->p.show_quiet)  in print_res()
1535 	pids = zalloc(g->p.nr_proc * sizeof(*pids));  in __bench_numa()
1538 	if (g->p.serialize_startup) {  in __bench_numa()
1545 	for (i = 0; i < g->p.nr_proc; i++) {  in __bench_numa()
1560 	if (g->p.serialize_startup) {  in __bench_numa()
1568 		pthread_mutex_lock(&g->startup_mutex);  in __bench_numa()
1569 		while (g->nr_tasks_started != g->p.nr_tasks)  in __bench_numa()
1570 			pthread_cond_wait(&g->startup_cond, &g->startup_mutex);  in __bench_numa()
1572 		pthread_mutex_unlock(&g->startup_mutex);  in __bench_numa()
1576 			pthread_mutex_lock(&g->start_work_mutex);  in __bench_numa()
1577 			threads_ready = (g->nr_tasks_working == g->p.nr_tasks);  in __bench_numa()
1578 			pthread_mutex_unlock(&g->start_work_mutex);  in __bench_numa()
1596 		pthread_mutex_lock(&g->start_work_mutex);  in __bench_numa()
1597 		g->start_work = true;  in __bench_numa()
1598 		pthread_mutex_unlock(&g->start_work_mutex);  in __bench_numa()
1599 		pthread_cond_broadcast(&g->start_work_cond);  in __bench_numa()
1607 	for (i = 0; i < g->p.nr_proc; i++) {  in __bench_numa()
1617 	for (t = 0; t < g->p.nr_tasks; t++) {  in __bench_numa()
1618 		u64 thread_runtime_ns = g->threads[t].runtime_ns;  in __bench_numa()
1638 	bytes = g->bytes_done;  in __bench_numa()
1639 	runtime_avg = (double)runtime_ns_sum / g->p.nr_tasks / NSEC_PER_SEC;  in __bench_numa()
1641 	if (g->p.measure_convergence) {  in __bench_numa()
1659 	print_res(name, bytes / g->p.nr_tasks / 1e9,  in __bench_numa()
1665 	print_res(name, runtime_sec_max * NSEC_PER_SEC / (bytes / g->p.nr_tasks),  in __bench_numa()
1668 	print_res(name, bytes / g->p.nr_tasks / 1e9 / runtime_sec_max,  in __bench_numa()
1674 	if (g->p.show_details >= 2) {  in __bench_numa()
1677 		for (p = 0; p < g->p.nr_proc; p++) {  in __bench_numa()
1678 			for (t = 0; t < g->p.nr_threads; t++) {  in __bench_numa()
1680 				td = g->threads + p*g->p.nr_threads + t;  in __bench_numa()