41 	struct priority_group *pg;	/* Owning PG */  member
169 	struct priority_group *pg;  in alloc_priority_group()  local
171 	pg = kzalloc(sizeof(*pg), GFP_KERNEL);  in alloc_priority_group()
173 	if (pg)  in alloc_priority_group()
174 		INIT_LIST_HEAD(&pg->pgpaths);  in alloc_priority_group()
176 	return pg;  in alloc_priority_group()
190 static void free_priority_group(struct priority_group *pg,  in free_priority_group()  argument
193 	struct path_selector *ps = &pg->ps;  in free_priority_group()
200 	free_pgpaths(&pg->pgpaths, ti);  in free_priority_group()
201 	kfree(pg);  in free_priority_group()
257 	struct priority_group *pg, *tmp;  in free_multipath()  local
259 	list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {  in free_multipath()
260 		list_del(&pg->list);  in free_multipath()
261 		free_priority_group(pg, m->ti);  in free_multipath()
352 static void __switch_pg(struct multipath *m, struct priority_group *pg)  in __switch_pg()  argument
356 	m->current_pg = pg;  in __switch_pg()
371 					struct priority_group *pg,  in choose_path_in_pg()  argument
378 	path = pg->ps.type->select_path(&pg->ps, nr_bytes);  in choose_path_in_pg()
384 	if (unlikely(READ_ONCE(m->current_pg) != pg)) {  in choose_path_in_pg()
388 		__switch_pg(m, pg);  in choose_path_in_pg()
398 	struct priority_group *pg;  in choose_pgpath()  local
412 		pg = m->next_pg;  in choose_pgpath()
413 		if (!pg) {  in choose_pgpath()
419 		pgpath = choose_path_in_pg(m, pg, nr_bytes);  in choose_pgpath()
426 	pg = READ_ONCE(m->current_pg);  in choose_pgpath()
427 	if (pg) {  in choose_pgpath()
428 		pgpath = choose_path_in_pg(m, pg, nr_bytes);  in choose_pgpath()
440 		list_for_each_entry(pg, &m->priority_groups, list) {  in choose_pgpath()
441 			if (pg->bypassed == !!bypassed)  in choose_pgpath()
443 			pgpath = choose_path_in_pg(m, pg, nr_bytes);  in choose_pgpath()
558 	if (pgpath->pg->ps.type->start_io)  in multipath_clone_and_map()
559 		pgpath->pg->ps.type->start_io(&pgpath->pg->ps,  in multipath_clone_and_map()
576 		if (pgpath && pgpath->pg->ps.type->end_io)  in multipath_release_clone()
577 			pgpath->pg->ps.type->end_io(&pgpath->pg->ps,  in multipath_release_clone()
652 	if (dm_ps_use_hr_timer(pgpath->pg->ps.type))  in __multipath_map_bio()
659 	if (pgpath->pg->ps.type->start_io)  in __multipath_map_bio()
660 		pgpath->pg->ps.type->start_io(&pgpath->pg->ps,  in __multipath_map_bio()
838 static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,  in parse_path_selector()  argument
861 	r = pst->create(&pg->ps, ps_argc, as->argv);  in parse_path_selector()
868 	pg->ps.type = pst;  in parse_path_selector()
987 	struct priority_group *pg;  in parse_priority_group()  local
996 	pg = alloc_priority_group();  in parse_priority_group()
997 	if (!pg) {  in parse_priority_group()
1001 	pg->m = m;  in parse_priority_group()
1003 	r = parse_path_selector(as, pg, ti);  in parse_priority_group()
1010 	r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error);  in parse_priority_group()
1019 	for (i = 0; i < pg->nr_pgpaths; i++) {  in parse_priority_group()
1032 		pgpath = parse_path(&path_args, &pg->ps, ti);  in parse_priority_group()
1038 		pgpath->pg = pg;  in parse_priority_group()
1039 		list_add_tail(&pgpath->list, &pg->pgpaths);  in parse_priority_group()
1043 	return pg;  in parse_priority_group()
1046 	free_priority_group(pg, ti);  in parse_priority_group()
1226 		struct priority_group *pg;  in multipath_ctr()  local
1229 		pg = parse_priority_group(&as, m);  in multipath_ctr()
1230 		if (IS_ERR(pg)) {  in multipath_ctr()
1231 			r = PTR_ERR(pg);  in multipath_ctr()
1235 		nr_valid_paths += pg->nr_pgpaths;  in multipath_ctr()
1238 		list_add_tail(&pg->list, &m->priority_groups);  in multipath_ctr()
1240 		pg->pg_num = pg_count;  in multipath_ctr()
1242 			m->next_pg = pg;  in multipath_ctr()
1327 	struct multipath *m = pgpath->pg->m;  in fail_path()
1338 	pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);  in fail_path()
1367 	struct multipath *m = pgpath->pg->m;  in reinstate_path()
1379 	r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);  in reinstate_path()
1389 	} else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {  in reinstate_path()
1420 	struct priority_group *pg;  in action_dev()  local
1422 	list_for_each_entry(pg, &m->priority_groups, list) {  in action_dev()
1423 		list_for_each_entry(pgpath, &pg->pgpaths, list) {  in action_dev()
1435 static void bypass_pg(struct multipath *m, struct priority_group *pg,  in bypass_pg()  argument
1442 	pg->bypassed = bypassed;  in bypass_pg()
1456 	struct priority_group *pg;  in switch_pg_num()  local
1468 	list_for_each_entry(pg, &m->priority_groups, list) {  in switch_pg_num()
1469 		pg->bypassed = false;  in switch_pg_num()
1475 		m->next_pg = pg;  in switch_pg_num()
1489 	struct priority_group *pg;  in bypass_pg_num()  local
1499 	list_for_each_entry(pg, &m->priority_groups, list) {  in bypass_pg_num()
1504 	bypass_pg(m, pg, bypassed);  in bypass_pg_num()
1532 	struct priority_group *pg = pgpath->pg;  in pg_init_done()  local
1533 	struct multipath *m = pg->m;  in pg_init_done()
1558 		bypass_pg(m, pg, true);  in pg_init_done()
1588 		pg->bypassed = false;  in pg_init_done()
1673 		struct path_selector *ps = &pgpath->pg->ps;  in multipath_end_io()
1717 		struct path_selector *ps = &pgpath->pg->ps;  in multipath_end_io_bio()
1797 	struct priority_group *pg;  in multipath_status()  local
1853 		list_for_each_entry(pg, &m->priority_groups, list) {  in multipath_status()
1854 			if (pg->bypassed)  in multipath_status()
1856 			else if (pg == m->current_pg)  in multipath_status()
1863 			if (pg->ps.type->status)  in multipath_status()
1864 				sz += pg->ps.type->status(&pg->ps, NULL, type,  in multipath_status()
1870 			DMEMIT("%u %u ", pg->nr_pgpaths,  in multipath_status()
1871 			       pg->ps.type->info_args);  in multipath_status()
1873 			list_for_each_entry(p, &pg->pgpaths, list) {  in multipath_status()
1877 				if (pg->ps.type->status)  in multipath_status()
1878 					sz += pg->ps.type->status(&pg->ps,  in multipath_status()
1886 		list_for_each_entry(pg, &m->priority_groups, list) {  in multipath_status()
1887 			DMEMIT("%s ", pg->ps.type->name);  in multipath_status()
1889 			if (pg->ps.type->status)  in multipath_status()
1890 				sz += pg->ps.type->status(&pg->ps, NULL, type,  in multipath_status()
1896 			DMEMIT("%u %u ", pg->nr_pgpaths,  in multipath_status()
1897 			       pg->ps.type->table_args);  in multipath_status()
1899 			list_for_each_entry(p, &pg->pgpaths, list) {  in multipath_status()
1901 				if (pg->ps.type->status)  in multipath_status()
1902 					sz += pg->ps.type->status(&pg->ps,  in multipath_status()
1916 		list_for_each_entry(pg, &m->priority_groups, list) {  in multipath_status()
1917 			if (pg->bypassed)  in multipath_status()
1919 			else if (pg == m->current_pg)  in multipath_status()
1924 			DMEMIT(",nr_pgpaths_%d=%u", pg_counter, pg->nr_pgpaths);  in multipath_status()
1925 			DMEMIT(",path_selector_name_%d=%s", pg_counter, pg->ps.type->name);  in multipath_status()
1928 			list_for_each_entry(p, &pg->pgpaths, list) {  in multipath_status()
1933 				if (pg->ps.type->status) {  in multipath_status()
1936 					sz += pg->ps.type->status(&pg->ps, &p->path,  in multipath_status()
2074 	struct priority_group *pg;  in multipath_iterate_devices()  local
2078 	list_for_each_entry(pg, &m->priority_groups, list) {  in multipath_iterate_devices()
2079 		list_for_each_entry(p, &pg->pgpaths, list) {  in multipath_iterate_devices()
2109 	struct priority_group *pg, *next_pg;  in multipath_busy()  local
2128 	pg = READ_ONCE(m->current_pg);  in multipath_busy()
2131 		pg = next_pg;  in multipath_busy()
2133 	if (!pg) {  in multipath_busy()
2149 	list_for_each_entry(pgpath, &pg->pgpaths, list) {  in multipath_busy()