99 static struct klp_object *klp_find_object(struct klp_patch *patch,  in klp_find_object()  argument
104 	klp_for_each_object(patch, obj) {  in klp_find_object()
309 static int __klp_disable_patch(struct klp_patch *patch);
314 	struct klp_patch *patch;  in enabled_store()  local
322 	patch = container_of(kobj, struct klp_patch, kobj);  in enabled_store()
326 	if (patch->enabled == enabled) {  in enabled_store()
339 	if (patch == klp_transition_patch)  in enabled_store()
342 		ret = __klp_disable_patch(patch);  in enabled_store()
357 	struct klp_patch *patch;  in enabled_show()  local
359 	patch = container_of(kobj, struct klp_patch, kobj);  in enabled_show()
360 	return snprintf(buf, PAGE_SIZE-1, "%d\n", patch->enabled);  in enabled_show()
366 	struct klp_patch *patch;  in transition_show()  local
368 	patch = container_of(kobj, struct klp_patch, kobj);  in transition_show()
370 			patch == klp_transition_patch);  in transition_show()
376 	struct klp_patch *patch;  in force_store()  local
389 	patch = container_of(kobj, struct klp_patch, kobj);  in force_store()
390 	if (patch != klp_transition_patch) {  in force_store()
421 static void klp_init_object_early(struct klp_patch *patch,
425 						   struct klp_patch *patch)  in klp_alloc_object_dynamic()  argument
441 	klp_init_object_early(patch, obj);  in klp_alloc_object_dynamic()
481 static int klp_add_object_nops(struct klp_patch *patch,  in klp_add_object_nops()  argument
487 	obj = klp_find_object(patch, old_obj);  in klp_add_object_nops()
490 		obj = klp_alloc_object_dynamic(old_obj->name, patch);  in klp_add_object_nops()
513 static int klp_add_nops(struct klp_patch *patch)  in klp_add_nops()  argument
522 			err = klp_add_object_nops(patch, old_obj);  in klp_add_nops()
533 	struct klp_patch *patch;  in klp_kobj_release_patch()  local
535 	patch = container_of(kobj, struct klp_patch, kobj);  in klp_kobj_release_patch()
536 	complete(&patch->finish);  in klp_kobj_release_patch()
603 static void __klp_free_objects(struct klp_patch *patch, bool nops_only)  in __klp_free_objects()  argument
607 	klp_for_each_object_safe(patch, obj, tmp_obj) {  in __klp_free_objects()
618 static void klp_free_objects(struct klp_patch *patch)  in klp_free_objects()  argument
620 	__klp_free_objects(patch, false);  in klp_free_objects()
623 static void klp_free_objects_dynamic(struct klp_patch *patch)  in klp_free_objects_dynamic()  argument
625 	__klp_free_objects(patch, true);  in klp_free_objects_dynamic()
635 void klp_free_patch_start(struct klp_patch *patch)  in klp_free_patch_start()  argument
637 	if (!list_empty(&patch->list))  in klp_free_patch_start()
638 		list_del(&patch->list);  in klp_free_patch_start()
640 	klp_free_objects(patch);  in klp_free_patch_start()
651 static void klp_free_patch_finish(struct klp_patch *patch)  in klp_free_patch_finish()  argument
659 	kobject_put(&patch->kobj);  in klp_free_patch_finish()
660 	wait_for_completion(&patch->finish);  in klp_free_patch_finish()
663 	if (!patch->forced)  in klp_free_patch_finish()
664 		module_put(patch->mod);  in klp_free_patch_finish()
674 	struct klp_patch *patch =  in klp_free_patch_work_fn()  local
677 	klp_free_patch_finish(patch);  in klp_free_patch_work_fn()
710 void __weak arch_klp_init_object_loaded(struct klp_patch *patch,  in arch_klp_init_object_loaded()  argument
716 static int klp_init_object_loaded(struct klp_patch *patch,  in klp_init_object_loaded()  argument
724 	module_disable_ro(patch->mod);  in klp_init_object_loaded()
725 	ret = klp_write_object_relocations(patch->mod, obj);  in klp_init_object_loaded()
727 		module_enable_ro(patch->mod, true);  in klp_init_object_loaded()
732 	arch_klp_init_object_loaded(patch, obj);  in klp_init_object_loaded()
733 	module_enable_ro(patch->mod, true);  in klp_init_object_loaded()
767 static int klp_init_object(struct klp_patch *patch, struct klp_object *obj)  in klp_init_object()  argument
782 	ret = kobject_add(&obj->kobj, &patch->kobj, "%s", name);  in klp_init_object()
793 		ret = klp_init_object_loaded(patch, obj);  in klp_init_object()
805 static void klp_init_object_early(struct klp_patch *patch,  in klp_init_object_early()  argument
810 	list_add_tail(&obj->node, &patch->obj_list);  in klp_init_object_early()
813 static int klp_init_patch_early(struct klp_patch *patch)  in klp_init_patch_early()  argument
818 	if (!patch->objs)  in klp_init_patch_early()
821 	INIT_LIST_HEAD(&patch->list);  in klp_init_patch_early()
822 	INIT_LIST_HEAD(&patch->obj_list);  in klp_init_patch_early()
823 	kobject_init(&patch->kobj, &klp_ktype_patch);  in klp_init_patch_early()
824 	patch->enabled = false;  in klp_init_patch_early()
825 	patch->forced = false;  in klp_init_patch_early()
826 	INIT_WORK(&patch->free_work, klp_free_patch_work_fn);  in klp_init_patch_early()
827 	init_completion(&patch->finish);  in klp_init_patch_early()
829 	klp_for_each_object_static(patch, obj) {  in klp_init_patch_early()
833 		klp_init_object_early(patch, obj);  in klp_init_patch_early()
840 	if (!try_module_get(patch->mod))  in klp_init_patch_early()
846 static int klp_init_patch(struct klp_patch *patch)  in klp_init_patch()  argument
851 	ret = kobject_add(&patch->kobj, klp_root_kobj, "%s", patch->mod->name);  in klp_init_patch()
855 	if (patch->replace) {  in klp_init_patch()
856 		ret = klp_add_nops(patch);  in klp_init_patch()
861 	klp_for_each_object(patch, obj) {  in klp_init_patch()
862 		ret = klp_init_object(patch, obj);  in klp_init_patch()
867 	list_add_tail(&patch->list, &klp_patches);  in klp_init_patch()
872 static int __klp_disable_patch(struct klp_patch *patch)  in __klp_disable_patch()  argument
876 	if (WARN_ON(!patch->enabled))  in __klp_disable_patch()
882 	klp_init_transition(patch, KLP_UNPATCHED);  in __klp_disable_patch()
884 	klp_for_each_object(patch, obj)  in __klp_disable_patch()
898 	patch->enabled = false;  in __klp_disable_patch()
904 static int __klp_enable_patch(struct klp_patch *patch)  in __klp_enable_patch()  argument
912 	if (WARN_ON(patch->enabled))  in __klp_enable_patch()
915 	pr_notice("enabling patch '%s'\n", patch->mod->name);  in __klp_enable_patch()
917 	klp_init_transition(patch, KLP_PATCHED);  in __klp_enable_patch()
928 	klp_for_each_object(patch, obj) {  in __klp_enable_patch()
948 	patch->enabled = true;  in __klp_enable_patch()
953 	pr_warn("failed to enable patch '%s'\n", patch->mod->name);  in __klp_enable_patch()
972 int klp_enable_patch(struct klp_patch *patch)  in klp_enable_patch()  argument
976 	if (!patch || !patch->mod)  in klp_enable_patch()
979 	if (!is_livepatch_module(patch->mod)) {  in klp_enable_patch()
981 		       patch->mod->name);  in klp_enable_patch()
995 	ret = klp_init_patch_early(patch);  in klp_enable_patch()
1001 	ret = klp_init_patch(patch);  in klp_enable_patch()
1005 	ret = __klp_enable_patch(patch);  in klp_enable_patch()
1014 	klp_free_patch_start(patch);  in klp_enable_patch()
1018 	klp_free_patch_finish(patch);  in klp_enable_patch()
1083 	struct klp_patch *patch;  in klp_cleanup_module_patches_limited()  local
1086 	klp_for_each_patch(patch) {  in klp_cleanup_module_patches_limited()
1087 		if (patch == limit)  in klp_cleanup_module_patches_limited()
1090 		klp_for_each_object(patch, obj) {  in klp_cleanup_module_patches_limited()
1094 			if (patch != klp_transition_patch)  in klp_cleanup_module_patches_limited()
1098 				  patch->mod->name, obj->mod->name);  in klp_cleanup_module_patches_limited()
1112 	struct klp_patch *patch;  in klp_module_coming()  local
1126 	klp_for_each_patch(patch) {  in klp_module_coming()
1127 		klp_for_each_object(patch, obj) {  in klp_module_coming()
1133 			ret = klp_init_object_loaded(patch, obj);  in klp_module_coming()
1136 					patch->mod->name, obj->mod->name, ret);  in klp_module_coming()
1141 				  patch->mod->name, obj->mod->name);  in klp_module_coming()
1153 					patch->mod->name, obj->mod->name, ret);  in klp_module_coming()
1159 			if (patch != klp_transition_patch)  in klp_module_coming()
1176 		patch->mod->name, obj->mod->name, obj->mod->name);  in klp_module_coming()
1179 	klp_cleanup_module_patches_limited(mod, patch);  in klp_module_coming()