48 					struct ap_matrix_mdev *matrix_mdev,  in vfio_ap_get_queue()  argument
54 	if (!test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm))  in vfio_ap_get_queue()
56 	if (!test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm))  in vfio_ap_get_queue()
64 	q->matrix_mdev = matrix_mdev;  in vfio_ap_get_queue()
122 	if (q->saved_isc != VFIO_AP_ISC_INVALID && q->matrix_mdev)  in vfio_ap_free_aqic_resources()
123 		kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc);  in vfio_ap_free_aqic_resources()
124 	if (q->saved_pfn && q->matrix_mdev)  in vfio_ap_free_aqic_resources()
125 		vfio_unpin_pages(mdev_dev(q->matrix_mdev->mdev),  in vfio_ap_free_aqic_resources()
180 	q->matrix_mdev = NULL;  in vfio_ap_irq_disable()
212 	ret = vfio_pin_pages(mdev_dev(q->matrix_mdev->mdev), &g_pfn, 1,  in vfio_ap_irq_enable()
222 	kvm = q->matrix_mdev->kvm;  in vfio_ap_irq_enable()
241 		vfio_unpin_pages(mdev_dev(q->matrix_mdev->mdev), &g_pfn, 1);  in vfio_ap_irq_enable()
282 	struct ap_matrix_mdev *matrix_mdev;  in handle_pqap()  local
293 	matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook,  in handle_pqap()
296 	q = vfio_ap_get_queue(matrix_mdev, apqn);  in handle_pqap()
326 	struct ap_matrix_mdev *matrix_mdev;  in vfio_ap_mdev_create()  local
331 	matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);  in vfio_ap_mdev_create()
332 	if (!matrix_mdev) {  in vfio_ap_mdev_create()
337 	matrix_mdev->mdev = mdev;  in vfio_ap_mdev_create()
338 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);  in vfio_ap_mdev_create()
339 	mdev_set_drvdata(mdev, matrix_mdev);  in vfio_ap_mdev_create()
340 	matrix_mdev->pqap_hook.hook = handle_pqap;  in vfio_ap_mdev_create()
341 	matrix_mdev->pqap_hook.owner = THIS_MODULE;  in vfio_ap_mdev_create()
343 	list_add(&matrix_mdev->node, &matrix_dev->mdev_list);  in vfio_ap_mdev_create()
351 	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);  in vfio_ap_mdev_remove()  local
353 	if (matrix_mdev->kvm)  in vfio_ap_mdev_remove()
358 	list_del(&matrix_mdev->node);  in vfio_ap_mdev_remove()
361 	kfree(matrix_mdev);  in vfio_ap_mdev_remove()
506 vfio_ap_mdev_verify_queues_reserved_for_apid(struct ap_matrix_mdev *matrix_mdev,  in vfio_ap_mdev_verify_queues_reserved_for_apid()  argument
511 	unsigned long nbits = matrix_mdev->matrix.aqm_max + 1;  in vfio_ap_mdev_verify_queues_reserved_for_apid()
513 	if (find_first_bit_inv(matrix_mdev->matrix.aqm, nbits) >= nbits)  in vfio_ap_mdev_verify_queues_reserved_for_apid()
516 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, nbits) {  in vfio_ap_mdev_verify_queues_reserved_for_apid()
536 static int vfio_ap_mdev_verify_no_sharing(struct ap_matrix_mdev *matrix_mdev)  in vfio_ap_mdev_verify_no_sharing()  argument
543 		if (matrix_mdev == lstdev)  in vfio_ap_mdev_verify_no_sharing()
553 		if (!bitmap_and(apm, matrix_mdev->matrix.apm,  in vfio_ap_mdev_verify_no_sharing()
557 		if (!bitmap_and(aqm, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_verify_no_sharing()
606 	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);  in assign_adapter_store()  local
609 	if (matrix_mdev->kvm)  in assign_adapter_store()
616 	if (apid > matrix_mdev->matrix.apm_max)  in assign_adapter_store()
626 	ret = vfio_ap_mdev_verify_queues_reserved_for_apid(matrix_mdev, apid);  in assign_adapter_store()
630 	set_bit_inv(apid, matrix_mdev->matrix.apm);  in assign_adapter_store()
632 	ret = vfio_ap_mdev_verify_no_sharing(matrix_mdev);  in assign_adapter_store()
640 	clear_bit_inv(apid, matrix_mdev->matrix.apm);  in assign_adapter_store()
672 	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);  in unassign_adapter_store()  local
675 	if (matrix_mdev->kvm)  in unassign_adapter_store()
682 	if (apid > matrix_mdev->matrix.apm_max)  in unassign_adapter_store()
686 	clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);  in unassign_adapter_store()
694 vfio_ap_mdev_verify_queues_reserved_for_apqi(struct ap_matrix_mdev *matrix_mdev,  in vfio_ap_mdev_verify_queues_reserved_for_apqi()  argument
699 	unsigned long nbits = matrix_mdev->matrix.apm_max + 1;  in vfio_ap_mdev_verify_queues_reserved_for_apqi()
701 	if (find_first_bit_inv(matrix_mdev->matrix.apm, nbits) >= nbits)  in vfio_ap_mdev_verify_queues_reserved_for_apqi()
704 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, nbits) {  in vfio_ap_mdev_verify_queues_reserved_for_apqi()
752 	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);  in assign_domain_store()  local
753 	unsigned long max_apqi = matrix_mdev->matrix.aqm_max;  in assign_domain_store()
756 	if (matrix_mdev->kvm)  in assign_domain_store()
767 	ret = vfio_ap_mdev_verify_queues_reserved_for_apqi(matrix_mdev, apqi);  in assign_domain_store()
771 	set_bit_inv(apqi, matrix_mdev->matrix.aqm);  in assign_domain_store()
773 	ret = vfio_ap_mdev_verify_no_sharing(matrix_mdev);  in assign_domain_store()
781 	clear_bit_inv(apqi, matrix_mdev->matrix.aqm);  in assign_domain_store()
814 	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);  in unassign_domain_store()  local
817 	if (matrix_mdev->kvm)  in unassign_domain_store()
824 	if (apqi > matrix_mdev->matrix.aqm_max)  in unassign_domain_store()
828 	clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);  in unassign_domain_store()
858 	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);  in assign_control_domain_store()  local
861 	if (matrix_mdev->kvm)  in assign_control_domain_store()
868 	if (id > matrix_mdev->matrix.adm_max)  in assign_control_domain_store()
877 	set_bit_inv(id, matrix_mdev->matrix.adm);  in assign_control_domain_store()
907 	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);  in unassign_control_domain_store()  local
908 	unsigned long max_domid =  matrix_mdev->matrix.adm_max;  in unassign_control_domain_store()
911 	if (matrix_mdev->kvm)  in unassign_control_domain_store()
921 	clear_bit_inv(domid, matrix_mdev->matrix.adm);  in unassign_control_domain_store()
937 	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);  in control_domains_show()  local
938 	unsigned long max_domid = matrix_mdev->matrix.adm_max;  in control_domains_show()
941 	for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) {  in control_domains_show()
956 	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);  in matrix_show()  local
962 	unsigned long napm_bits = matrix_mdev->matrix.apm_max + 1;  in matrix_show()
963 	unsigned long naqm_bits = matrix_mdev->matrix.aqm_max + 1;  in matrix_show()
967 	apid1 = find_first_bit_inv(matrix_mdev->matrix.apm, napm_bits);  in matrix_show()
968 	apqi1 = find_first_bit_inv(matrix_mdev->matrix.aqm, naqm_bits);  in matrix_show()
973 		for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, napm_bits) {  in matrix_show()
974 			for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,  in matrix_show()
983 		for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, napm_bits) {  in matrix_show()
989 		for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, naqm_bits) {  in matrix_show()
1035 static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev,  in vfio_ap_mdev_set_kvm()  argument
1043 		if ((m != matrix_mdev) && (m->kvm == kvm)) {  in vfio_ap_mdev_set_kvm()
1049 	matrix_mdev->kvm = kvm;  in vfio_ap_mdev_set_kvm()
1051 	kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;  in vfio_ap_mdev_set_kvm()
1071 	struct ap_matrix_mdev *matrix_mdev;  in vfio_ap_mdev_iommu_notifier()  local
1073 	matrix_mdev = container_of(nb, struct ap_matrix_mdev, iommu_notifier);  in vfio_ap_mdev_iommu_notifier()
1079 		vfio_unpin_pages(mdev_dev(matrix_mdev->mdev), &g_pfn, 1);  in vfio_ap_mdev_iommu_notifier()
1090 	struct ap_matrix_mdev *matrix_mdev;  in vfio_ap_mdev_group_notifier()  local
1095 	matrix_mdev = container_of(nb, struct ap_matrix_mdev, group_notifier);  in vfio_ap_mdev_group_notifier()
1098 		matrix_mdev->kvm = NULL;  in vfio_ap_mdev_group_notifier()
1102 	ret = vfio_ap_mdev_set_kvm(matrix_mdev, data);  in vfio_ap_mdev_group_notifier()
1107 	if (!matrix_mdev->kvm->arch.crypto.crycbd)  in vfio_ap_mdev_group_notifier()
1110 	kvm_arch_crypto_set_masks(matrix_mdev->kvm, matrix_mdev->matrix.apm,  in vfio_ap_mdev_group_notifier()
1111 				  matrix_mdev->matrix.aqm,  in vfio_ap_mdev_group_notifier()
1112 				  matrix_mdev->matrix.adm);  in vfio_ap_mdev_group_notifier()
1166 	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);  in vfio_ap_mdev_reset_queues()  local
1168 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm,  in vfio_ap_mdev_reset_queues()
1169 			     matrix_mdev->matrix.apm_max + 1) {  in vfio_ap_mdev_reset_queues()
1170 		for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_reset_queues()
1171 				     matrix_mdev->matrix.aqm_max + 1) {  in vfio_ap_mdev_reset_queues()
1189 	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);  in vfio_ap_mdev_open()  local
1197 	matrix_mdev->group_notifier.notifier_call = vfio_ap_mdev_group_notifier;  in vfio_ap_mdev_open()
1201 				     &events, &matrix_mdev->group_notifier);  in vfio_ap_mdev_open()
1207 	matrix_mdev->iommu_notifier.notifier_call = vfio_ap_mdev_iommu_notifier;  in vfio_ap_mdev_open()
1210 				     &events, &matrix_mdev->iommu_notifier);  in vfio_ap_mdev_open()
1215 				 &matrix_mdev->group_notifier);  in vfio_ap_mdev_open()
1222 	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);  in vfio_ap_mdev_release()  local
1225 	if (matrix_mdev->kvm) {  in vfio_ap_mdev_release()
1226 		kvm_arch_crypto_clear_masks(matrix_mdev->kvm);  in vfio_ap_mdev_release()
1227 		matrix_mdev->kvm->arch.crypto.pqap_hook = NULL;  in vfio_ap_mdev_release()
1229 		kvm_put_kvm(matrix_mdev->kvm);  in vfio_ap_mdev_release()
1230 		matrix_mdev->kvm = NULL;  in vfio_ap_mdev_release()
1235 				 &matrix_mdev->iommu_notifier);  in vfio_ap_mdev_release()
1237 				 &matrix_mdev->group_notifier);  in vfio_ap_mdev_release()