3  * Adjunct processor matrix VFIO device driver callbacks.
156 		if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) &&  in get_update_locks_by_apqn()
157 		    test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) {  in get_update_locks_by_apqn()
174  *			       KVM guest to which the matrix mdev linked to a
198  *			    hash table of queues assigned to a matrix mdev
199  * @matrix_mdev: the matrix mdev
629 				struct ap_matrix *matrix)  in vfio_ap_matrix_init()  argument
631 	matrix->apm_max = info->apxa ? info->na : 63;  in vfio_ap_matrix_init()
632 	matrix->aqm_max = info->apxa ? info->nd : 15;  in vfio_ap_matrix_init()
633 	matrix->adm_max = info->apxa ? info->nd : 15;  in vfio_ap_matrix_init()
650 	bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm,  in vfio_ap_mdev_filter_cdoms()
658  * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev
663  * @matrix_mdev: the matrix mdev whose matrix is to be filtered.
666  *	 driver, its APID will be filtered from the guest's APCB. The matrix
687 	 * from the matrix mdev, but only those that are assigned to the host's  in vfio_ap_mdev_filter_matrix()
690 	bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm,  in vfio_ap_mdev_filter_matrix()
692 	bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_filter_matrix()
727 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);  in vfio_ap_mdev_init_dev()
792 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {  in vfio_ap_mdev_unlink_fr_queues()
793 		for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_unlink_fr_queues()
836  * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs
842  * AP adapter IDs and AP queue indexes is not configured for any matrix
859 		if (mdev_apm == matrix_mdev->matrix.apm &&  in vfio_ap_mdev_verify_no_sharing()
860 		    mdev_aqm == matrix_mdev->matrix.aqm)  in vfio_ap_mdev_verify_no_sharing()
870 		if (!bitmap_and(apm, mdev_apm, matrix_mdev->matrix.apm,  in vfio_ap_mdev_verify_no_sharing()
874 		if (!bitmap_and(aqm, mdev_aqm, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_verify_no_sharing()
903 	if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm,  in vfio_ap_mdev_validate_masks()
904 					       matrix_mdev->matrix.aqm))  in vfio_ap_mdev_validate_masks()
907 	return vfio_ap_mdev_verify_no_sharing(matrix_mdev->matrix.apm,  in vfio_ap_mdev_validate_masks()
908 					      matrix_mdev->matrix.aqm);  in vfio_ap_mdev_validate_masks()
916 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS)  in vfio_ap_mdev_link_adapter()
923  * corresponding bit in the mediated matrix device's APM
925  * @dev:	the matrix device
926  * @attr:	the mediated matrix device's assign_adapter attribute
949  *	   matrix device
971 	if (apid > matrix_mdev->matrix.apm_max) {  in assign_adapter_store()
976 	if (test_bit_inv(apid, matrix_mdev->matrix.apm)) {  in assign_adapter_store()
981 	set_bit_inv(apid, matrix_mdev->matrix.apm);  in assign_adapter_store()
985 		clear_bit_inv(apid, matrix_mdev->matrix.apm);  in assign_adapter_store()
994 				       matrix_mdev->matrix.aqm, matrix_mdev))  in assign_adapter_store()
1013 	/* If the queue is assigned to the matrix mdev, unlink it. */  in vfio_ap_unlink_apqn_fr_mdev()
1022  *				 adapter from the matrix mdev to which the
1024  * @matrix_mdev: the matrix mediated device to which the adapter was assigned.
1035 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) {  in vfio_ap_mdev_unlink_adapter()
1074  * corresponding bit in the mediated matrix device's APM
1076  * @dev:	the matrix device
1077  * @attr:	the mediated matrix device's unassign_adapter attribute
1101 	if (apid > matrix_mdev->matrix.apm_max) {  in unassign_adapter_store()
1106 	if (!test_bit_inv(apid, matrix_mdev->matrix.apm)) {  in unassign_adapter_store()
1111 	clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);  in unassign_adapter_store()
1125 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES)  in vfio_ap_mdev_link_domain()
1132  * corresponding bit in the mediated matrix device's AQM
1134  * @dev:	the matrix device
1135  * @attr:	the mediated matrix device's assign_domain attribute
1158  *	   matrix device
1180 	if (apqi > matrix_mdev->matrix.aqm_max) {  in assign_domain_store()
1185 	if (test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {  in assign_domain_store()
1190 	set_bit_inv(apqi, matrix_mdev->matrix.aqm);  in assign_domain_store()
1194 		clear_bit_inv(apqi, matrix_mdev->matrix.aqm);  in assign_domain_store()
1202 	if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, aqm_delta,  in assign_domain_store()
1222 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {  in vfio_ap_mdev_unlink_domain()
1261  * corresponding bit in the mediated matrix device's AQM
1263  * @dev:	the matrix device
1264  * @attr:	the mediated matrix device's unassign_domain attribute
1288 	if (apqi > matrix_mdev->matrix.aqm_max) {  in unassign_domain_store()
1293 	if (!test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {  in unassign_domain_store()
1298 	clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);  in unassign_domain_store()
1310  * the corresponding bit in the mediated matrix device's ADM
1312  * @dev:	the matrix device
1313  * @attr:	the mediated matrix device's assign_control_domain attribute
1336 	if (id > matrix_mdev->matrix.adm_max) {  in assign_control_domain_store()
1341 	if (test_bit_inv(id, matrix_mdev->matrix.adm)) {  in assign_control_domain_store()
1351 	set_bit_inv(id, matrix_mdev->matrix.adm);  in assign_control_domain_store()
1364  * clears the corresponding bit in the mediated matrix device's ADM
1366  * @dev:	the matrix device
1367  * @attr:	the mediated matrix device's unassign_control_domain attribute
1390 	if (domid > matrix_mdev->matrix.adm_max) {  in unassign_control_domain_store()
1395 	if (!test_bit_inv(domid, matrix_mdev->matrix.adm)) {  in unassign_control_domain_store()
1400 	clear_bit_inv(domid, matrix_mdev->matrix.adm);  in unassign_control_domain_store()
1423 	unsigned long max_domid = matrix_mdev->matrix.adm_max;  in control_domains_show()
1426 	for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) {  in control_domains_show()
1437 static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf)  in vfio_ap_mdev_matrix_show()  argument
1444 	unsigned long napm_bits = matrix->apm_max + 1;  in vfio_ap_mdev_matrix_show()
1445 	unsigned long naqm_bits = matrix->aqm_max + 1;  in vfio_ap_mdev_matrix_show()
1449 	apid1 = find_first_bit_inv(matrix->apm, napm_bits);  in vfio_ap_mdev_matrix_show()
1450 	apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits);  in vfio_ap_mdev_matrix_show()
1453 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {  in vfio_ap_mdev_matrix_show()
1454 			for_each_set_bit_inv(apqi, matrix->aqm,  in vfio_ap_mdev_matrix_show()
1463 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {  in vfio_ap_mdev_matrix_show()
1469 		for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) {  in vfio_ap_mdev_matrix_show()
1486 	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf);  in matrix_show()
1491 static DEVICE_ATTR_RO(matrix);
1533  * @matrix_mdev: a mediated matrix device
1536  * Return: 0 if no other mediated matrix device has a reference to @kvm;
1597  * @matrix_mdev: a matrix mediated device
1965 		if (test_bit_inv(apid, matrix_mdev->matrix.apm) &&  in vfio_ap_mdev_for_queue()
1966 		    test_bit_inv(apqi, matrix_mdev->matrix.aqm))  in vfio_ap_mdev_for_queue()
2094 		if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm,  in vfio_ap_mdev_probe_queue()
2095 					       matrix_mdev->matrix.aqm,  in vfio_ap_mdev_probe_queue()
2238 					  matrix_mdev->matrix.apm,  in vfio_ap_mdev_cfg_remove()
2241 					  matrix_mdev->matrix.aqm,  in vfio_ap_mdev_cfg_remove()
2244 					     matrix_mdev->matrix.adm,  in vfio_ap_mdev_cfg_remove()
2356  *			  AP configuration for each matrix mdev to which they
2378 			   matrix_mdev->matrix.apm, apm_add, AP_DEVICES);  in vfio_ap_mdev_cfg_add()
2380 			   matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS);  in vfio_ap_mdev_cfg_add()
2382 			   matrix_mdev->matrix.adm, adm_add, AP_DEVICES);  in vfio_ap_mdev_cfg_add()
2455 	filter_adapters = bitmap_and(apm, matrix_mdev->matrix.apm,  in vfio_ap_mdev_hot_plug_cfg()
2457 	filter_domains = bitmap_and(aqm, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_hot_plug_cfg()
2472 	if (bitmap_intersects(matrix_mdev->matrix.adm, matrix_mdev->adm_add,  in vfio_ap_mdev_hot_plug_cfg()