3  * Adjunct processor matrix VFIO device driver callbacks.
154 		if (test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm) &&  in get_update_locks_by_apqn()
155 		    test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm)) {  in get_update_locks_by_apqn()
172  *			       KVM guest to which the matrix mdev linked to a
196  *			    hash table of queues assigned to a matrix mdev
197  * @matrix_mdev: the matrix mdev
595 				struct ap_matrix *matrix)  in vfio_ap_matrix_init()  argument
597 	matrix->apm_max = info->apxa ? info->Na : 63;  in vfio_ap_matrix_init()
598 	matrix->aqm_max = info->apxa ? info->Nd : 15;  in vfio_ap_matrix_init()
599 	matrix->adm_max = info->apxa ? info->Nd : 15;  in vfio_ap_matrix_init()
616 	bitmap_and(matrix_mdev->shadow_apcb.adm, matrix_mdev->matrix.adm,  in vfio_ap_mdev_filter_cdoms()
624  * vfio_ap_mdev_filter_matrix - filter the APQNs assigned to the matrix mdev
629  * @matrix_mdev: the matrix mdev whose matrix is to be filtered.
632  *	 driver, its APID will be filtered from the guest's APCB. The matrix
653 	 * from the matrix mdev, but only those that are assigned to the host's  in vfio_ap_mdev_filter_matrix()
656 	bitmap_and(matrix_mdev->shadow_apcb.apm, matrix_mdev->matrix.apm,  in vfio_ap_mdev_filter_matrix()
658 	bitmap_and(matrix_mdev->shadow_apcb.aqm, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_filter_matrix()
693 	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);  in vfio_ap_mdev_init_dev()
757 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {  in vfio_ap_mdev_unlink_fr_queues()
758 		for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_unlink_fr_queues()
806  * vfio_ap_mdev_verify_no_sharing - verify APQNs are not shared by matrix mdevs
812  * AP adapter IDs and AP queue indexes is not configured for any matrix
829 		if (mdev_apm == matrix_mdev->matrix.apm &&  in vfio_ap_mdev_verify_no_sharing()
830 		    mdev_aqm == matrix_mdev->matrix.aqm)  in vfio_ap_mdev_verify_no_sharing()
840 		if (!bitmap_and(apm, mdev_apm, matrix_mdev->matrix.apm,  in vfio_ap_mdev_verify_no_sharing()
844 		if (!bitmap_and(aqm, mdev_aqm, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_verify_no_sharing()
873 	if (ap_apqn_in_matrix_owned_by_def_drv(matrix_mdev->matrix.apm,  in vfio_ap_mdev_validate_masks()
874 					       matrix_mdev->matrix.aqm))  in vfio_ap_mdev_validate_masks()
877 	return vfio_ap_mdev_verify_no_sharing(matrix_mdev->matrix.apm,  in vfio_ap_mdev_validate_masks()
878 					      matrix_mdev->matrix.aqm);  in vfio_ap_mdev_validate_masks()
886 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS)  in vfio_ap_mdev_link_adapter()
893  * corresponding bit in the mediated matrix device's APM
895  * @dev:	the matrix device
896  * @attr:	the mediated matrix device's assign_adapter attribute
919  *	   matrix device
941 	if (apid > matrix_mdev->matrix.apm_max) {  in assign_adapter_store()
946 	if (test_bit_inv(apid, matrix_mdev->matrix.apm)) {  in assign_adapter_store()
951 	set_bit_inv(apid, matrix_mdev->matrix.apm);  in assign_adapter_store()
955 		clear_bit_inv(apid, matrix_mdev->matrix.apm);  in assign_adapter_store()
964 				       matrix_mdev->matrix.aqm, matrix_mdev))  in assign_adapter_store()
983 	/* If the queue is assigned to the matrix mdev, unlink it. */  in vfio_ap_unlink_apqn_fr_mdev()
992  *				 adapter from the matrix mdev to which the
994  * @matrix_mdev: the matrix mediated device to which the adapter was assigned.
1005 	for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, AP_DOMAINS) {  in vfio_ap_mdev_unlink_adapter()
1044  * corresponding bit in the mediated matrix device's APM
1046  * @dev:	the matrix device
1047  * @attr:	the mediated matrix device's unassign_adapter attribute
1071 	if (apid > matrix_mdev->matrix.apm_max) {  in unassign_adapter_store()
1076 	if (!test_bit_inv(apid, matrix_mdev->matrix.apm)) {  in unassign_adapter_store()
1081 	clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);  in unassign_adapter_store()
1095 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES)  in vfio_ap_mdev_link_domain()
1102  * corresponding bit in the mediated matrix device's AQM
1104  * @dev:	the matrix device
1105  * @attr:	the mediated matrix device's assign_domain attribute
1128  *	   matrix device
1150 	if (apqi > matrix_mdev->matrix.aqm_max) {  in assign_domain_store()
1155 	if (test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {  in assign_domain_store()
1160 	set_bit_inv(apqi, matrix_mdev->matrix.aqm);  in assign_domain_store()
1164 		clear_bit_inv(apqi, matrix_mdev->matrix.aqm);  in assign_domain_store()
1172 	if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm, aqm_delta,  in assign_domain_store()
1192 	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, AP_DEVICES) {  in vfio_ap_mdev_unlink_domain()
1231  * corresponding bit in the mediated matrix device's AQM
1233  * @dev:	the matrix device
1234  * @attr:	the mediated matrix device's unassign_domain attribute
1258 	if (apqi > matrix_mdev->matrix.aqm_max) {  in unassign_domain_store()
1263 	if (!test_bit_inv(apqi, matrix_mdev->matrix.aqm)) {  in unassign_domain_store()
1268 	clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);  in unassign_domain_store()
1280  * the corresponding bit in the mediated matrix device's ADM
1282  * @dev:	the matrix device
1283  * @attr:	the mediated matrix device's assign_control_domain attribute
1306 	if (id > matrix_mdev->matrix.adm_max) {  in assign_control_domain_store()
1311 	if (test_bit_inv(id, matrix_mdev->matrix.adm)) {  in assign_control_domain_store()
1321 	set_bit_inv(id, matrix_mdev->matrix.adm);  in assign_control_domain_store()
1334  * clears the corresponding bit in the mediated matrix device's ADM
1336  * @dev:	the matrix device
1337  * @attr:	the mediated matrix device's unassign_control_domain attribute
1360 	if (domid > matrix_mdev->matrix.adm_max) {  in unassign_control_domain_store()
1365 	if (!test_bit_inv(domid, matrix_mdev->matrix.adm)) {  in unassign_control_domain_store()
1370 	clear_bit_inv(domid, matrix_mdev->matrix.adm);  in unassign_control_domain_store()
1393 	unsigned long max_domid = matrix_mdev->matrix.adm_max;  in control_domains_show()
1396 	for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) {  in control_domains_show()
1407 static ssize_t vfio_ap_mdev_matrix_show(struct ap_matrix *matrix, char *buf)  in vfio_ap_mdev_matrix_show()  argument
1414 	unsigned long napm_bits = matrix->apm_max + 1;  in vfio_ap_mdev_matrix_show()
1415 	unsigned long naqm_bits = matrix->aqm_max + 1;  in vfio_ap_mdev_matrix_show()
1419 	apid1 = find_first_bit_inv(matrix->apm, napm_bits);  in vfio_ap_mdev_matrix_show()
1420 	apqi1 = find_first_bit_inv(matrix->aqm, naqm_bits);  in vfio_ap_mdev_matrix_show()
1423 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {  in vfio_ap_mdev_matrix_show()
1424 			for_each_set_bit_inv(apqi, matrix->aqm,  in vfio_ap_mdev_matrix_show()
1433 		for_each_set_bit_inv(apid, matrix->apm, napm_bits) {  in vfio_ap_mdev_matrix_show()
1439 		for_each_set_bit_inv(apqi, matrix->aqm, naqm_bits) {  in vfio_ap_mdev_matrix_show()
1456 	nchars = vfio_ap_mdev_matrix_show(&matrix_mdev->matrix, buf);  in matrix_show()
1461 static DEVICE_ATTR_RO(matrix);
1503  * @matrix_mdev: a mediated matrix device
1506  * Return: 0 if no other mediated matrix device has a reference to @kvm;
1551  * @matrix_mdev: a matrix mediated device
1736 		if (test_bit_inv(apid, matrix_mdev->matrix.apm) &&  in vfio_ap_mdev_for_queue()
1737 		    test_bit_inv(apqi, matrix_mdev->matrix.aqm))  in vfio_ap_mdev_for_queue()
1857 		if (vfio_ap_mdev_filter_matrix(matrix_mdev->matrix.apm,  in vfio_ap_mdev_probe_queue()
1858 					       matrix_mdev->matrix.aqm,  in vfio_ap_mdev_probe_queue()
1996 					  matrix_mdev->matrix.apm,  in vfio_ap_mdev_cfg_remove()
1999 					  matrix_mdev->matrix.aqm,  in vfio_ap_mdev_cfg_remove()
2002 					     matrix_mdev->matrix.adm,  in vfio_ap_mdev_cfg_remove()
2116  *			  AP configuration for each matrix mdev to which they
2138 			   matrix_mdev->matrix.apm, apm_add, AP_DEVICES);  in vfio_ap_mdev_cfg_add()
2140 			   matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS);  in vfio_ap_mdev_cfg_add()
2142 			   matrix_mdev->matrix.adm, adm_add, AP_DEVICES);  in vfio_ap_mdev_cfg_add()
2215 	filter_adapters = bitmap_and(apm, matrix_mdev->matrix.apm,  in vfio_ap_mdev_hot_plug_cfg()
2217 	filter_domains = bitmap_and(aqm, matrix_mdev->matrix.aqm,  in vfio_ap_mdev_hot_plug_cfg()
2232 	if (bitmap_intersects(matrix_mdev->matrix.adm, matrix_mdev->adm_add,  in vfio_ap_mdev_hot_plug_cfg()