227 mlx5vf_pci_save_device_data(struct mlx5vf_pci_core_device *mvdev)  in mlx5vf_pci_save_device_data()  argument
248 	mlx5_cmd_init_async_ctx(mvdev->mdev, &migf->async_ctx);  in mlx5vf_pci_save_device_data()
250 	ret = mlx5vf_cmd_query_vhca_migration_state(mvdev,  in mlx5vf_pci_save_device_data()
260 	migf->mvdev = mvdev;  in mlx5vf_pci_save_device_data()
261 	ret = mlx5vf_cmd_save_vhca_state(mvdev, migf);  in mlx5vf_pci_save_device_data()
345 mlx5vf_pci_resume_device_data(struct mlx5vf_pci_core_device *mvdev)  in mlx5vf_pci_resume_device_data()  argument
366 void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev)  in mlx5vf_disable_fds()  argument
368 	if (mvdev->resuming_migf) {  in mlx5vf_disable_fds()
369 		mlx5vf_disable_fd(mvdev->resuming_migf);  in mlx5vf_disable_fds()
370 		fput(mvdev->resuming_migf->filp);  in mlx5vf_disable_fds()
371 		mvdev->resuming_migf = NULL;  in mlx5vf_disable_fds()
373 	if (mvdev->saving_migf) {  in mlx5vf_disable_fds()
374 		mlx5_cmd_cleanup_async_ctx(&mvdev->saving_migf->async_ctx);  in mlx5vf_disable_fds()
375 		cancel_work_sync(&mvdev->saving_migf->async_data.work);  in mlx5vf_disable_fds()
376 		mlx5vf_disable_fd(mvdev->saving_migf);  in mlx5vf_disable_fds()
377 		fput(mvdev->saving_migf->filp);  in mlx5vf_disable_fds()
378 		mvdev->saving_migf = NULL;  in mlx5vf_disable_fds()
383 mlx5vf_pci_step_device_state_locked(struct mlx5vf_pci_core_device *mvdev,  in mlx5vf_pci_step_device_state_locked()  argument
386 	u32 cur = mvdev->mig_state;  in mlx5vf_pci_step_device_state_locked()
390 		ret = mlx5vf_cmd_suspend_vhca(mvdev,  in mlx5vf_pci_step_device_state_locked()
398 		ret = mlx5vf_cmd_resume_vhca(mvdev,  in mlx5vf_pci_step_device_state_locked()
406 		ret = mlx5vf_cmd_suspend_vhca(mvdev,  in mlx5vf_pci_step_device_state_locked()
414 		ret = mlx5vf_cmd_resume_vhca(mvdev,  in mlx5vf_pci_step_device_state_locked()
424 		migf = mlx5vf_pci_save_device_data(mvdev);  in mlx5vf_pci_step_device_state_locked()
428 		mvdev->saving_migf = migf;  in mlx5vf_pci_step_device_state_locked()
433 		mlx5vf_disable_fds(mvdev);  in mlx5vf_pci_step_device_state_locked()
440 		migf = mlx5vf_pci_resume_device_data(mvdev);  in mlx5vf_pci_step_device_state_locked()
444 		mvdev->resuming_migf = migf;  in mlx5vf_pci_step_device_state_locked()
449 		ret = mlx5vf_cmd_load_vhca_state(mvdev,  in mlx5vf_pci_step_device_state_locked()
450 						 mvdev->resuming_migf);  in mlx5vf_pci_step_device_state_locked()
453 		mlx5vf_disable_fds(mvdev);  in mlx5vf_pci_step_device_state_locked()
468 void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev)  in mlx5vf_state_mutex_unlock()  argument
471 	spin_lock(&mvdev->reset_lock);  in mlx5vf_state_mutex_unlock()
472 	if (mvdev->deferred_reset) {  in mlx5vf_state_mutex_unlock()
473 		mvdev->deferred_reset = false;  in mlx5vf_state_mutex_unlock()
474 		spin_unlock(&mvdev->reset_lock);  in mlx5vf_state_mutex_unlock()
475 		mvdev->mig_state = VFIO_DEVICE_STATE_RUNNING;  in mlx5vf_state_mutex_unlock()
476 		mlx5vf_disable_fds(mvdev);  in mlx5vf_state_mutex_unlock()
479 	mutex_unlock(&mvdev->state_mutex);  in mlx5vf_state_mutex_unlock()
480 	spin_unlock(&mvdev->reset_lock);  in mlx5vf_state_mutex_unlock()
487 	struct mlx5vf_pci_core_device *mvdev = container_of(  in mlx5vf_pci_set_device_state()  local
493 	mutex_lock(&mvdev->state_mutex);  in mlx5vf_pci_set_device_state()
494 	while (new_state != mvdev->mig_state) {  in mlx5vf_pci_set_device_state()
495 		ret = vfio_mig_get_next_state(vdev, mvdev->mig_state,  in mlx5vf_pci_set_device_state()
501 		res = mlx5vf_pci_step_device_state_locked(mvdev, next_state);  in mlx5vf_pci_set_device_state()
504 		mvdev->mig_state = next_state;  in mlx5vf_pci_set_device_state()
505 		if (WARN_ON(res && new_state != mvdev->mig_state)) {  in mlx5vf_pci_set_device_state()
511 	mlx5vf_state_mutex_unlock(mvdev);  in mlx5vf_pci_set_device_state()
518 	struct mlx5vf_pci_core_device *mvdev = container_of(  in mlx5vf_pci_get_device_state()  local
521 	mutex_lock(&mvdev->state_mutex);  in mlx5vf_pci_get_device_state()
522 	*curr_state = mvdev->mig_state;  in mlx5vf_pci_get_device_state()
523 	mlx5vf_state_mutex_unlock(mvdev);  in mlx5vf_pci_get_device_state()
529 	struct mlx5vf_pci_core_device *mvdev = mlx5vf_drvdata(pdev);  in mlx5vf_pci_aer_reset_done()  local
531 	if (!mvdev->migrate_cap)  in mlx5vf_pci_aer_reset_done()
541 	spin_lock(&mvdev->reset_lock);  in mlx5vf_pci_aer_reset_done()
542 	mvdev->deferred_reset = true;  in mlx5vf_pci_aer_reset_done()
543 	if (!mutex_trylock(&mvdev->state_mutex)) {  in mlx5vf_pci_aer_reset_done()
544 		spin_unlock(&mvdev->reset_lock);  in mlx5vf_pci_aer_reset_done()
547 	spin_unlock(&mvdev->reset_lock);  in mlx5vf_pci_aer_reset_done()
548 	mlx5vf_state_mutex_unlock(mvdev);  in mlx5vf_pci_aer_reset_done()
553 	struct mlx5vf_pci_core_device *mvdev = container_of(  in mlx5vf_pci_open_device()  local
555 	struct vfio_pci_core_device *vdev = &mvdev->core_device;  in mlx5vf_pci_open_device()
562 	if (mvdev->migrate_cap)  in mlx5vf_pci_open_device()
563 		mvdev->mig_state = VFIO_DEVICE_STATE_RUNNING;  in mlx5vf_pci_open_device()
570 	struct mlx5vf_pci_core_device *mvdev = container_of(  in mlx5vf_pci_close_device()  local
573 	mlx5vf_cmd_close_migratable(mvdev);  in mlx5vf_pci_close_device()
590 	struct mlx5vf_pci_core_device *mvdev = container_of(core_vdev,  in mlx5vf_pci_init_dev()  local
598 	mlx5vf_cmd_set_migratable(mvdev, &mlx5vf_pci_mig_ops,  in mlx5vf_pci_init_dev()
606 	struct mlx5vf_pci_core_device *mvdev = container_of(core_vdev,  in mlx5vf_pci_release_dev()  local
609 	mlx5vf_cmd_remove_migratable(mvdev);  in mlx5vf_pci_release_dev()
631 	struct mlx5vf_pci_core_device *mvdev;  in mlx5vf_pci_probe()  local
634 	mvdev = vfio_alloc_device(mlx5vf_pci_core_device, core_device.vdev,  in mlx5vf_pci_probe()
636 	if (IS_ERR(mvdev))  in mlx5vf_pci_probe()
637 		return PTR_ERR(mvdev);  in mlx5vf_pci_probe()
639 	dev_set_drvdata(&pdev->dev, &mvdev->core_device);  in mlx5vf_pci_probe()
640 	ret = vfio_pci_core_register_device(&mvdev->core_device);  in mlx5vf_pci_probe()
646 	vfio_put_device(&mvdev->core_device.vdev);  in mlx5vf_pci_probe()
652 	struct mlx5vf_pci_core_device *mvdev = mlx5vf_drvdata(pdev);  in mlx5vf_pci_remove()  local
654 	vfio_pci_core_unregister_device(&mvdev->core_device);  in mlx5vf_pci_remove()
655 	vfio_put_device(&mvdev->core_device.vdev);  in mlx5vf_pci_remove()