645 static void rbd_img_handle_request(struct rbd_img_request *img_req, int result);
1354 static bool rbd_img_is_write(struct rbd_img_request *img_req)  in rbd_img_is_write()  argument
1356 	switch (img_req->op_type) {  in rbd_img_is_write()
1582 static void rbd_img_capture_header(struct rbd_img_request *img_req)  in rbd_img_capture_header()  argument
1584 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_capture_header()
1588 	if (rbd_img_is_write(img_req))  in rbd_img_capture_header()
1589 		img_req->snapc = ceph_get_snap_context(rbd_dev->header.snapc);  in rbd_img_capture_header()
1591 		img_req->snap_id = rbd_dev->spec->snap_id;  in rbd_img_capture_header()
1594 		img_request_layered_set(img_req);  in rbd_img_capture_header()
2359 	struct rbd_img_request *img_req = obj_req->img_request;  in count_write_ops()  local
2361 	switch (img_req->op_type) {  in count_write_ops()
2363 		if (!use_object_map(img_req->rbd_dev) ||  in count_write_ops()
2406 static int __rbd_img_fill_request(struct rbd_img_request *img_req)  in __rbd_img_fill_request()  argument
2411 	for_each_obj_request_safe(img_req, obj_req, next_obj_req) {  in __rbd_img_fill_request()
2412 		switch (img_req->op_type) {  in __rbd_img_fill_request()
2431 			rbd_img_obj_request_del(img_req, obj_req);  in __rbd_img_fill_request()
2436 	img_req->state = RBD_IMG_START;  in __rbd_img_fill_request()
2456 	struct rbd_img_request *img_req = arg;  in alloc_object_extent()  local
2463 	rbd_img_obj_request_add(img_req, obj_req);  in alloc_object_extent()
2479 static int rbd_img_fill_request_nocopy(struct rbd_img_request *img_req,  in rbd_img_fill_request_nocopy()  argument
2487 	img_req->data_type = fctx->pos_type;  in rbd_img_fill_request_nocopy()
2495 		ret = ceph_file_to_extents(&img_req->rbd_dev->layout,  in rbd_img_fill_request_nocopy()
2498 					   &img_req->object_extents,  in rbd_img_fill_request_nocopy()
2499 					   alloc_object_extent, img_req,  in rbd_img_fill_request_nocopy()
2505 	return __rbd_img_fill_request(img_req);  in rbd_img_fill_request_nocopy()
2521 static int rbd_img_fill_request(struct rbd_img_request *img_req,  in rbd_img_fill_request()  argument
2526 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_fill_request()
2533 		return rbd_img_fill_request_nocopy(img_req, img_extents,  in rbd_img_fill_request()
2536 	img_req->data_type = OBJ_REQUEST_OWN_BVECS;  in rbd_img_fill_request()
2550 					   &img_req->object_extents,  in rbd_img_fill_request()
2551 					   alloc_object_extent, img_req,  in rbd_img_fill_request()
2557 	for_each_obj_request(img_req, obj_req) {  in rbd_img_fill_request()
2574 					   &img_req->object_extents,  in rbd_img_fill_request()
2580 	return __rbd_img_fill_request(img_req);  in rbd_img_fill_request()
2583 static int rbd_img_fill_nodata(struct rbd_img_request *img_req,  in rbd_img_fill_nodata()  argument
2593 	return rbd_img_fill_request(img_req, &ex, 1, &fctx);  in rbd_img_fill_nodata()
2633 static int __rbd_img_fill_from_bio(struct rbd_img_request *img_req,  in __rbd_img_fill_from_bio()  argument
2646 	return rbd_img_fill_request(img_req, img_extents, num_img_extents,  in __rbd_img_fill_from_bio()
2650 static int rbd_img_fill_from_bio(struct rbd_img_request *img_req,  in rbd_img_fill_from_bio()  argument
2656 	return __rbd_img_fill_from_bio(img_req, &ex, 1, &it);  in rbd_img_fill_from_bio()
2693 static int __rbd_img_fill_from_bvecs(struct rbd_img_request *img_req,  in __rbd_img_fill_from_bvecs()  argument
2706 	return rbd_img_fill_request(img_req, img_extents, num_img_extents,  in __rbd_img_fill_from_bvecs()
2710 static int rbd_img_fill_from_bvecs(struct rbd_img_request *img_req,  in rbd_img_fill_from_bvecs()  argument
2721 	return __rbd_img_fill_from_bvecs(img_req, img_extents, num_img_extents,  in rbd_img_fill_from_bvecs()
2727 	struct rbd_img_request *img_req =  in rbd_img_handle_request_work()  local
2730 	rbd_img_handle_request(img_req, img_req->work_result);  in rbd_img_handle_request_work()
2733 static void rbd_img_schedule(struct rbd_img_request *img_req, int result)  in rbd_img_schedule()  argument
2735 	INIT_WORK(&img_req->work, rbd_img_handle_request_work);  in rbd_img_schedule()
2736 	img_req->work_result = result;  in rbd_img_schedule()
2737 	queue_work(rbd_wq, &img_req->work);  in rbd_img_schedule()
2778 	struct rbd_img_request *img_req = obj_req->img_request;  in rbd_obj_read_from_parent()  local
2779 	struct rbd_device *parent = img_req->rbd_dev->parent;  in rbd_obj_read_from_parent()
2798 	if (!rbd_img_is_write(img_req)) {  in rbd_obj_read_from_parent()
2799 		switch (img_req->data_type) {  in rbd_obj_read_from_parent()
3370 	struct rbd_img_request *img_req = obj_req->img_request;  in __rbd_obj_handle_request()  local
3371 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in __rbd_obj_handle_request()
3375 	if (!rbd_img_is_write(img_req))  in __rbd_obj_handle_request()
3384 			 obj_op_name(img_req->op_type), obj_req->ex.oe_objno,  in __rbd_obj_handle_request()
3400 static bool need_exclusive_lock(struct rbd_img_request *img_req)  in need_exclusive_lock()  argument
3402 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in need_exclusive_lock()
3410 	rbd_assert(!test_bit(IMG_REQ_CHILD, &img_req->flags));  in need_exclusive_lock()
3415 	return rbd_img_is_write(img_req);  in need_exclusive_lock()
3418 static bool rbd_lock_add_request(struct rbd_img_request *img_req)  in rbd_lock_add_request()  argument
3420 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_lock_add_request()
3426 	rbd_assert(list_empty(&img_req->lock_item));  in rbd_lock_add_request()
3428 		list_add_tail(&img_req->lock_item, &rbd_dev->acquiring_list);  in rbd_lock_add_request()
3430 		list_add_tail(&img_req->lock_item, &rbd_dev->running_list);  in rbd_lock_add_request()
3435 static void rbd_lock_del_request(struct rbd_img_request *img_req)  in rbd_lock_del_request()  argument
3437 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_lock_del_request()
3442 	rbd_assert(!list_empty(&img_req->lock_item));  in rbd_lock_del_request()
3443 	list_del_init(&img_req->lock_item);  in rbd_lock_del_request()
3451 static int rbd_img_exclusive_lock(struct rbd_img_request *img_req)  in rbd_img_exclusive_lock()  argument
3453 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_exclusive_lock()
3455 	if (!need_exclusive_lock(img_req))  in rbd_img_exclusive_lock()
3458 	if (rbd_lock_add_request(img_req))  in rbd_img_exclusive_lock()
3475 static void rbd_img_object_requests(struct rbd_img_request *img_req)  in rbd_img_object_requests()  argument
3479 	rbd_assert(!img_req->pending.result && !img_req->pending.num_pending);  in rbd_img_object_requests()
3481 	for_each_obj_request(img_req, obj_req) {  in rbd_img_object_requests()
3486 				img_req->pending.result = result;  in rbd_img_object_requests()
3490 			img_req->pending.num_pending++;  in rbd_img_object_requests()
3495 static bool rbd_img_advance(struct rbd_img_request *img_req, int *result)  in rbd_img_advance()  argument
3497 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_advance()
3501 	switch (img_req->state) {  in rbd_img_advance()
3505 		ret = rbd_img_exclusive_lock(img_req);  in rbd_img_advance()
3510 		img_req->state = RBD_IMG_EXCLUSIVE_LOCK;  in rbd_img_advance()
3518 		rbd_assert(!need_exclusive_lock(img_req) ||  in rbd_img_advance()
3521 		rbd_img_object_requests(img_req);  in rbd_img_advance()
3522 		if (!img_req->pending.num_pending) {  in rbd_img_advance()
3523 			*result = img_req->pending.result;  in rbd_img_advance()
3524 			img_req->state = RBD_IMG_OBJECT_REQUESTS;  in rbd_img_advance()
3527 		img_req->state = __RBD_IMG_OBJECT_REQUESTS;  in rbd_img_advance()
3530 		if (!pending_result_dec(&img_req->pending, result))  in rbd_img_advance()
3543 static bool __rbd_img_handle_request(struct rbd_img_request *img_req,  in __rbd_img_handle_request()  argument
3546 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in __rbd_img_handle_request()
3549 	if (need_exclusive_lock(img_req)) {  in __rbd_img_handle_request()
3551 		mutex_lock(&img_req->state_mutex);  in __rbd_img_handle_request()
3552 		done = rbd_img_advance(img_req, result);  in __rbd_img_handle_request()
3554 			rbd_lock_del_request(img_req);  in __rbd_img_handle_request()
3555 		mutex_unlock(&img_req->state_mutex);  in __rbd_img_handle_request()
3558 		mutex_lock(&img_req->state_mutex);  in __rbd_img_handle_request()
3559 		done = rbd_img_advance(img_req, result);  in __rbd_img_handle_request()
3560 		mutex_unlock(&img_req->state_mutex);  in __rbd_img_handle_request()
3566 		      test_bit(IMG_REQ_CHILD, &img_req->flags) ? "child " : "",  in __rbd_img_handle_request()
3567 		      obj_op_name(img_req->op_type), *result);  in __rbd_img_handle_request()
3572 static void rbd_img_handle_request(struct rbd_img_request *img_req, int result)  in rbd_img_handle_request()  argument
3575 	if (!__rbd_img_handle_request(img_req, &result))  in rbd_img_handle_request()
3578 	if (test_bit(IMG_REQ_CHILD, &img_req->flags)) {  in rbd_img_handle_request()
3579 		struct rbd_obj_request *obj_req = img_req->obj_request;  in rbd_img_handle_request()
3581 		rbd_img_request_destroy(img_req);  in rbd_img_handle_request()
3583 			img_req = obj_req->img_request;  in rbd_img_handle_request()
3587 		struct request *rq = blk_mq_rq_from_pdu(img_req);  in rbd_img_handle_request()
3589 		rbd_img_request_destroy(img_req);  in rbd_img_handle_request()
3808 	struct rbd_img_request *img_req;  in wake_lock_waiters()  local
3822 	list_for_each_entry(img_req, &rbd_dev->acquiring_list, lock_item) {  in wake_lock_waiters()
3823 		mutex_lock(&img_req->state_mutex);  in wake_lock_waiters()
3824 		rbd_assert(img_req->state == RBD_IMG_EXCLUSIVE_LOCK);  in wake_lock_waiters()
3825 		rbd_img_schedule(img_req, result);  in wake_lock_waiters()
3826 		mutex_unlock(&img_req->state_mutex);  in wake_lock_waiters()
4694 	struct rbd_img_request *img_req = blk_mq_rq_to_pdu(bd->rq);  in rbd_queue_rq()  local
4715 	rbd_img_request_init(img_req, rbd_dev, op_type);  in rbd_queue_rq()
4717 	if (rbd_img_is_write(img_req)) {  in rbd_queue_rq()
4720 				 obj_op_name(img_req->op_type));  in rbd_queue_rq()
4726 	INIT_WORK(&img_req->work, rbd_queue_workfn);  in rbd_queue_rq()
4727 	queue_work(rbd_wq, &img_req->work);  in rbd_queue_rq()