644 static void rbd_img_handle_request(struct rbd_img_request *img_req, int result);
1372 static bool rbd_img_is_write(struct rbd_img_request *img_req)  in rbd_img_is_write()  argument
1374 	switch (img_req->op_type) {  in rbd_img_is_write()
1606 static void rbd_img_capture_header(struct rbd_img_request *img_req)  in rbd_img_capture_header()  argument
1608 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_capture_header()
1612 	if (!rbd_img_is_write(img_req))  in rbd_img_capture_header()
1613 		img_req->snap_id = rbd_dev->spec->snap_id;  in rbd_img_capture_header()
1616 		img_request_layered_set(img_req);  in rbd_img_capture_header()
2376 	struct rbd_img_request *img_req = obj_req->img_request;  in count_write_ops()  local
2378 	switch (img_req->op_type) {  in count_write_ops()
2380 		if (!use_object_map(img_req->rbd_dev) ||  in count_write_ops()
2423 static int __rbd_img_fill_request(struct rbd_img_request *img_req)  in __rbd_img_fill_request()  argument
2428 	for_each_obj_request_safe(img_req, obj_req, next_obj_req) {  in __rbd_img_fill_request()
2429 		switch (img_req->op_type) {  in __rbd_img_fill_request()
2448 			rbd_img_obj_request_del(img_req, obj_req);  in __rbd_img_fill_request()
2453 	img_req->state = RBD_IMG_START;  in __rbd_img_fill_request()
2473 	struct rbd_img_request *img_req = arg;  in alloc_object_extent()  local
2480 	rbd_img_obj_request_add(img_req, obj_req);  in alloc_object_extent()
2496 static int rbd_img_fill_request_nocopy(struct rbd_img_request *img_req,  in rbd_img_fill_request_nocopy()  argument
2504 	img_req->data_type = fctx->pos_type;  in rbd_img_fill_request_nocopy()
2512 		ret = ceph_file_to_extents(&img_req->rbd_dev->layout,  in rbd_img_fill_request_nocopy()
2515 					   &img_req->object_extents,  in rbd_img_fill_request_nocopy()
2516 					   alloc_object_extent, img_req,  in rbd_img_fill_request_nocopy()
2522 	return __rbd_img_fill_request(img_req);  in rbd_img_fill_request_nocopy()
2538 static int rbd_img_fill_request(struct rbd_img_request *img_req,  in rbd_img_fill_request()  argument
2543 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_fill_request()
2550 		return rbd_img_fill_request_nocopy(img_req, img_extents,  in rbd_img_fill_request()
2553 	img_req->data_type = OBJ_REQUEST_OWN_BVECS;  in rbd_img_fill_request()
2567 					   &img_req->object_extents,  in rbd_img_fill_request()
2568 					   alloc_object_extent, img_req,  in rbd_img_fill_request()
2574 	for_each_obj_request(img_req, obj_req) {  in rbd_img_fill_request()
2591 					   &img_req->object_extents,  in rbd_img_fill_request()
2597 	return __rbd_img_fill_request(img_req);  in rbd_img_fill_request()
2600 static int rbd_img_fill_nodata(struct rbd_img_request *img_req,  in rbd_img_fill_nodata()  argument
2610 	return rbd_img_fill_request(img_req, &ex, 1, &fctx);  in rbd_img_fill_nodata()
2650 static int __rbd_img_fill_from_bio(struct rbd_img_request *img_req,  in __rbd_img_fill_from_bio()  argument
2663 	return rbd_img_fill_request(img_req, img_extents, num_img_extents,  in __rbd_img_fill_from_bio()
2667 static int rbd_img_fill_from_bio(struct rbd_img_request *img_req,  in rbd_img_fill_from_bio()  argument
2673 	return __rbd_img_fill_from_bio(img_req, &ex, 1, &it);  in rbd_img_fill_from_bio()
2710 static int __rbd_img_fill_from_bvecs(struct rbd_img_request *img_req,  in __rbd_img_fill_from_bvecs()  argument
2723 	return rbd_img_fill_request(img_req, img_extents, num_img_extents,  in __rbd_img_fill_from_bvecs()
2727 static int rbd_img_fill_from_bvecs(struct rbd_img_request *img_req,  in rbd_img_fill_from_bvecs()  argument
2738 	return __rbd_img_fill_from_bvecs(img_req, img_extents, num_img_extents,  in rbd_img_fill_from_bvecs()
2744 	struct rbd_img_request *img_req =  in rbd_img_handle_request_work()  local
2747 	rbd_img_handle_request(img_req, img_req->work_result);  in rbd_img_handle_request_work()
2750 static void rbd_img_schedule(struct rbd_img_request *img_req, int result)  in rbd_img_schedule()  argument
2752 	INIT_WORK(&img_req->work, rbd_img_handle_request_work);  in rbd_img_schedule()
2753 	img_req->work_result = result;  in rbd_img_schedule()
2754 	queue_work(rbd_wq, &img_req->work);  in rbd_img_schedule()
2795 	struct rbd_img_request *img_req = obj_req->img_request;  in rbd_obj_read_from_parent()  local
2796 	struct rbd_device *parent = img_req->rbd_dev->parent;  in rbd_obj_read_from_parent()
2815 	if (!rbd_img_is_write(img_req)) {  in rbd_obj_read_from_parent()
2816 		switch (img_req->data_type) {  in rbd_obj_read_from_parent()
3387 	struct rbd_img_request *img_req = obj_req->img_request;  in __rbd_obj_handle_request()  local
3388 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in __rbd_obj_handle_request()
3392 	if (!rbd_img_is_write(img_req))  in __rbd_obj_handle_request()
3401 			 obj_op_name(img_req->op_type), obj_req->ex.oe_objno,  in __rbd_obj_handle_request()
3417 static bool need_exclusive_lock(struct rbd_img_request *img_req)  in need_exclusive_lock()  argument
3419 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in need_exclusive_lock()
3427 	rbd_assert(!test_bit(IMG_REQ_CHILD, &img_req->flags));  in need_exclusive_lock()
3432 	return rbd_img_is_write(img_req);  in need_exclusive_lock()
3435 static bool rbd_lock_add_request(struct rbd_img_request *img_req)  in rbd_lock_add_request()  argument
3437 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_lock_add_request()
3443 	rbd_assert(list_empty(&img_req->lock_item));  in rbd_lock_add_request()
3445 		list_add_tail(&img_req->lock_item, &rbd_dev->acquiring_list);  in rbd_lock_add_request()
3447 		list_add_tail(&img_req->lock_item, &rbd_dev->running_list);  in rbd_lock_add_request()
3452 static void rbd_lock_del_request(struct rbd_img_request *img_req)  in rbd_lock_del_request()  argument
3454 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_lock_del_request()
3459 	rbd_assert(!list_empty(&img_req->lock_item));  in rbd_lock_del_request()
3460 	list_del_init(&img_req->lock_item);  in rbd_lock_del_request()
3468 static int rbd_img_exclusive_lock(struct rbd_img_request *img_req)  in rbd_img_exclusive_lock()  argument
3470 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_exclusive_lock()
3472 	if (!need_exclusive_lock(img_req))  in rbd_img_exclusive_lock()
3475 	if (rbd_lock_add_request(img_req))  in rbd_img_exclusive_lock()
3492 static void rbd_img_object_requests(struct rbd_img_request *img_req)  in rbd_img_object_requests()  argument
3494 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_object_requests()
3497 	rbd_assert(!img_req->pending.result && !img_req->pending.num_pending);  in rbd_img_object_requests()
3498 	rbd_assert(!need_exclusive_lock(img_req) ||  in rbd_img_object_requests()
3501 	if (rbd_img_is_write(img_req)) {  in rbd_img_object_requests()
3502 		rbd_assert(!img_req->snapc);  in rbd_img_object_requests()
3504 		img_req->snapc = ceph_get_snap_context(rbd_dev->header.snapc);  in rbd_img_object_requests()
3508 	for_each_obj_request(img_req, obj_req) {  in rbd_img_object_requests()
3513 				img_req->pending.result = result;  in rbd_img_object_requests()
3517 			img_req->pending.num_pending++;  in rbd_img_object_requests()
3522 static bool rbd_img_advance(struct rbd_img_request *img_req, int *result)  in rbd_img_advance()  argument
3527 	switch (img_req->state) {  in rbd_img_advance()
3531 		ret = rbd_img_exclusive_lock(img_req);  in rbd_img_advance()
3536 		img_req->state = RBD_IMG_EXCLUSIVE_LOCK;  in rbd_img_advance()
3544 		rbd_img_object_requests(img_req);  in rbd_img_advance()
3545 		if (!img_req->pending.num_pending) {  in rbd_img_advance()
3546 			*result = img_req->pending.result;  in rbd_img_advance()
3547 			img_req->state = RBD_IMG_OBJECT_REQUESTS;  in rbd_img_advance()
3550 		img_req->state = __RBD_IMG_OBJECT_REQUESTS;  in rbd_img_advance()
3553 		if (!pending_result_dec(&img_req->pending, result))  in rbd_img_advance()
3566 static bool __rbd_img_handle_request(struct rbd_img_request *img_req,  in __rbd_img_handle_request()  argument
3569 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in __rbd_img_handle_request()
3572 	if (need_exclusive_lock(img_req)) {  in __rbd_img_handle_request()
3574 		mutex_lock(&img_req->state_mutex);  in __rbd_img_handle_request()
3575 		done = rbd_img_advance(img_req, result);  in __rbd_img_handle_request()
3577 			rbd_lock_del_request(img_req);  in __rbd_img_handle_request()
3578 		mutex_unlock(&img_req->state_mutex);  in __rbd_img_handle_request()
3581 		mutex_lock(&img_req->state_mutex);  in __rbd_img_handle_request()
3582 		done = rbd_img_advance(img_req, result);  in __rbd_img_handle_request()
3583 		mutex_unlock(&img_req->state_mutex);  in __rbd_img_handle_request()
3589 		      test_bit(IMG_REQ_CHILD, &img_req->flags) ? "child " : "",  in __rbd_img_handle_request()
3590 		      obj_op_name(img_req->op_type), *result);  in __rbd_img_handle_request()
3595 static void rbd_img_handle_request(struct rbd_img_request *img_req, int result)  in rbd_img_handle_request()  argument
3598 	if (!__rbd_img_handle_request(img_req, &result))  in rbd_img_handle_request()
3601 	if (test_bit(IMG_REQ_CHILD, &img_req->flags)) {  in rbd_img_handle_request()
3602 		struct rbd_obj_request *obj_req = img_req->obj_request;  in rbd_img_handle_request()
3604 		rbd_img_request_destroy(img_req);  in rbd_img_handle_request()
3606 			img_req = obj_req->img_request;  in rbd_img_handle_request()
3610 		struct request *rq = blk_mq_rq_from_pdu(img_req);  in rbd_img_handle_request()
3612 		rbd_img_request_destroy(img_req);  in rbd_img_handle_request()
3831 	struct rbd_img_request *img_req;  in wake_lock_waiters()  local
3845 	list_for_each_entry(img_req, &rbd_dev->acquiring_list, lock_item) {  in wake_lock_waiters()
3846 		mutex_lock(&img_req->state_mutex);  in wake_lock_waiters()
3847 		rbd_assert(img_req->state == RBD_IMG_EXCLUSIVE_LOCK);  in wake_lock_waiters()
3848 		rbd_img_schedule(img_req, result);  in wake_lock_waiters()
3849 		mutex_unlock(&img_req->state_mutex);  in wake_lock_waiters()
4772 	struct rbd_img_request *img_req = blk_mq_rq_to_pdu(bd->rq);  in rbd_queue_rq()  local
4793 	rbd_img_request_init(img_req, rbd_dev, op_type);  in rbd_queue_rq()
4795 	if (rbd_img_is_write(img_req)) {  in rbd_queue_rq()
4798 				 obj_op_name(img_req->op_type));  in rbd_queue_rq()
4804 	INIT_WORK(&img_req->work, rbd_queue_workfn);  in rbd_queue_rq()
4805 	queue_work(rbd_wq, &img_req->work);  in rbd_queue_rq()