645 static void rbd_img_handle_request(struct rbd_img_request *img_req, int result);
1353 static bool rbd_img_is_write(struct rbd_img_request *img_req)  in rbd_img_is_write()  argument
1355 	switch (img_req->op_type) {  in rbd_img_is_write()
1581 static void rbd_img_capture_header(struct rbd_img_request *img_req)  in rbd_img_capture_header()  argument
1583 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_capture_header()
1587 	if (rbd_img_is_write(img_req))  in rbd_img_capture_header()
1588 		img_req->snapc = ceph_get_snap_context(rbd_dev->header.snapc);  in rbd_img_capture_header()
1590 		img_req->snap_id = rbd_dev->spec->snap_id;  in rbd_img_capture_header()
1593 		img_request_layered_set(img_req);  in rbd_img_capture_header()
2358 	struct rbd_img_request *img_req = obj_req->img_request;  in count_write_ops()  local
2360 	switch (img_req->op_type) {  in count_write_ops()
2362 		if (!use_object_map(img_req->rbd_dev) ||  in count_write_ops()
2405 static int __rbd_img_fill_request(struct rbd_img_request *img_req)  in __rbd_img_fill_request()  argument
2410 	for_each_obj_request_safe(img_req, obj_req, next_obj_req) {  in __rbd_img_fill_request()
2411 		switch (img_req->op_type) {  in __rbd_img_fill_request()
2430 			rbd_img_obj_request_del(img_req, obj_req);  in __rbd_img_fill_request()
2435 	img_req->state = RBD_IMG_START;  in __rbd_img_fill_request()
2455 	struct rbd_img_request *img_req = arg;  in alloc_object_extent()  local
2462 	rbd_img_obj_request_add(img_req, obj_req);  in alloc_object_extent()
2478 static int rbd_img_fill_request_nocopy(struct rbd_img_request *img_req,  in rbd_img_fill_request_nocopy()  argument
2486 	img_req->data_type = fctx->pos_type;  in rbd_img_fill_request_nocopy()
2494 		ret = ceph_file_to_extents(&img_req->rbd_dev->layout,  in rbd_img_fill_request_nocopy()
2497 					   &img_req->object_extents,  in rbd_img_fill_request_nocopy()
2498 					   alloc_object_extent, img_req,  in rbd_img_fill_request_nocopy()
2504 	return __rbd_img_fill_request(img_req);  in rbd_img_fill_request_nocopy()
2520 static int rbd_img_fill_request(struct rbd_img_request *img_req,  in rbd_img_fill_request()  argument
2525 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_fill_request()
2532 		return rbd_img_fill_request_nocopy(img_req, img_extents,  in rbd_img_fill_request()
2535 	img_req->data_type = OBJ_REQUEST_OWN_BVECS;  in rbd_img_fill_request()
2549 					   &img_req->object_extents,  in rbd_img_fill_request()
2550 					   alloc_object_extent, img_req,  in rbd_img_fill_request()
2556 	for_each_obj_request(img_req, obj_req) {  in rbd_img_fill_request()
2573 					   &img_req->object_extents,  in rbd_img_fill_request()
2579 	return __rbd_img_fill_request(img_req);  in rbd_img_fill_request()
2582 static int rbd_img_fill_nodata(struct rbd_img_request *img_req,  in rbd_img_fill_nodata()  argument
2592 	return rbd_img_fill_request(img_req, &ex, 1, &fctx);  in rbd_img_fill_nodata()
2632 static int __rbd_img_fill_from_bio(struct rbd_img_request *img_req,  in __rbd_img_fill_from_bio()  argument
2645 	return rbd_img_fill_request(img_req, img_extents, num_img_extents,  in __rbd_img_fill_from_bio()
2649 static int rbd_img_fill_from_bio(struct rbd_img_request *img_req,  in rbd_img_fill_from_bio()  argument
2655 	return __rbd_img_fill_from_bio(img_req, &ex, 1, &it);  in rbd_img_fill_from_bio()
2692 static int __rbd_img_fill_from_bvecs(struct rbd_img_request *img_req,  in __rbd_img_fill_from_bvecs()  argument
2705 	return rbd_img_fill_request(img_req, img_extents, num_img_extents,  in __rbd_img_fill_from_bvecs()
2709 static int rbd_img_fill_from_bvecs(struct rbd_img_request *img_req,  in rbd_img_fill_from_bvecs()  argument
2720 	return __rbd_img_fill_from_bvecs(img_req, img_extents, num_img_extents,  in rbd_img_fill_from_bvecs()
2726 	struct rbd_img_request *img_req =  in rbd_img_handle_request_work()  local
2729 	rbd_img_handle_request(img_req, img_req->work_result);  in rbd_img_handle_request_work()
2732 static void rbd_img_schedule(struct rbd_img_request *img_req, int result)  in rbd_img_schedule()  argument
2734 	INIT_WORK(&img_req->work, rbd_img_handle_request_work);  in rbd_img_schedule()
2735 	img_req->work_result = result;  in rbd_img_schedule()
2736 	queue_work(rbd_wq, &img_req->work);  in rbd_img_schedule()
2777 	struct rbd_img_request *img_req = obj_req->img_request;  in rbd_obj_read_from_parent()  local
2778 	struct rbd_device *parent = img_req->rbd_dev->parent;  in rbd_obj_read_from_parent()
2797 	if (!rbd_img_is_write(img_req)) {  in rbd_obj_read_from_parent()
2798 		switch (img_req->data_type) {  in rbd_obj_read_from_parent()
3369 	struct rbd_img_request *img_req = obj_req->img_request;  in __rbd_obj_handle_request()  local
3370 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in __rbd_obj_handle_request()
3374 	if (!rbd_img_is_write(img_req))  in __rbd_obj_handle_request()
3383 			 obj_op_name(img_req->op_type), obj_req->ex.oe_objno,  in __rbd_obj_handle_request()
3399 static bool need_exclusive_lock(struct rbd_img_request *img_req)  in need_exclusive_lock()  argument
3401 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in need_exclusive_lock()
3409 	rbd_assert(!test_bit(IMG_REQ_CHILD, &img_req->flags));  in need_exclusive_lock()
3414 	return rbd_img_is_write(img_req);  in need_exclusive_lock()
3417 static bool rbd_lock_add_request(struct rbd_img_request *img_req)  in rbd_lock_add_request()  argument
3419 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_lock_add_request()
3425 	rbd_assert(list_empty(&img_req->lock_item));  in rbd_lock_add_request()
3427 		list_add_tail(&img_req->lock_item, &rbd_dev->acquiring_list);  in rbd_lock_add_request()
3429 		list_add_tail(&img_req->lock_item, &rbd_dev->running_list);  in rbd_lock_add_request()
3434 static void rbd_lock_del_request(struct rbd_img_request *img_req)  in rbd_lock_del_request()  argument
3436 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_lock_del_request()
3441 	rbd_assert(!list_empty(&img_req->lock_item));  in rbd_lock_del_request()
3442 	list_del_init(&img_req->lock_item);  in rbd_lock_del_request()
3450 static int rbd_img_exclusive_lock(struct rbd_img_request *img_req)  in rbd_img_exclusive_lock()  argument
3452 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_exclusive_lock()
3454 	if (!need_exclusive_lock(img_req))  in rbd_img_exclusive_lock()
3457 	if (rbd_lock_add_request(img_req))  in rbd_img_exclusive_lock()
3474 static void rbd_img_object_requests(struct rbd_img_request *img_req)  in rbd_img_object_requests()  argument
3478 	rbd_assert(!img_req->pending.result && !img_req->pending.num_pending);  in rbd_img_object_requests()
3480 	for_each_obj_request(img_req, obj_req) {  in rbd_img_object_requests()
3485 				img_req->pending.result = result;  in rbd_img_object_requests()
3489 			img_req->pending.num_pending++;  in rbd_img_object_requests()
3494 static bool rbd_img_advance(struct rbd_img_request *img_req, int *result)  in rbd_img_advance()  argument
3496 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in rbd_img_advance()
3500 	switch (img_req->state) {  in rbd_img_advance()
3504 		ret = rbd_img_exclusive_lock(img_req);  in rbd_img_advance()
3509 		img_req->state = RBD_IMG_EXCLUSIVE_LOCK;  in rbd_img_advance()
3517 		rbd_assert(!need_exclusive_lock(img_req) ||  in rbd_img_advance()
3520 		rbd_img_object_requests(img_req);  in rbd_img_advance()
3521 		if (!img_req->pending.num_pending) {  in rbd_img_advance()
3522 			*result = img_req->pending.result;  in rbd_img_advance()
3523 			img_req->state = RBD_IMG_OBJECT_REQUESTS;  in rbd_img_advance()
3526 		img_req->state = __RBD_IMG_OBJECT_REQUESTS;  in rbd_img_advance()
3529 		if (!pending_result_dec(&img_req->pending, result))  in rbd_img_advance()
3542 static bool __rbd_img_handle_request(struct rbd_img_request *img_req,  in __rbd_img_handle_request()  argument
3545 	struct rbd_device *rbd_dev = img_req->rbd_dev;  in __rbd_img_handle_request()
3548 	if (need_exclusive_lock(img_req)) {  in __rbd_img_handle_request()
3550 		mutex_lock(&img_req->state_mutex);  in __rbd_img_handle_request()
3551 		done = rbd_img_advance(img_req, result);  in __rbd_img_handle_request()
3553 			rbd_lock_del_request(img_req);  in __rbd_img_handle_request()
3554 		mutex_unlock(&img_req->state_mutex);  in __rbd_img_handle_request()
3557 		mutex_lock(&img_req->state_mutex);  in __rbd_img_handle_request()
3558 		done = rbd_img_advance(img_req, result);  in __rbd_img_handle_request()
3559 		mutex_unlock(&img_req->state_mutex);  in __rbd_img_handle_request()
3565 		      test_bit(IMG_REQ_CHILD, &img_req->flags) ? "child " : "",  in __rbd_img_handle_request()
3566 		      obj_op_name(img_req->op_type), *result);  in __rbd_img_handle_request()
3571 static void rbd_img_handle_request(struct rbd_img_request *img_req, int result)  in rbd_img_handle_request()  argument
3574 	if (!__rbd_img_handle_request(img_req, &result))  in rbd_img_handle_request()
3577 	if (test_bit(IMG_REQ_CHILD, &img_req->flags)) {  in rbd_img_handle_request()
3578 		struct rbd_obj_request *obj_req = img_req->obj_request;  in rbd_img_handle_request()
3580 		rbd_img_request_destroy(img_req);  in rbd_img_handle_request()
3582 			img_req = obj_req->img_request;  in rbd_img_handle_request()
3586 		struct request *rq = blk_mq_rq_from_pdu(img_req);  in rbd_img_handle_request()
3588 		rbd_img_request_destroy(img_req);  in rbd_img_handle_request()
3807 	struct rbd_img_request *img_req;  in wake_lock_waiters()  local
3821 	list_for_each_entry(img_req, &rbd_dev->acquiring_list, lock_item) {  in wake_lock_waiters()
3822 		mutex_lock(&img_req->state_mutex);  in wake_lock_waiters()
3823 		rbd_assert(img_req->state == RBD_IMG_EXCLUSIVE_LOCK);  in wake_lock_waiters()
3824 		rbd_img_schedule(img_req, result);  in wake_lock_waiters()
3825 		mutex_unlock(&img_req->state_mutex);  in wake_lock_waiters()
4693 	struct rbd_img_request *img_req = blk_mq_rq_to_pdu(bd->rq);  in rbd_queue_rq()  local
4714 	rbd_img_request_init(img_req, rbd_dev, op_type);  in rbd_queue_rq()
4716 	if (rbd_img_is_write(img_req)) {  in rbd_queue_rq()
4719 				 obj_op_name(img_req->op_type));  in rbd_queue_rq()
4725 	INIT_WORK(&img_req->work, rbd_queue_workfn);  in rbd_queue_rq()
4726 	queue_work(rbd_wq, &img_req->work);  in rbd_queue_rq()