1300 static void rbd_obj_zero_range(struct rbd_obj_request *obj_req, u32 off,  in rbd_obj_zero_range()  argument
1303 	switch (obj_req->img_request->data_type) {  in rbd_obj_zero_range()
1305 		zero_bios(&obj_req->bio_pos, off, bytes);  in rbd_obj_zero_range()
1309 		zero_bvecs(&obj_req->bvec_pos, off, bytes);  in rbd_obj_zero_range()
1397 static bool rbd_obj_is_entire(struct rbd_obj_request *obj_req)  in rbd_obj_is_entire()  argument
1399 	struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;  in rbd_obj_is_entire()
1401 	return !obj_req->ex.oe_off &&  in rbd_obj_is_entire()
1402 	       obj_req->ex.oe_len == rbd_dev->layout.object_size;  in rbd_obj_is_entire()
1405 static bool rbd_obj_is_tail(struct rbd_obj_request *obj_req)  in rbd_obj_is_tail()  argument
1407 	struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;  in rbd_obj_is_tail()
1409 	return obj_req->ex.oe_off + obj_req->ex.oe_len ==  in rbd_obj_is_tail()
1413 static u64 rbd_obj_img_extents_bytes(struct rbd_obj_request *obj_req)  in rbd_obj_img_extents_bytes()  argument
1415 	return ceph_file_extents_bytes(obj_req->img_extents,  in rbd_obj_img_extents_bytes()
1416 				       obj_req->num_img_extents);  in rbd_obj_img_extents_bytes()
1432 static void rbd_obj_handle_request(struct rbd_obj_request *obj_req);
1436 	struct rbd_obj_request *obj_req = osd_req->r_priv;  in rbd_osd_req_callback()  local
1439 	     osd_req->r_result, obj_req);  in rbd_osd_req_callback()
1440 	rbd_assert(osd_req == obj_req->osd_req);  in rbd_osd_req_callback()
1442 	obj_req->result = osd_req->r_result < 0 ? osd_req->r_result : 0;  in rbd_osd_req_callback()
1443 	if (!obj_req->result && !rbd_img_is_write(obj_req->img_request))  in rbd_osd_req_callback()
1444 		obj_req->xferred = osd_req->r_result;  in rbd_osd_req_callback()
1451 		obj_req->xferred = 0;  in rbd_osd_req_callback()
1453 	rbd_obj_handle_request(obj_req);  in rbd_osd_req_callback()
1474 rbd_osd_req_create(struct rbd_obj_request *obj_req, unsigned int num_ops)  in rbd_osd_req_create()  argument
1476 	struct rbd_img_request *img_req = obj_req->img_request;  in rbd_osd_req_create()
1490 	req->r_priv = obj_req;  in rbd_osd_req_create()
1500 			rbd_dev->header.object_prefix, obj_req->ex.oe_objno))  in rbd_osd_req_create()
1715 static int rbd_obj_calc_img_extents(struct rbd_obj_request *obj_req,  in rbd_obj_calc_img_extents()  argument
1718 	struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;  in rbd_obj_calc_img_extents()
1724 	ret = ceph_extent_to_file(&rbd_dev->layout, obj_req->ex.oe_objno,  in rbd_obj_calc_img_extents()
1725 				  entire ? 0 : obj_req->ex.oe_off,  in rbd_obj_calc_img_extents()
1727 							obj_req->ex.oe_len,  in rbd_obj_calc_img_extents()
1728 				  &obj_req->img_extents,  in rbd_obj_calc_img_extents()
1729 				  &obj_req->num_img_extents);  in rbd_obj_calc_img_extents()
1733 	prune_extents(obj_req->img_extents, &obj_req->num_img_extents,  in rbd_obj_calc_img_extents()
1738 static void rbd_osd_req_setup_data(struct rbd_obj_request *obj_req, u32 which)  in rbd_osd_req_setup_data()  argument
1740 	switch (obj_req->img_request->data_type) {  in rbd_osd_req_setup_data()
1742 		osd_req_op_extent_osd_data_bio(obj_req->osd_req, which,  in rbd_osd_req_setup_data()
1743 					       &obj_req->bio_pos,  in rbd_osd_req_setup_data()
1744 					       obj_req->ex.oe_len);  in rbd_osd_req_setup_data()
1748 		rbd_assert(obj_req->bvec_pos.iter.bi_size ==  in rbd_osd_req_setup_data()
1749 							obj_req->ex.oe_len);  in rbd_osd_req_setup_data()
1750 		rbd_assert(obj_req->bvec_idx == obj_req->bvec_count);  in rbd_osd_req_setup_data()
1751 		osd_req_op_extent_osd_data_bvec_pos(obj_req->osd_req, which,  in rbd_osd_req_setup_data()
1752 						    &obj_req->bvec_pos);  in rbd_osd_req_setup_data()
1759 static int rbd_obj_setup_read(struct rbd_obj_request *obj_req)  in rbd_obj_setup_read()  argument
1761 	obj_req->osd_req = rbd_osd_req_create(obj_req, 1);  in rbd_obj_setup_read()
1762 	if (!obj_req->osd_req)  in rbd_obj_setup_read()
1765 	osd_req_op_extent_init(obj_req->osd_req, 0, CEPH_OSD_OP_READ,  in rbd_obj_setup_read()
1766 			       obj_req->ex.oe_off, obj_req->ex.oe_len, 0, 0);  in rbd_obj_setup_read()
1767 	rbd_osd_req_setup_data(obj_req, 0);  in rbd_obj_setup_read()
1769 	rbd_osd_req_format_read(obj_req);  in rbd_obj_setup_read()
1773 static int __rbd_obj_setup_stat(struct rbd_obj_request *obj_req,  in __rbd_obj_setup_stat()  argument
1790 	osd_req_op_init(obj_req->osd_req, which, CEPH_OSD_OP_STAT, 0);  in __rbd_obj_setup_stat()
1791 	osd_req_op_raw_data_in_pages(obj_req->osd_req, which, pages,  in __rbd_obj_setup_stat()
1797 static void __rbd_obj_setup_write(struct rbd_obj_request *obj_req,  in __rbd_obj_setup_write()  argument
1800 	struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;  in __rbd_obj_setup_write()
1803 	osd_req_op_alloc_hint_init(obj_req->osd_req, which++,  in __rbd_obj_setup_write()
1807 	if (rbd_obj_is_entire(obj_req))  in __rbd_obj_setup_write()
1812 	osd_req_op_extent_init(obj_req->osd_req, which, opcode,  in __rbd_obj_setup_write()
1813 			       obj_req->ex.oe_off, obj_req->ex.oe_len, 0, 0);  in __rbd_obj_setup_write()
1814 	rbd_osd_req_setup_data(obj_req, which++);  in __rbd_obj_setup_write()
1816 	rbd_assert(which == obj_req->osd_req->r_num_ops);  in __rbd_obj_setup_write()
1817 	rbd_osd_req_format_write(obj_req);  in __rbd_obj_setup_write()
1820 static int rbd_obj_setup_write(struct rbd_obj_request *obj_req)  in rbd_obj_setup_write()  argument
1826 	ret = rbd_obj_calc_img_extents(obj_req, true);  in rbd_obj_setup_write()
1830 	if (obj_req->num_img_extents) {  in rbd_obj_setup_write()
1831 		obj_req->write_state = RBD_OBJ_WRITE_GUARD;  in rbd_obj_setup_write()
1834 		obj_req->write_state = RBD_OBJ_WRITE_FLAT;  in rbd_obj_setup_write()
1838 	obj_req->osd_req = rbd_osd_req_create(obj_req, num_osd_ops);  in rbd_obj_setup_write()
1839 	if (!obj_req->osd_req)  in rbd_obj_setup_write()
1842 	if (obj_req->num_img_extents) {  in rbd_obj_setup_write()
1843 		ret = __rbd_obj_setup_stat(obj_req, which++);  in rbd_obj_setup_write()
1848 	__rbd_obj_setup_write(obj_req, which);  in rbd_obj_setup_write()
1852 static void __rbd_obj_setup_discard(struct rbd_obj_request *obj_req,  in __rbd_obj_setup_discard()  argument
1857 	if (rbd_obj_is_entire(obj_req)) {  in __rbd_obj_setup_discard()
1858 		if (obj_req->num_img_extents) {  in __rbd_obj_setup_discard()
1859 			osd_req_op_init(obj_req->osd_req, which++,  in __rbd_obj_setup_discard()
1863 			osd_req_op_init(obj_req->osd_req, which++,  in __rbd_obj_setup_discard()
1867 	} else if (rbd_obj_is_tail(obj_req)) {  in __rbd_obj_setup_discard()
1874 		osd_req_op_extent_init(obj_req->osd_req, which++, opcode,  in __rbd_obj_setup_discard()
1875 				       obj_req->ex.oe_off, obj_req->ex.oe_len,  in __rbd_obj_setup_discard()
1878 	rbd_assert(which == obj_req->osd_req->r_num_ops);  in __rbd_obj_setup_discard()
1879 	rbd_osd_req_format_write(obj_req);  in __rbd_obj_setup_discard()
1882 static int rbd_obj_setup_discard(struct rbd_obj_request *obj_req)  in rbd_obj_setup_discard()  argument
1888 	ret = rbd_obj_calc_img_extents(obj_req, true);  in rbd_obj_setup_discard()
1892 	if (rbd_obj_is_entire(obj_req)) {  in rbd_obj_setup_discard()
1893 		obj_req->write_state = RBD_OBJ_WRITE_FLAT;  in rbd_obj_setup_discard()
1894 		if (obj_req->num_img_extents)  in rbd_obj_setup_discard()
1899 		if (obj_req->num_img_extents) {  in rbd_obj_setup_discard()
1900 			obj_req->write_state = RBD_OBJ_WRITE_GUARD;  in rbd_obj_setup_discard()
1903 			obj_req->write_state = RBD_OBJ_WRITE_FLAT;  in rbd_obj_setup_discard()
1908 	obj_req->osd_req = rbd_osd_req_create(obj_req, num_osd_ops);  in rbd_obj_setup_discard()
1909 	if (!obj_req->osd_req)  in rbd_obj_setup_discard()
1912 	if (!rbd_obj_is_entire(obj_req) && obj_req->num_img_extents) {  in rbd_obj_setup_discard()
1913 		ret = __rbd_obj_setup_stat(obj_req, which++);  in rbd_obj_setup_discard()
1918 	__rbd_obj_setup_discard(obj_req, which);  in rbd_obj_setup_discard()
1929 	struct rbd_obj_request *obj_req;  in __rbd_img_fill_request()  local
1932 	for_each_obj_request(img_req, obj_req) {  in __rbd_img_fill_request()
1935 			ret = rbd_obj_setup_read(obj_req);  in __rbd_img_fill_request()
1938 			ret = rbd_obj_setup_write(obj_req);  in __rbd_img_fill_request()
1941 			ret = rbd_obj_setup_discard(obj_req);  in __rbd_img_fill_request()
1970 	struct rbd_obj_request *obj_req;  in alloc_object_extent()  local
1972 	obj_req = rbd_obj_request_create();  in alloc_object_extent()
1973 	if (!obj_req)  in alloc_object_extent()
1976 	rbd_img_obj_request_add(img_req, obj_req);  in alloc_object_extent()
1977 	return &obj_req->ex;  in alloc_object_extent()
2040 	struct rbd_obj_request *obj_req;  in rbd_img_fill_request()  local
2070 	for_each_obj_request(img_req, obj_req) {  in rbd_img_fill_request()
2071 		obj_req->bvec_pos.bvecs = kmalloc_array(obj_req->bvec_count,  in rbd_img_fill_request()
2072 					      sizeof(*obj_req->bvec_pos.bvecs),  in rbd_img_fill_request()
2074 		if (!obj_req->bvec_pos.bvecs)  in rbd_img_fill_request()
2111 	struct rbd_obj_request *obj_req =  in set_bio_pos()  local
2116 	obj_req->bio_pos = *it;  in set_bio_pos()
2122 	struct rbd_obj_request *obj_req =  in count_bio_bvecs()  local
2128 		obj_req->bvec_count++;  in count_bio_bvecs()
2135 	struct rbd_obj_request *obj_req =  in copy_bio_bvecs()  local
2141 		obj_req->bvec_pos.bvecs[obj_req->bvec_idx++] = bv;  in copy_bio_bvecs()
2142 		obj_req->bvec_pos.iter.bi_size += bv.bv_len;  in copy_bio_bvecs()
2174 	struct rbd_obj_request *obj_req =  in set_bvec_pos()  local
2178 	obj_req->bvec_pos = *it;  in set_bvec_pos()
2179 	ceph_bvec_iter_shorten(&obj_req->bvec_pos, bytes);  in set_bvec_pos()
2185 	struct rbd_obj_request *obj_req =  in count_bvecs()  local
2190 		obj_req->bvec_count++;  in count_bvecs()
2196 	struct rbd_obj_request *obj_req =  in copy_bvecs()  local
2201 		obj_req->bvec_pos.bvecs[obj_req->bvec_idx++] = bv;  in copy_bvecs()
2202 		obj_req->bvec_pos.iter.bi_size += bv.bv_len;  in copy_bvecs()
2251 static int rbd_obj_read_from_parent(struct rbd_obj_request *obj_req)  in rbd_obj_read_from_parent()  argument
2253 	struct rbd_img_request *img_req = obj_req->img_request;  in rbd_obj_read_from_parent()
2263 	child_img_req->obj_request = obj_req;  in rbd_obj_read_from_parent()
2269 						      obj_req->img_extents,  in rbd_obj_read_from_parent()
2270 						      obj_req->num_img_extents,  in rbd_obj_read_from_parent()
2271 						      &obj_req->bio_pos);  in rbd_obj_read_from_parent()
2276 						      obj_req->img_extents,  in rbd_obj_read_from_parent()
2277 						      obj_req->num_img_extents,  in rbd_obj_read_from_parent()
2278 						      &obj_req->bvec_pos);  in rbd_obj_read_from_parent()
2285 					      obj_req->img_extents,  in rbd_obj_read_from_parent()
2286 					      obj_req->num_img_extents,  in rbd_obj_read_from_parent()
2287 					      obj_req->copyup_bvecs);  in rbd_obj_read_from_parent()
2298 static bool rbd_obj_handle_read(struct rbd_obj_request *obj_req)  in rbd_obj_handle_read()  argument
2300 	struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;  in rbd_obj_handle_read()
2303 	if (obj_req->result == -ENOENT &&  in rbd_obj_handle_read()
2304 	    rbd_dev->parent_overlap && !obj_req->tried_parent) {  in rbd_obj_handle_read()
2306 		ret = rbd_obj_calc_img_extents(obj_req, false);  in rbd_obj_handle_read()
2308 			obj_req->result = ret;  in rbd_obj_handle_read()
2312 		if (obj_req->num_img_extents) {  in rbd_obj_handle_read()
2313 			obj_req->tried_parent = true;  in rbd_obj_handle_read()
2314 			ret = rbd_obj_read_from_parent(obj_req);  in rbd_obj_handle_read()
2316 				obj_req->result = ret;  in rbd_obj_handle_read()
2329 	if (obj_req->result == -ENOENT ||  in rbd_obj_handle_read()
2330 	    (!obj_req->result && obj_req->xferred < obj_req->ex.oe_len)) {  in rbd_obj_handle_read()
2331 		rbd_assert(!obj_req->xferred || !obj_req->result);  in rbd_obj_handle_read()
2332 		rbd_obj_zero_range(obj_req, obj_req->xferred,  in rbd_obj_handle_read()
2333 				   obj_req->ex.oe_len - obj_req->xferred);  in rbd_obj_handle_read()
2334 		obj_req->result = 0;  in rbd_obj_handle_read()
2335 		obj_req->xferred = obj_req->ex.oe_len;  in rbd_obj_handle_read()
2359 static int rbd_obj_issue_copyup(struct rbd_obj_request *obj_req, u32 bytes)  in rbd_obj_issue_copyup()  argument
2361 	unsigned int num_osd_ops = obj_req->osd_req->r_num_ops;  in rbd_obj_issue_copyup()
2364 	dout("%s obj_req %p bytes %u\n", __func__, obj_req, bytes);  in rbd_obj_issue_copyup()
2365 	rbd_assert(obj_req->osd_req->r_ops[0].op == CEPH_OSD_OP_STAT);  in rbd_obj_issue_copyup()
2366 	rbd_osd_req_destroy(obj_req->osd_req);  in rbd_obj_issue_copyup()
2373 	obj_req->osd_req = rbd_osd_req_create(obj_req, num_osd_ops);  in rbd_obj_issue_copyup()
2374 	if (!obj_req->osd_req)  in rbd_obj_issue_copyup()
2377 	ret = osd_req_op_cls_init(obj_req->osd_req, 0, CEPH_OSD_OP_CALL, "rbd",  in rbd_obj_issue_copyup()
2387 	if (is_zero_bvecs(obj_req->copyup_bvecs, bytes)) {  in rbd_obj_issue_copyup()
2388 		dout("%s obj_req %p detected zeroes\n", __func__, obj_req);  in rbd_obj_issue_copyup()
2391 	osd_req_op_cls_request_data_bvecs(obj_req->osd_req, 0,  in rbd_obj_issue_copyup()
2392 					  obj_req->copyup_bvecs,  in rbd_obj_issue_copyup()
2393 					  obj_req->copyup_bvec_count,  in rbd_obj_issue_copyup()
2396 	switch (obj_req->img_request->op_type) {  in rbd_obj_issue_copyup()
2398 		__rbd_obj_setup_write(obj_req, 1);  in rbd_obj_issue_copyup()
2401 		rbd_assert(!rbd_obj_is_entire(obj_req));  in rbd_obj_issue_copyup()
2402 		__rbd_obj_setup_discard(obj_req, 1);  in rbd_obj_issue_copyup()
2408 	rbd_obj_request_submit(obj_req);  in rbd_obj_issue_copyup()
2412 static int setup_copyup_bvecs(struct rbd_obj_request *obj_req, u64 obj_overlap)  in setup_copyup_bvecs()  argument
2416 	rbd_assert(!obj_req->copyup_bvecs);  in setup_copyup_bvecs()
2417 	obj_req->copyup_bvec_count = calc_pages_for(0, obj_overlap);  in setup_copyup_bvecs()
2418 	obj_req->copyup_bvecs = kcalloc(obj_req->copyup_bvec_count,  in setup_copyup_bvecs()
2419 					sizeof(*obj_req->copyup_bvecs),  in setup_copyup_bvecs()
2421 	if (!obj_req->copyup_bvecs)  in setup_copyup_bvecs()
2424 	for (i = 0; i < obj_req->copyup_bvec_count; i++) {  in setup_copyup_bvecs()
2427 		obj_req->copyup_bvecs[i].bv_page = alloc_page(GFP_NOIO);  in setup_copyup_bvecs()
2428 		if (!obj_req->copyup_bvecs[i].bv_page)  in setup_copyup_bvecs()
2431 		obj_req->copyup_bvecs[i].bv_offset = 0;  in setup_copyup_bvecs()
2432 		obj_req->copyup_bvecs[i].bv_len = len;  in setup_copyup_bvecs()
2440 static int rbd_obj_handle_write_guard(struct rbd_obj_request *obj_req)  in rbd_obj_handle_write_guard()  argument
2442 	struct rbd_device *rbd_dev = obj_req->img_request->rbd_dev;  in rbd_obj_handle_write_guard()
2445 	rbd_assert(obj_req->num_img_extents);  in rbd_obj_handle_write_guard()
2446 	prune_extents(obj_req->img_extents, &obj_req->num_img_extents,  in rbd_obj_handle_write_guard()
2448 	if (!obj_req->num_img_extents) {  in rbd_obj_handle_write_guard()
2458 		obj_req->write_state = RBD_OBJ_WRITE_FLAT;  in rbd_obj_handle_write_guard()
2459 		return rbd_obj_issue_copyup(obj_req, 0);  in rbd_obj_handle_write_guard()
2462 	ret = setup_copyup_bvecs(obj_req, rbd_obj_img_extents_bytes(obj_req));  in rbd_obj_handle_write_guard()
2466 	obj_req->write_state = RBD_OBJ_WRITE_COPYUP;  in rbd_obj_handle_write_guard()
2467 	return rbd_obj_read_from_parent(obj_req);  in rbd_obj_handle_write_guard()
2470 static bool rbd_obj_handle_write(struct rbd_obj_request *obj_req)  in rbd_obj_handle_write()  argument
2475 	switch (obj_req->write_state) {  in rbd_obj_handle_write()
2477 		rbd_assert(!obj_req->xferred);  in rbd_obj_handle_write()
2478 		if (obj_req->result == -ENOENT) {  in rbd_obj_handle_write()
2484 			ret = rbd_obj_handle_write_guard(obj_req);  in rbd_obj_handle_write()
2486 				obj_req->result = ret;  in rbd_obj_handle_write()
2493 		if (!obj_req->result)  in rbd_obj_handle_write()
2498 			obj_req->xferred = obj_req->ex.oe_len;  in rbd_obj_handle_write()
2501 		obj_req->write_state = RBD_OBJ_WRITE_GUARD;  in rbd_obj_handle_write()
2502 		if (obj_req->result)  in rbd_obj_handle_write()
2505 		rbd_assert(obj_req->xferred);  in rbd_obj_handle_write()
2506 		ret = rbd_obj_issue_copyup(obj_req, obj_req->xferred);  in rbd_obj_handle_write()
2508 			obj_req->result = ret;  in rbd_obj_handle_write()
2520 static bool __rbd_obj_handle_request(struct rbd_obj_request *obj_req)  in __rbd_obj_handle_request()  argument
2522 	switch (obj_req->img_request->op_type) {  in __rbd_obj_handle_request()
2524 		return rbd_obj_handle_read(obj_req);  in __rbd_obj_handle_request()
2526 		return rbd_obj_handle_write(obj_req);  in __rbd_obj_handle_request()
2528 		if (rbd_obj_handle_write(obj_req)) {  in __rbd_obj_handle_request()
2533 			if (obj_req->result == -ENOENT) {  in __rbd_obj_handle_request()
2534 				obj_req->result = 0;  in __rbd_obj_handle_request()
2535 				obj_req->xferred = obj_req->ex.oe_len;  in __rbd_obj_handle_request()
2545 static void rbd_obj_end_request(struct rbd_obj_request *obj_req)  in rbd_obj_end_request()  argument
2547 	struct rbd_img_request *img_req = obj_req->img_request;  in rbd_obj_end_request()
2549 	rbd_assert((!obj_req->result &&  in rbd_obj_end_request()
2550 		    obj_req->xferred == obj_req->ex.oe_len) ||  in rbd_obj_end_request()
2551 		   (obj_req->result < 0 && !obj_req->xferred));  in rbd_obj_end_request()
2552 	if (!obj_req->result) {  in rbd_obj_end_request()
2553 		img_req->xferred += obj_req->xferred;  in rbd_obj_end_request()
2559 		 obj_op_name(img_req->op_type), obj_req->ex.oe_objno,  in rbd_obj_end_request()
2560 		 obj_req->ex.oe_off, obj_req->ex.oe_len, obj_req->result,  in rbd_obj_end_request()
2561 		 obj_req->xferred);  in rbd_obj_end_request()
2563 		img_req->result = obj_req->result;  in rbd_obj_end_request()
2570 	struct rbd_obj_request *obj_req = img_req->obj_request;  in rbd_img_end_child_request()  local
2574 		    img_req->xferred == rbd_obj_img_extents_bytes(obj_req)) ||  in rbd_img_end_child_request()
2577 	obj_req->result = img_req->result;  in rbd_img_end_child_request()
2578 	obj_req->xferred = img_req->xferred;  in rbd_img_end_child_request()
2594 static void rbd_obj_handle_request(struct rbd_obj_request *obj_req)  in rbd_obj_handle_request()  argument
2599 	if (!__rbd_obj_handle_request(obj_req))  in rbd_obj_handle_request()
2602 	img_req = obj_req->img_request;  in rbd_obj_handle_request()
2604 	rbd_obj_end_request(obj_req);  in rbd_obj_handle_request()
2613 		obj_req = img_req->obj_request;  in rbd_obj_handle_request()