Lines Matching +full:locality +full:- +full:specific
1 // SPDX-License-Identifier: GPL-2.0
71 WARN_ON(!rwsem_is_locked(&osdc->lock)); in verify_osdc_locked()
75 WARN_ON(!rwsem_is_wrlocked(&osdc->lock)); in verify_osdc_wrlocked()
79 struct ceph_osd_client *osdc = osd->o_osdc; in verify_osd_locked()
81 WARN_ON(!(mutex_is_locked(&osd->lock) && in verify_osd_locked()
82 rwsem_is_locked(&osdc->lock)) && in verify_osd_locked()
83 !rwsem_is_wrlocked(&osdc->lock)); in verify_osd_locked()
87 WARN_ON(!mutex_is_locked(&lreq->lock)); in verify_lreq_locked()
116 orig_len - *plen, off, *plen); in calc_layout()
126 osd_data->type = CEPH_OSD_DATA_TYPE_NONE; in ceph_osd_data_init()
136 osd_data->type = CEPH_OSD_DATA_TYPE_PAGES; in ceph_osd_data_pages_init()
137 osd_data->pages = pages; in ceph_osd_data_pages_init()
138 osd_data->length = length; in ceph_osd_data_pages_init()
139 osd_data->alignment = alignment; in ceph_osd_data_pages_init()
140 osd_data->pages_from_pool = pages_from_pool; in ceph_osd_data_pages_init()
141 osd_data->own_pages = own_pages; in ceph_osd_data_pages_init()
150 osd_data->type = CEPH_OSD_DATA_TYPE_PAGELIST; in ceph_osd_data_pagelist_init()
151 osd_data->pagelist = pagelist; in ceph_osd_data_pagelist_init()
159 osd_data->type = CEPH_OSD_DATA_TYPE_BIO; in ceph_osd_data_bio_init()
160 osd_data->bio_pos = *bio_pos; in ceph_osd_data_bio_init()
161 osd_data->bio_length = bio_length; in ceph_osd_data_bio_init()
169 osd_data->type = CEPH_OSD_DATA_TYPE_BVECS; in ceph_osd_data_bvecs_init()
170 osd_data->bvec_pos = *bvec_pos; in ceph_osd_data_bvecs_init()
171 osd_data->num_bvecs = num_bvecs; in ceph_osd_data_bvecs_init()
177 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_raw_data_in()
179 return &osd_req->r_ops[which].raw_data_in; in osd_req_op_raw_data_in()
285 osd_req->r_ops[which].cls.indata_len += pagelist->length; in osd_req_op_cls_request_data_pagelist()
286 osd_req->r_ops[which].indata_len += pagelist->length; in osd_req_op_cls_request_data_pagelist()
299 osd_req->r_ops[which].cls.indata_len += length; in osd_req_op_cls_request_data_pages()
300 osd_req->r_ops[which].indata_len += length; in osd_req_op_cls_request_data_pages()
317 osd_req->r_ops[which].cls.indata_len += bytes; in osd_req_op_cls_request_data_bvecs()
318 osd_req->r_ops[which].indata_len += bytes; in osd_req_op_cls_request_data_bvecs()
336 switch (osd_data->type) { in ceph_osd_data_length()
340 return osd_data->length; in ceph_osd_data_length()
342 return (u64)osd_data->pagelist->length; in ceph_osd_data_length()
345 return (u64)osd_data->bio_length; in ceph_osd_data_length()
348 return osd_data->bvec_pos.iter.bi_size; in ceph_osd_data_length()
350 WARN(true, "unrecognized data type %d\n", (int)osd_data->type); in ceph_osd_data_length()
357 if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES && osd_data->own_pages) { in ceph_osd_data_release()
360 num_pages = calc_pages_for((u64)osd_data->alignment, in ceph_osd_data_release()
361 (u64)osd_data->length); in ceph_osd_data_release()
362 ceph_release_page_vector(osd_data->pages, num_pages); in ceph_osd_data_release()
363 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGELIST) { in ceph_osd_data_release()
364 ceph_pagelist_release(osd_data->pagelist); in ceph_osd_data_release()
374 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_data_release()
375 op = &osd_req->r_ops[which]; in osd_req_op_data_release()
377 switch (op->op) { in osd_req_op_data_release()
381 ceph_osd_data_release(&op->extent.osd_data); in osd_req_op_data_release()
384 ceph_osd_data_release(&op->cls.request_info); in osd_req_op_data_release()
385 ceph_osd_data_release(&op->cls.request_data); in osd_req_op_data_release()
386 ceph_osd_data_release(&op->cls.response_data); in osd_req_op_data_release()
390 ceph_osd_data_release(&op->xattr.osd_data); in osd_req_op_data_release()
393 ceph_osd_data_release(&op->raw_data_in); in osd_req_op_data_release()
396 ceph_osd_data_release(&op->notify_ack.request_data); in osd_req_op_data_release()
399 ceph_osd_data_release(&op->notify.request_data); in osd_req_op_data_release()
400 ceph_osd_data_release(&op->notify.response_data); in osd_req_op_data_release()
403 ceph_osd_data_release(&op->list_watchers.response_data); in osd_req_op_data_release()
406 ceph_osd_data_release(&op->copy_from.osd_data); in osd_req_op_data_release()
414 * Assumes @t is zero-initialized.
418 ceph_oid_init(&t->base_oid); in target_init()
419 ceph_oloc_init(&t->base_oloc); in target_init()
420 ceph_oid_init(&t->target_oid); in target_init()
421 ceph_oloc_init(&t->target_oloc); in target_init()
423 ceph_osds_init(&t->acting); in target_init()
424 ceph_osds_init(&t->up); in target_init()
425 t->size = -1; in target_init()
426 t->min_size = -1; in target_init()
428 t->osd = CEPH_HOMELESS_OSD; in target_init()
434 ceph_oid_copy(&dest->base_oid, &src->base_oid); in target_copy()
435 ceph_oloc_copy(&dest->base_oloc, &src->base_oloc); in target_copy()
436 ceph_oid_copy(&dest->target_oid, &src->target_oid); in target_copy()
437 ceph_oloc_copy(&dest->target_oloc, &src->target_oloc); in target_copy()
439 dest->pgid = src->pgid; /* struct */ in target_copy()
440 dest->spgid = src->spgid; /* struct */ in target_copy()
441 dest->pg_num = src->pg_num; in target_copy()
442 dest->pg_num_mask = src->pg_num_mask; in target_copy()
443 ceph_osds_copy(&dest->acting, &src->acting); in target_copy()
444 ceph_osds_copy(&dest->up, &src->up); in target_copy()
445 dest->size = src->size; in target_copy()
446 dest->min_size = src->min_size; in target_copy()
447 dest->sort_bitwise = src->sort_bitwise; in target_copy()
448 dest->recovery_deletes = src->recovery_deletes; in target_copy()
450 dest->flags = src->flags; in target_copy()
451 dest->used_replica = src->used_replica; in target_copy()
452 dest->paused = src->paused; in target_copy()
454 dest->epoch = src->epoch; in target_copy()
455 dest->last_force_resend = src->last_force_resend; in target_copy()
457 dest->osd = src->osd; in target_copy()
462 ceph_oid_destroy(&t->base_oid); in target_destroy()
463 ceph_oloc_destroy(&t->base_oloc); in target_destroy()
464 ceph_oid_destroy(&t->target_oid); in target_destroy()
465 ceph_oloc_destroy(&t->target_oloc); in target_destroy()
473 WARN_ON(!RB_EMPTY_NODE(&req->r_node)); in request_release_checks()
474 WARN_ON(!RB_EMPTY_NODE(&req->r_mc_node)); in request_release_checks()
475 WARN_ON(!list_empty(&req->r_private_item)); in request_release_checks()
476 WARN_ON(req->r_osd); in request_release_checks()
486 req->r_request, req->r_reply); in ceph_osdc_release_request()
489 if (req->r_request) in ceph_osdc_release_request()
490 ceph_msg_put(req->r_request); in ceph_osdc_release_request()
491 if (req->r_reply) in ceph_osdc_release_request()
492 ceph_msg_put(req->r_reply); in ceph_osdc_release_request()
494 for (which = 0; which < req->r_num_ops; which++) in ceph_osdc_release_request()
497 target_destroy(&req->r_t); in ceph_osdc_release_request()
498 ceph_put_snap_context(req->r_snapc); in ceph_osdc_release_request()
500 if (req->r_mempool) in ceph_osdc_release_request()
501 mempool_free(req, req->r_osdc->req_mempool); in ceph_osdc_release_request()
502 else if (req->r_num_ops <= CEPH_OSD_SLAB_OPS) in ceph_osdc_release_request()
511 kref_read(&req->r_kref)); in ceph_osdc_get_request()
512 kref_get(&req->r_kref); in ceph_osdc_get_request()
520 kref_read(&req->r_kref)); in ceph_osdc_put_request()
521 kref_put(&req->r_kref, ceph_osdc_release_request); in ceph_osdc_put_request()
531 kref_init(&req->r_kref); in request_init()
532 init_completion(&req->r_completion); in request_init()
533 RB_CLEAR_NODE(&req->r_node); in request_init()
534 RB_CLEAR_NODE(&req->r_mc_node); in request_init()
535 INIT_LIST_HEAD(&req->r_private_item); in request_init()
537 target_init(&req->r_t); in request_init()
549 struct ceph_osd_client *osdc = req->r_osdc; in request_reinit()
550 bool mempool = req->r_mempool; in request_reinit()
551 unsigned int num_ops = req->r_num_ops; in request_reinit()
552 u64 snapid = req->r_snapid; in request_reinit()
553 struct ceph_snap_context *snapc = req->r_snapc; in request_reinit()
554 bool linger = req->r_linger; in request_reinit()
555 struct ceph_msg *request_msg = req->r_request; in request_reinit()
556 struct ceph_msg *reply_msg = req->r_reply; in request_reinit()
559 WARN_ON(kref_read(&req->r_kref) != 1); in request_reinit()
562 WARN_ON(kref_read(&request_msg->kref) != 1); in request_reinit()
563 WARN_ON(kref_read(&reply_msg->kref) != 1); in request_reinit()
564 target_destroy(&req->r_t); in request_reinit()
567 req->r_osdc = osdc; in request_reinit()
568 req->r_mempool = mempool; in request_reinit()
569 req->r_num_ops = num_ops; in request_reinit()
570 req->r_snapid = snapid; in request_reinit()
571 req->r_snapc = snapc; in request_reinit()
572 req->r_linger = linger; in request_reinit()
573 req->r_request = request_msg; in request_reinit()
574 req->r_reply = reply_msg; in request_reinit()
587 req = mempool_alloc(osdc->req_mempool, gfp_flags); in ceph_osdc_alloc_request()
598 req->r_osdc = osdc; in ceph_osdc_alloc_request()
599 req->r_mempool = use_mempool; in ceph_osdc_alloc_request()
600 req->r_num_ops = num_ops; in ceph_osdc_alloc_request()
601 req->r_snapid = CEPH_NOSNAP; in ceph_osdc_alloc_request()
602 req->r_snapc = ceph_get_snap_context(snapc); in ceph_osdc_alloc_request()
611 return 8 + 4 + 4 + 4 + (oloc->pool_ns ? oloc->pool_ns->len : 0); in ceph_oloc_encoding_size()
618 struct ceph_osd_client *osdc = req->r_osdc; in __ceph_osdc_alloc_messages()
622 WARN_ON(req->r_request || req->r_reply); in __ceph_osdc_alloc_messages()
623 WARN_ON(ceph_oid_empty(&req->r_base_oid)); in __ceph_osdc_alloc_messages()
624 WARN_ON(ceph_oloc_empty(&req->r_base_oloc)); in __ceph_osdc_alloc_messages()
635 ceph_oloc_encoding_size(&req->r_base_oloc); /* oloc */ in __ceph_osdc_alloc_messages()
636 msg_size += 4 + req->r_base_oid.name_len; /* oid */ in __ceph_osdc_alloc_messages()
637 msg_size += 2 + req->r_num_ops * sizeof(struct ceph_osd_op); in __ceph_osdc_alloc_messages()
640 msg_size += 4 + 8 * (req->r_snapc ? req->r_snapc->num_snaps : 0); in __ceph_osdc_alloc_messages()
643 if (req->r_mempool) in __ceph_osdc_alloc_messages()
644 msg = ceph_msgpool_get(&osdc->msgpool_op, msg_size, in __ceph_osdc_alloc_messages()
650 return -ENOMEM; in __ceph_osdc_alloc_messages()
652 memset(msg->front.iov_base, 0, msg->front.iov_len); in __ceph_osdc_alloc_messages()
653 req->r_request = msg; in __ceph_osdc_alloc_messages()
657 msg_size += req->r_base_oid.name_len; in __ceph_osdc_alloc_messages()
658 msg_size += req->r_num_ops * sizeof(struct ceph_osd_op); in __ceph_osdc_alloc_messages()
660 if (req->r_mempool) in __ceph_osdc_alloc_messages()
661 msg = ceph_msgpool_get(&osdc->msgpool_op_reply, msg_size, in __ceph_osdc_alloc_messages()
667 return -ENOMEM; in __ceph_osdc_alloc_messages()
669 req->r_reply = msg; in __ceph_osdc_alloc_messages()
694 for (op = req->r_ops; op != &req->r_ops[req->r_num_ops]; op++) { in get_num_data_items()
695 switch (op->op) { in get_num_data_items()
724 WARN_ON(!osd_req_opcode_valid(op->op)); in get_num_data_items()
755 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_init()
758 op = &osd_req->r_ops[which]; in osd_req_op_init()
760 op->op = opcode; in osd_req_op_init()
761 op->flags = flags; in osd_req_op_init()
780 op->extent.offset = offset; in osd_req_op_extent_init()
781 op->extent.length = length; in osd_req_op_extent_init()
782 op->extent.truncate_size = truncate_size; in osd_req_op_extent_init()
783 op->extent.truncate_seq = truncate_seq; in osd_req_op_extent_init()
787 op->indata_len = payload_len; in osd_req_op_extent_init()
797 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_extent_update()
798 op = &osd_req->r_ops[which]; in osd_req_op_extent_update()
799 previous = op->extent.length; in osd_req_op_extent_update()
805 op->extent.length = length; in osd_req_op_extent_update()
806 if (op->op == CEPH_OSD_OP_WRITE || op->op == CEPH_OSD_OP_WRITEFULL) in osd_req_op_extent_update()
807 op->indata_len -= previous - length; in osd_req_op_extent_update()
816 BUG_ON(which + 1 >= osd_req->r_num_ops); in osd_req_op_extent_dup_last()
818 prev_op = &osd_req->r_ops[which]; in osd_req_op_extent_dup_last()
819 op = osd_req_op_init(osd_req, which + 1, prev_op->op, prev_op->flags); in osd_req_op_extent_dup_last()
821 op->indata_len = prev_op->indata_len; in osd_req_op_extent_dup_last()
822 op->outdata_len = prev_op->outdata_len; in osd_req_op_extent_dup_last()
823 op->extent = prev_op->extent; in osd_req_op_extent_dup_last()
825 op->extent.offset += offset_inc; in osd_req_op_extent_dup_last()
826 op->extent.length -= offset_inc; in osd_req_op_extent_dup_last()
828 if (op->op == CEPH_OSD_OP_WRITE || op->op == CEPH_OSD_OP_WRITEFULL) in osd_req_op_extent_dup_last()
829 op->indata_len -= offset_inc; in osd_req_op_extent_dup_last()
846 return -ENOMEM; in osd_req_op_cls_init()
848 op->cls.class_name = class; in osd_req_op_cls_init()
851 op->cls.class_len = size; in osd_req_op_cls_init()
857 op->cls.method_name = method; in osd_req_op_cls_init()
860 op->cls.method_len = size; in osd_req_op_cls_init()
867 op->indata_len = payload_len; in osd_req_op_cls_init()
890 return -ENOMEM; in osd_req_op_xattr_init()
893 op->xattr.name_len = payload_len; in osd_req_op_xattr_init()
898 op->xattr.value_len = size; in osd_req_op_xattr_init()
904 op->xattr.cmp_op = cmp_op; in osd_req_op_xattr_init()
905 op->xattr.cmp_mode = cmp_mode; in osd_req_op_xattr_init()
907 ceph_osd_data_pagelist_init(&op->xattr.osd_data, pagelist); in osd_req_op_xattr_init()
908 op->indata_len = payload_len; in osd_req_op_xattr_init()
926 op->watch.cookie = cookie; in osd_req_op_watch_init()
927 op->watch.op = watch_opcode; in osd_req_op_watch_init()
928 op->watch.gen = 0; in osd_req_op_watch_init()
943 op->alloc_hint.expected_object_size = expected_object_size; in osd_req_op_alloc_hint_init()
944 op->alloc_hint.expected_write_size = expected_write_size; in osd_req_op_alloc_hint_init()
945 op->alloc_hint.flags = flags; in osd_req_op_alloc_hint_init()
949 * not worth a feature bit. Set FAILOK per-op flag to make in osd_req_op_alloc_hint_init()
952 op->flags |= CEPH_OSD_OP_FLAG_FAILOK; in osd_req_op_alloc_hint_init()
961 if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES) { in ceph_osdc_msg_data_add()
964 ceph_msg_data_add_pages(msg, osd_data->pages, in ceph_osdc_msg_data_add()
965 length, osd_data->alignment, false); in ceph_osdc_msg_data_add()
966 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGELIST) { in ceph_osdc_msg_data_add()
968 ceph_msg_data_add_pagelist(msg, osd_data->pagelist); in ceph_osdc_msg_data_add()
970 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_BIO) { in ceph_osdc_msg_data_add()
971 ceph_msg_data_add_bio(msg, &osd_data->bio_pos, length); in ceph_osdc_msg_data_add()
973 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_BVECS) { in ceph_osdc_msg_data_add()
974 ceph_msg_data_add_bvecs(msg, &osd_data->bvec_pos); in ceph_osdc_msg_data_add()
976 BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_NONE); in ceph_osdc_msg_data_add()
983 switch (src->op) { in osd_req_encode_op()
991 dst->extent.offset = cpu_to_le64(src->extent.offset); in osd_req_encode_op()
992 dst->extent.length = cpu_to_le64(src->extent.length); in osd_req_encode_op()
993 dst->extent.truncate_size = in osd_req_encode_op()
994 cpu_to_le64(src->extent.truncate_size); in osd_req_encode_op()
995 dst->extent.truncate_seq = in osd_req_encode_op()
996 cpu_to_le32(src->extent.truncate_seq); in osd_req_encode_op()
999 dst->cls.class_len = src->cls.class_len; in osd_req_encode_op()
1000 dst->cls.method_len = src->cls.method_len; in osd_req_encode_op()
1001 dst->cls.indata_len = cpu_to_le32(src->cls.indata_len); in osd_req_encode_op()
1004 dst->watch.cookie = cpu_to_le64(src->watch.cookie); in osd_req_encode_op()
1005 dst->watch.ver = cpu_to_le64(0); in osd_req_encode_op()
1006 dst->watch.op = src->watch.op; in osd_req_encode_op()
1007 dst->watch.gen = cpu_to_le32(src->watch.gen); in osd_req_encode_op()
1012 dst->notify.cookie = cpu_to_le64(src->notify.cookie); in osd_req_encode_op()
1017 dst->alloc_hint.expected_object_size = in osd_req_encode_op()
1018 cpu_to_le64(src->alloc_hint.expected_object_size); in osd_req_encode_op()
1019 dst->alloc_hint.expected_write_size = in osd_req_encode_op()
1020 cpu_to_le64(src->alloc_hint.expected_write_size); in osd_req_encode_op()
1021 dst->alloc_hint.flags = cpu_to_le32(src->alloc_hint.flags); in osd_req_encode_op()
1025 dst->xattr.name_len = cpu_to_le32(src->xattr.name_len); in osd_req_encode_op()
1026 dst->xattr.value_len = cpu_to_le32(src->xattr.value_len); in osd_req_encode_op()
1027 dst->xattr.cmp_op = src->xattr.cmp_op; in osd_req_encode_op()
1028 dst->xattr.cmp_mode = src->xattr.cmp_mode; in osd_req_encode_op()
1034 dst->copy_from.snapid = cpu_to_le64(src->copy_from.snapid); in osd_req_encode_op()
1035 dst->copy_from.src_version = in osd_req_encode_op()
1036 cpu_to_le64(src->copy_from.src_version); in osd_req_encode_op()
1037 dst->copy_from.flags = src->copy_from.flags; in osd_req_encode_op()
1038 dst->copy_from.src_fadvise_flags = in osd_req_encode_op()
1039 cpu_to_le32(src->copy_from.src_fadvise_flags); in osd_req_encode_op()
1043 ceph_osd_op_name(src->op)); in osd_req_encode_op()
1049 dst->op = cpu_to_le16(src->op); in osd_req_encode_op()
1050 dst->flags = cpu_to_le32(src->flags); in osd_req_encode_op()
1051 dst->payload_len = cpu_to_le32(src->indata_len); in osd_req_encode_op()
1053 return src->indata_len; in osd_req_encode_op()
1088 r = -ENOMEM; in ceph_osdc_new_request()
1100 u32 object_size = layout->object_size; in ceph_osdc_new_request()
1101 u32 object_base = off - objoff; in ceph_osdc_new_request()
1102 if (!(truncate_seq == 1 && truncate_size == -1ULL)) { in ceph_osdc_new_request()
1106 truncate_size -= object_base; in ceph_osdc_new_request()
1115 req->r_base_oloc.pool = layout->pool_id; in ceph_osdc_new_request()
1116 req->r_base_oloc.pool_ns = ceph_try_get_string(layout->pool_ns); in ceph_osdc_new_request()
1117 ceph_oid_printf(&req->r_base_oid, "%llx.%08llx", vino.ino, objnum); in ceph_osdc_new_request()
1118 req->r_flags = flags | osdc->client->options->read_from_replica; in ceph_osdc_new_request()
1120 req->r_snapid = vino.snap; in ceph_osdc_new_request()
1122 req->r_data_offset = off; in ceph_osdc_new_request()
1127 * also covers ceph_uninline_data(). If more multi-op request in ceph_osdc_new_request()
1145 * We keep osd requests in an rbtree, sorted by ->r_tid.
1159 for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { in DEFINE_RB_FUNCS()
1162 for (p = rb_first(&osd->o_requests); p; ) { in DEFINE_RB_FUNCS()
1172 for (p = rb_first(&osdc->homeless_osd.o_requests); p; ) { in DEFINE_RB_FUNCS()
1184 return osd->o_osd == CEPH_HOMELESS_OSD; in osd_homeless()
1189 verify_osdc_locked(osd->o_osdc); in osd_registered()
1191 return !RB_EMPTY_NODE(&osd->o_node); in osd_registered()
1195 * Assumes @osd is zero-initialized.
1199 refcount_set(&osd->o_ref, 1); in osd_init()
1200 RB_CLEAR_NODE(&osd->o_node); in osd_init()
1201 osd->o_requests = RB_ROOT; in osd_init()
1202 osd->o_linger_requests = RB_ROOT; in osd_init()
1203 osd->o_backoff_mappings = RB_ROOT; in osd_init()
1204 osd->o_backoffs_by_id = RB_ROOT; in osd_init()
1205 INIT_LIST_HEAD(&osd->o_osd_lru); in osd_init()
1206 INIT_LIST_HEAD(&osd->o_keepalive_item); in osd_init()
1207 osd->o_incarnation = 1; in osd_init()
1208 mutex_init(&osd->lock); in osd_init()
1213 WARN_ON(!RB_EMPTY_NODE(&osd->o_node)); in osd_cleanup()
1214 WARN_ON(!RB_EMPTY_ROOT(&osd->o_requests)); in osd_cleanup()
1215 WARN_ON(!RB_EMPTY_ROOT(&osd->o_linger_requests)); in osd_cleanup()
1216 WARN_ON(!RB_EMPTY_ROOT(&osd->o_backoff_mappings)); in osd_cleanup()
1217 WARN_ON(!RB_EMPTY_ROOT(&osd->o_backoffs_by_id)); in osd_cleanup()
1218 WARN_ON(!list_empty(&osd->o_osd_lru)); in osd_cleanup()
1219 WARN_ON(!list_empty(&osd->o_keepalive_item)); in osd_cleanup()
1221 if (osd->o_auth.authorizer) { in osd_cleanup()
1223 ceph_auth_destroy_authorizer(osd->o_auth.authorizer); in osd_cleanup()
1238 osd->o_osdc = osdc; in create_osd()
1239 osd->o_osd = onum; in create_osd()
1241 ceph_con_init(&osd->o_con, osd, &osd_con_ops, &osdc->client->msgr); in create_osd()
1248 if (refcount_inc_not_zero(&osd->o_ref)) { in get_osd()
1249 dout("get_osd %p %d -> %d\n", osd, refcount_read(&osd->o_ref)-1, in get_osd()
1250 refcount_read(&osd->o_ref)); in get_osd()
1260 dout("put_osd %p %d -> %d\n", osd, refcount_read(&osd->o_ref), in put_osd()
1261 refcount_read(&osd->o_ref) - 1); in put_osd()
1262 if (refcount_dec_and_test(&osd->o_ref)) { in put_osd()
1272 struct ceph_osd_client *osdc = osd->o_osdc; in DEFINE_RB_FUNCS()
1274 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in DEFINE_RB_FUNCS()
1275 BUG_ON(!list_empty(&osd->o_osd_lru)); in DEFINE_RB_FUNCS()
1277 spin_lock(&osdc->osd_lru_lock); in DEFINE_RB_FUNCS()
1278 list_add_tail(&osd->o_osd_lru, &osdc->osd_lru); in DEFINE_RB_FUNCS()
1279 spin_unlock(&osdc->osd_lru_lock); in DEFINE_RB_FUNCS()
1281 osd->lru_ttl = jiffies + osdc->client->options->osd_idle_ttl; in DEFINE_RB_FUNCS()
1286 if (RB_EMPTY_ROOT(&osd->o_requests) && in maybe_move_osd_to_lru()
1287 RB_EMPTY_ROOT(&osd->o_linger_requests)) in maybe_move_osd_to_lru()
1293 struct ceph_osd_client *osdc = osd->o_osdc; in __remove_osd_from_lru()
1295 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in __remove_osd_from_lru()
1297 spin_lock(&osdc->osd_lru_lock); in __remove_osd_from_lru()
1298 if (!list_empty(&osd->o_osd_lru)) in __remove_osd_from_lru()
1299 list_del_init(&osd->o_osd_lru); in __remove_osd_from_lru()
1300 spin_unlock(&osdc->osd_lru_lock); in __remove_osd_from_lru()
1309 struct ceph_osd_client *osdc = osd->o_osdc; in close_osd()
1313 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in close_osd()
1315 ceph_con_close(&osd->o_con); in close_osd()
1317 for (n = rb_first(&osd->o_requests); n; ) { in close_osd()
1323 dout(" reassigning req %p tid %llu\n", req, req->r_tid); in close_osd()
1325 link_request(&osdc->homeless_osd, req); in close_osd()
1327 for (n = rb_first(&osd->o_linger_requests); n; ) { in close_osd()
1334 lreq->linger_id); in close_osd()
1336 link_linger(&osdc->homeless_osd, lreq); in close_osd()
1341 erase_osd(&osdc->osds, osd); in close_osd()
1352 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in reopen_osd()
1354 if (RB_EMPTY_ROOT(&osd->o_requests) && in reopen_osd()
1355 RB_EMPTY_ROOT(&osd->o_linger_requests)) { in reopen_osd()
1357 return -ENODEV; in reopen_osd()
1360 peer_addr = &osd->o_osdc->osdmap->osd_addr[osd->o_osd]; in reopen_osd()
1361 if (!memcmp(peer_addr, &osd->o_con.peer_addr, sizeof (*peer_addr)) && in reopen_osd()
1362 !ceph_con_opened(&osd->o_con)) { in reopen_osd()
1368 for (n = rb_first(&osd->o_requests); n; n = rb_next(n)) { in reopen_osd()
1371 req->r_stamp = jiffies; in reopen_osd()
1374 return -EAGAIN; in reopen_osd()
1377 ceph_con_close(&osd->o_con); in reopen_osd()
1378 ceph_con_open(&osd->o_con, CEPH_ENTITY_TYPE_OSD, osd->o_osd, peer_addr); in reopen_osd()
1379 osd->o_incarnation++; in reopen_osd()
1395 osd = lookup_osd(&osdc->osds, o); in lookup_create_osd()
1397 osd = &osdc->homeless_osd; in lookup_create_osd()
1400 return ERR_PTR(-EAGAIN); in lookup_create_osd()
1403 insert_osd(&osdc->osds, osd); in lookup_create_osd()
1404 ceph_con_open(&osd->o_con, CEPH_ENTITY_TYPE_OSD, osd->o_osd, in lookup_create_osd()
1405 &osdc->osdmap->osd_addr[osd->o_osd]); in lookup_create_osd()
1408 dout("%s osdc %p osd%d -> osd %p\n", __func__, osdc, o, osd); in lookup_create_osd()
1413 * Create request <-> OSD session relation.
1420 WARN_ON(!req->r_tid || req->r_osd); in link_request()
1421 dout("%s osd %p osd%d req %p tid %llu\n", __func__, osd, osd->o_osd, in link_request()
1422 req, req->r_tid); in link_request()
1427 atomic_inc(&osd->o_osdc->num_homeless); in link_request()
1430 insert_request(&osd->o_requests, req); in link_request()
1431 req->r_osd = osd; in link_request()
1437 WARN_ON(req->r_osd != osd); in unlink_request()
1438 dout("%s osd %p osd%d req %p tid %llu\n", __func__, osd, osd->o_osd, in unlink_request()
1439 req, req->r_tid); in unlink_request()
1441 req->r_osd = NULL; in unlink_request()
1442 erase_request(&osd->o_requests, req); in unlink_request()
1448 atomic_dec(&osd->o_osdc->num_homeless); in unlink_request()
1453 return pi->flags & CEPH_POOL_FLAG_FULL; in __pool_full()
1460 for (n = rb_first(&osdc->osdmap->pg_pools); n; n = rb_next(n)) { in have_pool_full()
1475 pi = ceph_pg_pool_by_id(osdc->osdmap, pool_id); in pool_full()
1495 WARN_ON(pi->id != t->target_oloc.pool); in target_should_be_paused()
1496 return ((t->flags & CEPH_OSD_FLAG_READ) && pauserd) || in target_should_be_paused()
1497 ((t->flags & CEPH_OSD_FLAG_WRITE) && pausewr) || in target_should_be_paused()
1498 (osdc->osdmap->epoch < osdc->epoch_barrier); in target_should_be_paused()
1503 int i = prandom_u32() % acting->size; in pick_random_replica()
1506 acting->osds[i], acting->primary); in pick_random_replica()
1512 * crush_location option. Prefers the primary if the locality is
1518 struct ceph_options *opt = osdc->client->options; in pick_closest_replica()
1520 int i = 0, locality; in pick_closest_replica() local
1523 locality = ceph_get_crush_locality(osdc->osdmap, in pick_closest_replica()
1524 acting->osds[i], in pick_closest_replica()
1525 &opt->crush_locs); in pick_closest_replica()
1527 (locality >= 0 && best_locality < 0) || in pick_closest_replica()
1528 (locality >= 0 && best_locality >= 0 && in pick_closest_replica()
1529 locality < best_locality)) { in pick_closest_replica()
1531 best_locality = locality; in pick_closest_replica()
1533 } while (++i < acting->size); in pick_closest_replica()
1535 dout("%s picked osd%d with locality %d, primary osd%d\n", __func__, in pick_closest_replica()
1536 acting->osds[best_i], best_locality, acting->primary); in pick_closest_replica()
1553 bool is_read = t->flags & CEPH_OSD_FLAG_READ; in calc_target()
1554 bool is_write = t->flags & CEPH_OSD_FLAG_WRITE; in calc_target()
1564 t->epoch = osdc->osdmap->epoch; in calc_target()
1565 pi = ceph_pg_pool_by_id(osdc->osdmap, t->base_oloc.pool); in calc_target()
1567 t->osd = CEPH_HOMELESS_OSD; in calc_target()
1572 if (osdc->osdmap->epoch == pi->last_force_request_resend) { in calc_target()
1573 if (t->last_force_resend < pi->last_force_request_resend) { in calc_target()
1574 t->last_force_resend = pi->last_force_request_resend; in calc_target()
1576 } else if (t->last_force_resend == 0) { in calc_target()
1582 ceph_oid_copy(&t->target_oid, &t->base_oid); in calc_target()
1583 ceph_oloc_copy(&t->target_oloc, &t->base_oloc); in calc_target()
1584 if ((t->flags & CEPH_OSD_FLAG_IGNORE_OVERLAY) == 0) { in calc_target()
1585 if (is_read && pi->read_tier >= 0) in calc_target()
1586 t->target_oloc.pool = pi->read_tier; in calc_target()
1587 if (is_write && pi->write_tier >= 0) in calc_target()
1588 t->target_oloc.pool = pi->write_tier; in calc_target()
1590 pi = ceph_pg_pool_by_id(osdc->osdmap, t->target_oloc.pool); in calc_target()
1592 t->osd = CEPH_HOMELESS_OSD; in calc_target()
1598 __ceph_object_locator_to_pg(pi, &t->target_oid, &t->target_oloc, &pgid); in calc_target()
1600 last_pgid.seed = ceph_stable_mod(pgid.seed, t->pg_num, t->pg_num_mask); in calc_target()
1602 ceph_pg_to_up_acting_osds(osdc->osdmap, pi, &pgid, &up, &acting); in calc_target()
1604 ceph_is_new_interval(&t->acting, in calc_target()
1606 &t->up, in calc_target()
1608 t->size, in calc_target()
1609 pi->size, in calc_target()
1610 t->min_size, in calc_target()
1611 pi->min_size, in calc_target()
1612 t->pg_num, in calc_target()
1613 pi->pg_num, in calc_target()
1614 t->sort_bitwise, in calc_target()
1616 t->recovery_deletes, in calc_target()
1621 if (t->paused && !target_should_be_paused(osdc, t, pi)) { in calc_target()
1622 t->paused = false; in calc_target()
1625 legacy_change = ceph_pg_compare(&t->pgid, &pgid) || in calc_target()
1626 ceph_osds_changed(&t->acting, &acting, in calc_target()
1627 t->used_replica || any_change); in calc_target()
1628 if (t->pg_num) in calc_target()
1629 split = ceph_pg_is_split(&last_pgid, t->pg_num, pi->pg_num); in calc_target()
1632 t->pgid = pgid; /* struct */ in calc_target()
1633 ceph_pg_to_primary_shard(osdc->osdmap, pi, &pgid, &t->spgid); in calc_target()
1634 ceph_osds_copy(&t->acting, &acting); in calc_target()
1635 ceph_osds_copy(&t->up, &up); in calc_target()
1636 t->size = pi->size; in calc_target()
1637 t->min_size = pi->min_size; in calc_target()
1638 t->pg_num = pi->pg_num; in calc_target()
1639 t->pg_num_mask = pi->pg_num_mask; in calc_target()
1640 t->sort_bitwise = sort_bitwise; in calc_target()
1641 t->recovery_deletes = recovery_deletes; in calc_target()
1643 if ((t->flags & (CEPH_OSD_FLAG_BALANCE_READS | in calc_target()
1645 !is_write && pi->type == CEPH_POOL_TYPE_REP && in calc_target()
1650 if (t->flags & CEPH_OSD_FLAG_BALANCE_READS) { in calc_target()
1655 t->osd = acting.osds[pos]; in calc_target()
1656 t->used_replica = pos > 0; in calc_target()
1658 t->osd = acting.primary; in calc_target()
1659 t->used_replica = false; in calc_target()
1669 dout("%s t %p -> %d%d%d%d ct_res %d osd%d\n", __func__, t, unpaused, in calc_target()
1670 legacy_change, force_resend, split, ct_res, t->osd); in calc_target()
1682 RB_CLEAR_NODE(&spg->node); in alloc_spg_mapping()
1683 spg->backoffs = RB_ROOT; in alloc_spg_mapping()
1689 WARN_ON(!RB_EMPTY_NODE(&spg->node)); in free_spg_mapping()
1690 WARN_ON(!RB_EMPTY_ROOT(&spg->backoffs)); in free_spg_mapping()
1697 * ceph_pg_mapping. Used to track OSD backoffs -- a backoff [range] is
1698 * defined only within a specific spgid; it does not pass anything to
1706 return hoid->is_max ? 0x100000000ull : hoid->hash_reverse_bits; in DEFINE_RB_FUNCS2()
1712 if (hoid->key_len) { in hoid_get_effective_key()
1713 *pkey = hoid->key; in hoid_get_effective_key()
1714 *pkey_len = hoid->key_len; in hoid_get_effective_key()
1716 *pkey = hoid->oid; in hoid_get_effective_key()
1717 *pkey_len = hoid->oid_len; in hoid_get_effective_key()
1729 ret = -1; in compare_names()
1743 if (lhs->is_max < rhs->is_max) in hoid_compare()
1744 return -1; in hoid_compare()
1745 if (lhs->is_max > rhs->is_max) in hoid_compare()
1748 if (lhs->pool < rhs->pool) in hoid_compare()
1749 return -1; in hoid_compare()
1750 if (lhs->pool > rhs->pool) in hoid_compare()
1754 return -1; in hoid_compare()
1758 ret = compare_names(lhs->nspace, lhs->nspace_len, in hoid_compare()
1759 rhs->nspace, rhs->nspace_len); in hoid_compare()
1770 ret = compare_names(lhs->oid, lhs->oid_len, rhs->oid, rhs->oid_len); in hoid_compare()
1774 if (lhs->snapid < rhs->snapid) in hoid_compare()
1775 return -1; in hoid_compare()
1776 if (lhs->snapid > rhs->snapid) in hoid_compare()
1783 * For decoding ->begin and ->end of MOSDBackoff only -- no MIN/MAX
1786 * Assumes @hoid is zero-initialized.
1804 hoid->key = ceph_extract_encoded_string(p, end, &hoid->key_len, in decode_hoid()
1806 if (IS_ERR(hoid->key)) { in decode_hoid()
1807 ret = PTR_ERR(hoid->key); in decode_hoid()
1808 hoid->key = NULL; in decode_hoid()
1812 hoid->oid = ceph_extract_encoded_string(p, end, &hoid->oid_len, in decode_hoid()
1814 if (IS_ERR(hoid->oid)) { in decode_hoid()
1815 ret = PTR_ERR(hoid->oid); in decode_hoid()
1816 hoid->oid = NULL; in decode_hoid()
1820 ceph_decode_64_safe(p, end, hoid->snapid, e_inval); in decode_hoid()
1821 ceph_decode_32_safe(p, end, hoid->hash, e_inval); in decode_hoid()
1822 ceph_decode_8_safe(p, end, hoid->is_max, e_inval); in decode_hoid()
1824 hoid->nspace = ceph_extract_encoded_string(p, end, &hoid->nspace_len, in decode_hoid()
1826 if (IS_ERR(hoid->nspace)) { in decode_hoid()
1827 ret = PTR_ERR(hoid->nspace); in decode_hoid()
1828 hoid->nspace = NULL; in decode_hoid()
1832 ceph_decode_64_safe(p, end, hoid->pool, e_inval); in decode_hoid()
1838 return -EINVAL; in decode_hoid()
1844 4 + hoid->key_len + 4 + hoid->oid_len + 4 + hoid->nspace_len; in hoid_encoding_size()
1850 ceph_encode_string(p, end, hoid->key, hoid->key_len); in encode_hoid()
1851 ceph_encode_string(p, end, hoid->oid, hoid->oid_len); in encode_hoid()
1852 ceph_encode_64(p, hoid->snapid); in encode_hoid()
1853 ceph_encode_32(p, hoid->hash); in encode_hoid()
1854 ceph_encode_8(p, hoid->is_max); in encode_hoid()
1855 ceph_encode_string(p, end, hoid->nspace, hoid->nspace_len); in encode_hoid()
1856 ceph_encode_64(p, hoid->pool); in encode_hoid()
1862 kfree(hoid->key); in free_hoid()
1863 kfree(hoid->oid); in free_hoid()
1864 kfree(hoid->nspace); in free_hoid()
1877 RB_CLEAR_NODE(&backoff->spg_node); in alloc_backoff()
1878 RB_CLEAR_NODE(&backoff->id_node); in alloc_backoff()
1884 WARN_ON(!RB_EMPTY_NODE(&backoff->spg_node)); in free_backoff()
1885 WARN_ON(!RB_EMPTY_NODE(&backoff->id_node)); in free_backoff()
1887 free_hoid(backoff->begin); in free_backoff()
1888 free_hoid(backoff->end); in free_backoff()
1893 * Within a specific spgid, backoffs are managed by ->begin hoid.
1901 struct rb_node *n = root->rb_node; in lookup_containing_backoff()
1908 cmp = hoid_compare(hoid, cur->begin); in lookup_containing_backoff()
1910 n = n->rb_left; in lookup_containing_backoff()
1912 if (hoid_compare(hoid, cur->end) < 0) in lookup_containing_backoff()
1915 n = n->rb_right; in lookup_containing_backoff()
1931 while (!RB_EMPTY_ROOT(&osd->o_backoff_mappings)) { in DEFINE_RB_FUNCS()
1933 rb_entry(rb_first(&osd->o_backoff_mappings), in DEFINE_RB_FUNCS()
1936 while (!RB_EMPTY_ROOT(&spg->backoffs)) { in DEFINE_RB_FUNCS()
1938 rb_entry(rb_first(&spg->backoffs), in DEFINE_RB_FUNCS()
1941 erase_backoff(&spg->backoffs, backoff); in DEFINE_RB_FUNCS()
1942 erase_backoff_by_id(&osd->o_backoffs_by_id, backoff); in DEFINE_RB_FUNCS()
1945 erase_spg_mapping(&osd->o_backoff_mappings, spg); in DEFINE_RB_FUNCS()
1951 * Set up a temporary, non-owning view into @t.
1956 hoid->key = NULL; in hoid_fill_from_target()
1957 hoid->key_len = 0; in hoid_fill_from_target()
1958 hoid->oid = t->target_oid.name; in hoid_fill_from_target()
1959 hoid->oid_len = t->target_oid.name_len; in hoid_fill_from_target()
1960 hoid->snapid = CEPH_NOSNAP; in hoid_fill_from_target()
1961 hoid->hash = t->pgid.seed; in hoid_fill_from_target()
1962 hoid->is_max = false; in hoid_fill_from_target()
1963 if (t->target_oloc.pool_ns) { in hoid_fill_from_target()
1964 hoid->nspace = t->target_oloc.pool_ns->str; in hoid_fill_from_target()
1965 hoid->nspace_len = t->target_oloc.pool_ns->len; in hoid_fill_from_target()
1967 hoid->nspace = NULL; in hoid_fill_from_target()
1968 hoid->nspace_len = 0; in hoid_fill_from_target()
1970 hoid->pool = t->target_oloc.pool; in hoid_fill_from_target()
1976 struct ceph_osd *osd = req->r_osd; in should_plug_request()
1981 spg = lookup_spg_mapping(&osd->o_backoff_mappings, &req->r_t.spgid); in should_plug_request()
1985 hoid_fill_from_target(&hoid, &req->r_t); in should_plug_request()
1986 backoff = lookup_containing_backoff(&spg->backoffs, &hoid); in should_plug_request()
1991 __func__, req, req->r_tid, osd->o_osd, backoff->spgid.pgid.pool, in should_plug_request()
1992 backoff->spgid.pgid.seed, backoff->spgid.shard, backoff->id); in should_plug_request()
2001 struct ceph_msg *request_msg = req->r_request; in setup_request_data()
2002 struct ceph_msg *reply_msg = req->r_reply; in setup_request_data()
2005 if (req->r_request->num_data_items || req->r_reply->num_data_items) in setup_request_data()
2008 WARN_ON(request_msg->data_length || reply_msg->data_length); in setup_request_data()
2009 for (op = req->r_ops; op != &req->r_ops[req->r_num_ops]; op++) { in setup_request_data()
2010 switch (op->op) { in setup_request_data()
2014 WARN_ON(op->indata_len != op->extent.length); in setup_request_data()
2016 &op->extent.osd_data); in setup_request_data()
2020 WARN_ON(op->indata_len != op->xattr.name_len + in setup_request_data()
2021 op->xattr.value_len); in setup_request_data()
2023 &op->xattr.osd_data); in setup_request_data()
2027 &op->notify_ack.request_data); in setup_request_data()
2031 &op->copy_from.osd_data); in setup_request_data()
2037 &op->raw_data_in); in setup_request_data()
2041 &op->extent.osd_data); in setup_request_data()
2045 &op->list_watchers.response_data); in setup_request_data()
2050 WARN_ON(op->indata_len != op->cls.class_len + in setup_request_data()
2051 op->cls.method_len + in setup_request_data()
2052 op->cls.indata_len); in setup_request_data()
2054 &op->cls.request_info); in setup_request_data()
2057 &op->cls.request_data); in setup_request_data()
2060 &op->cls.response_data); in setup_request_data()
2064 &op->notify.request_data); in setup_request_data()
2066 &op->notify.response_data); in setup_request_data()
2075 ceph_encode_64(p, pgid->pool); in encode_pgid()
2076 ceph_encode_32(p, pgid->seed); in encode_pgid()
2077 ceph_encode_32(p, -1); /* preferred */ in encode_pgid()
2083 encode_pgid(p, &spgid->pgid); in encode_spgid()
2084 ceph_encode_8(p, spgid->shard); in encode_spgid()
2091 ceph_encode_64(p, oloc->pool); in encode_oloc()
2092 ceph_encode_32(p, -1); /* preferred */ in encode_oloc()
2094 if (oloc->pool_ns) in encode_oloc()
2095 ceph_encode_string(p, end, oloc->pool_ns->str, in encode_oloc()
2096 oloc->pool_ns->len); in encode_oloc()
2104 void *p = msg->front.iov_base; in encode_request_partial()
2105 void *const end = p + msg->front_alloc_len; in encode_request_partial()
2109 if (req->r_flags & CEPH_OSD_FLAG_WRITE) { in encode_request_partial()
2111 WARN_ON(req->r_snapid != CEPH_NOSNAP); in encode_request_partial()
2113 WARN_ON(req->r_mtime.tv_sec || req->r_mtime.tv_nsec || in encode_request_partial()
2114 req->r_data_offset || req->r_snapc); in encode_request_partial()
2119 encode_spgid(&p, &req->r_t.spgid); /* actual spg */ in encode_request_partial()
2120 ceph_encode_32(&p, req->r_t.pgid.seed); /* raw hash */ in encode_request_partial()
2121 ceph_encode_32(&p, req->r_osdc->osdmap->epoch); in encode_request_partial()
2122 ceph_encode_32(&p, req->r_flags); in encode_request_partial()
2134 ceph_encode_timespec64(p, &req->r_mtime); in encode_request_partial()
2137 encode_oloc(&p, end, &req->r_t.target_oloc); in encode_request_partial()
2138 ceph_encode_string(&p, end, req->r_t.target_oid.name, in encode_request_partial()
2139 req->r_t.target_oid.name_len); in encode_request_partial()
2142 ceph_encode_16(&p, req->r_num_ops); in encode_request_partial()
2143 for (i = 0; i < req->r_num_ops; i++) { in encode_request_partial()
2144 data_len += osd_req_encode_op(p, &req->r_ops[i]); in encode_request_partial()
2148 ceph_encode_64(&p, req->r_snapid); /* snapid */ in encode_request_partial()
2149 if (req->r_snapc) { in encode_request_partial()
2150 ceph_encode_64(&p, req->r_snapc->seq); in encode_request_partial()
2151 ceph_encode_32(&p, req->r_snapc->num_snaps); in encode_request_partial()
2152 for (i = 0; i < req->r_snapc->num_snaps; i++) in encode_request_partial()
2153 ceph_encode_64(&p, req->r_snapc->snaps[i]); in encode_request_partial()
2159 ceph_encode_32(&p, req->r_attempts); /* retry_attempt */ in encode_request_partial()
2160 BUG_ON(p > end - 8); /* space for features */ in encode_request_partial()
2162 msg->hdr.version = cpu_to_le16(8); /* MOSDOp v8 */ in encode_request_partial()
2164 msg->front.iov_len = p - msg->front.iov_base; in encode_request_partial()
2165 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); in encode_request_partial()
2166 msg->hdr.data_len = cpu_to_le32(data_len); in encode_request_partial()
2170 * need to re-copy it before writing it to disk (direct I/O). in encode_request_partial()
2172 msg->hdr.data_off = cpu_to_le16(req->r_data_offset); in encode_request_partial()
2175 req->r_t.target_oid.name, req->r_t.target_oid.name_len); in encode_request_partial()
2180 void *p = msg->front.iov_base; in encode_request_finish()
2181 void *const partial_end = p + msg->front.iov_len; in encode_request_finish()
2182 void *const end = p + msg->front_alloc_len; in encode_request_finish()
2184 if (CEPH_HAVE_FEATURE(msg->con->peer_features, RESEND_ON_SPLIT)) { in encode_request_finish()
2185 /* luminous OSD -- encode features and be done */ in encode_request_finish()
2187 ceph_encode_64(&p, msg->con->peer_features); in encode_request_finish()
2207 * Pre-luminous OSD -- reencode v8 into v4 using @head in encode_request_finish()
2221 oloc_len = p - oloc; in encode_request_finish()
2226 oid_len = p - oid; in encode_request_finish()
2229 tail_len = partial_end - p; in encode_request_finish()
2231 p = msg->front.iov_base; in encode_request_finish()
2252 /* tail -- ops, snapid, snapc, retry_attempt */ in encode_request_finish()
2257 msg->hdr.version = cpu_to_le16(4); /* MOSDOp v4 */ in encode_request_finish()
2261 msg->front.iov_len = p - msg->front.iov_base; in encode_request_finish()
2262 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); in encode_request_finish()
2265 le64_to_cpu(msg->hdr.tid), le32_to_cpu(msg->hdr.front_len), in encode_request_finish()
2266 le32_to_cpu(msg->hdr.middle_len), le32_to_cpu(msg->hdr.data_len), in encode_request_finish()
2267 le16_to_cpu(msg->hdr.version)); in encode_request_finish()
2275 struct ceph_osd *osd = req->r_osd; in send_request()
2278 WARN_ON(osd->o_osd != req->r_t.osd); in send_request()
2288 if (req->r_sent) in send_request()
2289 ceph_msg_revoke(req->r_request); in send_request()
2291 req->r_flags |= CEPH_OSD_FLAG_KNOWN_REDIR; in send_request()
2292 if (req->r_attempts) in send_request()
2293 req->r_flags |= CEPH_OSD_FLAG_RETRY; in send_request()
2295 WARN_ON(req->r_flags & CEPH_OSD_FLAG_RETRY); in send_request()
2297 encode_request_partial(req, req->r_request); in send_request()
2300 __func__, req, req->r_tid, req->r_t.pgid.pool, req->r_t.pgid.seed, in send_request()
2301 req->r_t.spgid.pgid.pool, req->r_t.spgid.pgid.seed, in send_request()
2302 req->r_t.spgid.shard, osd->o_osd, req->r_t.epoch, req->r_flags, in send_request()
2303 req->r_attempts); in send_request()
2305 req->r_t.paused = false; in send_request()
2306 req->r_stamp = jiffies; in send_request()
2307 req->r_attempts++; in send_request()
2309 req->r_sent = osd->o_incarnation; in send_request()
2310 req->r_request->hdr.tid = cpu_to_le64(req->r_tid); in send_request()
2311 ceph_con_send(&osd->o_con, ceph_msg_get(req->r_request)); in send_request()
2319 WARN_ON(!osdc->osdmap->epoch); in maybe_request_map()
2330 if (ceph_monc_want_map(&osdc->client->monc, CEPH_SUB_OSDMAP, in maybe_request_map()
2331 osdc->osdmap->epoch + 1, continuous)) in maybe_request_map()
2332 ceph_monc_renew_subs(&osdc->client->monc); in maybe_request_map()
2340 struct ceph_osd_client *osdc = req->r_osdc; in __submit_request()
2347 WARN_ON(req->r_tid); in __submit_request()
2351 ct_res = calc_target(osdc, &req->r_t, false); in __submit_request()
2355 osd = lookup_create_osd(osdc, req->r_t.osd, wrlocked); in __submit_request()
2357 WARN_ON(PTR_ERR(osd) != -EAGAIN || wrlocked); in __submit_request()
2361 if (osdc->abort_err) { in __submit_request()
2362 dout("req %p abort_err %d\n", req, osdc->abort_err); in __submit_request()
2363 err = osdc->abort_err; in __submit_request()
2364 } else if (osdc->osdmap->epoch < osdc->epoch_barrier) { in __submit_request()
2365 dout("req %p epoch %u barrier %u\n", req, osdc->osdmap->epoch, in __submit_request()
2366 osdc->epoch_barrier); in __submit_request()
2367 req->r_t.paused = true; in __submit_request()
2369 } else if ((req->r_flags & CEPH_OSD_FLAG_WRITE) && in __submit_request()
2372 req->r_t.paused = true; in __submit_request()
2374 } else if ((req->r_flags & CEPH_OSD_FLAG_READ) && in __submit_request()
2377 req->r_t.paused = true; in __submit_request()
2379 } else if ((req->r_flags & CEPH_OSD_FLAG_WRITE) && in __submit_request()
2380 !(req->r_flags & (CEPH_OSD_FLAG_FULL_TRY | in __submit_request()
2383 pool_full(osdc, req->r_t.base_oloc.pool))) { in __submit_request()
2385 if (ceph_test_opt(osdc->client, ABORT_ON_FULL)) { in __submit_request()
2386 err = -ENOSPC; in __submit_request()
2389 req->r_t.paused = true; in __submit_request()
2398 mutex_lock(&osd->lock); in __submit_request()
2404 req->r_tid = atomic64_inc_return(&osdc->last_tid); in __submit_request()
2410 mutex_unlock(&osd->lock); in __submit_request()
2416 downgrade_write(&osdc->lock); in __submit_request()
2420 up_read(&osdc->lock); in __submit_request()
2421 down_write(&osdc->lock); in __submit_request()
2429 WARN_ON(req->r_flags & (CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK)); in account_request()
2430 WARN_ON(!(req->r_flags & (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_WRITE))); in account_request()
2432 req->r_flags |= CEPH_OSD_FLAG_ONDISK; in account_request()
2433 atomic_inc(&req->r_osdc->num_requests); in account_request()
2435 req->r_start_stamp = jiffies; in account_request()
2436 req->r_start_latency = ktime_get(); in account_request()
2448 struct ceph_osd_client *osdc = req->r_osdc; in finish_request()
2450 WARN_ON(lookup_request_mc(&osdc->map_checks, req->r_tid)); in finish_request()
2451 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in finish_request()
2453 req->r_end_latency = ktime_get(); in finish_request()
2455 if (req->r_osd) in finish_request()
2456 unlink_request(req->r_osd, req); in finish_request()
2457 atomic_dec(&osdc->num_requests); in finish_request()
2465 ceph_msg_revoke(req->r_request); in finish_request()
2466 ceph_msg_revoke_incoming(req->r_reply); in finish_request()
2472 req->r_tid, req->r_callback, req->r_result); in __complete_request()
2474 if (req->r_callback) in __complete_request()
2475 req->r_callback(req); in __complete_request()
2476 complete_all(&req->r_completion); in __complete_request()
2489 * This is open-coded in handle_reply().
2493 dout("%s req %p tid %llu err %d\n", __func__, req, req->r_tid, err); in complete_request()
2495 req->r_result = err; in complete_request()
2498 INIT_WORK(&req->r_complete_work, complete_request_workfn); in complete_request()
2499 queue_work(req->r_osdc->completion_wq, &req->r_complete_work); in complete_request()
2504 struct ceph_osd_client *osdc = req->r_osdc; in cancel_map_check()
2509 lookup_req = lookup_request_mc(&osdc->map_checks, req->r_tid); in cancel_map_check()
2514 erase_request_mc(&osdc->map_checks, req); in cancel_map_check()
2520 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in cancel_request()
2524 complete_all(&req->r_completion); in cancel_request()
2530 dout("%s req %p tid %llu err %d\n", __func__, req, req->r_tid, err); in abort_request()
2545 * Abort all in-flight requests with @err and arrange for all future
2551 down_write(&osdc->lock); in ceph_osdc_abort_requests()
2553 osdc->abort_err = err; in ceph_osdc_abort_requests()
2554 up_write(&osdc->lock); in ceph_osdc_abort_requests()
2560 down_write(&osdc->lock); in ceph_osdc_clear_abort_err()
2561 osdc->abort_err = 0; in ceph_osdc_clear_abort_err()
2562 up_write(&osdc->lock); in ceph_osdc_clear_abort_err()
2568 if (likely(eb > osdc->epoch_barrier)) { in update_epoch_barrier()
2570 osdc->epoch_barrier, eb); in update_epoch_barrier()
2571 osdc->epoch_barrier = eb; in update_epoch_barrier()
2573 if (eb > osdc->osdmap->epoch) in update_epoch_barrier()
2580 down_read(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2581 if (unlikely(eb > osdc->epoch_barrier)) { in ceph_osdc_update_epoch_barrier()
2582 up_read(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2583 down_write(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2585 up_write(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2587 up_read(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2600 struct ceph_osd_client *osdc = req->r_osdc; in abort_on_full_fn()
2603 if ((req->r_flags & CEPH_OSD_FLAG_WRITE) && in abort_on_full_fn()
2605 pool_full(osdc, req->r_t.base_oloc.pool))) { in abort_on_full_fn()
2607 update_epoch_barrier(osdc, osdc->osdmap->epoch); in abort_on_full_fn()
2610 abort_request(req, -ENOSPC); in abort_on_full_fn()
2619 * osdc->epoch_barrier to the latest map epoch that we've seen if any were
2626 if (ceph_test_opt(osdc->client, ABORT_ON_FULL) && in ceph_osdc_abort_on_full()
2633 struct ceph_osd_client *osdc = req->r_osdc; in check_pool_dne()
2634 struct ceph_osdmap *map = osdc->osdmap; in check_pool_dne()
2637 WARN_ON(!map->epoch); in check_pool_dne()
2639 if (req->r_attempts) { in check_pool_dne()
2645 req->r_map_dne_bound = map->epoch; in check_pool_dne()
2647 req->r_tid); in check_pool_dne()
2650 req, req->r_tid, req->r_map_dne_bound, map->epoch); in check_pool_dne()
2653 if (req->r_map_dne_bound) { in check_pool_dne()
2654 if (map->epoch >= req->r_map_dne_bound) { in check_pool_dne()
2657 req->r_tid); in check_pool_dne()
2658 complete_request(req, -ENOENT); in check_pool_dne()
2667 struct ceph_osd_client *osdc = &greq->monc->client->osdc; in map_check_cb()
2669 u64 tid = greq->private_data; in map_check_cb()
2671 WARN_ON(greq->result || !greq->u.newest); in map_check_cb()
2673 down_write(&osdc->lock); in map_check_cb()
2674 req = lookup_request_mc(&osdc->map_checks, tid); in map_check_cb()
2681 req, req->r_tid, req->r_map_dne_bound, greq->u.newest); in map_check_cb()
2682 if (!req->r_map_dne_bound) in map_check_cb()
2683 req->r_map_dne_bound = greq->u.newest; in map_check_cb()
2684 erase_request_mc(&osdc->map_checks, req); in map_check_cb()
2689 up_write(&osdc->lock); in map_check_cb()
2694 struct ceph_osd_client *osdc = req->r_osdc; in send_map_check()
2700 lookup_req = lookup_request_mc(&osdc->map_checks, req->r_tid); in send_map_check()
2707 insert_request_mc(&osdc->map_checks, req); in send_map_check()
2708 ret = ceph_monc_get_version_async(&osdc->client->monc, "osdmap", in send_map_check()
2709 map_check_cb, req->r_tid); in send_map_check()
2722 lreq->reg_req, lreq->ping_req); in linger_release()
2723 WARN_ON(!RB_EMPTY_NODE(&lreq->node)); in linger_release()
2724 WARN_ON(!RB_EMPTY_NODE(&lreq->osdc_node)); in linger_release()
2725 WARN_ON(!RB_EMPTY_NODE(&lreq->mc_node)); in linger_release()
2726 WARN_ON(!list_empty(&lreq->scan_item)); in linger_release()
2727 WARN_ON(!list_empty(&lreq->pending_lworks)); in linger_release()
2728 WARN_ON(lreq->osd); in linger_release()
2730 if (lreq->reg_req) in linger_release()
2731 ceph_osdc_put_request(lreq->reg_req); in linger_release()
2732 if (lreq->ping_req) in linger_release()
2733 ceph_osdc_put_request(lreq->ping_req); in linger_release()
2734 target_destroy(&lreq->t); in linger_release()
2741 kref_put(&lreq->kref, linger_release); in linger_put()
2747 kref_get(&lreq->kref); in linger_get()
2760 kref_init(&lreq->kref); in linger_alloc()
2761 mutex_init(&lreq->lock); in linger_alloc()
2762 RB_CLEAR_NODE(&lreq->node); in linger_alloc()
2763 RB_CLEAR_NODE(&lreq->osdc_node); in linger_alloc()
2764 RB_CLEAR_NODE(&lreq->mc_node); in linger_alloc()
2765 INIT_LIST_HEAD(&lreq->scan_item); in linger_alloc()
2766 INIT_LIST_HEAD(&lreq->pending_lworks); in linger_alloc()
2767 init_completion(&lreq->reg_commit_wait); in linger_alloc()
2768 init_completion(&lreq->notify_finish_wait); in linger_alloc()
2770 lreq->osdc = osdc; in linger_alloc()
2771 target_init(&lreq->t); in linger_alloc()
2782 * Create linger request <-> OSD session relation. in DEFINE_RB_INSDEL_FUNCS()
2790 WARN_ON(!lreq->linger_id || lreq->osd); in DEFINE_RB_INSDEL_FUNCS()
2792 osd->o_osd, lreq, lreq->linger_id); in DEFINE_RB_INSDEL_FUNCS()
2797 atomic_inc(&osd->o_osdc->num_homeless); in DEFINE_RB_INSDEL_FUNCS()
2800 insert_linger(&osd->o_linger_requests, lreq); in DEFINE_RB_INSDEL_FUNCS()
2801 lreq->osd = osd; in DEFINE_RB_INSDEL_FUNCS()
2808 WARN_ON(lreq->osd != osd); in unlink_linger()
2810 osd->o_osd, lreq, lreq->linger_id); in unlink_linger()
2812 lreq->osd = NULL; in unlink_linger()
2813 erase_linger(&osd->o_linger_requests, lreq); in unlink_linger()
2819 atomic_dec(&osd->o_osdc->num_homeless); in unlink_linger()
2824 verify_osdc_locked(lreq->osdc); in __linger_registered()
2826 return !RB_EMPTY_NODE(&lreq->osdc_node); in __linger_registered()
2831 struct ceph_osd_client *osdc = lreq->osdc; in linger_registered()
2834 down_read(&osdc->lock); in linger_registered()
2836 up_read(&osdc->lock); in linger_registered()
2843 struct ceph_osd_client *osdc = lreq->osdc; in linger_register()
2846 WARN_ON(lreq->linger_id); in linger_register()
2849 lreq->linger_id = ++osdc->last_linger_id; in linger_register()
2850 insert_linger_osdc(&osdc->linger_requests, lreq); in linger_register()
2855 struct ceph_osd_client *osdc = lreq->osdc; in linger_unregister()
2859 erase_linger_osdc(&osdc->linger_requests, lreq); in linger_unregister()
2865 struct ceph_osd_linger_request *lreq = req->r_priv; in cancel_linger_request()
2867 WARN_ON(!req->r_linger); in cancel_linger_request()
2902 INIT_WORK(&lwork->work, workfn); in lwork_alloc()
2903 INIT_LIST_HEAD(&lwork->pending_item); in lwork_alloc()
2904 lwork->lreq = linger_get(lreq); in lwork_alloc()
2911 struct ceph_osd_linger_request *lreq = lwork->lreq; in lwork_free()
2913 mutex_lock(&lreq->lock); in lwork_free()
2914 list_del(&lwork->pending_item); in lwork_free()
2915 mutex_unlock(&lreq->lock); in lwork_free()
2923 struct ceph_osd_linger_request *lreq = lwork->lreq; in lwork_queue()
2924 struct ceph_osd_client *osdc = lreq->osdc; in lwork_queue()
2927 WARN_ON(!list_empty(&lwork->pending_item)); in lwork_queue()
2929 lwork->queued_stamp = jiffies; in lwork_queue()
2930 list_add_tail(&lwork->pending_item, &lreq->pending_lworks); in lwork_queue()
2931 queue_work(osdc->notify_wq, &lwork->work); in lwork_queue()
2937 struct ceph_osd_linger_request *lreq = lwork->lreq; in do_watch_notify()
2944 WARN_ON(!lreq->is_watch); in do_watch_notify()
2946 __func__, lreq, lwork->notify.notify_id, lwork->notify.notifier_id, in do_watch_notify()
2947 lwork->notify.payload_len); in do_watch_notify()
2948 lreq->wcb(lreq->data, lwork->notify.notify_id, lreq->linger_id, in do_watch_notify()
2949 lwork->notify.notifier_id, lwork->notify.payload, in do_watch_notify()
2950 lwork->notify.payload_len); in do_watch_notify()
2953 ceph_msg_put(lwork->notify.msg); in do_watch_notify()
2960 struct ceph_osd_linger_request *lreq = lwork->lreq; in do_watch_error()
2967 dout("%s lreq %p err %d\n", __func__, lreq, lwork->error.err); in do_watch_error()
2968 lreq->errcb(lreq->data, lreq->linger_id, lwork->error.err); in do_watch_error()
2980 pr_err("failed to allocate error-lwork\n"); in queue_watch_error()
2984 lwork->error.err = lreq->last_error; in queue_watch_error()
2991 if (!completion_done(&lreq->reg_commit_wait)) { in linger_reg_commit_complete()
2992 lreq->reg_commit_error = (result <= 0 ? result : 0); in linger_reg_commit_complete()
2993 complete_all(&lreq->reg_commit_wait); in linger_reg_commit_complete()
2999 struct ceph_osd_linger_request *lreq = req->r_priv; in linger_commit_cb()
3001 mutex_lock(&lreq->lock); in linger_commit_cb()
3003 lreq->linger_id, req->r_result); in linger_commit_cb()
3004 linger_reg_commit_complete(lreq, req->r_result); in linger_commit_cb()
3005 lreq->committed = true; in linger_commit_cb()
3007 if (!lreq->is_watch) { in linger_commit_cb()
3010 void *p = page_address(osd_data->pages[0]); in linger_commit_cb()
3012 WARN_ON(req->r_ops[0].op != CEPH_OSD_OP_NOTIFY || in linger_commit_cb()
3013 osd_data->type != CEPH_OSD_DATA_TYPE_PAGES); in linger_commit_cb()
3016 if (req->r_ops[0].outdata_len >= sizeof(u64)) { in linger_commit_cb()
3017 lreq->notify_id = ceph_decode_64(&p); in linger_commit_cb()
3019 lreq->notify_id); in linger_commit_cb()
3025 mutex_unlock(&lreq->lock); in linger_commit_cb()
3032 * Translate ENOENT -> ENOTCONN so that a delete->disconnection in normalize_watch_error()
3036 if (err == -ENOENT) in normalize_watch_error()
3037 err = -ENOTCONN; in normalize_watch_error()
3044 struct ceph_osd_linger_request *lreq = req->r_priv; in linger_reconnect_cb()
3046 mutex_lock(&lreq->lock); in linger_reconnect_cb()
3048 lreq, lreq->linger_id, req->r_result, lreq->last_error); in linger_reconnect_cb()
3049 if (req->r_result < 0) { in linger_reconnect_cb()
3050 if (!lreq->last_error) { in linger_reconnect_cb()
3051 lreq->last_error = normalize_watch_error(req->r_result); in linger_reconnect_cb()
3056 mutex_unlock(&lreq->lock); in linger_reconnect_cb()
3062 struct ceph_osd_request *req = lreq->reg_req; in send_linger()
3063 struct ceph_osd_req_op *op = &req->r_ops[0]; in send_linger()
3065 verify_osdc_wrlocked(req->r_osdc); in send_linger()
3066 dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id); in send_linger()
3068 if (req->r_osd) in send_linger()
3072 target_copy(&req->r_t, &lreq->t); in send_linger()
3073 req->r_mtime = lreq->mtime; in send_linger()
3075 mutex_lock(&lreq->lock); in send_linger()
3076 if (lreq->is_watch && lreq->committed) { in send_linger()
3077 WARN_ON(op->op != CEPH_OSD_OP_WATCH || in send_linger()
3078 op->watch.cookie != lreq->linger_id); in send_linger()
3079 op->watch.op = CEPH_OSD_WATCH_OP_RECONNECT; in send_linger()
3080 op->watch.gen = ++lreq->register_gen; in send_linger()
3082 op->watch.gen); in send_linger()
3083 req->r_callback = linger_reconnect_cb; in send_linger()
3085 if (!lreq->is_watch) in send_linger()
3086 lreq->notify_id = 0; in send_linger()
3088 WARN_ON(op->watch.op != CEPH_OSD_WATCH_OP_WATCH); in send_linger()
3090 req->r_callback = linger_commit_cb; in send_linger()
3092 mutex_unlock(&lreq->lock); in send_linger()
3094 req->r_priv = linger_get(lreq); in send_linger()
3095 req->r_linger = true; in send_linger()
3102 struct ceph_osd_linger_request *lreq = req->r_priv; in linger_ping_cb()
3104 mutex_lock(&lreq->lock); in linger_ping_cb()
3106 __func__, lreq, lreq->linger_id, req->r_result, lreq->ping_sent, in linger_ping_cb()
3107 lreq->last_error); in linger_ping_cb()
3108 if (lreq->register_gen == req->r_ops[0].watch.gen) { in linger_ping_cb()
3109 if (!req->r_result) { in linger_ping_cb()
3110 lreq->watch_valid_thru = lreq->ping_sent; in linger_ping_cb()
3111 } else if (!lreq->last_error) { in linger_ping_cb()
3112 lreq->last_error = normalize_watch_error(req->r_result); in linger_ping_cb()
3117 lreq->register_gen, req->r_ops[0].watch.gen); in linger_ping_cb()
3120 mutex_unlock(&lreq->lock); in linger_ping_cb()
3126 struct ceph_osd_client *osdc = lreq->osdc; in send_linger_ping()
3127 struct ceph_osd_request *req = lreq->ping_req; in send_linger_ping()
3128 struct ceph_osd_req_op *op = &req->r_ops[0]; in send_linger_ping()
3135 lreq->ping_sent = jiffies; in send_linger_ping()
3137 __func__, lreq, lreq->linger_id, lreq->ping_sent, in send_linger_ping()
3138 lreq->register_gen); in send_linger_ping()
3140 if (req->r_osd) in send_linger_ping()
3144 target_copy(&req->r_t, &lreq->t); in send_linger_ping()
3146 WARN_ON(op->op != CEPH_OSD_OP_WATCH || in send_linger_ping()
3147 op->watch.cookie != lreq->linger_id || in send_linger_ping()
3148 op->watch.op != CEPH_OSD_WATCH_OP_PING); in send_linger_ping()
3149 op->watch.gen = lreq->register_gen; in send_linger_ping()
3150 req->r_callback = linger_ping_cb; in send_linger_ping()
3151 req->r_priv = linger_get(lreq); in send_linger_ping()
3152 req->r_linger = true; in send_linger_ping()
3156 req->r_tid = atomic64_inc_return(&osdc->last_tid); in send_linger_ping()
3157 link_request(lreq->osd, req); in send_linger_ping()
3163 struct ceph_osd_client *osdc = lreq->osdc; in linger_submit()
3166 down_write(&osdc->lock); in linger_submit()
3168 if (lreq->is_watch) { in linger_submit()
3169 lreq->reg_req->r_ops[0].watch.cookie = lreq->linger_id; in linger_submit()
3170 lreq->ping_req->r_ops[0].watch.cookie = lreq->linger_id; in linger_submit()
3172 lreq->reg_req->r_ops[0].notify.cookie = lreq->linger_id; in linger_submit()
3175 calc_target(osdc, &lreq->t, false); in linger_submit()
3176 osd = lookup_create_osd(osdc, lreq->t.osd, true); in linger_submit()
3180 up_write(&osdc->lock); in linger_submit()
3185 struct ceph_osd_client *osdc = lreq->osdc; in cancel_linger_map_check()
3190 lookup_lreq = lookup_linger_mc(&osdc->linger_map_checks, in cancel_linger_map_check()
3191 lreq->linger_id); in cancel_linger_map_check()
3196 erase_linger_mc(&osdc->linger_map_checks, lreq); in cancel_linger_map_check()
3205 if (lreq->is_watch && lreq->ping_req->r_osd) in __linger_cancel()
3206 cancel_linger_request(lreq->ping_req); in __linger_cancel()
3207 if (lreq->reg_req->r_osd) in __linger_cancel()
3208 cancel_linger_request(lreq->reg_req); in __linger_cancel()
3210 unlink_linger(lreq->osd, lreq); in __linger_cancel()
3216 struct ceph_osd_client *osdc = lreq->osdc; in linger_cancel()
3218 down_write(&osdc->lock); in linger_cancel()
3221 up_write(&osdc->lock); in linger_cancel()
3228 struct ceph_osd_client *osdc = lreq->osdc; in check_linger_pool_dne()
3229 struct ceph_osdmap *map = osdc->osdmap; in check_linger_pool_dne()
3232 WARN_ON(!map->epoch); in check_linger_pool_dne()
3234 if (lreq->register_gen) { in check_linger_pool_dne()
3235 lreq->map_dne_bound = map->epoch; in check_linger_pool_dne()
3237 lreq, lreq->linger_id); in check_linger_pool_dne()
3240 __func__, lreq, lreq->linger_id, lreq->map_dne_bound, in check_linger_pool_dne()
3241 map->epoch); in check_linger_pool_dne()
3244 if (lreq->map_dne_bound) { in check_linger_pool_dne()
3245 if (map->epoch >= lreq->map_dne_bound) { in check_linger_pool_dne()
3248 lreq->linger_id); in check_linger_pool_dne()
3249 linger_reg_commit_complete(lreq, -ENOENT); in check_linger_pool_dne()
3259 struct ceph_osd_client *osdc = &greq->monc->client->osdc; in linger_map_check_cb()
3261 u64 linger_id = greq->private_data; in linger_map_check_cb()
3263 WARN_ON(greq->result || !greq->u.newest); in linger_map_check_cb()
3265 down_write(&osdc->lock); in linger_map_check_cb()
3266 lreq = lookup_linger_mc(&osdc->linger_map_checks, linger_id); in linger_map_check_cb()
3273 __func__, lreq, lreq->linger_id, lreq->map_dne_bound, in linger_map_check_cb()
3274 greq->u.newest); in linger_map_check_cb()
3275 if (!lreq->map_dne_bound) in linger_map_check_cb()
3276 lreq->map_dne_bound = greq->u.newest; in linger_map_check_cb()
3277 erase_linger_mc(&osdc->linger_map_checks, lreq); in linger_map_check_cb()
3282 up_write(&osdc->lock); in linger_map_check_cb()
3287 struct ceph_osd_client *osdc = lreq->osdc; in send_linger_map_check()
3293 lookup_lreq = lookup_linger_mc(&osdc->linger_map_checks, in send_linger_map_check()
3294 lreq->linger_id); in send_linger_map_check()
3301 insert_linger_mc(&osdc->linger_map_checks, lreq); in send_linger_map_check()
3302 ret = ceph_monc_get_version_async(&osdc->client->monc, "osdmap", in send_linger_map_check()
3303 linger_map_check_cb, lreq->linger_id); in send_linger_map_check()
3311 dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id); in linger_reg_commit_wait()
3312 ret = wait_for_completion_interruptible(&lreq->reg_commit_wait); in linger_reg_commit_wait()
3313 return ret ?: lreq->reg_commit_error; in linger_reg_commit_wait()
3320 dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id); in linger_notify_finish_wait()
3321 ret = wait_for_completion_interruptible(&lreq->notify_finish_wait); in linger_notify_finish_wait()
3322 return ret ?: lreq->notify_finish_error; in linger_notify_finish_wait()
3335 struct ceph_options *opts = osdc->client->options; in handle_timeout()
3336 unsigned long cutoff = jiffies - opts->osd_keepalive_timeout; in handle_timeout()
3337 unsigned long expiry_cutoff = jiffies - opts->osd_request_timeout; in handle_timeout()
3342 down_write(&osdc->lock); in handle_timeout()
3349 for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { in handle_timeout()
3353 for (p = rb_first(&osd->o_requests); p; ) { in handle_timeout()
3359 if (time_before(req->r_stamp, cutoff)) { in handle_timeout()
3361 req, req->r_tid, osd->o_osd); in handle_timeout()
3364 if (opts->osd_request_timeout && in handle_timeout()
3365 time_before(req->r_start_stamp, expiry_cutoff)) { in handle_timeout()
3367 req->r_tid, osd->o_osd); in handle_timeout()
3368 abort_request(req, -ETIMEDOUT); in handle_timeout()
3371 for (p = rb_first(&osd->o_linger_requests); p; p = rb_next(p)) { in handle_timeout()
3376 lreq, lreq->linger_id, osd->o_osd); in handle_timeout()
3379 mutex_lock(&lreq->lock); in handle_timeout()
3380 if (lreq->is_watch && lreq->committed && !lreq->last_error) in handle_timeout()
3382 mutex_unlock(&lreq->lock); in handle_timeout()
3386 list_move_tail(&osd->o_keepalive_item, &slow_osds); in handle_timeout()
3389 if (opts->osd_request_timeout) { in handle_timeout()
3390 for (p = rb_first(&osdc->homeless_osd.o_requests); p; ) { in handle_timeout()
3396 if (time_before(req->r_start_stamp, expiry_cutoff)) { in handle_timeout()
3398 req->r_tid, osdc->homeless_osd.o_osd); in handle_timeout()
3399 abort_request(req, -ETIMEDOUT); in handle_timeout()
3404 if (atomic_read(&osdc->num_homeless) || !list_empty(&slow_osds)) in handle_timeout()
3411 list_del_init(&osd->o_keepalive_item); in handle_timeout()
3412 ceph_con_keepalive(&osd->o_con); in handle_timeout()
3415 up_write(&osdc->lock); in handle_timeout()
3416 schedule_delayed_work(&osdc->timeout_work, in handle_timeout()
3417 osdc->client->options->osd_keepalive_timeout); in handle_timeout()
3425 unsigned long delay = osdc->client->options->osd_idle_ttl / 4; in handle_osds_timeout()
3429 down_write(&osdc->lock); in handle_osds_timeout()
3430 list_for_each_entry_safe(osd, nosd, &osdc->osd_lru, o_osd_lru) { in handle_osds_timeout()
3431 if (time_before(jiffies, osd->lru_ttl)) in handle_osds_timeout()
3434 WARN_ON(!RB_EMPTY_ROOT(&osd->o_requests)); in handle_osds_timeout()
3435 WARN_ON(!RB_EMPTY_ROOT(&osd->o_linger_requests)); in handle_osds_timeout()
3439 up_write(&osdc->lock); in handle_osds_timeout()
3440 schedule_delayed_work(&osdc->osds_timeout_work, in handle_osds_timeout()
3469 oloc->pool = ceph_decode_64(p); in ceph_oloc_decode()
3484 if (!oloc->pool_ns || in ceph_oloc_decode()
3485 ceph_compare_string(oloc->pool_ns, *p, len)) in ceph_oloc_decode()
3489 if (oloc->pool_ns) in ceph_oloc_decode()
3501 if (hash != -1) { in ceph_oloc_decode()
3513 ret = -EINVAL; in ceph_oloc_decode()
3537 ret = ceph_oloc_decode(p, end, &redir->oloc); in ceph_redirect_decode()
3553 ret = -EINVAL; in ceph_redirect_decode()
3573 void *p = msg->front.iov_base; in decode_MOSDOpReply()
3574 void *const end = p + msg->front.iov_len; in decode_MOSDOpReply()
3575 u16 version = le16_to_cpu(msg->hdr.version); in decode_MOSDOpReply()
3586 ret = ceph_decode_pgid(&p, end, &m->pgid); in decode_MOSDOpReply()
3590 ceph_decode_64_safe(&p, end, m->flags, e_inval); in decode_MOSDOpReply()
3591 ceph_decode_32_safe(&p, end, m->result, e_inval); in decode_MOSDOpReply()
3595 ceph_decode_32_safe(&p, end, m->epoch, e_inval); in decode_MOSDOpReply()
3597 ceph_decode_32_safe(&p, end, m->num_ops, e_inval); in decode_MOSDOpReply()
3598 if (m->num_ops > ARRAY_SIZE(m->outdata_len)) in decode_MOSDOpReply()
3601 ceph_decode_need(&p, end, m->num_ops * sizeof(struct ceph_osd_op), in decode_MOSDOpReply()
3603 for (i = 0; i < m->num_ops; i++) { in decode_MOSDOpReply()
3606 m->outdata_len[i] = le32_to_cpu(op->payload_len); in decode_MOSDOpReply()
3610 ceph_decode_32_safe(&p, end, m->retry_attempt, e_inval); in decode_MOSDOpReply()
3611 for (i = 0; i < m->num_ops; i++) in decode_MOSDOpReply()
3612 ceph_decode_32_safe(&p, end, m->rval[i], e_inval); in decode_MOSDOpReply()
3615 ceph_decode_need(&p, end, sizeof(m->replay_version), e_inval); in decode_MOSDOpReply()
3616 memcpy(&m->replay_version, p, sizeof(m->replay_version)); in decode_MOSDOpReply()
3617 p += sizeof(m->replay_version); in decode_MOSDOpReply()
3618 ceph_decode_64_safe(&p, end, m->user_version, e_inval); in decode_MOSDOpReply()
3620 m->replay_version = bad_replay_version; /* struct */ in decode_MOSDOpReply()
3621 m->user_version = le64_to_cpu(m->replay_version.version); in decode_MOSDOpReply()
3634 ret = ceph_redirect_decode(&p, end, &m->redirect); in decode_MOSDOpReply()
3638 ceph_oloc_init(&m->redirect.oloc); in decode_MOSDOpReply()
3644 return -EINVAL; in decode_MOSDOpReply()
3648 * Handle MOSDOpReply. Set ->r_result and call the callback if it is
3653 struct ceph_osd_client *osdc = osd->o_osdc; in handle_reply()
3656 u64 tid = le64_to_cpu(msg->hdr.tid); in handle_reply()
3663 down_read(&osdc->lock); in handle_reply()
3665 dout("%s osd%d unknown\n", __func__, osd->o_osd); in handle_reply()
3668 WARN_ON(osd->o_osd != le64_to_cpu(msg->hdr.src.num)); in handle_reply()
3670 mutex_lock(&osd->lock); in handle_reply()
3671 req = lookup_request(&osd->o_requests, tid); in handle_reply()
3673 dout("%s osd%d tid %llu unknown\n", __func__, osd->o_osd, tid); in handle_reply()
3677 m.redirect.oloc.pool_ns = req->r_t.target_oloc.pool_ns; in handle_reply()
3682 req->r_tid, ret); in handle_reply()
3687 __func__, req, req->r_tid, m.flags, m.pgid.pool, m.pgid.seed, in handle_reply()
3692 if (m.retry_attempt != req->r_attempts - 1) { in handle_reply()
3694 req, req->r_tid, m.retry_attempt, in handle_reply()
3695 req->r_attempts - 1); in handle_reply()
3703 dout("req %p tid %llu redirect pool %lld\n", req, req->r_tid, in handle_reply()
3706 mutex_unlock(&osd->lock); in handle_reply()
3709 * Not ceph_oloc_copy() - changing pool_ns is not in handle_reply()
3712 req->r_t.target_oloc.pool = m.redirect.oloc.pool; in handle_reply()
3713 req->r_flags |= CEPH_OSD_FLAG_REDIRECTED | in handle_reply()
3716 req->r_tid = 0; in handle_reply()
3721 if (m.result == -EAGAIN) { in handle_reply()
3722 dout("req %p tid %llu EAGAIN\n", req, req->r_tid); in handle_reply()
3724 mutex_unlock(&osd->lock); in handle_reply()
3731 req->r_t.pgid.pool = 0; in handle_reply()
3732 req->r_t.pgid.seed = 0; in handle_reply()
3733 WARN_ON(!req->r_t.used_replica); in handle_reply()
3734 req->r_flags &= ~(CEPH_OSD_FLAG_BALANCE_READS | in handle_reply()
3736 req->r_tid = 0; in handle_reply()
3741 if (m.num_ops != req->r_num_ops) { in handle_reply()
3743 req->r_num_ops, req->r_tid); in handle_reply()
3746 for (i = 0; i < req->r_num_ops; i++) { in handle_reply()
3748 req->r_tid, i, m.rval[i], m.outdata_len[i]); in handle_reply()
3749 req->r_ops[i].rval = m.rval[i]; in handle_reply()
3750 req->r_ops[i].outdata_len = m.outdata_len[i]; in handle_reply()
3753 if (data_len != le32_to_cpu(msg->hdr.data_len)) { in handle_reply()
3755 le32_to_cpu(msg->hdr.data_len), req->r_tid); in handle_reply()
3759 req, req->r_tid, m.result, data_len); in handle_reply()
3766 req->r_result = m.result ?: data_len; in handle_reply()
3768 mutex_unlock(&osd->lock); in handle_reply()
3769 up_read(&osdc->lock); in handle_reply()
3775 complete_request(req, -EIO); in handle_reply()
3777 mutex_unlock(&osd->lock); in handle_reply()
3779 up_read(&osdc->lock); in handle_reply()
3786 for (n = rb_first(&osdc->osdmap->pg_pools); n; n = rb_next(n)) { in set_pool_was_full()
3790 pi->was_full = __pool_full(pi); in set_pool_was_full()
3798 pi = ceph_pg_pool_by_id(osdc->osdmap, pool_id); in pool_cleared_full()
3802 return pi->was_full && !__pool_full(pi); in pool_cleared_full()
3808 struct ceph_osd_client *osdc = lreq->osdc; in recalc_linger_target()
3811 ct_res = calc_target(osdc, &lreq->t, true); in recalc_linger_target()
3815 osd = lookup_create_osd(osdc, lreq->t.osd, true); in recalc_linger_target()
3816 if (osd != lreq->osd) { in recalc_linger_target()
3817 unlink_linger(lreq->osd, lreq); in recalc_linger_target()
3835 struct ceph_osd_client *osdc = osd->o_osdc; in scan_requests()
3839 for (n = rb_first(&osd->o_linger_requests); n; ) { in scan_requests()
3847 lreq->linger_id); in scan_requests()
3853 pool_cleared_full(osdc, lreq->t.base_oloc.pool)); in scan_requests()
3865 if (list_empty(&lreq->scan_item)) in scan_requests()
3866 list_add_tail(&lreq->scan_item, need_resend_linger); in scan_requests()
3869 list_del_init(&lreq->scan_item); in scan_requests()
3875 for (n = rb_first(&osd->o_requests); n; ) { in scan_requests()
3882 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in scan_requests()
3883 ct_res = calc_target(osdc, &req->r_t, false); in scan_requests()
3888 pool_cleared_full(osdc, req->r_t.base_oloc.pool)); in scan_requests()
3890 (!(req->r_flags & CEPH_OSD_FLAG_WRITE) || in scan_requests()
3921 newmap = osdmap_apply_incremental(&p, end, osdc->osdmap); in handle_one_map()
3927 if (newmap != osdc->osdmap) { in handle_one_map()
3929 * Preserve ->was_full before destroying the old map. in handle_one_map()
3930 * For pools that weren't in the old map, ->was_full in handle_one_map()
3933 for (n = rb_first(&newmap->pg_pools); n; n = rb_next(n)) { in handle_one_map()
3938 old_pi = ceph_pg_pool_by_id(osdc->osdmap, pi->id); in handle_one_map()
3940 pi->was_full = old_pi->was_full; in handle_one_map()
3942 WARN_ON(pi->was_full); in handle_one_map()
3945 if (osdc->osdmap->epoch && in handle_one_map()
3946 osdc->osdmap->epoch + 1 < newmap->epoch) { in handle_one_map()
3951 ceph_osdmap_destroy(osdc->osdmap); in handle_one_map()
3952 osdc->osdmap = newmap; in handle_one_map()
3956 scan_requests(&osdc->homeless_osd, skipped_map, was_full, true, in handle_one_map()
3959 for (n = rb_first(&osdc->osds); n; ) { in handle_one_map()
3966 if (!ceph_osd_is_up(osdc->osdmap, osd->o_osd) || in handle_one_map()
3967 memcmp(&osd->o_con.peer_addr, in handle_one_map()
3968 ceph_osd_addr(osdc->osdmap, osd->o_osd), in handle_one_map()
3991 if (req->r_t.epoch < osdc->osdmap->epoch) { in kick_requests()
3992 ct_res = calc_target(osdc, &req->r_t, false); in kick_requests()
4008 osd = lookup_create_osd(osdc, req->r_t.osd, true); in kick_requests()
4010 if (!req->r_linger) { in kick_requests()
4011 if (!osd_homeless(osd) && !req->r_t.paused) in kick_requests()
4019 if (!osd_homeless(lreq->osd)) in kick_requests()
4022 list_del_init(&lreq->scan_item); in kick_requests()
4035 void *p = msg->front.iov_base; in ceph_osdc_handle_map()
4036 void *const end = p + msg->front.iov_len; in ceph_osdc_handle_map()
4047 dout("%s have %u\n", __func__, osdc->osdmap->epoch); in ceph_osdc_handle_map()
4048 down_write(&osdc->lock); in ceph_osdc_handle_map()
4053 if (ceph_check_fsid(osdc->client, &fsid) < 0) in ceph_osdc_handle_map()
4069 if (osdc->osdmap->epoch && in ceph_osdc_handle_map()
4070 osdc->osdmap->epoch + 1 == epoch) { in ceph_osdc_handle_map()
4083 nr_maps--; in ceph_osdc_handle_map()
4097 dout("skipping non-latest full map %u len %d\n", in ceph_osdc_handle_map()
4099 } else if (osdc->osdmap->epoch >= epoch) { in ceph_osdc_handle_map()
4102 osdc->osdmap->epoch); in ceph_osdc_handle_map()
4111 nr_maps--; in ceph_osdc_handle_map()
4125 osdc->osdmap->epoch < osdc->epoch_barrier) in ceph_osdc_handle_map()
4131 ceph_monc_got_map(&osdc->client->monc, CEPH_SUB_OSDMAP, in ceph_osdc_handle_map()
4132 osdc->osdmap->epoch); in ceph_osdc_handle_map()
4133 up_write(&osdc->lock); in ceph_osdc_handle_map()
4134 wake_up_all(&osdc->client->auth_wq); in ceph_osdc_handle_map()
4140 up_write(&osdc->lock); in ceph_osdc_handle_map()
4152 for (n = rb_first(&osd->o_requests); n; ) { in kick_osd_requests()
4158 if (!req->r_linger) { in kick_osd_requests()
4159 if (!req->r_t.paused) in kick_osd_requests()
4165 for (n = rb_first(&osd->o_linger_requests); n; n = rb_next(n)) { in kick_osd_requests()
4178 struct ceph_osd *osd = con->private; in osd_fault()
4179 struct ceph_osd_client *osdc = osd->o_osdc; in osd_fault()
4181 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in osd_fault()
4183 down_write(&osdc->lock); in osd_fault()
4185 dout("%s osd%d unknown\n", __func__, osd->o_osd); in osd_fault()
4194 up_write(&osdc->lock); in osd_fault()
4208 void *p = msg->front.iov_base; in decode_MOSDBackoff()
4209 void *const end = p + msg->front.iov_len; in decode_MOSDBackoff()
4218 ret = ceph_decode_pgid(&p, end, &m->spgid.pgid); in decode_MOSDBackoff()
4222 ceph_decode_8_safe(&p, end, m->spgid.shard, e_inval); in decode_MOSDBackoff()
4223 ceph_decode_32_safe(&p, end, m->map_epoch, e_inval); in decode_MOSDBackoff()
4224 ceph_decode_8_safe(&p, end, m->op, e_inval); in decode_MOSDBackoff()
4225 ceph_decode_64_safe(&p, end, m->id, e_inval); in decode_MOSDBackoff()
4227 m->begin = kzalloc(sizeof(*m->begin), GFP_NOIO); in decode_MOSDBackoff()
4228 if (!m->begin) in decode_MOSDBackoff()
4229 return -ENOMEM; in decode_MOSDBackoff()
4231 ret = decode_hoid(&p, end, m->begin); in decode_MOSDBackoff()
4233 free_hoid(m->begin); in decode_MOSDBackoff()
4237 m->end = kzalloc(sizeof(*m->end), GFP_NOIO); in decode_MOSDBackoff()
4238 if (!m->end) { in decode_MOSDBackoff()
4239 free_hoid(m->begin); in decode_MOSDBackoff()
4240 return -ENOMEM; in decode_MOSDBackoff()
4243 ret = decode_hoid(&p, end, m->end); in decode_MOSDBackoff()
4245 free_hoid(m->begin); in decode_MOSDBackoff()
4246 free_hoid(m->end); in decode_MOSDBackoff()
4253 return -EINVAL; in decode_MOSDBackoff()
4268 hoid_encoding_size(backoff->begin); in create_backoff_message()
4270 hoid_encoding_size(backoff->end); in create_backoff_message()
4276 p = msg->front.iov_base; in create_backoff_message()
4277 end = p + msg->front_alloc_len; in create_backoff_message()
4279 encode_spgid(&p, &backoff->spgid); in create_backoff_message()
4282 ceph_encode_64(&p, backoff->id); in create_backoff_message()
4283 encode_hoid(&p, end, backoff->begin); in create_backoff_message()
4284 encode_hoid(&p, end, backoff->end); in create_backoff_message()
4287 msg->front.iov_len = p - msg->front.iov_base; in create_backoff_message()
4288 msg->hdr.version = cpu_to_le16(1); /* MOSDBackoff v1 */ in create_backoff_message()
4289 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); in create_backoff_message()
4300 dout("%s osd%d spgid %llu.%xs%d id %llu\n", __func__, osd->o_osd, in handle_backoff_block()
4301 m->spgid.pgid.pool, m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_block()
4303 spg = lookup_spg_mapping(&osd->o_backoff_mappings, &m->spgid); in handle_backoff_block()
4310 spg->spgid = m->spgid; /* struct */ in handle_backoff_block()
4311 insert_spg_mapping(&osd->o_backoff_mappings, spg); in handle_backoff_block()
4319 backoff->spgid = m->spgid; /* struct */ in handle_backoff_block()
4320 backoff->id = m->id; in handle_backoff_block()
4321 backoff->begin = m->begin; in handle_backoff_block()
4322 m->begin = NULL; /* backoff now owns this */ in handle_backoff_block()
4323 backoff->end = m->end; in handle_backoff_block()
4324 m->end = NULL; /* ditto */ in handle_backoff_block()
4326 insert_backoff(&spg->backoffs, backoff); in handle_backoff_block()
4327 insert_backoff_by_id(&osd->o_backoffs_by_id, backoff); in handle_backoff_block()
4333 msg = create_backoff_message(backoff, m->map_epoch); in handle_backoff_block()
4338 ceph_con_send(&osd->o_con, msg); in handle_backoff_block()
4360 dout("%s osd%d spgid %llu.%xs%d id %llu\n", __func__, osd->o_osd, in handle_backoff_unblock()
4361 m->spgid.pgid.pool, m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_unblock()
4363 backoff = lookup_backoff_by_id(&osd->o_backoffs_by_id, m->id); in handle_backoff_unblock()
4366 __func__, osd->o_osd, m->spgid.pgid.pool, in handle_backoff_unblock()
4367 m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_unblock()
4371 if (hoid_compare(backoff->begin, m->begin) && in handle_backoff_unblock()
4372 hoid_compare(backoff->end, m->end)) { in handle_backoff_unblock()
4374 __func__, osd->o_osd, m->spgid.pgid.pool, in handle_backoff_unblock()
4375 m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_unblock()
4379 spg = lookup_spg_mapping(&osd->o_backoff_mappings, &backoff->spgid); in handle_backoff_unblock()
4382 erase_backoff(&spg->backoffs, backoff); in handle_backoff_unblock()
4383 erase_backoff_by_id(&osd->o_backoffs_by_id, backoff); in handle_backoff_unblock()
4386 if (RB_EMPTY_ROOT(&spg->backoffs)) { in handle_backoff_unblock()
4387 erase_spg_mapping(&osd->o_backoff_mappings, spg); in handle_backoff_unblock()
4391 for (n = rb_first(&osd->o_requests); n; n = rb_next(n)) { in handle_backoff_unblock()
4395 if (!ceph_spg_compare(&req->r_t.spgid, &m->spgid)) { in handle_backoff_unblock()
4397 * Match against @m, not @backoff -- the PG may in handle_backoff_unblock()
4400 if (target_contained_by(&req->r_t, m->begin, m->end)) { in handle_backoff_unblock()
4413 struct ceph_osd_client *osdc = osd->o_osdc; in handle_backoff()
4417 down_read(&osdc->lock); in handle_backoff()
4419 dout("%s osd%d unknown\n", __func__, osd->o_osd); in handle_backoff()
4420 up_read(&osdc->lock); in handle_backoff()
4423 WARN_ON(osd->o_osd != le64_to_cpu(msg->hdr.src.num)); in handle_backoff()
4425 mutex_lock(&osd->lock); in handle_backoff()
4441 pr_err("%s osd%d unknown op %d\n", __func__, osd->o_osd, m.op); in handle_backoff()
4448 mutex_unlock(&osd->lock); in handle_backoff()
4449 up_read(&osdc->lock); in handle_backoff()
4458 void *p = msg->front.iov_base; in handle_watch_notify()
4459 void *const end = p + msg->front.iov_len; in handle_watch_notify()
4482 if (le16_to_cpu(msg->hdr.version) >= 2) in handle_watch_notify()
4485 if (le16_to_cpu(msg->hdr.version) >= 3) in handle_watch_notify()
4488 down_read(&osdc->lock); in handle_watch_notify()
4489 lreq = lookup_linger_osdc(&osdc->linger_requests, cookie); in handle_watch_notify()
4496 mutex_lock(&lreq->lock); in handle_watch_notify()
4498 opcode, cookie, lreq, lreq->is_watch); in handle_watch_notify()
4500 if (!lreq->last_error) { in handle_watch_notify()
4501 lreq->last_error = -ENOTCONN; in handle_watch_notify()
4504 } else if (!lreq->is_watch) { in handle_watch_notify()
4506 if (lreq->notify_id && lreq->notify_id != notify_id) { in handle_watch_notify()
4508 lreq->notify_id, notify_id); in handle_watch_notify()
4509 } else if (!completion_done(&lreq->notify_finish_wait)) { in handle_watch_notify()
4511 msg->num_data_items ? &msg->data[0] : NULL; in handle_watch_notify()
4514 if (lreq->preply_pages) { in handle_watch_notify()
4515 WARN_ON(data->type != in handle_watch_notify()
4517 *lreq->preply_pages = data->pages; in handle_watch_notify()
4518 *lreq->preply_len = data->length; in handle_watch_notify()
4519 data->own_pages = false; in handle_watch_notify()
4522 lreq->notify_finish_error = return_code; in handle_watch_notify()
4523 complete_all(&lreq->notify_finish_wait); in handle_watch_notify()
4529 pr_err("failed to allocate notify-lwork\n"); in handle_watch_notify()
4533 lwork->notify.notify_id = notify_id; in handle_watch_notify()
4534 lwork->notify.notifier_id = notifier_id; in handle_watch_notify()
4535 lwork->notify.payload = payload; in handle_watch_notify()
4536 lwork->notify.payload_len = payload_len; in handle_watch_notify()
4537 lwork->notify.msg = ceph_msg_get(msg); in handle_watch_notify()
4542 mutex_unlock(&lreq->lock); in handle_watch_notify()
4544 up_read(&osdc->lock); in handle_watch_notify()
4558 down_read(&osdc->lock); in ceph_osdc_start_request()
4560 up_read(&osdc->lock); in ceph_osdc_start_request()
4568 * ->r_result isn't set and __complete_request() isn't called.
4572 struct ceph_osd_client *osdc = req->r_osdc; in ceph_osdc_cancel_request()
4574 down_write(&osdc->lock); in ceph_osdc_cancel_request()
4575 if (req->r_osd) in ceph_osdc_cancel_request()
4577 up_write(&osdc->lock); in ceph_osdc_cancel_request()
4589 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in wait_request_timeout()
4590 left = wait_for_completion_killable_timeout(&req->r_completion, in wait_request_timeout()
4593 left = left ?: -ETIMEDOUT; in wait_request_timeout()
4596 left = req->r_result; /* completed */ in wait_request_timeout()
4613 * sync - wait for all in-flight requests to flush. avoid starvation.
4618 u64 last_tid = atomic64_read(&osdc->last_tid); in ceph_osdc_sync()
4621 down_read(&osdc->lock); in ceph_osdc_sync()
4622 for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { in ceph_osdc_sync()
4625 mutex_lock(&osd->lock); in ceph_osdc_sync()
4626 for (p = rb_first(&osd->o_requests); p; p = rb_next(p)) { in ceph_osdc_sync()
4630 if (req->r_tid > last_tid) in ceph_osdc_sync()
4633 if (!(req->r_flags & CEPH_OSD_FLAG_WRITE)) in ceph_osdc_sync()
4637 mutex_unlock(&osd->lock); in ceph_osdc_sync()
4638 up_read(&osdc->lock); in ceph_osdc_sync()
4640 __func__, req, req->r_tid, last_tid); in ceph_osdc_sync()
4641 wait_for_completion(&req->r_completion); in ceph_osdc_sync()
4646 mutex_unlock(&osd->lock); in ceph_osdc_sync()
4649 up_read(&osdc->lock); in ceph_osdc_sync()
4659 req = ceph_osdc_alloc_request(lreq->osdc, NULL, 1, false, GFP_NOIO); in alloc_linger_request()
4663 ceph_oid_copy(&req->r_base_oid, &lreq->t.base_oid); in alloc_linger_request()
4664 ceph_oloc_copy(&req->r_base_oloc, &lreq->t.base_oloc); in alloc_linger_request()
4707 return ERR_PTR(-ENOMEM); in ceph_osdc_watch()
4709 lreq->is_watch = true; in ceph_osdc_watch()
4710 lreq->wcb = wcb; in ceph_osdc_watch()
4711 lreq->errcb = errcb; in ceph_osdc_watch()
4712 lreq->data = data; in ceph_osdc_watch()
4713 lreq->watch_valid_thru = jiffies; in ceph_osdc_watch()
4715 ceph_oid_copy(&lreq->t.base_oid, oid); in ceph_osdc_watch()
4716 ceph_oloc_copy(&lreq->t.base_oloc, oloc); in ceph_osdc_watch()
4717 lreq->t.flags = CEPH_OSD_FLAG_WRITE; in ceph_osdc_watch()
4718 ktime_get_real_ts64(&lreq->mtime); in ceph_osdc_watch()
4720 lreq->reg_req = alloc_watch_request(lreq, CEPH_OSD_WATCH_OP_WATCH); in ceph_osdc_watch()
4721 if (!lreq->reg_req) { in ceph_osdc_watch()
4722 ret = -ENOMEM; in ceph_osdc_watch()
4726 lreq->ping_req = alloc_watch_request(lreq, CEPH_OSD_WATCH_OP_PING); in ceph_osdc_watch()
4727 if (!lreq->ping_req) { in ceph_osdc_watch()
4728 ret = -ENOMEM; in ceph_osdc_watch()
4757 struct ceph_options *opts = osdc->client->options; in ceph_osdc_unwatch()
4763 return -ENOMEM; in ceph_osdc_unwatch()
4765 ceph_oid_copy(&req->r_base_oid, &lreq->t.base_oid); in ceph_osdc_unwatch()
4766 ceph_oloc_copy(&req->r_base_oloc, &lreq->t.base_oloc); in ceph_osdc_unwatch()
4767 req->r_flags = CEPH_OSD_FLAG_WRITE; in ceph_osdc_unwatch()
4768 ktime_get_real_ts64(&req->r_mtime); in ceph_osdc_unwatch()
4769 osd_req_op_watch_init(req, 0, lreq->linger_id, in ceph_osdc_unwatch()
4779 ret = wait_request_timeout(req, opts->mount_timeout); in ceph_osdc_unwatch()
4799 return -ENOMEM; in osd_req_op_notify_ack_init()
4811 return -ENOMEM; in osd_req_op_notify_ack_init()
4814 ceph_osd_data_pagelist_init(&op->notify_ack.request_data, pl); in osd_req_op_notify_ack_init()
4815 op->indata_len = pl->length; in osd_req_op_notify_ack_init()
4832 return -ENOMEM; in ceph_osdc_notify_ack()
4834 ceph_oid_copy(&req->r_base_oid, oid); in ceph_osdc_notify_ack()
4835 ceph_oloc_copy(&req->r_base_oloc, oloc); in ceph_osdc_notify_ack()
4836 req->r_flags = CEPH_OSD_FLAG_READ; in ceph_osdc_notify_ack()
4865 op->notify.cookie = cookie; in osd_req_op_notify_init()
4869 return -ENOMEM; in osd_req_op_notify_init()
4877 return -ENOMEM; in osd_req_op_notify_init()
4880 ceph_osd_data_pagelist_init(&op->notify.request_data, pl); in osd_req_op_notify_init()
4881 op->indata_len = pl->length; in osd_req_op_notify_init()
4914 return -ENOMEM; in ceph_osdc_notify()
4916 lreq->preply_pages = preply_pages; in ceph_osdc_notify()
4917 lreq->preply_len = preply_len; in ceph_osdc_notify()
4919 ceph_oid_copy(&lreq->t.base_oid, oid); in ceph_osdc_notify()
4920 ceph_oloc_copy(&lreq->t.base_oloc, oloc); in ceph_osdc_notify()
4921 lreq->t.flags = CEPH_OSD_FLAG_READ; in ceph_osdc_notify()
4923 lreq->reg_req = alloc_linger_request(lreq); in ceph_osdc_notify()
4924 if (!lreq->reg_req) { in ceph_osdc_notify()
4925 ret = -ENOMEM; in ceph_osdc_notify()
4933 ret = osd_req_op_notify_init(lreq->reg_req, 0, 0, 1, timeout, in ceph_osdc_notify()
4944 ceph_osd_data_pages_init(osd_req_op_data(lreq->reg_req, 0, notify, in ceph_osdc_notify()
4948 ret = ceph_osdc_alloc_messages(lreq->reg_req, GFP_NOIO); in ceph_osdc_notify()
4977 down_read(&osdc->lock); in ceph_osdc_watch_check()
4978 mutex_lock(&lreq->lock); in ceph_osdc_watch_check()
4979 stamp = lreq->watch_valid_thru; in ceph_osdc_watch_check()
4980 if (!list_empty(&lreq->pending_lworks)) { in ceph_osdc_watch_check()
4982 list_first_entry(&lreq->pending_lworks, in ceph_osdc_watch_check()
4986 if (time_before(lwork->queued_stamp, stamp)) in ceph_osdc_watch_check()
4987 stamp = lwork->queued_stamp; in ceph_osdc_watch_check()
4989 age = jiffies - stamp; in ceph_osdc_watch_check()
4991 lreq, lreq->linger_id, age, lreq->last_error); in ceph_osdc_watch_check()
4993 ret = lreq->last_error ?: 1 + jiffies_to_msecs(age); in ceph_osdc_watch_check()
4995 mutex_unlock(&lreq->lock); in ceph_osdc_watch_check()
4996 up_read(&osdc->lock); in ceph_osdc_watch_check()
5011 ret = -EINVAL; in decode_watcher()
5012 ceph_decode_copy_safe(p, end, &item->name, sizeof(item->name), bad); in decode_watcher()
5013 ceph_decode_64_safe(p, end, item->cookie, bad); in decode_watcher()
5017 ret = ceph_decode_entity_addr(p, end, &item->addr); in decode_watcher()
5025 ENTITY_NAME(item->name), item->cookie, in decode_watcher()
5026 ceph_pr_addr(&item->addr)); in decode_watcher()
5048 return -ENOMEM; in decode_watchers()
5078 return -ENOMEM; in ceph_osdc_list_watchers()
5080 ceph_oid_copy(&req->r_base_oid, oid); in ceph_osdc_list_watchers()
5081 ceph_oloc_copy(&req->r_base_oloc, oloc); in ceph_osdc_list_watchers()
5082 req->r_flags = CEPH_OSD_FLAG_READ; in ceph_osdc_list_watchers()
5103 void *const end = p + req->r_ops[0].outdata_len; in ceph_osdc_list_watchers()
5115 * Call all pending notify callbacks - for use after a watch is
5121 flush_workqueue(osdc->notify_wq); in ceph_osdc_flush_notifies()
5127 down_read(&osdc->lock); in ceph_osdc_maybe_request_map()
5129 up_read(&osdc->lock); in ceph_osdc_maybe_request_map()
5151 return -E2BIG; in ceph_osdc_call()
5155 return -ENOMEM; in ceph_osdc_call()
5157 ceph_oid_copy(&req->r_base_oid, oid); in ceph_osdc_call()
5158 ceph_oloc_copy(&req->r_base_oloc, oloc); in ceph_osdc_call()
5159 req->r_flags = flags; in ceph_osdc_call()
5179 ret = req->r_ops[0].rval; in ceph_osdc_call()
5181 *resp_len = req->r_ops[0].outdata_len; in ceph_osdc_call()
5197 down_write(&osdc->lock); in ceph_osdc_reopen_osds()
5198 for (n = rb_first(&osdc->osds); n; ) { in ceph_osdc_reopen_osds()
5205 up_write(&osdc->lock); in ceph_osdc_reopen_osds()
5216 osdc->client = client; in ceph_osdc_init()
5217 init_rwsem(&osdc->lock); in ceph_osdc_init()
5218 osdc->osds = RB_ROOT; in ceph_osdc_init()
5219 INIT_LIST_HEAD(&osdc->osd_lru); in ceph_osdc_init()
5220 spin_lock_init(&osdc->osd_lru_lock); in ceph_osdc_init()
5221 osd_init(&osdc->homeless_osd); in ceph_osdc_init()
5222 osdc->homeless_osd.o_osdc = osdc; in ceph_osdc_init()
5223 osdc->homeless_osd.o_osd = CEPH_HOMELESS_OSD; in ceph_osdc_init()
5224 osdc->last_linger_id = CEPH_LINGER_ID_START; in ceph_osdc_init()
5225 osdc->linger_requests = RB_ROOT; in ceph_osdc_init()
5226 osdc->map_checks = RB_ROOT; in ceph_osdc_init()
5227 osdc->linger_map_checks = RB_ROOT; in ceph_osdc_init()
5228 INIT_DELAYED_WORK(&osdc->timeout_work, handle_timeout); in ceph_osdc_init()
5229 INIT_DELAYED_WORK(&osdc->osds_timeout_work, handle_osds_timeout); in ceph_osdc_init()
5231 err = -ENOMEM; in ceph_osdc_init()
5232 osdc->osdmap = ceph_osdmap_alloc(); in ceph_osdc_init()
5233 if (!osdc->osdmap) in ceph_osdc_init()
5236 osdc->req_mempool = mempool_create_slab_pool(10, in ceph_osdc_init()
5238 if (!osdc->req_mempool) in ceph_osdc_init()
5241 err = ceph_msgpool_init(&osdc->msgpool_op, CEPH_MSG_OSD_OP, in ceph_osdc_init()
5245 err = ceph_msgpool_init(&osdc->msgpool_op_reply, CEPH_MSG_OSD_OPREPLY, in ceph_osdc_init()
5251 err = -ENOMEM; in ceph_osdc_init()
5252 osdc->notify_wq = create_singlethread_workqueue("ceph-watch-notify"); in ceph_osdc_init()
5253 if (!osdc->notify_wq) in ceph_osdc_init()
5256 osdc->completion_wq = create_singlethread_workqueue("ceph-completion"); in ceph_osdc_init()
5257 if (!osdc->completion_wq) in ceph_osdc_init()
5260 schedule_delayed_work(&osdc->timeout_work, in ceph_osdc_init()
5261 osdc->client->options->osd_keepalive_timeout); in ceph_osdc_init()
5262 schedule_delayed_work(&osdc->osds_timeout_work, in ceph_osdc_init()
5263 round_jiffies_relative(osdc->client->options->osd_idle_ttl)); in ceph_osdc_init()
5268 destroy_workqueue(osdc->notify_wq); in ceph_osdc_init()
5270 ceph_msgpool_destroy(&osdc->msgpool_op_reply); in ceph_osdc_init()
5272 ceph_msgpool_destroy(&osdc->msgpool_op); in ceph_osdc_init()
5274 mempool_destroy(osdc->req_mempool); in ceph_osdc_init()
5276 ceph_osdmap_destroy(osdc->osdmap); in ceph_osdc_init()
5283 destroy_workqueue(osdc->completion_wq); in ceph_osdc_stop()
5284 destroy_workqueue(osdc->notify_wq); in ceph_osdc_stop()
5285 cancel_delayed_work_sync(&osdc->timeout_work); in ceph_osdc_stop()
5286 cancel_delayed_work_sync(&osdc->osds_timeout_work); in ceph_osdc_stop()
5288 down_write(&osdc->lock); in ceph_osdc_stop()
5289 while (!RB_EMPTY_ROOT(&osdc->osds)) { in ceph_osdc_stop()
5290 struct ceph_osd *osd = rb_entry(rb_first(&osdc->osds), in ceph_osdc_stop()
5294 up_write(&osdc->lock); in ceph_osdc_stop()
5295 WARN_ON(refcount_read(&osdc->homeless_osd.o_ref) != 1); in ceph_osdc_stop()
5296 osd_cleanup(&osdc->homeless_osd); in ceph_osdc_stop()
5298 WARN_ON(!list_empty(&osdc->osd_lru)); in ceph_osdc_stop()
5299 WARN_ON(!RB_EMPTY_ROOT(&osdc->linger_requests)); in ceph_osdc_stop()
5300 WARN_ON(!RB_EMPTY_ROOT(&osdc->map_checks)); in ceph_osdc_stop()
5301 WARN_ON(!RB_EMPTY_ROOT(&osdc->linger_map_checks)); in ceph_osdc_stop()
5302 WARN_ON(atomic_read(&osdc->num_requests)); in ceph_osdc_stop()
5303 WARN_ON(atomic_read(&osdc->num_homeless)); in ceph_osdc_stop()
5305 ceph_osdmap_destroy(osdc->osdmap); in ceph_osdc_stop()
5306 mempool_destroy(osdc->req_mempool); in ceph_osdc_stop()
5307 ceph_msgpool_destroy(&osdc->msgpool_op); in ceph_osdc_stop()
5308 ceph_msgpool_destroy(&osdc->msgpool_op_reply); in ceph_osdc_stop()
5330 op->copy_from.snapid = src_snapid; in osd_req_op_copy_from_init()
5331 op->copy_from.src_version = src_version; in osd_req_op_copy_from_init()
5332 op->copy_from.flags = copy_from_flags; in osd_req_op_copy_from_init()
5333 op->copy_from.src_fadvise_flags = src_fadvise_flags; in osd_req_op_copy_from_init()
5337 ceph_encode_string(&p, end, src_oid->name, src_oid->name_len); in osd_req_op_copy_from_init()
5341 op->indata_len = PAGE_SIZE - (end - p); in osd_req_op_copy_from_init()
5343 ceph_osd_data_pages_init(&op->copy_from.osd_data, pages, in osd_req_op_copy_from_init()
5344 op->indata_len, 0, false, true); in osd_req_op_copy_from_init()
5364 return -ENOMEM; in ceph_osdc_copy_from()
5366 req->r_flags = CEPH_OSD_FLAG_WRITE; in ceph_osdc_copy_from()
5368 ceph_oloc_copy(&req->r_t.base_oloc, dst_oloc); in ceph_osdc_copy_from()
5369 ceph_oid_copy(&req->r_t.base_oid, dst_oid); in ceph_osdc_copy_from()
5400 return ceph_osd_request_cache ? 0 : -ENOMEM; in ceph_osdc_setup()
5415 struct ceph_osd *osd = con->private; in dispatch()
5416 struct ceph_osd_client *osdc = osd->o_osdc; in dispatch()
5417 int type = le16_to_cpu(msg->hdr.type); in dispatch()
5444 * message at the moment - for now, just skip the message.
5450 struct ceph_osd *osd = con->private; in get_reply()
5451 struct ceph_osd_client *osdc = osd->o_osdc; in get_reply()
5454 int front_len = le32_to_cpu(hdr->front_len); in get_reply()
5455 int data_len = le32_to_cpu(hdr->data_len); in get_reply()
5456 u64 tid = le64_to_cpu(hdr->tid); in get_reply()
5458 down_read(&osdc->lock); in get_reply()
5460 dout("%s osd%d unknown, skipping\n", __func__, osd->o_osd); in get_reply()
5464 WARN_ON(osd->o_osd != le64_to_cpu(hdr->src.num)); in get_reply()
5466 mutex_lock(&osd->lock); in get_reply()
5467 req = lookup_request(&osd->o_requests, tid); in get_reply()
5470 osd->o_osd, tid); in get_reply()
5475 ceph_msg_revoke_incoming(req->r_reply); in get_reply()
5477 if (front_len > req->r_reply->front_alloc_len) { in get_reply()
5479 __func__, osd->o_osd, req->r_tid, front_len, in get_reply()
5480 req->r_reply->front_alloc_len); in get_reply()
5485 ceph_msg_put(req->r_reply); in get_reply()
5486 req->r_reply = m; in get_reply()
5489 if (data_len > req->r_reply->data_length) { in get_reply()
5491 __func__, osd->o_osd, req->r_tid, data_len, in get_reply()
5492 req->r_reply->data_length); in get_reply()
5498 m = ceph_msg_get(req->r_reply); in get_reply()
5502 mutex_unlock(&osd->lock); in get_reply()
5504 up_read(&osdc->lock); in get_reply()
5511 int type = le16_to_cpu(hdr->type); in alloc_msg_with_page_vector()
5512 u32 front_len = le32_to_cpu(hdr->front_len); in alloc_msg_with_page_vector()
5513 u32 data_len = le32_to_cpu(hdr->data_len); in alloc_msg_with_page_vector()
5539 struct ceph_osd *osd = con->private; in alloc_msg()
5540 int type = le16_to_cpu(hdr->type); in alloc_msg()
5552 osd->o_osd, type); in alloc_msg()
5563 struct ceph_osd *osd = con->private; in get_osd_con()
5571 struct ceph_osd *osd = con->private; in put_osd_con()
5585 struct ceph_osd *o = con->private; in get_authorizer()
5586 struct ceph_osd_client *osdc = o->o_osdc; in get_authorizer()
5587 struct ceph_auth_client *ac = osdc->client->monc.auth; in get_authorizer()
5588 struct ceph_auth_handshake *auth = &o->o_auth; in get_authorizer()
5590 if (force_new && auth->authorizer) { in get_authorizer()
5591 ceph_auth_destroy_authorizer(auth->authorizer); in get_authorizer()
5592 auth->authorizer = NULL; in get_authorizer()
5594 if (!auth->authorizer) { in get_authorizer()
5605 *proto = ac->protocol; in get_authorizer()
5613 struct ceph_osd *o = con->private; in add_authorizer_challenge()
5614 struct ceph_osd_client *osdc = o->o_osdc; in add_authorizer_challenge()
5615 struct ceph_auth_client *ac = osdc->client->monc.auth; in add_authorizer_challenge()
5617 return ceph_auth_add_authorizer_challenge(ac, o->o_auth.authorizer, in add_authorizer_challenge()
5623 struct ceph_osd *o = con->private; in verify_authorizer_reply()
5624 struct ceph_osd_client *osdc = o->o_osdc; in verify_authorizer_reply()
5625 struct ceph_auth_client *ac = osdc->client->monc.auth; in verify_authorizer_reply()
5627 return ceph_auth_verify_authorizer_reply(ac, o->o_auth.authorizer); in verify_authorizer_reply()
5632 struct ceph_osd *o = con->private; in invalidate_authorizer()
5633 struct ceph_osd_client *osdc = o->o_osdc; in invalidate_authorizer()
5634 struct ceph_auth_client *ac = osdc->client->monc.auth; in invalidate_authorizer()
5637 return ceph_monc_validate_auth(&osdc->client->monc); in invalidate_authorizer()
5642 int type = le16_to_cpu(msg->hdr.type); in osd_reencode_message()
5650 struct ceph_osd *o = msg->con->private; in osd_sign_message()
5651 struct ceph_auth_handshake *auth = &o->o_auth; in osd_sign_message()
5658 struct ceph_osd *o = msg->con->private; in osd_check_message_signature()
5659 struct ceph_auth_handshake *auth = &o->o_auth; in osd_check_message_signature()