Lines Matching refs:gl
57 struct gfs2_glock *gl; /* current glock struct */ member
61 typedef void (*glock_examiner) (struct gfs2_glock * gl);
63 static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target);
65 static void handle_callback(struct gfs2_glock *gl, unsigned int state,
121 static void wake_up_glock(struct gfs2_glock *gl) in wake_up_glock() argument
123 wait_queue_head_t *wq = glock_waitqueue(&gl->gl_name); in wake_up_glock()
126 __wake_up(wq, TASK_NORMAL, 1, &gl->gl_name); in wake_up_glock()
131 struct gfs2_glock *gl = container_of(rcu, struct gfs2_glock, gl_rcu); in gfs2_glock_dealloc() local
133 kfree(gl->gl_lksb.sb_lvbptr); in gfs2_glock_dealloc()
134 if (gl->gl_ops->go_flags & GLOF_ASPACE) { in gfs2_glock_dealloc()
136 container_of(gl, struct gfs2_glock_aspace, glock); in gfs2_glock_dealloc()
139 kmem_cache_free(gfs2_glock_cachep, gl); in gfs2_glock_dealloc()
155 static bool glock_blocked_by_withdraw(struct gfs2_glock *gl) in glock_blocked_by_withdraw() argument
157 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; in glock_blocked_by_withdraw()
161 if (gl->gl_ops->go_flags & GLOF_NONDISK) in glock_blocked_by_withdraw()
164 gl->gl_name.ln_number == sdp->sd_jdesc->jd_no_addr) in glock_blocked_by_withdraw()
169 void gfs2_glock_free(struct gfs2_glock *gl) in gfs2_glock_free() argument
171 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; in gfs2_glock_free()
173 gfs2_glock_assert_withdraw(gl, atomic_read(&gl->gl_revokes) == 0); in gfs2_glock_free()
174 rhashtable_remove_fast(&gl_hash_table, &gl->gl_node, ht_parms); in gfs2_glock_free()
176 wake_up_glock(gl); in gfs2_glock_free()
177 call_rcu(&gl->gl_rcu, gfs2_glock_dealloc); in gfs2_glock_free()
188 struct gfs2_glock *gfs2_glock_hold(struct gfs2_glock *gl) in gfs2_glock_hold() argument
190 GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref)); in gfs2_glock_hold()
191 lockref_get(&gl->gl_lockref); in gfs2_glock_hold()
192 return gl; in gfs2_glock_hold()
202 static int demote_ok(const struct gfs2_glock *gl) in demote_ok() argument
204 const struct gfs2_glock_operations *glops = gl->gl_ops; in demote_ok()
206 if (gl->gl_state == LM_ST_UNLOCKED) in demote_ok()
208 if (!list_empty(&gl->gl_holders)) in demote_ok()
211 return glops->go_demote_ok(gl); in demote_ok()
216 void gfs2_glock_add_to_lru(struct gfs2_glock *gl) in gfs2_glock_add_to_lru() argument
218 if (!(gl->gl_ops->go_flags & GLOF_LRU)) in gfs2_glock_add_to_lru()
223 list_move_tail(&gl->gl_lru, &lru_list); in gfs2_glock_add_to_lru()
225 if (!test_bit(GLF_LRU, &gl->gl_flags)) { in gfs2_glock_add_to_lru()
226 set_bit(GLF_LRU, &gl->gl_flags); in gfs2_glock_add_to_lru()
233 static void gfs2_glock_remove_from_lru(struct gfs2_glock *gl) in gfs2_glock_remove_from_lru() argument
235 if (!(gl->gl_ops->go_flags & GLOF_LRU)) in gfs2_glock_remove_from_lru()
239 if (test_bit(GLF_LRU, &gl->gl_flags)) { in gfs2_glock_remove_from_lru()
240 list_del_init(&gl->gl_lru); in gfs2_glock_remove_from_lru()
242 clear_bit(GLF_LRU, &gl->gl_flags); in gfs2_glock_remove_from_lru()
251 static void __gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) { in __gfs2_glock_queue_work() argument
252 if (!queue_delayed_work(glock_workqueue, &gl->gl_work, delay)) { in __gfs2_glock_queue_work()
259 GLOCK_BUG_ON(gl, gl->gl_lockref.count < 2); in __gfs2_glock_queue_work()
260 gl->gl_lockref.count--; in __gfs2_glock_queue_work()
264 static void gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) { in gfs2_glock_queue_work() argument
265 spin_lock(&gl->gl_lockref.lock); in gfs2_glock_queue_work()
266 __gfs2_glock_queue_work(gl, delay); in gfs2_glock_queue_work()
267 spin_unlock(&gl->gl_lockref.lock); in gfs2_glock_queue_work()
270 static void __gfs2_glock_put(struct gfs2_glock *gl) in __gfs2_glock_put() argument
272 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; in __gfs2_glock_put()
273 struct address_space *mapping = gfs2_glock2aspace(gl); in __gfs2_glock_put()
275 lockref_mark_dead(&gl->gl_lockref); in __gfs2_glock_put()
276 spin_unlock(&gl->gl_lockref.lock); in __gfs2_glock_put()
277 gfs2_glock_remove_from_lru(gl); in __gfs2_glock_put()
278 GLOCK_BUG_ON(gl, !list_empty(&gl->gl_holders)); in __gfs2_glock_put()
282 GLOCK_BUG_ON(gl, !mapping_empty(mapping)); in __gfs2_glock_put()
284 trace_gfs2_glock_put(gl); in __gfs2_glock_put()
285 sdp->sd_lockstruct.ls_ops->lm_put_lock(gl); in __gfs2_glock_put()
291 void gfs2_glock_queue_put(struct gfs2_glock *gl) in gfs2_glock_queue_put() argument
293 gfs2_glock_queue_work(gl, 0); in gfs2_glock_queue_put()
302 void gfs2_glock_put(struct gfs2_glock *gl) in gfs2_glock_put() argument
304 if (lockref_put_or_lock(&gl->gl_lockref)) in gfs2_glock_put()
307 __gfs2_glock_put(gl); in gfs2_glock_put()
324 static inline bool may_grant(struct gfs2_glock *gl, in may_grant() argument
329 GLOCK_BUG_ON(gl, !test_bit(HIF_HOLDER, ¤t_gh->gh_iflags)); in may_grant()
353 if (gl->gl_state == gh->gh_state) in may_grant()
357 if (gl->gl_state == LM_ST_EXCLUSIVE) { in may_grant()
362 return gl->gl_state != LM_ST_UNLOCKED; in may_grant()
384 static void do_error(struct gfs2_glock *gl, const int ret) in do_error() argument
388 list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) { in do_error()
408 static inline struct gfs2_holder *find_first_holder(const struct gfs2_glock *gl) in find_first_holder() argument
412 if (!list_empty(&gl->gl_holders)) { in find_first_holder()
413 gh = list_first_entry(&gl->gl_holders, struct gfs2_holder, in find_first_holder()
429 struct gfs2_glock *gl = gh->gh_gl; in gfs2_instantiate() local
430 const struct gfs2_glock_operations *glops = gl->gl_ops; in gfs2_instantiate()
434 if (!test_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags)) in gfs2_instantiate()
441 if (test_and_set_bit(GLF_INSTANTIATE_IN_PROG, &gl->gl_flags)) { in gfs2_instantiate()
442 wait_on_bit(&gl->gl_flags, GLF_INSTANTIATE_IN_PROG, in gfs2_instantiate()
454 ret = glops->go_instantiate(gl); in gfs2_instantiate()
456 clear_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags); in gfs2_instantiate()
457 clear_and_wake_up_bit(GLF_INSTANTIATE_IN_PROG, &gl->gl_flags); in gfs2_instantiate()
474 static bool do_promote(struct gfs2_glock *gl) in do_promote() argument
478 current_gh = find_first_holder(gl); in do_promote()
479 list_for_each_entry(gh, &gl->gl_holders, gh_list) { in do_promote()
482 if (!may_grant(gl, current_gh, gh)) { in do_promote()
489 if (list_is_first(&gh->gh_list, &gl->gl_holders)) in do_promote()
491 do_error(gl, 0); in do_promote()
508 static inline struct gfs2_holder *find_first_waiter(const struct gfs2_glock *gl) in find_first_waiter() argument
512 list_for_each_entry(gh, &gl->gl_holders, gh_list) { in find_first_waiter()
525 static void state_change(struct gfs2_glock *gl, unsigned int new_state) in state_change() argument
529 held1 = (gl->gl_state != LM_ST_UNLOCKED); in state_change()
533 GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref)); in state_change()
535 gl->gl_lockref.count++; in state_change()
537 gl->gl_lockref.count--; in state_change()
539 if (new_state != gl->gl_target) in state_change()
541 gl->gl_hold_time = max(gl->gl_hold_time - GL_GLOCK_HOLD_DECR, in state_change()
543 gl->gl_state = new_state; in state_change()
544 gl->gl_tchange = jiffies; in state_change()
547 static void gfs2_set_demote(struct gfs2_glock *gl) in gfs2_set_demote() argument
549 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; in gfs2_set_demote()
551 set_bit(GLF_DEMOTE, &gl->gl_flags); in gfs2_set_demote()
556 static void gfs2_demote_wake(struct gfs2_glock *gl) in gfs2_demote_wake() argument
558 gl->gl_demote_state = LM_ST_EXCLUSIVE; in gfs2_demote_wake()
559 clear_bit(GLF_DEMOTE, &gl->gl_flags); in gfs2_demote_wake()
561 wake_up_bit(&gl->gl_flags, GLF_DEMOTE); in gfs2_demote_wake()
571 static void finish_xmote(struct gfs2_glock *gl, unsigned int ret) in finish_xmote() argument
573 const struct gfs2_glock_operations *glops = gl->gl_ops; in finish_xmote()
577 spin_lock(&gl->gl_lockref.lock); in finish_xmote()
578 trace_gfs2_glock_state_change(gl, state); in finish_xmote()
579 state_change(gl, state); in finish_xmote()
580 gh = find_first_waiter(gl); in finish_xmote()
583 if (test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) && in finish_xmote()
584 state != LM_ST_UNLOCKED && gl->gl_demote_state == LM_ST_UNLOCKED) in finish_xmote()
585 gl->gl_target = LM_ST_UNLOCKED; in finish_xmote()
588 if (unlikely(state != gl->gl_target)) { in finish_xmote()
591 if (gh && !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) { in finish_xmote()
594 list_move_tail(&gh->gh_list, &gl->gl_holders); in finish_xmote()
595 gh = find_first_waiter(gl); in finish_xmote()
596 gl->gl_target = gh->gh_state; in finish_xmote()
597 if (do_promote(gl)) in finish_xmote()
604 gl->gl_target = gl->gl_state; in finish_xmote()
605 do_error(gl, ret); in finish_xmote()
613 do_xmote(gl, gh, gl->gl_target); in finish_xmote()
618 do_xmote(gl, gh, LM_ST_UNLOCKED); in finish_xmote()
621 fs_err(gl->gl_name.ln_sbd, "wanted %u got %u\n", in finish_xmote()
622 gl->gl_target, state); in finish_xmote()
623 GLOCK_BUG_ON(gl, 1); in finish_xmote()
625 spin_unlock(&gl->gl_lockref.lock); in finish_xmote()
630 if (test_and_clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) in finish_xmote()
631 gfs2_demote_wake(gl); in finish_xmote()
636 spin_unlock(&gl->gl_lockref.lock); in finish_xmote()
637 rv = glops->go_xmote_bh(gl); in finish_xmote()
638 spin_lock(&gl->gl_lockref.lock); in finish_xmote()
640 do_error(gl, rv); in finish_xmote()
644 do_promote(gl); in finish_xmote()
647 clear_bit(GLF_LOCK, &gl->gl_flags); in finish_xmote()
648 spin_unlock(&gl->gl_lockref.lock); in finish_xmote()
651 static bool is_system_glock(struct gfs2_glock *gl) in is_system_glock() argument
653 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; in is_system_glock()
656 if (gl == m_ip->i_gl) in is_system_glock()
669 static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, in do_xmote() argument
671 __releases(&gl->gl_lockref.lock) in do_xmote()
672 __acquires(&gl->gl_lockref.lock) in do_xmote()
674 const struct gfs2_glock_operations *glops = gl->gl_ops; in do_xmote()
675 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; in do_xmote()
679 if (target != LM_ST_UNLOCKED && glock_blocked_by_withdraw(gl) && in do_xmote()
684 GLOCK_BUG_ON(gl, gl->gl_state == target); in do_xmote()
685 GLOCK_BUG_ON(gl, gl->gl_state == gl->gl_target); in do_xmote()
694 &gl->gl_flags)) in do_xmote()
696 do_error(gl, 0); /* Fail queued try locks */ in do_xmote()
698 gl->gl_req = target; in do_xmote()
699 set_bit(GLF_BLOCKING, &gl->gl_flags); in do_xmote()
700 if ((gl->gl_req == LM_ST_UNLOCKED) || in do_xmote()
701 (gl->gl_state == LM_ST_EXCLUSIVE) || in do_xmote()
703 clear_bit(GLF_BLOCKING, &gl->gl_flags); in do_xmote()
704 spin_unlock(&gl->gl_lockref.lock); in do_xmote()
706 ret = glops->go_sync(gl); in do_xmote()
714 gfs2_dump_glock(NULL, gl, true); in do_xmote()
719 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags)) { in do_xmote()
727 if ((atomic_read(&gl->gl_ail_count) != 0) && in do_xmote()
729 gfs2_glock_assert_warn(gl, in do_xmote()
730 !atomic_read(&gl->gl_ail_count)); in do_xmote()
731 gfs2_dump_glock(NULL, gl, true); in do_xmote()
733 glops->go_inval(gl, target == LM_ST_DEFERRED ? 0 : DIO_METADATA); in do_xmote()
734 clear_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags); in do_xmote()
738 gfs2_glock_hold(gl); in do_xmote()
762 if (glock_blocked_by_withdraw(gl) && in do_xmote()
765 if (!is_system_glock(gl)) { in do_xmote()
766 handle_callback(gl, LM_ST_UNLOCKED, 0, false); /* sets demote */ in do_xmote()
773 state_change(gl, LM_ST_UNLOCKED); in do_xmote()
778 clear_bit(GLF_LOCK, &gl->gl_flags); in do_xmote()
779 clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags); in do_xmote()
780 gfs2_glock_queue_work(gl, GL_GLOCK_DFT_HOLD); in do_xmote()
783 clear_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags); in do_xmote()
789 ret = sdp->sd_lockstruct.ls_ops->lm_lock(gl, target, lck_flags); in do_xmote()
790 if (ret == -EINVAL && gl->gl_target == LM_ST_UNLOCKED && in do_xmote()
793 finish_xmote(gl, target); in do_xmote()
794 gfs2_glock_queue_work(gl, 0); in do_xmote()
797 GLOCK_BUG_ON(gl, !gfs2_withdrawn(sdp)); in do_xmote()
800 finish_xmote(gl, target); in do_xmote()
801 gfs2_glock_queue_work(gl, 0); in do_xmote()
804 spin_lock(&gl->gl_lockref.lock); in do_xmote()
814 static void run_queue(struct gfs2_glock *gl, const int nonblock) in run_queue() argument
815 __releases(&gl->gl_lockref.lock) in run_queue()
816 __acquires(&gl->gl_lockref.lock) in run_queue()
820 if (test_and_set_bit(GLF_LOCK, &gl->gl_flags)) in run_queue()
823 GLOCK_BUG_ON(gl, test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)); in run_queue()
825 if (test_bit(GLF_DEMOTE, &gl->gl_flags) && in run_queue()
826 gl->gl_demote_state != gl->gl_state) { in run_queue()
827 if (find_first_holder(gl)) in run_queue()
831 set_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags); in run_queue()
832 GLOCK_BUG_ON(gl, gl->gl_demote_state == LM_ST_EXCLUSIVE); in run_queue()
833 gl->gl_target = gl->gl_demote_state; in run_queue()
835 if (test_bit(GLF_DEMOTE, &gl->gl_flags)) in run_queue()
836 gfs2_demote_wake(gl); in run_queue()
837 if (do_promote(gl)) in run_queue()
839 gh = find_first_waiter(gl); in run_queue()
840 gl->gl_target = gh->gh_state; in run_queue()
842 do_error(gl, 0); /* Fail queued try locks */ in run_queue()
844 do_xmote(gl, gh, gl->gl_target); in run_queue()
848 clear_bit(GLF_LOCK, &gl->gl_flags); in run_queue()
850 gl->gl_lockref.count++; in run_queue()
851 __gfs2_glock_queue_work(gl, 0); in run_queue()
855 clear_bit(GLF_LOCK, &gl->gl_flags); in run_queue()
865 void glock_set_object(struct gfs2_glock *gl, void *object) in glock_set_object() argument
869 spin_lock(&gl->gl_lockref.lock); in glock_set_object()
870 prev_object = gl->gl_object; in glock_set_object()
871 gl->gl_object = object; in glock_set_object()
872 spin_unlock(&gl->gl_lockref.lock); in glock_set_object()
873 if (gfs2_assert_warn(gl->gl_name.ln_sbd, prev_object == NULL)) { in glock_set_object()
875 gl->gl_name.ln_type, in glock_set_object()
876 (unsigned long long)gl->gl_name.ln_number); in glock_set_object()
877 gfs2_dump_glock(NULL, gl, true); in glock_set_object()
886 void glock_clear_object(struct gfs2_glock *gl, void *object) in glock_clear_object() argument
890 spin_lock(&gl->gl_lockref.lock); in glock_clear_object()
891 prev_object = gl->gl_object; in glock_clear_object()
892 gl->gl_object = NULL; in glock_clear_object()
893 spin_unlock(&gl->gl_lockref.lock); in glock_clear_object()
894 if (gfs2_assert_warn(gl->gl_name.ln_sbd, prev_object == object)) { in glock_clear_object()
896 gl->gl_name.ln_type, in glock_clear_object()
897 (unsigned long long)gl->gl_name.ln_number); in glock_clear_object()
898 gfs2_dump_glock(NULL, gl, true); in glock_clear_object()
902 void gfs2_inode_remember_delete(struct gfs2_glock *gl, u64 generation) in gfs2_inode_remember_delete() argument
904 struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr; in gfs2_inode_remember_delete()
912 bool gfs2_inode_already_deleted(struct gfs2_glock *gl, u64 generation) in gfs2_inode_already_deleted() argument
914 struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr; in gfs2_inode_already_deleted()
921 static void gfs2_glock_poke(struct gfs2_glock *gl) in gfs2_glock_poke() argument
927 __gfs2_holder_init(gl, LM_ST_SHARED, flags, &gh, _RET_IP_); in gfs2_glock_poke()
934 static bool gfs2_try_evict(struct gfs2_glock *gl) in gfs2_try_evict() argument
949 spin_lock(&gl->gl_lockref.lock); in gfs2_try_evict()
950 ip = gl->gl_object; in gfs2_try_evict()
953 spin_unlock(&gl->gl_lockref.lock); in gfs2_try_evict()
955 gl->gl_no_formal_ino = ip->i_no_formal_ino; in gfs2_try_evict()
961 spin_lock(&gl->gl_lockref.lock); in gfs2_try_evict()
962 ip = gl->gl_object; in gfs2_try_evict()
968 spin_unlock(&gl->gl_lockref.lock); in gfs2_try_evict()
978 bool gfs2_queue_try_to_evict(struct gfs2_glock *gl) in gfs2_queue_try_to_evict() argument
980 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; in gfs2_queue_try_to_evict()
982 if (test_and_set_bit(GLF_TRY_TO_EVICT, &gl->gl_flags)) in gfs2_queue_try_to_evict()
985 &gl->gl_delete, 0); in gfs2_queue_try_to_evict()
988 static bool gfs2_queue_verify_evict(struct gfs2_glock *gl) in gfs2_queue_verify_evict() argument
990 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; in gfs2_queue_verify_evict()
992 if (test_and_set_bit(GLF_VERIFY_EVICT, &gl->gl_flags)) in gfs2_queue_verify_evict()
995 &gl->gl_delete, 5 * HZ); in gfs2_queue_verify_evict()
1001 struct gfs2_glock *gl = container_of(dwork, struct gfs2_glock, gl_delete); in delete_work_func() local
1002 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; in delete_work_func()
1004 u64 no_addr = gl->gl_name.ln_number; in delete_work_func()
1006 if (test_and_clear_bit(GLF_TRY_TO_EVICT, &gl->gl_flags)) { in delete_work_func()
1024 if (gfs2_try_evict(gl)) { in delete_work_func()
1027 if (gfs2_queue_verify_evict(gl)) in delete_work_func()
1033 if (test_and_clear_bit(GLF_VERIFY_EVICT, &gl->gl_flags)) { in delete_work_func()
1034 inode = gfs2_lookup_by_inum(sdp, no_addr, gl->gl_no_formal_ino, in delete_work_func()
1039 gfs2_queue_verify_evict(gl)) in delete_work_func()
1048 gfs2_glock_put(gl); in delete_work_func()
1054 struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_work.work); in glock_work_func() local
1057 if (test_and_clear_bit(GLF_REPLY_PENDING, &gl->gl_flags)) { in glock_work_func()
1058 finish_xmote(gl, gl->gl_reply); in glock_work_func()
1061 spin_lock(&gl->gl_lockref.lock); in glock_work_func()
1062 if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) && in glock_work_func()
1063 gl->gl_state != LM_ST_UNLOCKED && in glock_work_func()
1064 gl->gl_demote_state != LM_ST_EXCLUSIVE) { in glock_work_func()
1067 holdtime = gl->gl_tchange + gl->gl_hold_time; in glock_work_func()
1072 clear_bit(GLF_PENDING_DEMOTE, &gl->gl_flags); in glock_work_func()
1073 gfs2_set_demote(gl); in glock_work_func()
1076 run_queue(gl, 0); in glock_work_func()
1080 if (gl->gl_name.ln_type != LM_TYPE_INODE) in glock_work_func()
1082 __gfs2_glock_queue_work(gl, delay); in glock_work_func()
1090 gl->gl_lockref.count -= drop_refs; in glock_work_func()
1091 if (!gl->gl_lockref.count) { in glock_work_func()
1092 __gfs2_glock_put(gl); in glock_work_func()
1095 spin_unlock(&gl->gl_lockref.lock); in glock_work_func()
1103 struct gfs2_glock *gl; in find_insert_glock() local
1113 gl = rhashtable_lookup_get_insert_fast(&gl_hash_table, in find_insert_glock()
1115 if (IS_ERR(gl)) in find_insert_glock()
1118 gl = rhashtable_lookup_fast(&gl_hash_table, in find_insert_glock()
1121 if (gl && !lockref_get_not_dead(&gl->gl_lockref)) { in find_insert_glock()
1129 return gl; in find_insert_glock()
1153 struct gfs2_glock *gl, *tmp; in gfs2_glock_get() local
1157 gl = find_insert_glock(&name, NULL); in gfs2_glock_get()
1158 if (gl) { in gfs2_glock_get()
1159 *glp = gl; in gfs2_glock_get()
1170 gl = &gla->glock; in gfs2_glock_get()
1172 gl = kmem_cache_alloc(gfs2_glock_cachep, GFP_NOFS); in gfs2_glock_get()
1173 if (!gl) in gfs2_glock_get()
1176 memset(&gl->gl_lksb, 0, sizeof(struct dlm_lksb)); in gfs2_glock_get()
1177 gl->gl_ops = glops; in gfs2_glock_get()
1180 gl->gl_lksb.sb_lvbptr = kzalloc(GDLM_LVB_SIZE, GFP_NOFS); in gfs2_glock_get()
1181 if (!gl->gl_lksb.sb_lvbptr) { in gfs2_glock_get()
1182 gfs2_glock_dealloc(&gl->gl_rcu); in gfs2_glock_get()
1188 gl->gl_node.next = NULL; in gfs2_glock_get()
1189 gl->gl_flags = glops->go_instantiate ? BIT(GLF_INSTANTIATE_NEEDED) : 0; in gfs2_glock_get()
1190 gl->gl_name = name; in gfs2_glock_get()
1191 lockdep_set_subclass(&gl->gl_lockref.lock, glops->go_subclass); in gfs2_glock_get()
1192 gl->gl_lockref.count = 1; in gfs2_glock_get()
1193 gl->gl_state = LM_ST_UNLOCKED; in gfs2_glock_get()
1194 gl->gl_target = LM_ST_UNLOCKED; in gfs2_glock_get()
1195 gl->gl_demote_state = LM_ST_EXCLUSIVE; in gfs2_glock_get()
1196 gl->gl_dstamp = 0; in gfs2_glock_get()
1199 gl->gl_stats = this_cpu_ptr(sdp->sd_lkstats)->lkstats[glops->go_type]; in gfs2_glock_get()
1201 gl->gl_stats.stats[GFS2_LKS_DCOUNT] = 0; in gfs2_glock_get()
1202 gl->gl_stats.stats[GFS2_LKS_QCOUNT] = 0; in gfs2_glock_get()
1203 gl->gl_tchange = jiffies; in gfs2_glock_get()
1204 gl->gl_object = NULL; in gfs2_glock_get()
1205 gl->gl_hold_time = GL_GLOCK_DFT_HOLD; in gfs2_glock_get()
1206 INIT_DELAYED_WORK(&gl->gl_work, glock_work_func); in gfs2_glock_get()
1207 if (gl->gl_name.ln_type == LM_TYPE_IOPEN) in gfs2_glock_get()
1208 INIT_DELAYED_WORK(&gl->gl_delete, delete_work_func); in gfs2_glock_get()
1210 mapping = gfs2_glock2aspace(gl); in gfs2_glock_get()
1220 tmp = find_insert_glock(&name, gl); in gfs2_glock_get()
1222 *glp = gl; in gfs2_glock_get()
1232 gfs2_glock_dealloc(&gl->gl_rcu); in gfs2_glock_get()
1249 void __gfs2_holder_init(struct gfs2_glock *gl, unsigned int state, u16 flags, in __gfs2_holder_init() argument
1253 gh->gh_gl = gfs2_glock_hold(gl); in __gfs2_holder_init()
1295 static void gfs2_glock_update_hold_time(struct gfs2_glock *gl, in gfs2_glock_update_hold_time() argument
1301 gl->gl_hold_time = min(gl->gl_hold_time + GL_GLOCK_HOLD_INCR, in gfs2_glock_update_hold_time()
1417 static void handle_callback(struct gfs2_glock *gl, unsigned int state, in handle_callback() argument
1421 set_bit(GLF_PENDING_DEMOTE, &gl->gl_flags); in handle_callback()
1423 gfs2_set_demote(gl); in handle_callback()
1424 if (gl->gl_demote_state == LM_ST_EXCLUSIVE) { in handle_callback()
1425 gl->gl_demote_state = state; in handle_callback()
1426 gl->gl_demote_time = jiffies; in handle_callback()
1427 } else if (gl->gl_demote_state != LM_ST_UNLOCKED && in handle_callback()
1428 gl->gl_demote_state != state) { in handle_callback()
1429 gl->gl_demote_state = LM_ST_UNLOCKED; in handle_callback()
1431 if (gl->gl_ops->go_callback) in handle_callback()
1432 gl->gl_ops->go_callback(gl, remote); in handle_callback()
1433 trace_gfs2_demote_rq(gl, remote); in handle_callback()
1475 __releases(&gl->gl_lockref.lock) in add_to_queue()
1476 __acquires(&gl->gl_lockref.lock) in add_to_queue()
1478 struct gfs2_glock *gl = gh->gh_gl; in add_to_queue() local
1479 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; in add_to_queue()
1484 GLOCK_BUG_ON(gl, gh->gh_owner_pid == NULL); in add_to_queue()
1486 GLOCK_BUG_ON(gl, true); in add_to_queue()
1489 if (test_bit(GLF_LOCK, &gl->gl_flags)) { in add_to_queue()
1492 current_gh = find_first_holder(gl); in add_to_queue()
1493 try_futile = !may_grant(gl, current_gh, gh); in add_to_queue()
1495 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags)) in add_to_queue()
1499 list_for_each_entry(gh2, &gl->gl_holders, gh_list) { in add_to_queue()
1508 list_for_each_entry(gh2, &gl->gl_holders, gh_list) { in add_to_queue()
1520 gfs2_glstats_inc(gl, GFS2_LKS_QCOUNT); in add_to_queue()
1521 gfs2_sbstats_inc(gl, GFS2_LKS_QCOUNT); in add_to_queue()
1523 list_add_tail(&gh->gh_list, &gl->gl_holders); in add_to_queue()
1527 gh = list_first_entry(&gl->gl_holders, struct gfs2_holder, gh_list); in add_to_queue()
1528 spin_unlock(&gl->gl_lockref.lock); in add_to_queue()
1530 sdp->sd_lockstruct.ls_ops->lm_cancel(gl); in add_to_queue()
1531 spin_lock(&gl->gl_lockref.lock); in add_to_queue()
1543 gfs2_dump_glock(NULL, gl, true); in add_to_queue()
1558 struct gfs2_glock *gl = gh->gh_gl; in gfs2_glock_nq() local
1561 if (glock_blocked_by_withdraw(gl) && !(gh->gh_flags & LM_FLAG_NOEXP)) in gfs2_glock_nq()
1564 if (test_bit(GLF_LRU, &gl->gl_flags)) in gfs2_glock_nq()
1565 gfs2_glock_remove_from_lru(gl); in gfs2_glock_nq()
1568 spin_lock(&gl->gl_lockref.lock); in gfs2_glock_nq()
1571 test_and_clear_bit(GLF_FROZEN, &gl->gl_flags))) { in gfs2_glock_nq()
1572 set_bit(GLF_REPLY_PENDING, &gl->gl_flags); in gfs2_glock_nq()
1573 gl->gl_lockref.count++; in gfs2_glock_nq()
1574 __gfs2_glock_queue_work(gl, 0); in gfs2_glock_nq()
1576 run_queue(gl, 1); in gfs2_glock_nq()
1577 spin_unlock(&gl->gl_lockref.lock); in gfs2_glock_nq()
1597 static inline bool needs_demote(struct gfs2_glock *gl) in needs_demote() argument
1599 return (test_bit(GLF_DEMOTE, &gl->gl_flags) || in needs_demote()
1600 test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags)); in needs_demote()
1605 struct gfs2_glock *gl = gh->gh_gl; in __gfs2_glock_dq() local
1615 handle_callback(gl, LM_ST_UNLOCKED, 0, false); in __gfs2_glock_dq()
1625 if (!needs_demote(gl)) { in __gfs2_glock_dq()
1626 if (list_empty(&gl->gl_holders)) in __gfs2_glock_dq()
1630 if (!test_bit(GLF_LFLUSH, &gl->gl_flags) && demote_ok(gl)) in __gfs2_glock_dq()
1631 gfs2_glock_add_to_lru(gl); in __gfs2_glock_dq()
1634 gl->gl_lockref.count++; in __gfs2_glock_dq()
1635 if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) && in __gfs2_glock_dq()
1636 !test_bit(GLF_DEMOTE, &gl->gl_flags) && in __gfs2_glock_dq()
1637 gl->gl_name.ln_type == LM_TYPE_INODE) in __gfs2_glock_dq()
1638 delay = gl->gl_hold_time; in __gfs2_glock_dq()
1639 __gfs2_glock_queue_work(gl, delay); in __gfs2_glock_dq()
1650 struct gfs2_glock *gl = gh->gh_gl; in gfs2_glock_dq() local
1651 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; in gfs2_glock_dq()
1653 spin_lock(&gl->gl_lockref.lock); in gfs2_glock_dq()
1662 if (list_is_first(&gh->gh_list, &gl->gl_holders) && in gfs2_glock_dq()
1664 spin_unlock(&gl->gl_lockref.lock); in gfs2_glock_dq()
1665 gl->gl_name.ln_sbd->sd_lockstruct.ls_ops->lm_cancel(gl); in gfs2_glock_dq()
1667 spin_lock(&gl->gl_lockref.lock); in gfs2_glock_dq()
1678 glock_blocked_by_withdraw(gl) && in gfs2_glock_dq()
1681 spin_unlock(&gl->gl_lockref.lock); in gfs2_glock_dq()
1685 spin_lock(&gl->gl_lockref.lock); in gfs2_glock_dq()
1690 spin_unlock(&gl->gl_lockref.lock); in gfs2_glock_dq()
1695 struct gfs2_glock *gl = gh->gh_gl; in gfs2_glock_dq_wait() local
1698 wait_on_bit(&gl->gl_flags, GLF_DEMOTE, TASK_UNINTERRUPTIBLE); in gfs2_glock_dq_wait()
1729 struct gfs2_glock *gl; in gfs2_glock_nq_num() local
1732 error = gfs2_glock_get(sdp, number, glops, CREATE, &gl); in gfs2_glock_nq_num()
1734 error = gfs2_glock_nq_init(gl, state, flags, gh); in gfs2_glock_nq_num()
1735 gfs2_glock_put(gl); in gfs2_glock_nq_num()
1846 void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state) in gfs2_glock_cb() argument
1852 gfs2_glock_hold(gl); in gfs2_glock_cb()
1853 spin_lock(&gl->gl_lockref.lock); in gfs2_glock_cb()
1854 holdtime = gl->gl_tchange + gl->gl_hold_time; in gfs2_glock_cb()
1855 if (!list_empty(&gl->gl_holders) && in gfs2_glock_cb()
1856 gl->gl_name.ln_type == LM_TYPE_INODE) { in gfs2_glock_cb()
1859 if (test_bit(GLF_REPLY_PENDING, &gl->gl_flags)) in gfs2_glock_cb()
1860 delay = gl->gl_hold_time; in gfs2_glock_cb()
1862 handle_callback(gl, state, delay, true); in gfs2_glock_cb()
1863 __gfs2_glock_queue_work(gl, delay); in gfs2_glock_cb()
1864 spin_unlock(&gl->gl_lockref.lock); in gfs2_glock_cb()
1878 static int gfs2_should_freeze(const struct gfs2_glock *gl) in gfs2_should_freeze() argument
1882 if (gl->gl_reply & ~LM_OUT_ST_MASK) in gfs2_should_freeze()
1884 if (gl->gl_target == LM_ST_UNLOCKED) in gfs2_should_freeze()
1887 list_for_each_entry(gh, &gl->gl_holders, gh_list) { in gfs2_should_freeze()
1906 void gfs2_glock_complete(struct gfs2_glock *gl, int ret) in gfs2_glock_complete() argument
1908 struct lm_lockstruct *ls = &gl->gl_name.ln_sbd->sd_lockstruct; in gfs2_glock_complete()
1910 spin_lock(&gl->gl_lockref.lock); in gfs2_glock_complete()
1911 gl->gl_reply = ret; in gfs2_glock_complete()
1914 if (gfs2_should_freeze(gl)) { in gfs2_glock_complete()
1915 set_bit(GLF_FROZEN, &gl->gl_flags); in gfs2_glock_complete()
1916 spin_unlock(&gl->gl_lockref.lock); in gfs2_glock_complete()
1921 gl->gl_lockref.count++; in gfs2_glock_complete()
1922 set_bit(GLF_REPLY_PENDING, &gl->gl_flags); in gfs2_glock_complete()
1923 __gfs2_glock_queue_work(gl, 0); in gfs2_glock_complete()
1924 spin_unlock(&gl->gl_lockref.lock); in gfs2_glock_complete()
1961 struct gfs2_glock *gl; in gfs2_dispose_glock_lru() local
1966 gl = list_first_entry(list, struct gfs2_glock, gl_lru); in gfs2_dispose_glock_lru()
1967 list_del_init(&gl->gl_lru); in gfs2_dispose_glock_lru()
1968 clear_bit(GLF_LRU, &gl->gl_flags); in gfs2_dispose_glock_lru()
1969 if (!spin_trylock(&gl->gl_lockref.lock)) { in gfs2_dispose_glock_lru()
1971 list_add(&gl->gl_lru, &lru_list); in gfs2_dispose_glock_lru()
1972 set_bit(GLF_LRU, &gl->gl_flags); in gfs2_dispose_glock_lru()
1976 if (test_and_set_bit(GLF_LOCK, &gl->gl_flags)) { in gfs2_dispose_glock_lru()
1977 spin_unlock(&gl->gl_lockref.lock); in gfs2_dispose_glock_lru()
1980 gl->gl_lockref.count++; in gfs2_dispose_glock_lru()
1981 if (demote_ok(gl)) in gfs2_dispose_glock_lru()
1982 handle_callback(gl, LM_ST_UNLOCKED, 0, false); in gfs2_dispose_glock_lru()
1983 WARN_ON(!test_and_clear_bit(GLF_LOCK, &gl->gl_flags)); in gfs2_dispose_glock_lru()
1984 __gfs2_glock_queue_work(gl, 0); in gfs2_dispose_glock_lru()
1985 spin_unlock(&gl->gl_lockref.lock); in gfs2_dispose_glock_lru()
2001 struct gfs2_glock *gl, *next; in gfs2_scan_glock_lru() local
2006 list_for_each_entry_safe(gl, next, &lru_list, gl_lru) { in gfs2_scan_glock_lru()
2010 if (!test_bit(GLF_LOCK, &gl->gl_flags)) { in gfs2_scan_glock_lru()
2011 if (!spin_trylock(&gl->gl_lockref.lock)) in gfs2_scan_glock_lru()
2013 if (gl->gl_lockref.count <= 1 && in gfs2_scan_glock_lru()
2014 (gl->gl_state == LM_ST_UNLOCKED || in gfs2_scan_glock_lru()
2015 demote_ok(gl))) { in gfs2_scan_glock_lru()
2016 list_move(&gl->gl_lru, &dispose); in gfs2_scan_glock_lru()
2020 spin_unlock(&gl->gl_lockref.lock); in gfs2_scan_glock_lru()
2062 struct gfs2_glock *gl; in glock_hash_walk() local
2070 while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl)) { in glock_hash_walk()
2071 if (gl->gl_name.ln_sbd == sdp) in glock_hash_walk()
2072 examiner(gl); in glock_hash_walk()
2076 } while (cond_resched(), gl == ERR_PTR(-EAGAIN)); in glock_hash_walk()
2081 void gfs2_cancel_delete_work(struct gfs2_glock *gl) in gfs2_cancel_delete_work() argument
2083 clear_bit(GLF_TRY_TO_EVICT, &gl->gl_flags); in gfs2_cancel_delete_work()
2084 clear_bit(GLF_VERIFY_EVICT, &gl->gl_flags); in gfs2_cancel_delete_work()
2085 if (cancel_delayed_work(&gl->gl_delete)) in gfs2_cancel_delete_work()
2086 gfs2_glock_put(gl); in gfs2_cancel_delete_work()
2089 static void flush_delete_work(struct gfs2_glock *gl) in flush_delete_work() argument
2091 if (gl->gl_name.ln_type == LM_TYPE_IOPEN) { in flush_delete_work()
2092 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; in flush_delete_work()
2094 if (cancel_delayed_work(&gl->gl_delete)) { in flush_delete_work()
2096 &gl->gl_delete, 0); in flush_delete_work()
2113 static void thaw_glock(struct gfs2_glock *gl) in thaw_glock() argument
2115 if (!test_and_clear_bit(GLF_FROZEN, &gl->gl_flags)) in thaw_glock()
2117 if (!lockref_get_not_dead(&gl->gl_lockref)) in thaw_glock()
2119 set_bit(GLF_REPLY_PENDING, &gl->gl_flags); in thaw_glock()
2120 gfs2_glock_queue_work(gl, 0); in thaw_glock()
2129 static void clear_glock(struct gfs2_glock *gl) in clear_glock() argument
2131 gfs2_glock_remove_from_lru(gl); in clear_glock()
2133 spin_lock(&gl->gl_lockref.lock); in clear_glock()
2134 if (!__lockref_is_dead(&gl->gl_lockref)) { in clear_glock()
2135 gl->gl_lockref.count++; in clear_glock()
2136 if (gl->gl_state != LM_ST_UNLOCKED) in clear_glock()
2137 handle_callback(gl, LM_ST_UNLOCKED, 0, false); in clear_glock()
2138 __gfs2_glock_queue_work(gl, 0); in clear_glock()
2140 spin_unlock(&gl->gl_lockref.lock); in clear_glock()
2154 static void dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid) in dump_glock() argument
2156 spin_lock(&gl->gl_lockref.lock); in dump_glock()
2157 gfs2_dump_glock(seq, gl, fsid); in dump_glock()
2158 spin_unlock(&gl->gl_lockref.lock); in dump_glock()
2161 static void dump_glock_func(struct gfs2_glock *gl) in dump_glock_func() argument
2163 dump_glock(NULL, gl, true); in dump_glock_func()
2166 static void withdraw_dq(struct gfs2_glock *gl) in withdraw_dq() argument
2168 spin_lock(&gl->gl_lockref.lock); in withdraw_dq()
2169 if (!__lockref_is_dead(&gl->gl_lockref) && in withdraw_dq()
2170 glock_blocked_by_withdraw(gl)) in withdraw_dq()
2171 do_error(gl, LM_OUT_ERROR); /* remove pending waiters */ in withdraw_dq()
2172 spin_unlock(&gl->gl_lockref.lock); in withdraw_dq()
2275 static const char *gflags2str(char *buf, const struct gfs2_glock *gl) in gflags2str() argument
2277 const unsigned long *gflags = &gl->gl_flags; in gflags2str()
2300 if (!list_empty(&gl->gl_holders)) in gflags2str()
2304 if (gl->gl_object) in gflags2str()
2340 void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid) in gfs2_dump_glock() argument
2342 const struct gfs2_glock_operations *glops = gl->gl_ops; in gfs2_dump_glock()
2346 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd; in gfs2_dump_glock()
2350 if (gl->gl_ops->go_flags & GLOF_ASPACE) { in gfs2_dump_glock()
2351 struct address_space *mapping = gfs2_glock2aspace(gl); in gfs2_dump_glock()
2358 dtime = jiffies - gl->gl_demote_time; in gfs2_dump_glock()
2360 if (!test_bit(GLF_DEMOTE, &gl->gl_flags)) in gfs2_dump_glock()
2364 fs_id_buf, state2str(gl->gl_state), in gfs2_dump_glock()
2365 gl->gl_name.ln_type, in gfs2_dump_glock()
2366 (unsigned long long)gl->gl_name.ln_number, in gfs2_dump_glock()
2367 gflags2str(gflags_buf, gl), in gfs2_dump_glock()
2368 state2str(gl->gl_target), in gfs2_dump_glock()
2369 state2str(gl->gl_demote_state), dtime, in gfs2_dump_glock()
2370 atomic_read(&gl->gl_ail_count), in gfs2_dump_glock()
2371 atomic_read(&gl->gl_revokes), in gfs2_dump_glock()
2372 (int)gl->gl_lockref.count, gl->gl_hold_time, nrpages); in gfs2_dump_glock()
2374 list_for_each_entry(gh, &gl->gl_holders, gh_list) in gfs2_dump_glock()
2377 if (gl->gl_state != LM_ST_UNLOCKED && glops->go_dump) in gfs2_dump_glock()
2378 glops->go_dump(seq, gl, fs_id_buf); in gfs2_dump_glock()
2383 struct gfs2_glock *gl = iter_ptr; in gfs2_glstats_seq_show() local
2386 gl->gl_name.ln_type, in gfs2_glstats_seq_show()
2387 (unsigned long long)gl->gl_name.ln_number, in gfs2_glstats_seq_show()
2388 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTT], in gfs2_glstats_seq_show()
2389 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVAR], in gfs2_glstats_seq_show()
2390 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTB], in gfs2_glstats_seq_show()
2391 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVARB], in gfs2_glstats_seq_show()
2392 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRT], in gfs2_glstats_seq_show()
2393 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRTVAR], in gfs2_glstats_seq_show()
2394 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_DCOUNT], in gfs2_glstats_seq_show()
2395 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_QCOUNT]); in gfs2_glstats_seq_show()
2490 struct gfs2_glock *gl = gi->gl; in gfs2_glock_iter_next() local
2492 if (gl) { in gfs2_glock_iter_next()
2495 if (!lockref_put_not_zero(&gl->gl_lockref)) in gfs2_glock_iter_next()
2496 gfs2_glock_queue_put(gl); in gfs2_glock_iter_next()
2499 gl = rhashtable_walk_next(&gi->hti); in gfs2_glock_iter_next()
2500 if (IS_ERR_OR_NULL(gl)) { in gfs2_glock_iter_next()
2501 if (gl == ERR_PTR(-EAGAIN)) { in gfs2_glock_iter_next()
2505 gl = NULL; in gfs2_glock_iter_next()
2508 if (gl->gl_name.ln_sbd != gi->sdp) in gfs2_glock_iter_next()
2511 if (!lockref_get_not_dead(&gl->gl_lockref)) in gfs2_glock_iter_next()
2515 if (__lockref_is_dead(&gl->gl_lockref)) in gfs2_glock_iter_next()
2520 gi->gl = gl; in gfs2_glock_iter_next()
2545 return gi->gl; in gfs2_glock_seq_start()
2556 return gi->gl; in gfs2_glock_seq_next()
2635 gi->gl = NULL; in __gfs2_glocks_open()
2651 if (gi->gl) in gfs2_glocks_release()
2652 gfs2_glock_put(gi->gl); in gfs2_glocks_release()
2802 struct gfs2_glock *gl; in gfs2_glockfd_seq_show() local
2805 gl = GFS2_I(inode)->i_iopen_gh.gh_gl; in gfs2_glockfd_seq_show()
2806 if (gl) { in gfs2_glockfd_seq_show()
2808 i->tgid, i->fd, gl->gl_name.ln_type, in gfs2_glockfd_seq_show()
2809 (unsigned long long)gl->gl_name.ln_number); in gfs2_glockfd_seq_show()