63 static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target);
280 static inline int may_grant(const struct gfs2_glock *gl, const struct gfs2_holder *gh)  in may_grant()  argument
283 	if ((gh->gh_state == LM_ST_EXCLUSIVE ||  in may_grant()
284 	     gh_head->gh_state == LM_ST_EXCLUSIVE) && gh != gh_head)  in may_grant()
286 	if (gl->gl_state == gh->gh_state)  in may_grant()
288 	if (gh->gh_flags & GL_EXACT)  in may_grant()
291 		if (gh->gh_state == LM_ST_SHARED && gh_head->gh_state == LM_ST_SHARED)  in may_grant()
293 		if (gh->gh_state == LM_ST_DEFERRED && gh_head->gh_state == LM_ST_DEFERRED)  in may_grant()
296 	if (gl->gl_state != LM_ST_UNLOCKED && (gh->gh_flags & LM_FLAG_ANY))  in may_grant()
301 static void gfs2_holder_wake(struct gfs2_holder *gh)  in gfs2_holder_wake()  argument
303 	clear_bit(HIF_WAIT, &gh->gh_iflags);  in gfs2_holder_wake()
305 	wake_up_bit(&gh->gh_iflags, HIF_WAIT);  in gfs2_holder_wake()
315 	struct gfs2_holder *gh, *tmp;  in do_error()  local
317 	list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {  in do_error()
318 		if (test_bit(HIF_HOLDER, &gh->gh_iflags))  in do_error()
321 			gh->gh_error = -EIO;  in do_error()
322 		else if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))  in do_error()
323 			gh->gh_error = GLR_TRYFAILED;  in do_error()
326 		list_del_init(&gh->gh_list);  in do_error()
327 		trace_gfs2_glock_queue(gh, 0);  in do_error()
328 		gfs2_holder_wake(gh);  in do_error()
345 	struct gfs2_holder *gh, *tmp;  in do_promote()  local
349 	list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {  in do_promote()
350 		if (test_bit(HIF_HOLDER, &gh->gh_iflags))  in do_promote()
352 		if (may_grant(gl, gh)) {  in do_promote()
353 			if (gh->gh_list.prev == &gl->gl_holders &&  in do_promote()
357 				ret = glops->go_lock(gh);  in do_promote()
362 					gh->gh_error = ret;  in do_promote()
363 					list_del_init(&gh->gh_list);  in do_promote()
364 					trace_gfs2_glock_queue(gh, 0);  in do_promote()
365 					gfs2_holder_wake(gh);  in do_promote()
368 				set_bit(HIF_HOLDER, &gh->gh_iflags);  in do_promote()
369 				trace_gfs2_promote(gh, 1);  in do_promote()
370 				gfs2_holder_wake(gh);  in do_promote()
373 			set_bit(HIF_HOLDER, &gh->gh_iflags);  in do_promote()
374 			trace_gfs2_promote(gh, 0);  in do_promote()
375 			gfs2_holder_wake(gh);  in do_promote()
378 		if (gh->gh_list.prev == &gl->gl_holders)  in do_promote()
393 	struct gfs2_holder *gh;  in find_first_waiter()  local
395 	list_for_each_entry(gh, &gl->gl_holders, gh_list) {  in find_first_waiter()
396 		if (!test_bit(HIF_HOLDER, &gh->gh_iflags))  in find_first_waiter()
397 			return gh;  in find_first_waiter()
452 	struct gfs2_holder *gh;  in finish_xmote()  local
459 	gh = find_first_waiter(gl);  in finish_xmote()
468 		if (gh && !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) {  in finish_xmote()
471 				if ((gh->gh_flags & LM_FLAG_PRIORITY) == 0)  in finish_xmote()
472 					list_move_tail(&gh->gh_list, &gl->gl_holders);  in finish_xmote()
473 				gh = find_first_waiter(gl);  in finish_xmote()
474 				gl->gl_target = gh->gh_state;  in finish_xmote()
479 			    (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {  in finish_xmote()
489 			do_xmote(gl, gh, gl->gl_target);  in finish_xmote()
494 			do_xmote(gl, gh, LM_ST_UNLOCKED);  in finish_xmote()
510 			rv = glops->go_xmote_bh(gl, gh);  in finish_xmote()
535 static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target)  in do_xmote()  argument
541 	unsigned int lck_flags = (unsigned int)(gh ? gh->gh_flags : 0);  in do_xmote()
599 	struct gfs2_holder *gh;  in find_first_holder()  local
602 		gh = list_entry(gl->gl_holders.next, struct gfs2_holder, gh_list);  in find_first_holder()
603 		if (test_bit(HIF_HOLDER, &gh->gh_iflags))  in find_first_holder()
604 			return gh;  in find_first_holder()
620 	struct gfs2_holder *gh = NULL;  in run_queue()  local
645 		gh = find_first_waiter(gl);  in run_queue()
646 		gl->gl_target = gh->gh_state;  in run_queue()
647 		if (!(gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)))  in run_queue()
650 	do_xmote(gl, gh, gl->gl_target);  in run_queue()
884 		      struct gfs2_holder *gh)  in gfs2_holder_init()  argument
886 	INIT_LIST_HEAD(&gh->gh_list);  in gfs2_holder_init()
887 	gh->gh_gl = gl;  in gfs2_holder_init()
888 	gh->gh_ip = _RET_IP_;  in gfs2_holder_init()
889 	gh->gh_owner_pid = get_pid(task_pid(current));  in gfs2_holder_init()
890 	gh->gh_state = state;  in gfs2_holder_init()
891 	gh->gh_flags = flags;  in gfs2_holder_init()
892 	gh->gh_error = 0;  in gfs2_holder_init()
893 	gh->gh_iflags = 0;  in gfs2_holder_init()
907 void gfs2_holder_reinit(unsigned int state, u16 flags, struct gfs2_holder *gh)  in gfs2_holder_reinit()  argument
909 	gh->gh_state = state;  in gfs2_holder_reinit()
910 	gh->gh_flags = flags;  in gfs2_holder_reinit()
911 	gh->gh_iflags = 0;  in gfs2_holder_reinit()
912 	gh->gh_ip = _RET_IP_;  in gfs2_holder_reinit()
913 	put_pid(gh->gh_owner_pid);  in gfs2_holder_reinit()
914 	gh->gh_owner_pid = get_pid(task_pid(current));  in gfs2_holder_reinit()
923 void gfs2_holder_uninit(struct gfs2_holder *gh)  in gfs2_holder_uninit()  argument
925 	put_pid(gh->gh_owner_pid);  in gfs2_holder_uninit()
926 	gfs2_glock_put(gh->gh_gl);  in gfs2_holder_uninit()
927 	gfs2_holder_mark_uninitialized(gh);  in gfs2_holder_uninit()
928 	gh->gh_ip = 0;  in gfs2_holder_uninit()
938 int gfs2_glock_wait(struct gfs2_holder *gh)  in gfs2_glock_wait()  argument
943 	wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE);  in gfs2_glock_wait()
946 		gh->gh_gl->gl_hold_time = min(gh->gh_gl->gl_hold_time +  in gfs2_glock_wait()
949 	return gh->gh_error;  in gfs2_glock_wait()
1008 static inline void add_to_queue(struct gfs2_holder *gh)  in add_to_queue()  argument
1012 	struct gfs2_glock *gl = gh->gh_gl;  in add_to_queue()
1018 	BUG_ON(gh->gh_owner_pid == NULL);  in add_to_queue()
1019 	if (test_and_set_bit(HIF_WAIT, &gh->gh_iflags))  in add_to_queue()
1022 	if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) {  in add_to_queue()
1024 			try_futile = !may_grant(gl, gh);  in add_to_queue()
1030 		if (unlikely(gh2->gh_owner_pid == gh->gh_owner_pid &&  in add_to_queue()
1031 		    (gh->gh_gl->gl_ops->go_type != LM_TYPE_FLOCK)))  in add_to_queue()
1036 			gh->gh_error = GLR_TRYFAILED;  in add_to_queue()
1037 			gfs2_holder_wake(gh);  in add_to_queue()
1042 		if (unlikely((gh->gh_flags & LM_FLAG_PRIORITY) && !insert_pt))  in add_to_queue()
1046 	trace_gfs2_glock_queue(gh, 1);  in add_to_queue()
1050 		list_add_tail(&gh->gh_list, &gl->gl_holders);  in add_to_queue()
1051 		if (unlikely(gh->gh_flags & LM_FLAG_PRIORITY))  in add_to_queue()
1055 	list_add_tail(&gh->gh_list, insert_pt);  in add_to_queue()
1057 	gh = list_entry(gl->gl_holders.next, struct gfs2_holder, gh_list);  in add_to_queue()
1058 	if (!(gh->gh_flags & LM_FLAG_PRIORITY)) {  in add_to_queue()
1071 	pr_err("new: %pSR\n", (void *)gh->gh_ip);  in add_to_queue()
1072 	pr_err("pid: %d\n", pid_nr(gh->gh_owner_pid));  in add_to_queue()
1074 	       gh->gh_gl->gl_name.ln_type, gh->gh_state);  in add_to_queue()
1088 int gfs2_glock_nq(struct gfs2_holder *gh)  in gfs2_glock_nq()  argument
1090 	struct gfs2_glock *gl = gh->gh_gl;  in gfs2_glock_nq()
1101 	add_to_queue(gh);  in gfs2_glock_nq()
1102 	if (unlikely((LM_FLAG_NOEXP & gh->gh_flags) &&  in gfs2_glock_nq()
1111 	if (!(gh->gh_flags & GL_ASYNC))  in gfs2_glock_nq()
1112 		error = gfs2_glock_wait(gh);  in gfs2_glock_nq()
1124 int gfs2_glock_poll(struct gfs2_holder *gh)  in gfs2_glock_poll()  argument
1126 	return test_bit(HIF_WAIT, &gh->gh_iflags) ? 0 : 1;  in gfs2_glock_poll()
1135 void gfs2_glock_dq(struct gfs2_holder *gh)  in gfs2_glock_dq()  argument
1137 	struct gfs2_glock *gl = gh->gh_gl;  in gfs2_glock_dq()
1143 	if (gh->gh_flags & GL_NOCACHE)  in gfs2_glock_dq()
1146 	list_del_init(&gh->gh_list);  in gfs2_glock_dq()
1147 	clear_bit(HIF_HOLDER, &gh->gh_iflags);  in gfs2_glock_dq()
1152 			glops->go_unlock(gh);  in gfs2_glock_dq()
1165 	trace_gfs2_glock_queue(gh, 0);  in gfs2_glock_dq()
1177 void gfs2_glock_dq_wait(struct gfs2_holder *gh)  in gfs2_glock_dq_wait()  argument
1179 	struct gfs2_glock *gl = gh->gh_gl;  in gfs2_glock_dq_wait()
1180 	gfs2_glock_dq(gh);  in gfs2_glock_dq_wait()
1191 void gfs2_glock_dq_uninit(struct gfs2_holder *gh)  in gfs2_glock_dq_uninit()  argument
1193 	gfs2_glock_dq(gh);  in gfs2_glock_dq_uninit()
1194 	gfs2_holder_uninit(gh);  in gfs2_glock_dq_uninit()
1211 		      unsigned int state, u16 flags, struct gfs2_holder *gh)  in gfs2_glock_nq_num()  argument
1218 		error = gfs2_glock_nq_init(gl, state, flags, gh);  in gfs2_glock_nq_num()
1368 	const struct gfs2_holder *gh;  in gfs2_should_freeze()  local
1375 	list_for_each_entry(gh, &gl->gl_holders, gh_list) {  in gfs2_should_freeze()
1376 		if (test_bit(HIF_HOLDER, &gh->gh_iflags))  in gfs2_should_freeze()
1378 		if (LM_FLAG_NOEXP & gh->gh_flags)  in gfs2_should_freeze()
1707 static void dump_holder(struct seq_file *seq, const struct gfs2_holder *gh)  in dump_holder()  argument
1713 	if (gh->gh_owner_pid)  in dump_holder()
1714 		gh_owner = pid_task(gh->gh_owner_pid, PIDTYPE_PID);  in dump_holder()
1716 		       state2str(gh->gh_state),  in dump_holder()
1717 		       hflags2str(flags_buf, gh->gh_flags, gh->gh_iflags),  in dump_holder()
1718 		       gh->gh_error,  in dump_holder()
1719 		       gh->gh_owner_pid ? (long)pid_nr(gh->gh_owner_pid) : -1,  in dump_holder()
1721 		       (void *)gh->gh_ip);  in dump_holder()
1783 	const struct gfs2_holder *gh;  in gfs2_dump_glock()  local
1801 	list_for_each_entry(gh, &gl->gl_holders, gh_list)  in gfs2_dump_glock()
1802 		dump_holder(seq, gh);  in gfs2_dump_glock()