191 static inline struct lock_class *hlock_class(struct held_lock *hlock)  in hlock_class()  argument
193 	unsigned int class_idx = hlock->class_idx;  in hlock_class()
312 static void lock_release_holdtime(struct held_lock *hlock)  in lock_release_holdtime()  argument
320 	holdtime = lockstat_clock() - hlock->holdtime_stamp;  in lock_release_holdtime()
322 	stats = get_lock_stats(hlock_class(hlock));  in lock_release_holdtime()
323 	if (hlock->read)  in lock_release_holdtime()
329 static inline void lock_release_holdtime(struct held_lock *hlock)  in lock_release_holdtime()  argument
396 static inline u16 hlock_id(struct held_lock *hlock)  in hlock_id()  argument
400 	return (hlock->class_idx | (hlock->read << MAX_LOCKDEP_KEYS_BITS));  in hlock_id()
725 static void print_lock(struct held_lock *hlock)  in print_lock()  argument
737 	struct lock_class *lock = hlock_class(hlock);  in print_lock()
744 	printk(KERN_CONT "%px", hlock->instance);  in print_lock()
746 	printk(KERN_CONT ", at: %pS\n", (void *)hlock->acquire_ip);  in print_lock()
1615 				 struct held_lock *hlock)  in bfs_init_root()  argument
1617 	__bfs_init_root(lock, hlock_class(hlock));  in bfs_init_root()
1618 	lock->only_xr = (hlock->read == 2);  in bfs_init_root()
1629 				  struct held_lock *hlock)  in bfs_init_rootb()  argument
1631 	__bfs_init_root(lock, hlock_class(hlock));  in bfs_init_rootb()
1632 	lock->only_xr = (hlock->read != 0);  in bfs_init_rootb()
1926 	struct held_lock *hlock = (struct held_lock *)data;  in hlock_equal()  local
1928 	return hlock_class(hlock) == entry->class && /* Found A -> .. -> B */  in hlock_equal()
1929 	       (hlock->read == 2 ||  /* A -> B is -(*R)-> */  in hlock_equal()
1953 	struct held_lock *hlock = (struct held_lock *)data;  in hlock_conflict()  local
1955 	return hlock_class(hlock) == entry->class && /* Found A -> .. -> B */  in hlock_conflict()
1956 	       (hlock->read == 0 || /* B -> A is -(E*)-> */  in hlock_conflict()
2971 	struct held_lock *hlock;  in check_prevs_add()  local
2990 		hlock = curr->held_locks + depth - 1;  in check_prevs_add()
2992 		if (hlock->check) {  in check_prevs_add()
2993 			int ret = check_prev_add(curr, hlock, next, distance, &trace);  in check_prevs_add()
3003 			if (!hlock->trylock)  in check_prevs_add()
3279 					struct held_lock *hlock)  in get_first_held_lock()  argument
3286 		if (hlock_curr->irq_context != hlock->irq_context)  in get_first_held_lock()
3311 	struct held_lock *hlock;  in print_chain_keys_held_locks()  local
3319 		hlock = curr->held_locks + i;  in print_chain_keys_held_locks()
3320 		chain_key = print_chain_key_iteration(hlock_id(hlock), chain_key);  in print_chain_keys_held_locks()
3322 		print_lock(hlock);  in print_chain_keys_held_locks()
3375 			struct held_lock *hlock,  in check_no_collision()  argument
3381 	i = get_first_held_lock(curr, hlock);  in check_no_collision()
3384 		print_collision(curr, hlock, chain);  in check_no_collision()
3392 			print_collision(curr, hlock, chain);  in check_no_collision()
3435 				  struct held_lock *hlock,  in add_chain_cache()  argument
3460 	chain->irq_context = hlock->irq_context;  in add_chain_cache()
3461 	i = get_first_held_lock(curr, hlock);  in add_chain_cache()
3484 	chain_hlocks[chain->base + j] = hlock_id(hlock);  in add_chain_cache()
3517 					 struct held_lock *hlock,  in lookup_chain_cache_add()  argument
3520 	struct lock_class *class = hlock_class(hlock);  in lookup_chain_cache_add()
3525 		if (!check_no_collision(curr, hlock, chain))  in lookup_chain_cache_add()
3555 	if (!add_chain_cache(curr, hlock, chain_key))  in lookup_chain_cache_add()
3562 			  struct held_lock *hlock,  in validate_chain()  argument
3575 	if (!hlock->trylock && hlock->check &&  in validate_chain()
3576 	    lookup_chain_cache_add(curr, hlock, chain_key)) {  in validate_chain()
3595 		int ret = check_deadlock(curr, hlock);  in validate_chain()
3608 			if (!check_prevs_add(curr, hlock))  in validate_chain()
3623 				 struct held_lock *hlock,  in validate_chain()  argument
3639 	struct held_lock *hlock, *prev_hlock = NULL;  in check_chain_key()  local
3644 		hlock = curr->held_locks + i;  in check_chain_key()
3645 		if (chain_key != hlock->prev_chain_key) {  in check_chain_key()
3654 				(unsigned long long)hlock->prev_chain_key);  in check_chain_key()
3662 		if (DEBUG_LOCKS_WARN_ON(!test_bit(hlock->class_idx, lock_classes_in_use)))  in check_chain_key()
3666 							hlock->irq_context))  in check_chain_key()
3668 		chain_key = iterate_chain_key(chain_key, hlock_id(hlock));  in check_chain_key()
3669 		prev_hlock = hlock;  in check_chain_key()
3997 	struct held_lock *hlock;  in mark_held_locks()  local
4002 		hlock = curr->held_locks + i;  in mark_held_locks()
4004 		if (hlock->read)  in mark_held_locks()
4009 		if (!hlock->check)  in mark_held_locks()
4012 		if (!mark_lock(curr, hlock, hlock_bit))  in mark_held_locks()
4282 mark_usage(struct task_struct *curr, struct held_lock *hlock, int check)  in mark_usage()  argument
4291 	if (!hlock->trylock) {  in mark_usage()
4292 		if (hlock->read) {  in mark_usage()
4294 				if (!mark_lock(curr, hlock,  in mark_usage()
4298 				if (!mark_lock(curr, hlock,  in mark_usage()
4303 				if (!mark_lock(curr, hlock, LOCK_USED_IN_HARDIRQ))  in mark_usage()
4306 				if (!mark_lock(curr, hlock, LOCK_USED_IN_SOFTIRQ))  in mark_usage()
4310 	if (!hlock->hardirqs_off) {  in mark_usage()
4311 		if (hlock->read) {  in mark_usage()
4312 			if (!mark_lock(curr, hlock,  in mark_usage()
4316 				if (!mark_lock(curr, hlock,  in mark_usage()
4320 			if (!mark_lock(curr, hlock,  in mark_usage()
4324 				if (!mark_lock(curr, hlock,  in mark_usage()
4332 	if (!mark_lock(curr, hlock, LOCK_USED))  in mark_usage()
4345 		struct held_lock *hlock)  in separate_irq_context()  argument
4361 		if (prev_hlock->irq_context != hlock->irq_context)  in separate_irq_context()
4458 				struct held_lock *hlock)  in print_lock_invalid_wait_context()  argument
4474 	print_lock(hlock);  in print_lock_invalid_wait_context()
4553 mark_usage(struct task_struct *curr, struct held_lock *hlock, int check)  in mark_usage()  argument
4564 		struct held_lock *hlock)  in separate_irq_context()  argument
4646 				struct held_lock *hlock,  in print_lock_nested_lock_not_held()  argument
4661 	print_lock(hlock);  in print_lock_nested_lock_not_held()
4664 	pr_warn("%s\n", hlock->nest_lock->name);  in print_lock_nested_lock_not_held()
4693 	struct held_lock *hlock;  in __lock_acquire()  local
4741 		hlock = curr->held_locks + depth - 1;  in __lock_acquire()
4742 		if (hlock->class_idx == class_idx && nest_lock) {  in __lock_acquire()
4746 			if (!hlock->references)  in __lock_acquire()
4747 				hlock->references++;  in __lock_acquire()
4749 			hlock->references += references;  in __lock_acquire()
4752 			if (DEBUG_LOCKS_WARN_ON(hlock->references < references))  in __lock_acquire()
4759 	hlock = curr->held_locks + depth;  in __lock_acquire()
4766 	hlock->class_idx = class_idx;  in __lock_acquire()
4767 	hlock->acquire_ip = ip;  in __lock_acquire()
4768 	hlock->instance = lock;  in __lock_acquire()
4769 	hlock->nest_lock = nest_lock;  in __lock_acquire()
4770 	hlock->irq_context = task_irq_context(curr);  in __lock_acquire()
4771 	hlock->trylock = trylock;  in __lock_acquire()
4772 	hlock->read = read;  in __lock_acquire()
4773 	hlock->check = check;  in __lock_acquire()
4774 	hlock->hardirqs_off = !!hardirqs_off;  in __lock_acquire()
4775 	hlock->references = references;  in __lock_acquire()
4777 	hlock->waittime_stamp = 0;  in __lock_acquire()
4778 	hlock->holdtime_stamp = lockstat_clock();  in __lock_acquire()
4780 	hlock->pin_count = pin_count;  in __lock_acquire()
4782 	if (check_wait_context(curr, hlock))  in __lock_acquire()
4786 	if (!mark_usage(curr, hlock, check))  in __lock_acquire()
4815 	hlock->prev_chain_key = chain_key;  in __lock_acquire()
4816 	if (separate_irq_context(curr, hlock)) {  in __lock_acquire()
4820 	chain_key = iterate_chain_key(chain_key, hlock_id(hlock));  in __lock_acquire()
4823 		print_lock_nested_lock_not_held(curr, hlock, ip);  in __lock_acquire()
4828 		WARN_ON_ONCE(depth && !hlock_class(hlock - 1)->key);  in __lock_acquire()
4829 		WARN_ON_ONCE(!hlock_class(hlock)->key);  in __lock_acquire()
4832 	if (!validate_chain(curr, hlock, chain_head, chain_key))  in __lock_acquire()
4888 static noinstr int match_held_lock(const struct held_lock *hlock,  in match_held_lock()  argument
4891 	if (hlock->instance == lock)  in match_held_lock()
4894 	if (hlock->references) {  in match_held_lock()
4914 		if (DEBUG_LOCKS_WARN_ON(!hlock->nest_lock))  in match_held_lock()
4917 		if (hlock->class_idx == class - lock_classes)  in match_held_lock()
4929 	struct held_lock *ret, *hlock, *prev_hlock;  in find_held_lock()  local
4933 	hlock = curr->held_locks + i;  in find_held_lock()
4934 	ret = hlock;  in find_held_lock()
4935 	if (match_held_lock(hlock, lock))  in find_held_lock()
4939 	for (i--, prev_hlock = hlock--;  in find_held_lock()
4941 	     i--, prev_hlock = hlock--) {  in find_held_lock()
4945 		if (prev_hlock->irq_context != hlock->irq_context) {  in find_held_lock()
4949 		if (match_held_lock(hlock, lock)) {  in find_held_lock()
4950 			ret = hlock;  in find_held_lock()
4963 	struct held_lock *hlock;  in reacquire_held_locks()  local
4969 	for (hlock = curr->held_locks + idx; idx < depth; idx++, hlock++) {  in reacquire_held_locks()
4970 		switch (__lock_acquire(hlock->instance,  in reacquire_held_locks()
4971 				    hlock_class(hlock)->subclass,  in reacquire_held_locks()
4972 				    hlock->trylock,  in reacquire_held_locks()
4973 				    hlock->read, hlock->check,  in reacquire_held_locks()
4974 				    hlock->hardirqs_off,  in reacquire_held_locks()
4975 				    hlock->nest_lock, hlock->acquire_ip,  in reacquire_held_locks()
4976 				    hlock->references, hlock->pin_count)) {  in reacquire_held_locks()
4999 	struct held_lock *hlock;  in __lock_set_class()  local
5014 	hlock = find_held_lock(curr, lock, depth, &i);  in __lock_set_class()
5015 	if (!hlock) {  in __lock_set_class()
5024 	hlock->class_idx = class - lock_classes;  in __lock_set_class()
5027 	curr->curr_chain_key = hlock->prev_chain_key;  in __lock_set_class()
5045 	struct held_lock *hlock;  in __lock_downgrade()  local
5059 	hlock = find_held_lock(curr, lock, depth, &i);  in __lock_downgrade()
5060 	if (!hlock) {  in __lock_downgrade()
5066 	curr->curr_chain_key = hlock->prev_chain_key;  in __lock_downgrade()
5068 	WARN(hlock->read, "downgrading a read lock");  in __lock_downgrade()
5069 	hlock->read = 1;  in __lock_downgrade()
5070 	hlock->acquire_ip = ip;  in __lock_downgrade()
5099 	struct held_lock *hlock;  in __lock_release()  local
5119 	hlock = find_held_lock(curr, lock, depth, &i);  in __lock_release()
5120 	if (!hlock) {  in __lock_release()
5125 	if (hlock->instance == lock)  in __lock_release()
5126 		lock_release_holdtime(hlock);  in __lock_release()
5128 	WARN(hlock->pin_count, "releasing a pinned lock\n");  in __lock_release()
5130 	if (hlock->references) {  in __lock_release()
5131 		hlock->references--;  in __lock_release()
5132 		if (hlock->references) {  in __lock_release()
5149 	curr->curr_chain_key = hlock->prev_chain_key;  in __lock_release()
5183 		struct held_lock *hlock = curr->held_locks + i;  in __lock_is_held()  local
5185 		if (match_held_lock(hlock, lock)) {  in __lock_is_held()
5186 			if (read == -1 || hlock->read == read)  in __lock_is_held()
5206 		struct held_lock *hlock = curr->held_locks + i;  in __lock_pin_lock()  local
5208 		if (match_held_lock(hlock, lock)) {  in __lock_pin_lock()
5215 			hlock->pin_count += cookie.val;  in __lock_pin_lock()
5233 		struct held_lock *hlock = curr->held_locks + i;  in __lock_repin_lock()  local
5235 		if (match_held_lock(hlock, lock)) {  in __lock_repin_lock()
5236 			hlock->pin_count += cookie.val;  in __lock_repin_lock()
5253 		struct held_lock *hlock = curr->held_locks + i;  in __lock_unpin_lock()  local
5255 		if (match_held_lock(hlock, lock)) {  in __lock_unpin_lock()
5256 			if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n"))  in __lock_unpin_lock()
5259 			hlock->pin_count -= cookie.val;  in __lock_unpin_lock()
5261 			if (WARN((int)hlock->pin_count < 0, "pin count corrupted\n"))  in __lock_unpin_lock()
5262 				hlock->pin_count = 0;  in __lock_unpin_lock()
5347 static void verify_lock_unused(struct lockdep_map *lock, struct held_lock *hlock, int subclass)  in verify_lock_unused()  argument
5361 	if (!hlock->read)  in verify_lock_unused()
5367 	hlock->class_idx = class - lock_classes;  in verify_lock_unused()
5369 	print_usage_bug(current, hlock, LOCK_USED, LOCK_USAGE_STATES);  in verify_lock_unused()
5416 			struct held_lock hlock;  in lock_acquire()  local
5418 			hlock.acquire_ip = ip;  in lock_acquire()
5419 			hlock.instance = lock;  in lock_acquire()
5420 			hlock.nest_lock = nest_lock;  in lock_acquire()
5421 			hlock.irq_context = 2; // XXX  in lock_acquire()
5422 			hlock.trylock = trylock;  in lock_acquire()
5423 			hlock.read = read;  in lock_acquire()
5424 			hlock.check = check;  in lock_acquire()
5425 			hlock.hardirqs_off = true;  in lock_acquire()
5426 			hlock.references = 0;  in lock_acquire()
5428 			verify_lock_unused(lock, &hlock, subclass);  in lock_acquire()
5571 	struct held_lock *hlock;  in __lock_contended()  local
5584 	hlock = find_held_lock(curr, lock, depth, &i);  in __lock_contended()
5585 	if (!hlock) {  in __lock_contended()
5590 	if (hlock->instance != lock)  in __lock_contended()
5593 	hlock->waittime_stamp = lockstat_clock();  in __lock_contended()
5595 	contention_point = lock_point(hlock_class(hlock)->contention_point, ip);  in __lock_contended()
5596 	contending_point = lock_point(hlock_class(hlock)->contending_point,  in __lock_contended()
5599 	stats = get_lock_stats(hlock_class(hlock));  in __lock_contended()
5605 		stats->bounces[bounce_contended + !!hlock->read]++;  in __lock_contended()
5612 	struct held_lock *hlock;  in __lock_acquired()  local
5626 	hlock = find_held_lock(curr, lock, depth, &i);  in __lock_acquired()
5627 	if (!hlock) {  in __lock_acquired()
5632 	if (hlock->instance != lock)  in __lock_acquired()
5636 	if (hlock->waittime_stamp) {  in __lock_acquired()
5638 		waittime = now - hlock->waittime_stamp;  in __lock_acquired()
5639 		hlock->holdtime_stamp = now;  in __lock_acquired()
5642 	stats = get_lock_stats(hlock_class(hlock));  in __lock_acquired()
5644 		if (hlock->read)  in __lock_acquired()
5650 		stats->bounces[bounce_acquired + !!hlock->read]++;  in __lock_acquired()
6166 		     const void *mem_to, struct held_lock *hlock)  in print_freed_lock_bug()  argument
6180 	print_lock(hlock);  in print_freed_lock_bug()
6202 	struct held_lock *hlock;  in debug_check_no_locks_freed()  local
6211 		hlock = curr->held_locks + i;  in debug_check_no_locks_freed()
6213 		if (not_in_range(mem_from, mem_len, hlock->instance,  in debug_check_no_locks_freed()
6214 					sizeof(*hlock->instance)))  in debug_check_no_locks_freed()
6217 		print_freed_lock_bug(curr, mem_from, mem_from + mem_len, hlock);  in debug_check_no_locks_freed()