220 static inline struct lock_class *hlock_class(struct held_lock *hlock)  in hlock_class()  argument
222 	unsigned int class_idx = hlock->class_idx;  in hlock_class()
341 static void lock_release_holdtime(struct held_lock *hlock)  in lock_release_holdtime()  argument
349 	holdtime = lockstat_clock() - hlock->holdtime_stamp;  in lock_release_holdtime()
351 	stats = get_lock_stats(hlock_class(hlock));  in lock_release_holdtime()
352 	if (hlock->read)  in lock_release_holdtime()
358 static inline void lock_release_holdtime(struct held_lock *hlock)  in lock_release_holdtime()  argument
425 static inline u16 hlock_id(struct held_lock *hlock)  in hlock_id()  argument
429 	return (hlock->class_idx | (hlock->read << MAX_LOCKDEP_KEYS_BITS));  in hlock_id()
754 static void print_lock(struct held_lock *hlock)  in print_lock()  argument
766 	struct lock_class *lock = hlock_class(hlock);  in print_lock()
773 	printk(KERN_CONT "%px", hlock->instance);  in print_lock()
775 	printk(KERN_CONT ", at: %pS\n", (void *)hlock->acquire_ip);  in print_lock()
1666 				 struct held_lock *hlock)  in bfs_init_root()  argument
1668 	__bfs_init_root(lock, hlock_class(hlock));  in bfs_init_root()
1669 	lock->only_xr = (hlock->read == 2);  in bfs_init_root()
1680 				  struct held_lock *hlock)  in bfs_init_rootb()  argument
1682 	__bfs_init_root(lock, hlock_class(hlock));  in bfs_init_rootb()
1683 	lock->only_xr = (hlock->read != 0);  in bfs_init_rootb()
1985 	struct held_lock *hlock = (struct held_lock *)data;  in hlock_equal()  local
1987 	return hlock_class(hlock) == entry->class && /* Found A -> .. -> B */  in hlock_equal()
1988 	       (hlock->read == 2 ||  /* A -> B is -(*R)-> */  in hlock_equal()
2012 	struct held_lock *hlock = (struct held_lock *)data;  in hlock_conflict()  local
2014 	return hlock_class(hlock) == entry->class && /* Found A -> .. -> B */  in hlock_conflict()
2015 	       (hlock->read == 0 || /* B -> A is -(E*)-> */  in hlock_conflict()
3194 	struct held_lock *hlock;  in check_prevs_add()  local
3213 		hlock = curr->held_locks + depth - 1;  in check_prevs_add()
3215 		if (hlock->check) {  in check_prevs_add()
3216 			int ret = check_prev_add(curr, hlock, next, distance, &trace);  in check_prevs_add()
3226 			if (!hlock->trylock)  in check_prevs_add()
3502 					struct held_lock *hlock)  in get_first_held_lock()  argument
3509 		if (hlock_curr->irq_context != hlock->irq_context)  in get_first_held_lock()
3534 	struct held_lock *hlock;  in print_chain_keys_held_locks()  local
3542 		hlock = curr->held_locks + i;  in print_chain_keys_held_locks()
3543 		chain_key = print_chain_key_iteration(hlock_id(hlock), chain_key);  in print_chain_keys_held_locks()
3545 		print_lock(hlock);  in print_chain_keys_held_locks()
3598 			struct held_lock *hlock,  in check_no_collision()  argument
3604 	i = get_first_held_lock(curr, hlock);  in check_no_collision()
3607 		print_collision(curr, hlock, chain);  in check_no_collision()
3615 			print_collision(curr, hlock, chain);  in check_no_collision()
3658 				  struct held_lock *hlock,  in add_chain_cache()  argument
3683 	chain->irq_context = hlock->irq_context;  in add_chain_cache()
3684 	i = get_first_held_lock(curr, hlock);  in add_chain_cache()
3707 	chain_hlocks[chain->base + j] = hlock_id(hlock);  in add_chain_cache()
3740 					 struct held_lock *hlock,  in lookup_chain_cache_add()  argument
3743 	struct lock_class *class = hlock_class(hlock);  in lookup_chain_cache_add()
3748 		if (!check_no_collision(curr, hlock, chain))  in lookup_chain_cache_add()
3778 	if (!add_chain_cache(curr, hlock, chain_key))  in lookup_chain_cache_add()
3785 			  struct held_lock *hlock,  in validate_chain()  argument
3798 	if (!hlock->trylock && hlock->check &&  in validate_chain()
3799 	    lookup_chain_cache_add(curr, hlock, chain_key)) {  in validate_chain()
3818 		int ret = check_deadlock(curr, hlock);  in validate_chain()
3831 			if (!check_prevs_add(curr, hlock))  in validate_chain()
3846 				 struct held_lock *hlock,  in validate_chain()  argument
3862 	struct held_lock *hlock, *prev_hlock = NULL;  in check_chain_key()  local
3867 		hlock = curr->held_locks + i;  in check_chain_key()
3868 		if (chain_key != hlock->prev_chain_key) {  in check_chain_key()
3877 				(unsigned long long)hlock->prev_chain_key);  in check_chain_key()
3885 		if (DEBUG_LOCKS_WARN_ON(!test_bit(hlock->class_idx, lock_classes_in_use)))  in check_chain_key()
3889 							hlock->irq_context))  in check_chain_key()
3891 		chain_key = iterate_chain_key(chain_key, hlock_id(hlock));  in check_chain_key()
3892 		prev_hlock = hlock;  in check_chain_key()
4221 	struct held_lock *hlock;  in mark_held_locks()  local
4226 		hlock = curr->held_locks + i;  in mark_held_locks()
4228 		if (hlock->read)  in mark_held_locks()
4233 		if (!hlock->check)  in mark_held_locks()
4236 		if (!mark_lock(curr, hlock, hlock_bit))  in mark_held_locks()
4505 mark_usage(struct task_struct *curr, struct held_lock *hlock, int check)  in mark_usage()  argument
4514 	if (!hlock->trylock) {  in mark_usage()
4515 		if (hlock->read) {  in mark_usage()
4517 				if (!mark_lock(curr, hlock,  in mark_usage()
4521 				if (!mark_lock(curr, hlock,  in mark_usage()
4526 				if (!mark_lock(curr, hlock, LOCK_USED_IN_HARDIRQ))  in mark_usage()
4529 				if (!mark_lock(curr, hlock, LOCK_USED_IN_SOFTIRQ))  in mark_usage()
4533 	if (!hlock->hardirqs_off) {  in mark_usage()
4534 		if (hlock->read) {  in mark_usage()
4535 			if (!mark_lock(curr, hlock,  in mark_usage()
4539 				if (!mark_lock(curr, hlock,  in mark_usage()
4543 			if (!mark_lock(curr, hlock,  in mark_usage()
4547 				if (!mark_lock(curr, hlock,  in mark_usage()
4555 	if (!mark_lock(curr, hlock, LOCK_USED))  in mark_usage()
4568 		struct held_lock *hlock)  in separate_irq_context()  argument
4584 		if (prev_hlock->irq_context != hlock->irq_context)  in separate_irq_context()
4681 				struct held_lock *hlock)  in print_lock_invalid_wait_context()  argument
4697 	print_lock(hlock);  in print_lock_invalid_wait_context()
4776 mark_usage(struct task_struct *curr, struct held_lock *hlock, int check)  in mark_usage()  argument
4787 		struct held_lock *hlock)  in separate_irq_context()  argument
4870 				struct held_lock *hlock)  in print_lock_nested_lock_not_held()  argument
4884 	print_lock(hlock);  in print_lock_nested_lock_not_held()
4887 	pr_warn("%s\n", hlock->nest_lock->name);  in print_lock_nested_lock_not_held()
4916 	struct held_lock *hlock;  in __lock_acquire()  local
4964 		hlock = curr->held_locks + depth - 1;  in __lock_acquire()
4965 		if (hlock->class_idx == class_idx && nest_lock) {  in __lock_acquire()
4969 			if (!hlock->references)  in __lock_acquire()
4970 				hlock->references++;  in __lock_acquire()
4972 			hlock->references += references;  in __lock_acquire()
4975 			if (DEBUG_LOCKS_WARN_ON(hlock->references < references))  in __lock_acquire()
4982 	hlock = curr->held_locks + depth;  in __lock_acquire()
4989 	hlock->class_idx = class_idx;  in __lock_acquire()
4990 	hlock->acquire_ip = ip;  in __lock_acquire()
4991 	hlock->instance = lock;  in __lock_acquire()
4992 	hlock->nest_lock = nest_lock;  in __lock_acquire()
4993 	hlock->irq_context = task_irq_context(curr);  in __lock_acquire()
4994 	hlock->trylock = trylock;  in __lock_acquire()
4995 	hlock->read = read;  in __lock_acquire()
4996 	hlock->check = check;  in __lock_acquire()
4997 	hlock->hardirqs_off = !!hardirqs_off;  in __lock_acquire()
4998 	hlock->references = references;  in __lock_acquire()
5000 	hlock->waittime_stamp = 0;  in __lock_acquire()
5001 	hlock->holdtime_stamp = lockstat_clock();  in __lock_acquire()
5003 	hlock->pin_count = pin_count;  in __lock_acquire()
5005 	if (check_wait_context(curr, hlock))  in __lock_acquire()
5009 	if (!mark_usage(curr, hlock, check))  in __lock_acquire()
5038 	hlock->prev_chain_key = chain_key;  in __lock_acquire()
5039 	if (separate_irq_context(curr, hlock)) {  in __lock_acquire()
5043 	chain_key = iterate_chain_key(chain_key, hlock_id(hlock));  in __lock_acquire()
5046 		print_lock_nested_lock_not_held(curr, hlock);  in __lock_acquire()
5051 		WARN_ON_ONCE(depth && !hlock_class(hlock - 1)->key);  in __lock_acquire()
5052 		WARN_ON_ONCE(!hlock_class(hlock)->key);  in __lock_acquire()
5055 	if (!validate_chain(curr, hlock, chain_head, chain_key))  in __lock_acquire()
5111 static noinstr int match_held_lock(const struct held_lock *hlock,  in match_held_lock()  argument
5114 	if (hlock->instance == lock)  in match_held_lock()
5117 	if (hlock->references) {  in match_held_lock()
5137 		if (DEBUG_LOCKS_WARN_ON(!hlock->nest_lock))  in match_held_lock()
5140 		if (hlock->class_idx == class - lock_classes)  in match_held_lock()
5152 	struct held_lock *ret, *hlock, *prev_hlock;  in find_held_lock()  local
5156 	hlock = curr->held_locks + i;  in find_held_lock()
5157 	ret = hlock;  in find_held_lock()
5158 	if (match_held_lock(hlock, lock))  in find_held_lock()
5162 	for (i--, prev_hlock = hlock--;  in find_held_lock()
5164 	     i--, prev_hlock = hlock--) {  in find_held_lock()
5168 		if (prev_hlock->irq_context != hlock->irq_context) {  in find_held_lock()
5172 		if (match_held_lock(hlock, lock)) {  in find_held_lock()
5173 			ret = hlock;  in find_held_lock()
5186 	struct held_lock *hlock;  in reacquire_held_locks()  local
5192 	for (hlock = curr->held_locks + idx; idx < depth; idx++, hlock++) {  in reacquire_held_locks()
5193 		switch (__lock_acquire(hlock->instance,  in reacquire_held_locks()
5194 				    hlock_class(hlock)->subclass,  in reacquire_held_locks()
5195 				    hlock->trylock,  in reacquire_held_locks()
5196 				    hlock->read, hlock->check,  in reacquire_held_locks()
5197 				    hlock->hardirqs_off,  in reacquire_held_locks()
5198 				    hlock->nest_lock, hlock->acquire_ip,  in reacquire_held_locks()
5199 				    hlock->references, hlock->pin_count)) {  in reacquire_held_locks()
5222 	struct held_lock *hlock;  in __lock_set_class()  local
5237 	hlock = find_held_lock(curr, lock, depth, &i);  in __lock_set_class()
5238 	if (!hlock) {  in __lock_set_class()
5248 	hlock->class_idx = class - lock_classes;  in __lock_set_class()
5251 	curr->curr_chain_key = hlock->prev_chain_key;  in __lock_set_class()
5269 	struct held_lock *hlock;  in __lock_downgrade()  local
5283 	hlock = find_held_lock(curr, lock, depth, &i);  in __lock_downgrade()
5284 	if (!hlock) {  in __lock_downgrade()
5290 	curr->curr_chain_key = hlock->prev_chain_key;  in __lock_downgrade()
5292 	WARN(hlock->read, "downgrading a read lock");  in __lock_downgrade()
5293 	hlock->read = 1;  in __lock_downgrade()
5294 	hlock->acquire_ip = ip;  in __lock_downgrade()
5323 	struct held_lock *hlock;  in __lock_release()  local
5343 	hlock = find_held_lock(curr, lock, depth, &i);  in __lock_release()
5344 	if (!hlock) {  in __lock_release()
5349 	if (hlock->instance == lock)  in __lock_release()
5350 		lock_release_holdtime(hlock);  in __lock_release()
5352 	WARN(hlock->pin_count, "releasing a pinned lock\n");  in __lock_release()
5354 	if (hlock->references) {  in __lock_release()
5355 		hlock->references--;  in __lock_release()
5356 		if (hlock->references) {  in __lock_release()
5373 	curr->curr_chain_key = hlock->prev_chain_key;  in __lock_release()
5407 		struct held_lock *hlock = curr->held_locks + i;  in __lock_is_held()  local
5409 		if (match_held_lock(hlock, lock)) {  in __lock_is_held()
5410 			if (read == -1 || !!hlock->read == read)  in __lock_is_held()
5430 		struct held_lock *hlock = curr->held_locks + i;  in __lock_pin_lock()  local
5432 		if (match_held_lock(hlock, lock)) {  in __lock_pin_lock()
5439 			hlock->pin_count += cookie.val;  in __lock_pin_lock()
5457 		struct held_lock *hlock = curr->held_locks + i;  in __lock_repin_lock()  local
5459 		if (match_held_lock(hlock, lock)) {  in __lock_repin_lock()
5460 			hlock->pin_count += cookie.val;  in __lock_repin_lock()
5477 		struct held_lock *hlock = curr->held_locks + i;  in __lock_unpin_lock()  local
5479 		if (match_held_lock(hlock, lock)) {  in __lock_unpin_lock()
5480 			if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n"))  in __lock_unpin_lock()
5483 			hlock->pin_count -= cookie.val;  in __lock_unpin_lock()
5485 			if (WARN((int)hlock->pin_count < 0, "pin count corrupted\n"))  in __lock_unpin_lock()
5486 				hlock->pin_count = 0;  in __lock_unpin_lock()
5578 static void verify_lock_unused(struct lockdep_map *lock, struct held_lock *hlock, int subclass)  in verify_lock_unused()  argument
5592 	if (!hlock->read)  in verify_lock_unused()
5598 	hlock->class_idx = class - lock_classes;  in verify_lock_unused()
5600 	print_usage_bug(current, hlock, LOCK_USED, LOCK_USAGE_STATES);  in verify_lock_unused()
5647 			struct held_lock hlock;  in lock_acquire()  local
5649 			hlock.acquire_ip = ip;  in lock_acquire()
5650 			hlock.instance = lock;  in lock_acquire()
5651 			hlock.nest_lock = nest_lock;  in lock_acquire()
5652 			hlock.irq_context = 2; // XXX  in lock_acquire()
5653 			hlock.trylock = trylock;  in lock_acquire()
5654 			hlock.read = read;  in lock_acquire()
5655 			hlock.check = check;  in lock_acquire()
5656 			hlock.hardirqs_off = true;  in lock_acquire()
5657 			hlock.references = 0;  in lock_acquire()
5659 			verify_lock_unused(lock, &hlock, subclass);  in lock_acquire()
5806 	struct held_lock *hlock;  in __lock_contended()  local
5819 	hlock = find_held_lock(curr, lock, depth, &i);  in __lock_contended()
5820 	if (!hlock) {  in __lock_contended()
5825 	if (hlock->instance != lock)  in __lock_contended()
5828 	hlock->waittime_stamp = lockstat_clock();  in __lock_contended()
5830 	contention_point = lock_point(hlock_class(hlock)->contention_point, ip);  in __lock_contended()
5831 	contending_point = lock_point(hlock_class(hlock)->contending_point,  in __lock_contended()
5834 	stats = get_lock_stats(hlock_class(hlock));  in __lock_contended()
5840 		stats->bounces[bounce_contended + !!hlock->read]++;  in __lock_contended()
5847 	struct held_lock *hlock;  in __lock_acquired()  local
5861 	hlock = find_held_lock(curr, lock, depth, &i);  in __lock_acquired()
5862 	if (!hlock) {  in __lock_acquired()
5867 	if (hlock->instance != lock)  in __lock_acquired()
5871 	if (hlock->waittime_stamp) {  in __lock_acquired()
5873 		waittime = now - hlock->waittime_stamp;  in __lock_acquired()
5874 		hlock->holdtime_stamp = now;  in __lock_acquired()
5877 	stats = get_lock_stats(hlock_class(hlock));  in __lock_acquired()
5879 		if (hlock->read)  in __lock_acquired()
5885 		stats->bounces[bounce_acquired + !!hlock->read]++;  in __lock_acquired()
6406 		     const void *mem_to, struct held_lock *hlock)  in print_freed_lock_bug()  argument
6420 	print_lock(hlock);  in print_freed_lock_bug()
6442 	struct held_lock *hlock;  in debug_check_no_locks_freed()  local
6451 		hlock = curr->held_locks + i;  in debug_check_no_locks_freed()
6453 		if (not_in_range(mem_from, mem_len, hlock->instance,  in debug_check_no_locks_freed()
6454 					sizeof(*hlock->instance)))  in debug_check_no_locks_freed()
6457 		print_freed_lock_bug(curr, mem_from, mem_from + mem_len, hlock);  in debug_check_no_locks_freed()