17 static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb)  in btrfs_assert_spinning_writers_get()  argument
19 	WARN_ON(eb->spinning_writers);  in btrfs_assert_spinning_writers_get()
20 	eb->spinning_writers++;  in btrfs_assert_spinning_writers_get()
23 static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb)  in btrfs_assert_spinning_writers_put()  argument
25 	WARN_ON(eb->spinning_writers != 1);  in btrfs_assert_spinning_writers_put()
26 	eb->spinning_writers--;  in btrfs_assert_spinning_writers_put()
29 static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb)  in btrfs_assert_no_spinning_writers()  argument
31 	WARN_ON(eb->spinning_writers);  in btrfs_assert_no_spinning_writers()
34 static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb)  in btrfs_assert_spinning_readers_get()  argument
36 	atomic_inc(&eb->spinning_readers);  in btrfs_assert_spinning_readers_get()
39 static void btrfs_assert_spinning_readers_put(struct extent_buffer *eb)  in btrfs_assert_spinning_readers_put()  argument
41 	WARN_ON(atomic_read(&eb->spinning_readers) == 0);  in btrfs_assert_spinning_readers_put()
42 	atomic_dec(&eb->spinning_readers);  in btrfs_assert_spinning_readers_put()
45 static void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb)  in btrfs_assert_tree_read_locks_get()  argument
47 	atomic_inc(&eb->read_locks);  in btrfs_assert_tree_read_locks_get()
50 static void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb)  in btrfs_assert_tree_read_locks_put()  argument
52 	atomic_dec(&eb->read_locks);  in btrfs_assert_tree_read_locks_put()
55 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb)  in btrfs_assert_tree_read_locked()  argument
57 	BUG_ON(!atomic_read(&eb->read_locks));  in btrfs_assert_tree_read_locked()
60 static void btrfs_assert_tree_write_locks_get(struct extent_buffer *eb)  in btrfs_assert_tree_write_locks_get()  argument
62 	eb->write_locks++;  in btrfs_assert_tree_write_locks_get()
65 static void btrfs_assert_tree_write_locks_put(struct extent_buffer *eb)  in btrfs_assert_tree_write_locks_put()  argument
67 	eb->write_locks--;  in btrfs_assert_tree_write_locks_put()
70 void btrfs_assert_tree_locked(struct extent_buffer *eb)  in btrfs_assert_tree_locked()  argument
72 	BUG_ON(!eb->write_locks);  in btrfs_assert_tree_locked()
76 static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb) { }  in btrfs_assert_spinning_writers_get()  argument
77 static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb) { }  in btrfs_assert_spinning_writers_put()  argument
78 static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb) { }  in btrfs_assert_no_spinning_writers()  argument
79 static void btrfs_assert_spinning_readers_put(struct extent_buffer *eb) { }  in btrfs_assert_spinning_readers_put()  argument
80 static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb) { }  in btrfs_assert_spinning_readers_get()  argument
81 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb) { }  in btrfs_assert_tree_read_locked()  argument
82 static void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb) { }  in btrfs_assert_tree_read_locks_get()  argument
83 static void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb) { }  in btrfs_assert_tree_read_locks_put()  argument
84 void btrfs_assert_tree_locked(struct extent_buffer *eb) { }  in btrfs_assert_tree_locked()  argument
85 static void btrfs_assert_tree_write_locks_get(struct extent_buffer *eb) { }  in btrfs_assert_tree_write_locks_get()  argument
86 static void btrfs_assert_tree_write_locks_put(struct extent_buffer *eb) { }  in btrfs_assert_tree_write_locks_put()  argument
89 void btrfs_set_lock_blocking_read(struct extent_buffer *eb)  in btrfs_set_lock_blocking_read()  argument
91 	trace_btrfs_set_lock_blocking_read(eb);  in btrfs_set_lock_blocking_read()
97 	if (eb->lock_nested && current->pid == eb->lock_owner)  in btrfs_set_lock_blocking_read()
99 	btrfs_assert_tree_read_locked(eb);  in btrfs_set_lock_blocking_read()
100 	atomic_inc(&eb->blocking_readers);  in btrfs_set_lock_blocking_read()
101 	btrfs_assert_spinning_readers_put(eb);  in btrfs_set_lock_blocking_read()
102 	read_unlock(&eb->lock);  in btrfs_set_lock_blocking_read()
105 void btrfs_set_lock_blocking_write(struct extent_buffer *eb)  in btrfs_set_lock_blocking_write()  argument
107 	trace_btrfs_set_lock_blocking_write(eb);  in btrfs_set_lock_blocking_write()
113 	if (eb->lock_nested && current->pid == eb->lock_owner)  in btrfs_set_lock_blocking_write()
115 	if (eb->blocking_writers == 0) {  in btrfs_set_lock_blocking_write()
116 		btrfs_assert_spinning_writers_put(eb);  in btrfs_set_lock_blocking_write()
117 		btrfs_assert_tree_locked(eb);  in btrfs_set_lock_blocking_write()
118 		eb->blocking_writers++;  in btrfs_set_lock_blocking_write()
119 		write_unlock(&eb->lock);  in btrfs_set_lock_blocking_write()
127 void btrfs_tree_read_lock(struct extent_buffer *eb)  in btrfs_tree_read_lock()  argument
134 	read_lock(&eb->lock);  in btrfs_tree_read_lock()
135 	BUG_ON(eb->blocking_writers == 0 &&  in btrfs_tree_read_lock()
136 	       current->pid == eb->lock_owner);  in btrfs_tree_read_lock()
137 	if (eb->blocking_writers && current->pid == eb->lock_owner) {  in btrfs_tree_read_lock()
144 		BUG_ON(eb->lock_nested);  in btrfs_tree_read_lock()
145 		eb->lock_nested = true;  in btrfs_tree_read_lock()
146 		read_unlock(&eb->lock);  in btrfs_tree_read_lock()
147 		trace_btrfs_tree_read_lock(eb, start_ns);  in btrfs_tree_read_lock()
150 	if (eb->blocking_writers) {  in btrfs_tree_read_lock()
151 		read_unlock(&eb->lock);  in btrfs_tree_read_lock()
152 		wait_event(eb->write_lock_wq,  in btrfs_tree_read_lock()
153 			   eb->blocking_writers == 0);  in btrfs_tree_read_lock()
156 	btrfs_assert_tree_read_locks_get(eb);  in btrfs_tree_read_lock()
157 	btrfs_assert_spinning_readers_get(eb);  in btrfs_tree_read_lock()
158 	trace_btrfs_tree_read_lock(eb, start_ns);  in btrfs_tree_read_lock()
166 int btrfs_tree_read_lock_atomic(struct extent_buffer *eb)  in btrfs_tree_read_lock_atomic()  argument
168 	if (eb->blocking_writers)  in btrfs_tree_read_lock_atomic()
171 	read_lock(&eb->lock);  in btrfs_tree_read_lock_atomic()
172 	if (eb->blocking_writers) {  in btrfs_tree_read_lock_atomic()
173 		read_unlock(&eb->lock);  in btrfs_tree_read_lock_atomic()
176 	btrfs_assert_tree_read_locks_get(eb);  in btrfs_tree_read_lock_atomic()
177 	btrfs_assert_spinning_readers_get(eb);  in btrfs_tree_read_lock_atomic()
178 	trace_btrfs_tree_read_lock_atomic(eb);  in btrfs_tree_read_lock_atomic()
186 int btrfs_try_tree_read_lock(struct extent_buffer *eb)  in btrfs_try_tree_read_lock()  argument
188 	if (eb->blocking_writers)  in btrfs_try_tree_read_lock()
191 	if (!read_trylock(&eb->lock))  in btrfs_try_tree_read_lock()
194 	if (eb->blocking_writers) {  in btrfs_try_tree_read_lock()
195 		read_unlock(&eb->lock);  in btrfs_try_tree_read_lock()
198 	btrfs_assert_tree_read_locks_get(eb);  in btrfs_try_tree_read_lock()
199 	btrfs_assert_spinning_readers_get(eb);  in btrfs_try_tree_read_lock()
200 	trace_btrfs_try_tree_read_lock(eb);  in btrfs_try_tree_read_lock()
208 int btrfs_try_tree_write_lock(struct extent_buffer *eb)  in btrfs_try_tree_write_lock()  argument
210 	if (eb->blocking_writers || atomic_read(&eb->blocking_readers))  in btrfs_try_tree_write_lock()
213 	write_lock(&eb->lock);  in btrfs_try_tree_write_lock()
214 	if (eb->blocking_writers || atomic_read(&eb->blocking_readers)) {  in btrfs_try_tree_write_lock()
215 		write_unlock(&eb->lock);  in btrfs_try_tree_write_lock()
218 	btrfs_assert_tree_write_locks_get(eb);  in btrfs_try_tree_write_lock()
219 	btrfs_assert_spinning_writers_get(eb);  in btrfs_try_tree_write_lock()
220 	eb->lock_owner = current->pid;  in btrfs_try_tree_write_lock()
221 	trace_btrfs_try_tree_write_lock(eb);  in btrfs_try_tree_write_lock()
228 void btrfs_tree_read_unlock(struct extent_buffer *eb)  in btrfs_tree_read_unlock()  argument
230 	trace_btrfs_tree_read_unlock(eb);  in btrfs_tree_read_unlock()
237 	if (eb->lock_nested && current->pid == eb->lock_owner) {  in btrfs_tree_read_unlock()
238 		eb->lock_nested = false;  in btrfs_tree_read_unlock()
241 	btrfs_assert_tree_read_locked(eb);  in btrfs_tree_read_unlock()
242 	btrfs_assert_spinning_readers_put(eb);  in btrfs_tree_read_unlock()
243 	btrfs_assert_tree_read_locks_put(eb);  in btrfs_tree_read_unlock()
244 	read_unlock(&eb->lock);  in btrfs_tree_read_unlock()
250 void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb)  in btrfs_tree_read_unlock_blocking()  argument
252 	trace_btrfs_tree_read_unlock_blocking(eb);  in btrfs_tree_read_unlock_blocking()
259 	if (eb->lock_nested && current->pid == eb->lock_owner) {  in btrfs_tree_read_unlock_blocking()
260 		eb->lock_nested = false;  in btrfs_tree_read_unlock_blocking()
263 	btrfs_assert_tree_read_locked(eb);  in btrfs_tree_read_unlock_blocking()
264 	WARN_ON(atomic_read(&eb->blocking_readers) == 0);  in btrfs_tree_read_unlock_blocking()
266 	if (atomic_dec_and_test(&eb->blocking_readers))  in btrfs_tree_read_unlock_blocking()
267 		cond_wake_up_nomb(&eb->read_lock_wq);  in btrfs_tree_read_unlock_blocking()
268 	btrfs_assert_tree_read_locks_put(eb);  in btrfs_tree_read_unlock_blocking()
275 void btrfs_tree_lock(struct extent_buffer *eb)  in btrfs_tree_lock()  argument
282 	WARN_ON(eb->lock_owner == current->pid);  in btrfs_tree_lock()
284 	wait_event(eb->read_lock_wq, atomic_read(&eb->blocking_readers) == 0);  in btrfs_tree_lock()
285 	wait_event(eb->write_lock_wq, eb->blocking_writers == 0);  in btrfs_tree_lock()
286 	write_lock(&eb->lock);  in btrfs_tree_lock()
287 	if (atomic_read(&eb->blocking_readers) || eb->blocking_writers) {  in btrfs_tree_lock()
288 		write_unlock(&eb->lock);  in btrfs_tree_lock()
291 	btrfs_assert_spinning_writers_get(eb);  in btrfs_tree_lock()
292 	btrfs_assert_tree_write_locks_get(eb);  in btrfs_tree_lock()
293 	eb->lock_owner = current->pid;  in btrfs_tree_lock()
294 	trace_btrfs_tree_lock(eb, start_ns);  in btrfs_tree_lock()
300 void btrfs_tree_unlock(struct extent_buffer *eb)  in btrfs_tree_unlock()  argument
302 	int blockers = eb->blocking_writers;  in btrfs_tree_unlock()
306 	btrfs_assert_tree_locked(eb);  in btrfs_tree_unlock()
307 	trace_btrfs_tree_unlock(eb);  in btrfs_tree_unlock()
308 	eb->lock_owner = 0;  in btrfs_tree_unlock()
309 	btrfs_assert_tree_write_locks_put(eb);  in btrfs_tree_unlock()
312 		btrfs_assert_no_spinning_writers(eb);  in btrfs_tree_unlock()
313 		eb->blocking_writers--;  in btrfs_tree_unlock()
319 		cond_wake_up(&eb->write_lock_wq);  in btrfs_tree_unlock()
321 		btrfs_assert_spinning_writers_put(eb);  in btrfs_tree_unlock()
322 		write_unlock(&eb->lock);  in btrfs_tree_unlock()