24 	struct percpu_counter *fbc = addr;  in percpu_counter_fixup_free()  local
28 		percpu_counter_destroy(fbc);  in percpu_counter_fixup_free()
29 		debug_object_free(fbc, &percpu_counter_debug_descr);  in percpu_counter_fixup_free()
41 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)  in debug_percpu_counter_activate()  argument
43 	debug_object_init(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_activate()
44 	debug_object_activate(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_activate()
47 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)  in debug_percpu_counter_deactivate()  argument
49 	debug_object_deactivate(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_deactivate()
50 	debug_object_free(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_deactivate()
54 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)  in debug_percpu_counter_activate()  argument
56 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)  in debug_percpu_counter_deactivate()  argument
60 void percpu_counter_set(struct percpu_counter *fbc, s64 amount)  in percpu_counter_set()  argument
65 	raw_spin_lock_irqsave(&fbc->lock, flags);  in percpu_counter_set()
67 		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);  in percpu_counter_set()
70 	fbc->count = amount;  in percpu_counter_set()
71 	raw_spin_unlock_irqrestore(&fbc->lock, flags);  in percpu_counter_set()
87 void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)  in percpu_counter_add_batch()  argument
93 	count = __this_cpu_read(*fbc->counters) + amount;  in percpu_counter_add_batch()
95 		raw_spin_lock(&fbc->lock);  in percpu_counter_add_batch()
96 		fbc->count += count;  in percpu_counter_add_batch()
97 		__this_cpu_sub(*fbc->counters, count - amount);  in percpu_counter_add_batch()
98 		raw_spin_unlock(&fbc->lock);  in percpu_counter_add_batch()
100 		this_cpu_add(*fbc->counters, amount);  in percpu_counter_add_batch()
112 void percpu_counter_sync(struct percpu_counter *fbc)  in percpu_counter_sync()  argument
117 	raw_spin_lock_irqsave(&fbc->lock, flags);  in percpu_counter_sync()
118 	count = __this_cpu_read(*fbc->counters);  in percpu_counter_sync()
119 	fbc->count += count;  in percpu_counter_sync()
120 	__this_cpu_sub(*fbc->counters, count);  in percpu_counter_sync()
121 	raw_spin_unlock_irqrestore(&fbc->lock, flags);  in percpu_counter_sync()
137 s64 __percpu_counter_sum(struct percpu_counter *fbc)  in __percpu_counter_sum()  argument
143 	raw_spin_lock_irqsave(&fbc->lock, flags);  in __percpu_counter_sum()
144 	ret = fbc->count;  in __percpu_counter_sum()
146 		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);  in __percpu_counter_sum()
149 	raw_spin_unlock_irqrestore(&fbc->lock, flags);  in __percpu_counter_sum()
154 int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount,  in __percpu_counter_init_many()  argument
167 		fbc[0].counters = NULL;  in __percpu_counter_init_many()
172 		raw_spin_lock_init(&fbc[i].lock);  in __percpu_counter_init_many()
173 		lockdep_set_class(&fbc[i].lock, key);  in __percpu_counter_init_many()
175 		INIT_LIST_HEAD(&fbc[i].list);  in __percpu_counter_init_many()
177 		fbc[i].count = amount;  in __percpu_counter_init_many()
178 		fbc[i].counters = (void *)counters + (i * counter_size);  in __percpu_counter_init_many()
180 		debug_percpu_counter_activate(&fbc[i]);  in __percpu_counter_init_many()
186 		list_add(&fbc[i].list, &percpu_counters);  in __percpu_counter_init_many()
193 void percpu_counter_destroy_many(struct percpu_counter *fbc, u32 nr_counters)  in percpu_counter_destroy_many()  argument
198 	if (WARN_ON_ONCE(!fbc))  in percpu_counter_destroy_many()
201 	if (!fbc[0].counters)  in percpu_counter_destroy_many()
205 		debug_percpu_counter_deactivate(&fbc[i]);  in percpu_counter_destroy_many()
210 		list_del(&fbc[i].list);  in percpu_counter_destroy_many()
214 	free_percpu(fbc[0].counters);  in percpu_counter_destroy_many()
217 		fbc[i].counters = NULL;  in percpu_counter_destroy_many()
235 	struct percpu_counter *fbc;  in percpu_counter_cpu_dead()  local
240 	list_for_each_entry(fbc, &percpu_counters, list) {  in percpu_counter_cpu_dead()
243 		raw_spin_lock(&fbc->lock);  in percpu_counter_cpu_dead()
244 		pcount = per_cpu_ptr(fbc->counters, cpu);  in percpu_counter_cpu_dead()
245 		fbc->count += *pcount;  in percpu_counter_cpu_dead()
247 		raw_spin_unlock(&fbc->lock);  in percpu_counter_cpu_dead()
258 int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch)  in __percpu_counter_compare()  argument
262 	count = percpu_counter_read(fbc);  in __percpu_counter_compare()
271 	count = percpu_counter_sum(fbc);  in __percpu_counter_compare()