24 	struct percpu_counter *fbc = addr;  in percpu_counter_fixup_free()  local
28 		percpu_counter_destroy(fbc);  in percpu_counter_fixup_free()
29 		debug_object_free(fbc, &percpu_counter_debug_descr);  in percpu_counter_fixup_free()
41 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)  in debug_percpu_counter_activate()  argument
43 	debug_object_init(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_activate()
44 	debug_object_activate(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_activate()
47 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)  in debug_percpu_counter_deactivate()  argument
49 	debug_object_deactivate(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_deactivate()
50 	debug_object_free(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_deactivate()
54 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)  in debug_percpu_counter_activate()  argument
56 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)  in debug_percpu_counter_deactivate()  argument
60 void percpu_counter_set(struct percpu_counter *fbc, s64 amount)  in percpu_counter_set()  argument
64 	raw_spin_lock(&fbc->lock);  in percpu_counter_set()
66 		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);  in percpu_counter_set()
69 	fbc->count = amount;  in percpu_counter_set()
70 	raw_spin_unlock(&fbc->lock);  in percpu_counter_set()
74 void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)  in __percpu_counter_add()  argument
79 	count = __this_cpu_read(*fbc->counters) + amount;  in __percpu_counter_add()
81 		raw_spin_lock(&fbc->lock);  in __percpu_counter_add()
82 		fbc->count += count;  in __percpu_counter_add()
83 		__this_cpu_write(*fbc->counters, 0);  in __percpu_counter_add()
84 		raw_spin_unlock(&fbc->lock);  in __percpu_counter_add()
86 		__this_cpu_write(*fbc->counters, count);  in __percpu_counter_add()
96 s64 __percpu_counter_sum(struct percpu_counter *fbc)  in __percpu_counter_sum()  argument
101 	raw_spin_lock(&fbc->lock);  in __percpu_counter_sum()
102 	ret = fbc->count;  in __percpu_counter_sum()
104 		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);  in __percpu_counter_sum()
107 	raw_spin_unlock(&fbc->lock);  in __percpu_counter_sum()
112 int __percpu_counter_init(struct percpu_counter *fbc, s64 amount,  in __percpu_counter_init()  argument
115 	raw_spin_lock_init(&fbc->lock);  in __percpu_counter_init()
116 	lockdep_set_class(&fbc->lock, key);  in __percpu_counter_init()
117 	fbc->count = amount;  in __percpu_counter_init()
118 	fbc->counters = alloc_percpu(s32);  in __percpu_counter_init()
119 	if (!fbc->counters)  in __percpu_counter_init()
122 	debug_percpu_counter_activate(fbc);  in __percpu_counter_init()
125 	INIT_LIST_HEAD(&fbc->list);  in __percpu_counter_init()
127 	list_add(&fbc->list, &percpu_counters);  in __percpu_counter_init()
134 void percpu_counter_destroy(struct percpu_counter *fbc)  in percpu_counter_destroy()  argument
136 	if (!fbc->counters)  in percpu_counter_destroy()
139 	debug_percpu_counter_deactivate(fbc);  in percpu_counter_destroy()
143 	list_del(&fbc->list);  in percpu_counter_destroy()
146 	free_percpu(fbc->counters);  in percpu_counter_destroy()
147 	fbc->counters = NULL;  in percpu_counter_destroy()
166 	struct percpu_counter *fbc;  in percpu_counter_hotcpu_callback()  local
174 	list_for_each_entry(fbc, &percpu_counters, list) {  in percpu_counter_hotcpu_callback()
178 		raw_spin_lock_irqsave(&fbc->lock, flags);  in percpu_counter_hotcpu_callback()
179 		pcount = per_cpu_ptr(fbc->counters, cpu);  in percpu_counter_hotcpu_callback()
180 		fbc->count += *pcount;  in percpu_counter_hotcpu_callback()
182 		raw_spin_unlock_irqrestore(&fbc->lock, flags);  in percpu_counter_hotcpu_callback()
193 int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs)  in percpu_counter_compare()  argument
197 	count = percpu_counter_read(fbc);  in percpu_counter_compare()
206 	count = percpu_counter_sum(fbc);  in percpu_counter_compare()