76 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay);
122 static void init_srcu_struct_data(struct srcu_struct *ssp)  in init_srcu_struct_data()  argument
134 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_data()
138 		sdp->srcu_gp_seq_needed = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
139 		sdp->srcu_gp_seq_needed_exp = ssp->srcu_sup->srcu_gp_seq;  in init_srcu_struct_data()
144 		sdp->ssp = ssp;  in init_srcu_struct_data()
164 static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)  in init_srcu_struct_nodes()  argument
176 	ssp->srcu_sup->node = kcalloc(rcu_num_nodes, sizeof(*ssp->srcu_sup->node), gfp_flags);  in init_srcu_struct_nodes()
177 	if (!ssp->srcu_sup->node)  in init_srcu_struct_nodes()
181 	ssp->srcu_sup->level[0] = &ssp->srcu_sup->node[0];  in init_srcu_struct_nodes()
183 		ssp->srcu_sup->level[i] = ssp->srcu_sup->level[i - 1] + num_rcu_lvl[i - 1];  in init_srcu_struct_nodes()
187 	srcu_for_each_node_breadth_first(ssp, snp) {  in init_srcu_struct_nodes()
198 		if (snp == &ssp->srcu_sup->node[0]) {  in init_srcu_struct_nodes()
205 		if (snp == ssp->srcu_sup->level[level + 1])  in init_srcu_struct_nodes()
207 		snp->srcu_parent = ssp->srcu_sup->level[level - 1] +  in init_srcu_struct_nodes()
208 				   (snp - ssp->srcu_sup->level[level]) /  in init_srcu_struct_nodes()
217 	snp_first = ssp->srcu_sup->level[level];  in init_srcu_struct_nodes()
219 		sdp = per_cpu_ptr(ssp->sda, cpu);  in init_srcu_struct_nodes()
228 	smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_WAIT_BARRIER);  in init_srcu_struct_nodes()
237 static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)  in init_srcu_struct_fields()  argument
240 		ssp->srcu_sup = kzalloc(sizeof(*ssp->srcu_sup), GFP_KERNEL);  in init_srcu_struct_fields()
241 	if (!ssp->srcu_sup)  in init_srcu_struct_fields()
244 		spin_lock_init(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in init_srcu_struct_fields()
245 	ssp->srcu_sup->srcu_size_state = SRCU_SIZE_SMALL;  in init_srcu_struct_fields()
246 	ssp->srcu_sup->node = NULL;  in init_srcu_struct_fields()
247 	mutex_init(&ssp->srcu_sup->srcu_cb_mutex);  in init_srcu_struct_fields()
248 	mutex_init(&ssp->srcu_sup->srcu_gp_mutex);  in init_srcu_struct_fields()
249 	ssp->srcu_idx = 0;  in init_srcu_struct_fields()
250 	ssp->srcu_sup->srcu_gp_seq = 0;  in init_srcu_struct_fields()
251 	ssp->srcu_sup->srcu_barrier_seq = 0;  in init_srcu_struct_fields()
252 	mutex_init(&ssp->srcu_sup->srcu_barrier_mutex);  in init_srcu_struct_fields()
253 	atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0);  in init_srcu_struct_fields()
254 	INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu);  in init_srcu_struct_fields()
255 	ssp->srcu_sup->sda_is_static = is_static;  in init_srcu_struct_fields()
257 		ssp->sda = alloc_percpu(struct srcu_data);  in init_srcu_struct_fields()
258 	if (!ssp->sda) {  in init_srcu_struct_fields()
260 			kfree(ssp->srcu_sup);  in init_srcu_struct_fields()
263 	init_srcu_struct_data(ssp);  in init_srcu_struct_fields()
264 	ssp->srcu_sup->srcu_gp_seq_needed_exp = 0;  in init_srcu_struct_fields()
265 	ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns();  in init_srcu_struct_fields()
266 	if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) {  in init_srcu_struct_fields()
267 		if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC)) {  in init_srcu_struct_fields()
268 			if (!ssp->srcu_sup->sda_is_static) {  in init_srcu_struct_fields()
269 				free_percpu(ssp->sda);  in init_srcu_struct_fields()
270 				ssp->sda = NULL;  in init_srcu_struct_fields()
271 				kfree(ssp->srcu_sup);  in init_srcu_struct_fields()
275 			WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG);  in init_srcu_struct_fields()
278 	ssp->srcu_sup->srcu_ssp = ssp;  in init_srcu_struct_fields()
279 	smp_store_release(&ssp->srcu_sup->srcu_gp_seq_needed, 0); /* Init done. */  in init_srcu_struct_fields()
285 int __init_srcu_struct(struct srcu_struct *ssp, const char *name,  in __init_srcu_struct()  argument
289 	debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));  in __init_srcu_struct()
290 	lockdep_init_map(&ssp->dep_map, name, key, 0);  in __init_srcu_struct()
291 	return init_srcu_struct_fields(ssp, false);  in __init_srcu_struct()
305 int init_srcu_struct(struct srcu_struct *ssp)  in init_srcu_struct()  argument
307 	return init_srcu_struct_fields(ssp, false);  in init_srcu_struct()
316 static void __srcu_transition_to_big(struct srcu_struct *ssp)  in __srcu_transition_to_big()  argument
318 	lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in __srcu_transition_to_big()
319 	smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_ALLOC);  in __srcu_transition_to_big()
325 static void srcu_transition_to_big(struct srcu_struct *ssp)  in srcu_transition_to_big()  argument
330 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL)  in srcu_transition_to_big()
332 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
333 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL) {  in srcu_transition_to_big()
334 		spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
337 	__srcu_transition_to_big(ssp);  in srcu_transition_to_big()
338 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_transition_to_big()
345 static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp)  in spin_lock_irqsave_check_contention()  argument
349 	if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_sup->srcu_size_state)  in spin_lock_irqsave_check_contention()
352 	if (ssp->srcu_sup->srcu_size_jiffies != j) {  in spin_lock_irqsave_check_contention()
353 		ssp->srcu_sup->srcu_size_jiffies = j;  in spin_lock_irqsave_check_contention()
354 		ssp->srcu_sup->srcu_n_lock_retries = 0;  in spin_lock_irqsave_check_contention()
356 	if (++ssp->srcu_sup->srcu_n_lock_retries <= small_contention_lim)  in spin_lock_irqsave_check_contention()
358 	__srcu_transition_to_big(ssp);  in spin_lock_irqsave_check_contention()
369 	struct srcu_struct *ssp = sdp->ssp;  in spin_lock_irqsave_sdp_contention()  local
373 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);  in spin_lock_irqsave_sdp_contention()
374 	spin_lock_irqsave_check_contention(ssp);  in spin_lock_irqsave_sdp_contention()
375 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, *flags);  in spin_lock_irqsave_sdp_contention()
385 static void spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsigned long *flags)  in spin_lock_irqsave_ssp_contention()  argument
387 	if (spin_trylock_irqsave_rcu_node(ssp->srcu_sup, *flags))  in spin_lock_irqsave_ssp_contention()
389 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, *flags);  in spin_lock_irqsave_ssp_contention()
390 	spin_lock_irqsave_check_contention(ssp);  in spin_lock_irqsave_ssp_contention()
401 static void check_init_srcu_struct(struct srcu_struct *ssp)  in check_init_srcu_struct()  argument
406 	if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed))) /*^^^*/  in check_init_srcu_struct()
408 	spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
409 	if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq_needed)) {  in check_init_srcu_struct()
410 		spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
413 	init_srcu_struct_fields(ssp, true);  in check_init_srcu_struct()
414 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in check_init_srcu_struct()
421 static unsigned long srcu_readers_lock_idx(struct srcu_struct *ssp, int idx)  in srcu_readers_lock_idx()  argument
427 		struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_lock_idx()
438 static unsigned long srcu_readers_unlock_idx(struct srcu_struct *ssp, int idx)  in srcu_readers_unlock_idx()  argument
445 		struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_unlock_idx()
452 		  "Mixed NMI-safe readers for srcu_struct at %ps.\n", ssp);  in srcu_readers_unlock_idx()
460 static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx)  in srcu_readers_active_idx_check()  argument
464 	unlocks = srcu_readers_unlock_idx(ssp, idx);  in srcu_readers_active_idx_check()
535 	return srcu_readers_lock_idx(ssp, idx) == unlocks;  in srcu_readers_active_idx_check()
547 static bool srcu_readers_active(struct srcu_struct *ssp)  in srcu_readers_active()  argument
553 		struct srcu_data *cpuc = per_cpu_ptr(ssp->sda, cpu);  in srcu_readers_active()
614 static unsigned long srcu_get_delay(struct srcu_struct *ssp)  in srcu_get_delay()  argument
619 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_get_delay()
644 void cleanup_srcu_struct(struct srcu_struct *ssp)  in cleanup_srcu_struct()  argument
647 	struct srcu_usage *sup = ssp->srcu_sup;  in cleanup_srcu_struct()
649 	if (WARN_ON(!srcu_get_delay(ssp)))  in cleanup_srcu_struct()
651 	if (WARN_ON(srcu_readers_active(ssp)))  in cleanup_srcu_struct()
655 		struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu);  in cleanup_srcu_struct()
664 	    WARN_ON(srcu_readers_active(ssp))) {  in cleanup_srcu_struct()
666 			__func__, ssp, rcu_seq_state(READ_ONCE(sup->srcu_gp_seq)),  in cleanup_srcu_struct()
674 		free_percpu(ssp->sda);  in cleanup_srcu_struct()
675 		ssp->sda = NULL;  in cleanup_srcu_struct()
677 		ssp->srcu_sup = NULL;  in cleanup_srcu_struct()
686 void srcu_check_nmi_safety(struct srcu_struct *ssp, bool nmi_safe)  in srcu_check_nmi_safety()  argument
694 	sdp = raw_cpu_ptr(ssp->sda);  in srcu_check_nmi_safety()
710 int __srcu_read_lock(struct srcu_struct *ssp)  in __srcu_read_lock()  argument
714 	idx = READ_ONCE(ssp->srcu_idx) & 0x1;  in __srcu_read_lock()
715 	this_cpu_inc(ssp->sda->srcu_lock_count[idx].counter);  in __srcu_read_lock()
726 void __srcu_read_unlock(struct srcu_struct *ssp, int idx)  in __srcu_read_unlock()  argument
729 	this_cpu_inc(ssp->sda->srcu_unlock_count[idx].counter);  in __srcu_read_unlock()
740 int __srcu_read_lock_nmisafe(struct srcu_struct *ssp)  in __srcu_read_lock_nmisafe()  argument
743 	struct srcu_data *sdp = raw_cpu_ptr(ssp->sda);  in __srcu_read_lock_nmisafe()
745 	idx = READ_ONCE(ssp->srcu_idx) & 0x1;  in __srcu_read_lock_nmisafe()
757 void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx)  in __srcu_read_unlock_nmisafe()  argument
759 	struct srcu_data *sdp = raw_cpu_ptr(ssp->sda);  in __srcu_read_unlock_nmisafe()
771 static void srcu_gp_start(struct srcu_struct *ssp)  in srcu_gp_start()  argument
776 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)  in srcu_gp_start()
777 		sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id());  in srcu_gp_start()
779 		sdp = this_cpu_ptr(ssp->sda);  in srcu_gp_start()
780 	lockdep_assert_held(&ACCESS_PRIVATE(ssp->srcu_sup, lock));  in srcu_gp_start()
781 	WARN_ON_ONCE(ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed));  in srcu_gp_start()
784 			      rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq));  in srcu_gp_start()
787 	WRITE_ONCE(ssp->srcu_sup->srcu_gp_start, jiffies);  in srcu_gp_start()
788 	WRITE_ONCE(ssp->srcu_sup->srcu_n_exp_nodelay, 0);  in srcu_gp_start()
790 	rcu_seq_start(&ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start()
791 	state = rcu_seq_state(ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start()
829 static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_schedule_cbs_snp()  argument
837 		srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay);  in srcu_schedule_cbs_snp()
850 static void srcu_gp_end(struct srcu_struct *ssp)  in srcu_gp_end()  argument
864 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_gp_end()
888 		srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, get_boot_cpu_id()),  in srcu_gp_end()
892 		srcu_for_each_node_breadth_first(ssp, snp) {  in srcu_gp_end()
910 				srcu_schedule_cbs_snp(ssp, snp, mask, cbdelay);  in srcu_gp_end()
917 			sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_gp_end()
934 		srcu_gp_start(ssp);  in srcu_gp_end()
936 		srcu_reschedule(ssp, 0);  in srcu_gp_end()
944 			init_srcu_struct_nodes(ssp, GFP_KERNEL);  in srcu_gp_end()
957 static void srcu_funnel_exp_start(struct srcu_struct *ssp, struct srcu_node *snp,  in srcu_funnel_exp_start()  argument
966 			if (WARN_ON_ONCE(rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, s)) ||  in srcu_funnel_exp_start()
978 	spin_lock_irqsave_ssp_contention(ssp, &flags);  in srcu_funnel_exp_start()
979 	if (ULONG_CMP_LT(ssp->srcu_sup->srcu_gp_seq_needed_exp, s))  in srcu_funnel_exp_start()
980 		WRITE_ONCE(ssp->srcu_sup->srcu_gp_seq_needed_exp, s);  in srcu_funnel_exp_start()
981 	spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);  in srcu_funnel_exp_start()
997 static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,  in srcu_funnel_gp_start()  argument
1006 	struct srcu_usage *sup = ssp->srcu_sup;  in srcu_funnel_gp_start()
1030 					srcu_funnel_exp_start(ssp, snp, s);  in srcu_funnel_gp_start()
1043 	spin_lock_irqsave_ssp_contention(ssp, &flags);  in srcu_funnel_gp_start()
1058 		srcu_gp_start(ssp);  in srcu_funnel_gp_start()
1067 					   !!srcu_get_delay(ssp));  in srcu_funnel_gp_start()
1079 static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount)  in try_check_zero()  argument
1083 	curdelay = !srcu_get_delay(ssp);  in try_check_zero()
1086 		if (srcu_readers_active_idx_check(ssp, idx))  in try_check_zero()
1099 static void srcu_flip(struct srcu_struct *ssp)  in srcu_flip()  argument
1131 	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); // Flip the counter.  in srcu_flip()
1165 static bool srcu_might_be_idle(struct srcu_struct *ssp)  in srcu_might_be_idle()  argument
1173 	check_init_srcu_struct(ssp);  in srcu_might_be_idle()
1175 	sdp = raw_cpu_ptr(ssp->sda);  in srcu_might_be_idle()
1191 	tlast = READ_ONCE(ssp->srcu_sup->srcu_last_gp_end);  in srcu_might_be_idle()
1197 	curseq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq);  in srcu_might_be_idle()
1199 	if (ULONG_CMP_LT(curseq, READ_ONCE(ssp->srcu_sup->srcu_gp_seq_needed)))  in srcu_might_be_idle()
1202 	if (curseq != rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq))  in srcu_might_be_idle()
1217 static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,  in srcu_gp_start_if_needed()  argument
1229 	check_init_srcu_struct(ssp);  in srcu_gp_start_if_needed()
1235 	idx = __srcu_read_lock_nmisafe(ssp);  in srcu_gp_start_if_needed()
1236 	ss_state = smp_load_acquire(&ssp->srcu_sup->srcu_size_state);  in srcu_gp_start_if_needed()
1238 		sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id());  in srcu_gp_start_if_needed()
1240 		sdp = raw_cpu_ptr(ssp->sda);  in srcu_gp_start_if_needed()
1271 	s = rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq);  in srcu_gp_start_if_needed()
1273 			      rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq));  in srcu_gp_start_if_needed()
1292 		srcu_funnel_gp_start(ssp, sdp, s, do_norm);  in srcu_gp_start_if_needed()
1294 		srcu_funnel_exp_start(ssp, sdp_mynode, s);  in srcu_gp_start_if_needed()
1295 	__srcu_read_unlock_nmisafe(ssp, idx);  in srcu_gp_start_if_needed()
1327 static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,  in __call_srcu()  argument
1337 	(void)srcu_gp_start_if_needed(ssp, rhp, do_norm);  in __call_srcu()
1357 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,  in call_srcu()  argument
1360 	__call_srcu(ssp, rhp, func, true);  in call_srcu()
1367 static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm)  in __synchronize_srcu()  argument
1371 	srcu_lock_sync(&ssp->dep_map);  in __synchronize_srcu()
1373 	RCU_LOCKDEP_WARN(lockdep_is_held(ssp) ||  in __synchronize_srcu()
1382 	check_init_srcu_struct(ssp);  in __synchronize_srcu()
1385 	__call_srcu(ssp, &rcu.head, wakeme_after_rcu, do_norm);  in __synchronize_srcu()
1409 void synchronize_srcu_expedited(struct srcu_struct *ssp)  in synchronize_srcu_expedited()  argument
1411 	__synchronize_srcu(ssp, rcu_gp_is_normal());  in synchronize_srcu_expedited()
1462 void synchronize_srcu(struct srcu_struct *ssp)  in synchronize_srcu()  argument
1464 	if (srcu_might_be_idle(ssp) || rcu_gp_is_expedited())  in synchronize_srcu()
1465 		synchronize_srcu_expedited(ssp);  in synchronize_srcu()
1467 		__synchronize_srcu(ssp, true);  in synchronize_srcu()
1481 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)  in get_state_synchronize_srcu()  argument
1486 	return rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq);  in get_state_synchronize_srcu()
1500 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)  in start_poll_synchronize_srcu()  argument
1502 	return srcu_gp_start_if_needed(ssp, NULL, true);  in start_poll_synchronize_srcu()
1531 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)  in poll_state_synchronize_srcu()  argument
1533 	if (!rcu_seq_done(&ssp->srcu_sup->srcu_gp_seq, cookie))  in poll_state_synchronize_srcu()
1548 	struct srcu_struct *ssp;  in srcu_barrier_cb()  local
1551 	ssp = sdp->ssp;  in srcu_barrier_cb()
1552 	if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt))  in srcu_barrier_cb()
1553 		complete(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier_cb()
1564 static void srcu_barrier_one_cpu(struct srcu_struct *ssp, struct srcu_data *sdp)  in srcu_barrier_one_cpu()  argument
1567 	atomic_inc(&ssp->srcu_sup->srcu_barrier_cpu_cnt);  in srcu_barrier_one_cpu()
1573 		atomic_dec(&ssp->srcu_sup->srcu_barrier_cpu_cnt);  in srcu_barrier_one_cpu()
1582 void srcu_barrier(struct srcu_struct *ssp)  in srcu_barrier()  argument
1586 	unsigned long s = rcu_seq_snap(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1588 	check_init_srcu_struct(ssp);  in srcu_barrier()
1589 	mutex_lock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1590 	if (rcu_seq_done(&ssp->srcu_sup->srcu_barrier_seq, s)) {  in srcu_barrier()
1592 		mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1595 	rcu_seq_start(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1596 	init_completion(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1599 	atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 1);  in srcu_barrier()
1601 	idx = __srcu_read_lock_nmisafe(ssp);  in srcu_barrier()
1602 	if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)  in srcu_barrier()
1603 		srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda,	get_boot_cpu_id()));  in srcu_barrier()
1606 			srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda, cpu));  in srcu_barrier()
1607 	__srcu_read_unlock_nmisafe(ssp, idx);  in srcu_barrier()
1610 	if (atomic_dec_and_test(&ssp->srcu_sup->srcu_barrier_cpu_cnt))  in srcu_barrier()
1611 		complete(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1612 	wait_for_completion(&ssp->srcu_sup->srcu_barrier_completion);  in srcu_barrier()
1614 	rcu_seq_end(&ssp->srcu_sup->srcu_barrier_seq);  in srcu_barrier()
1615 	mutex_unlock(&ssp->srcu_sup->srcu_barrier_mutex);  in srcu_barrier()
1626 unsigned long srcu_batches_completed(struct srcu_struct *ssp)  in srcu_batches_completed()  argument
1628 	return READ_ONCE(ssp->srcu_idx);  in srcu_batches_completed()
1637 static void srcu_advance_state(struct srcu_struct *ssp)  in srcu_advance_state()  argument
1641 	mutex_lock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1653 	idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq)); /* ^^^ */  in srcu_advance_state()
1655 		spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1656 		if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {  in srcu_advance_state()
1657 			WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq));  in srcu_advance_state()
1658 			spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1659 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1662 		idx = rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq));  in srcu_advance_state()
1664 			srcu_gp_start(ssp);  in srcu_advance_state()
1665 		spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1667 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1672 	if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN1) {  in srcu_advance_state()
1673 		idx = 1 ^ (ssp->srcu_idx & 1);  in srcu_advance_state()
1674 		if (!try_check_zero(ssp, idx, 1)) {  in srcu_advance_state()
1675 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1678 		srcu_flip(ssp);  in srcu_advance_state()
1679 		spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1680 		rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2);  in srcu_advance_state()
1681 		ssp->srcu_sup->srcu_n_exp_nodelay = 0;  in srcu_advance_state()
1682 		spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_advance_state()
1685 	if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN2) {  in srcu_advance_state()
1691 		idx = 1 ^ (ssp->srcu_idx & 1);  in srcu_advance_state()
1692 		if (!try_check_zero(ssp, idx, 2)) {  in srcu_advance_state()
1693 			mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);  in srcu_advance_state()
1696 		ssp->srcu_sup->srcu_n_exp_nodelay = 0;  in srcu_advance_state()
1697 		srcu_gp_end(ssp);  /* Releases ->srcu_gp_mutex. */  in srcu_advance_state()
1714 	struct srcu_struct *ssp;  in srcu_invoke_callbacks()  local
1718 	ssp = sdp->ssp;  in srcu_invoke_callbacks()
1723 			      rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq));  in srcu_invoke_callbacks()
1761 static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay)  in srcu_reschedule()  argument
1765 	spin_lock_irq_rcu_node(ssp->srcu_sup);  in srcu_reschedule()
1766 	if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {  in srcu_reschedule()
1767 		if (!WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq))) {  in srcu_reschedule()
1771 	} else if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)) {  in srcu_reschedule()
1773 		srcu_gp_start(ssp);  in srcu_reschedule()
1775 	spin_unlock_irq_rcu_node(ssp->srcu_sup);  in srcu_reschedule()
1778 		queue_delayed_work(rcu_gp_wq, &ssp->srcu_sup->work, delay);  in srcu_reschedule()
1788 	struct srcu_struct *ssp;  in process_srcu()  local
1792 	ssp = sup->srcu_ssp;  in process_srcu()
1794 	srcu_advance_state(ssp);  in process_srcu()
1795 	curdelay = srcu_get_delay(ssp);  in process_srcu()
1809 	srcu_reschedule(ssp, curdelay);  in process_srcu()
1813 			     struct srcu_struct *ssp, int *flags,  in srcutorture_get_gp_data()  argument
1819 	*gp_seq = rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq);  in srcutorture_get_gp_data()
1836 void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf)  in srcu_torture_stats_print()  argument
1841 	int ss_state = READ_ONCE(ssp->srcu_sup->srcu_size_state);  in srcu_torture_stats_print()
1844 	idx = ssp->srcu_idx & 0x1;  in srcu_torture_stats_print()
1848 		 tt, tf, rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq), ss_state,  in srcu_torture_stats_print()
1850 	if (!ssp->sda) {  in srcu_torture_stats_print()
1861 			sdp = per_cpu_ptr(ssp->sda, cpu);  in srcu_torture_stats_print()
1885 		srcu_transition_to_big(ssp);  in srcu_torture_stats_print()
1941 	struct srcu_struct *ssp;  in srcu_module_coming()  local
1945 		ssp = *(sspp++);  in srcu_module_coming()
1946 		ssp->sda = alloc_percpu(struct srcu_data);  in srcu_module_coming()
1947 		if (WARN_ON_ONCE(!ssp->sda))  in srcu_module_coming()
1957 	struct srcu_struct *ssp;  in srcu_module_going()  local
1961 		ssp = *(sspp++);  in srcu_module_going()
1962 		if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed)) &&  in srcu_module_going()
1963 		    !WARN_ON_ONCE(!ssp->srcu_sup->sda_is_static))  in srcu_module_going()
1964 			cleanup_srcu_struct(ssp);  in srcu_module_going()
1965 		if (!WARN_ON(srcu_readers_active(ssp)))  in srcu_module_going()
1966 			free_percpu(ssp->sda);  in srcu_module_going()