27 static int init_srcu_struct_fields(struct srcu_struct *ssp)  in init_srcu_struct_fields()  argument
29 	ssp->srcu_lock_nesting[0] = 0;  in init_srcu_struct_fields()
30 	ssp->srcu_lock_nesting[1] = 0;  in init_srcu_struct_fields()
31 	init_swait_queue_head(&ssp->srcu_wq);  in init_srcu_struct_fields()
32 	ssp->srcu_cb_head = NULL;  in init_srcu_struct_fields()
33 	ssp->srcu_cb_tail = &ssp->srcu_cb_head;  in init_srcu_struct_fields()
34 	ssp->srcu_gp_running = false;  in init_srcu_struct_fields()
35 	ssp->srcu_gp_waiting = false;  in init_srcu_struct_fields()
36 	ssp->srcu_idx = 0;  in init_srcu_struct_fields()
37 	ssp->srcu_idx_max = 0;  in init_srcu_struct_fields()
38 	INIT_WORK(&ssp->srcu_work, srcu_drive_gp);  in init_srcu_struct_fields()
39 	INIT_LIST_HEAD(&ssp->srcu_work.entry);  in init_srcu_struct_fields()
45 int __init_srcu_struct(struct srcu_struct *ssp, const char *name,  in __init_srcu_struct()  argument
49 	debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));  in __init_srcu_struct()
50 	lockdep_init_map(&ssp->dep_map, name, key, 0);  in __init_srcu_struct()
51 	return init_srcu_struct_fields(ssp);  in __init_srcu_struct()
65 int init_srcu_struct(struct srcu_struct *ssp)  in init_srcu_struct()  argument
67 	return init_srcu_struct_fields(ssp);  in init_srcu_struct()
80 void cleanup_srcu_struct(struct srcu_struct *ssp)  in cleanup_srcu_struct()  argument
82 	WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]);  in cleanup_srcu_struct()
83 	flush_work(&ssp->srcu_work);  in cleanup_srcu_struct()
84 	WARN_ON(ssp->srcu_gp_running);  in cleanup_srcu_struct()
85 	WARN_ON(ssp->srcu_gp_waiting);  in cleanup_srcu_struct()
86 	WARN_ON(ssp->srcu_cb_head);  in cleanup_srcu_struct()
87 	WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail);  in cleanup_srcu_struct()
88 	WARN_ON(ssp->srcu_idx != ssp->srcu_idx_max);  in cleanup_srcu_struct()
89 	WARN_ON(ssp->srcu_idx & 0x1);  in cleanup_srcu_struct()
97 void __srcu_read_unlock(struct srcu_struct *ssp, int idx)  in __srcu_read_unlock()  argument
99 	int newval = READ_ONCE(ssp->srcu_lock_nesting[idx]) - 1;  in __srcu_read_unlock()
101 	WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval);  in __srcu_read_unlock()
102 	if (!newval && READ_ONCE(ssp->srcu_gp_waiting) && in_task())  in __srcu_read_unlock()
103 		swake_up_one(&ssp->srcu_wq);  in __srcu_read_unlock()
117 	struct srcu_struct *ssp;  in srcu_drive_gp()  local
119 	ssp = container_of(wp, struct srcu_struct, srcu_work);  in srcu_drive_gp()
120 	if (ssp->srcu_gp_running || ULONG_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))  in srcu_drive_gp()
124 	WRITE_ONCE(ssp->srcu_gp_running, true);  in srcu_drive_gp()
126 	lh = ssp->srcu_cb_head;  in srcu_drive_gp()
127 	ssp->srcu_cb_head = NULL;  in srcu_drive_gp()
128 	ssp->srcu_cb_tail = &ssp->srcu_cb_head;  in srcu_drive_gp()
130 	idx = (ssp->srcu_idx & 0x2) / 2;  in srcu_drive_gp()
131 	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);  in srcu_drive_gp()
132 	WRITE_ONCE(ssp->srcu_gp_waiting, true);  /* srcu_read_unlock() wakes! */  in srcu_drive_gp()
133 	swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx]));  in srcu_drive_gp()
134 	WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */  in srcu_drive_gp()
135 	WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);  in srcu_drive_gp()
152 	WRITE_ONCE(ssp->srcu_gp_running, false);  in srcu_drive_gp()
153 	if (ULONG_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))  in srcu_drive_gp()
154 		schedule_work(&ssp->srcu_work);  in srcu_drive_gp()
158 static void srcu_gp_start_if_needed(struct srcu_struct *ssp)  in srcu_gp_start_if_needed()  argument
162 	cookie = get_state_synchronize_srcu(ssp);  in srcu_gp_start_if_needed()
163 	if (ULONG_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie))  in srcu_gp_start_if_needed()
165 	WRITE_ONCE(ssp->srcu_idx_max, cookie);  in srcu_gp_start_if_needed()
166 	if (!READ_ONCE(ssp->srcu_gp_running)) {  in srcu_gp_start_if_needed()
168 			schedule_work(&ssp->srcu_work);  in srcu_gp_start_if_needed()
169 		else if (list_empty(&ssp->srcu_work.entry))  in srcu_gp_start_if_needed()
170 			list_add(&ssp->srcu_work.entry, &srcu_boot_list);  in srcu_gp_start_if_needed()
178 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,  in call_srcu()  argument
186 	*ssp->srcu_cb_tail = rhp;  in call_srcu()
187 	ssp->srcu_cb_tail = &rhp->next;  in call_srcu()
189 	srcu_gp_start_if_needed(ssp);  in call_srcu()
196 void synchronize_srcu(struct srcu_struct *ssp)  in synchronize_srcu()  argument
200 	srcu_lock_sync(&ssp->dep_map);  in synchronize_srcu()
202 	RCU_LOCKDEP_WARN(lockdep_is_held(ssp) ||  in synchronize_srcu()
214 	call_srcu(ssp, &rs.head, wakeme_after_rcu);  in synchronize_srcu()
223 unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)  in get_state_synchronize_srcu()  argument
228 	ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1;  in get_state_synchronize_srcu()
241 unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)  in start_poll_synchronize_srcu()  argument
243 	unsigned long ret = get_state_synchronize_srcu(ssp);  in start_poll_synchronize_srcu()
245 	srcu_gp_start_if_needed(ssp);  in start_poll_synchronize_srcu()
253 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)  in poll_state_synchronize_srcu()  argument
255 	unsigned long cur_s = READ_ONCE(ssp->srcu_idx);  in poll_state_synchronize_srcu()
275 	struct srcu_struct *ssp;  in srcu_init()  local
279 		ssp = list_first_entry(&srcu_boot_list,  in srcu_init()
281 		list_del_init(&ssp->srcu_work.entry);  in srcu_init()
282 		schedule_work(&ssp->srcu_work);  in srcu_init()