82 static void rcu_segcblist_set_len(struct rcu_segcblist *rsclp, long v)  in rcu_segcblist_set_len()  argument
85 	atomic_long_set(&rsclp->len, v);  in rcu_segcblist_set_len()
87 	WRITE_ONCE(rsclp->len, v);  in rcu_segcblist_set_len()
92 static long rcu_segcblist_get_seglen(struct rcu_segcblist *rsclp, int seg)  in rcu_segcblist_get_seglen()  argument
94 	return READ_ONCE(rsclp->seglen[seg]);  in rcu_segcblist_get_seglen()
98 long rcu_segcblist_n_segment_cbs(struct rcu_segcblist *rsclp)  in rcu_segcblist_n_segment_cbs()  argument
104 		len += rcu_segcblist_get_seglen(rsclp, i);  in rcu_segcblist_n_segment_cbs()
110 static void rcu_segcblist_set_seglen(struct rcu_segcblist *rsclp, int seg, long v)  in rcu_segcblist_set_seglen()  argument
112 	WRITE_ONCE(rsclp->seglen[seg], v);  in rcu_segcblist_set_seglen()
116 static void rcu_segcblist_add_seglen(struct rcu_segcblist *rsclp, int seg, long v)  in rcu_segcblist_add_seglen()  argument
118 	WRITE_ONCE(rsclp->seglen[seg], rsclp->seglen[seg] + v);  in rcu_segcblist_add_seglen()
122 static void rcu_segcblist_move_seglen(struct rcu_segcblist *rsclp, int from, int to)  in rcu_segcblist_move_seglen()  argument
129 	len = rcu_segcblist_get_seglen(rsclp, from);  in rcu_segcblist_move_seglen()
133 	rcu_segcblist_add_seglen(rsclp, to, len);  in rcu_segcblist_move_seglen()
134 	rcu_segcblist_set_seglen(rsclp, from, 0);  in rcu_segcblist_move_seglen()
138 static void rcu_segcblist_inc_seglen(struct rcu_segcblist *rsclp, int seg)  in rcu_segcblist_inc_seglen()  argument
140 	rcu_segcblist_add_seglen(rsclp, seg, 1);  in rcu_segcblist_inc_seglen()
210 void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v)  in rcu_segcblist_add_len()  argument
214 	atomic_long_add(v, &rsclp->len);  in rcu_segcblist_add_len()
218 	WRITE_ONCE(rsclp->len, rsclp->len + v);  in rcu_segcblist_add_len()
229 void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp)  in rcu_segcblist_inc_len()  argument
231 	rcu_segcblist_add_len(rsclp, 1);  in rcu_segcblist_inc_len()
237 void rcu_segcblist_init(struct rcu_segcblist *rsclp)  in rcu_segcblist_init()  argument
241 	BUILD_BUG_ON(RCU_NEXT_TAIL + 1 != ARRAY_SIZE(rsclp->gp_seq));  in rcu_segcblist_init()
242 	BUILD_BUG_ON(ARRAY_SIZE(rsclp->tails) != ARRAY_SIZE(rsclp->gp_seq));  in rcu_segcblist_init()
243 	rsclp->head = NULL;  in rcu_segcblist_init()
245 		rsclp->tails[i] = &rsclp->head;  in rcu_segcblist_init()
246 		rcu_segcblist_set_seglen(rsclp, i, 0);  in rcu_segcblist_init()
248 	rcu_segcblist_set_len(rsclp, 0);  in rcu_segcblist_init()
249 	rcu_segcblist_set_flags(rsclp, SEGCBLIST_ENABLED);  in rcu_segcblist_init()
256 void rcu_segcblist_disable(struct rcu_segcblist *rsclp)  in rcu_segcblist_disable()  argument
258 	WARN_ON_ONCE(!rcu_segcblist_empty(rsclp));  in rcu_segcblist_disable()
259 	WARN_ON_ONCE(rcu_segcblist_n_cbs(rsclp));  in rcu_segcblist_disable()
260 	rcu_segcblist_clear_flags(rsclp, SEGCBLIST_ENABLED);  in rcu_segcblist_disable()
266 void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload)  in rcu_segcblist_offload()  argument
269 		rcu_segcblist_set_flags(rsclp, SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED);  in rcu_segcblist_offload()
271 		rcu_segcblist_clear_flags(rsclp, SEGCBLIST_OFFLOADED);  in rcu_segcblist_offload()
278 bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp)  in rcu_segcblist_ready_cbs()  argument
280 	return rcu_segcblist_is_enabled(rsclp) &&  in rcu_segcblist_ready_cbs()
281 	       &rsclp->head != READ_ONCE(rsclp->tails[RCU_DONE_TAIL]);  in rcu_segcblist_ready_cbs()
288 bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp)  in rcu_segcblist_pend_cbs()  argument
290 	return rcu_segcblist_is_enabled(rsclp) &&  in rcu_segcblist_pend_cbs()
291 	       !rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL);  in rcu_segcblist_pend_cbs()
298 struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp)  in rcu_segcblist_first_cb()  argument
300 	if (rcu_segcblist_is_enabled(rsclp))  in rcu_segcblist_first_cb()
301 		return rsclp->head;  in rcu_segcblist_first_cb()
312 struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp)  in rcu_segcblist_first_pend_cb()  argument
314 	if (rcu_segcblist_is_enabled(rsclp))  in rcu_segcblist_first_pend_cb()
315 		return *rsclp->tails[RCU_DONE_TAIL];  in rcu_segcblist_first_pend_cb()
323 bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp)  in rcu_segcblist_nextgp()  argument
325 	if (!rcu_segcblist_pend_cbs(rsclp))  in rcu_segcblist_nextgp()
327 	*lp = rsclp->gp_seq[RCU_WAIT_TAIL];  in rcu_segcblist_nextgp()
340 void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,  in rcu_segcblist_enqueue()  argument
343 	rcu_segcblist_inc_len(rsclp);  in rcu_segcblist_enqueue()
344 	rcu_segcblist_inc_seglen(rsclp, RCU_NEXT_TAIL);  in rcu_segcblist_enqueue()
346 	WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rhp);  in rcu_segcblist_enqueue()
347 	WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], &rhp->next);  in rcu_segcblist_enqueue()
360 bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,  in rcu_segcblist_entrain()  argument
365 	if (rcu_segcblist_n_cbs(rsclp) == 0)  in rcu_segcblist_entrain()
367 	rcu_segcblist_inc_len(rsclp);  in rcu_segcblist_entrain()
371 		if (rsclp->tails[i] != rsclp->tails[i - 1])  in rcu_segcblist_entrain()
373 	rcu_segcblist_inc_seglen(rsclp, i);  in rcu_segcblist_entrain()
374 	WRITE_ONCE(*rsclp->tails[i], rhp);  in rcu_segcblist_entrain()
376 		WRITE_ONCE(rsclp->tails[i], &rhp->next);  in rcu_segcblist_entrain()
385 void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,  in rcu_segcblist_extract_done_cbs()  argument
390 	if (!rcu_segcblist_ready_cbs(rsclp))  in rcu_segcblist_extract_done_cbs()
392 	rclp->len = rcu_segcblist_get_seglen(rsclp, RCU_DONE_TAIL);  in rcu_segcblist_extract_done_cbs()
393 	*rclp->tail = rsclp->head;  in rcu_segcblist_extract_done_cbs()
394 	WRITE_ONCE(rsclp->head, *rsclp->tails[RCU_DONE_TAIL]);  in rcu_segcblist_extract_done_cbs()
395 	WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);  in rcu_segcblist_extract_done_cbs()
396 	rclp->tail = rsclp->tails[RCU_DONE_TAIL];  in rcu_segcblist_extract_done_cbs()
398 		if (rsclp->tails[i] == rsclp->tails[RCU_DONE_TAIL])  in rcu_segcblist_extract_done_cbs()
399 			WRITE_ONCE(rsclp->tails[i], &rsclp->head);  in rcu_segcblist_extract_done_cbs()
400 	rcu_segcblist_set_seglen(rsclp, RCU_DONE_TAIL, 0);  in rcu_segcblist_extract_done_cbs()
410 void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,  in rcu_segcblist_extract_pend_cbs()  argument
415 	if (!rcu_segcblist_pend_cbs(rsclp))  in rcu_segcblist_extract_pend_cbs()
418 	*rclp->tail = *rsclp->tails[RCU_DONE_TAIL];  in rcu_segcblist_extract_pend_cbs()
419 	rclp->tail = rsclp->tails[RCU_NEXT_TAIL];  in rcu_segcblist_extract_pend_cbs()
420 	WRITE_ONCE(*rsclp->tails[RCU_DONE_TAIL], NULL);  in rcu_segcblist_extract_pend_cbs()
422 		rclp->len += rcu_segcblist_get_seglen(rsclp, i);  in rcu_segcblist_extract_pend_cbs()
423 		WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_DONE_TAIL]);  in rcu_segcblist_extract_pend_cbs()
424 		rcu_segcblist_set_seglen(rsclp, i, 0);  in rcu_segcblist_extract_pend_cbs()
432 void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,  in rcu_segcblist_insert_count()  argument
435 	rcu_segcblist_add_len(rsclp, rclp->len);  in rcu_segcblist_insert_count()
442 void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,  in rcu_segcblist_insert_done_cbs()  argument
449 	rcu_segcblist_add_seglen(rsclp, RCU_DONE_TAIL, rclp->len);  in rcu_segcblist_insert_done_cbs()
450 	*rclp->tail = rsclp->head;  in rcu_segcblist_insert_done_cbs()
451 	WRITE_ONCE(rsclp->head, rclp->head);  in rcu_segcblist_insert_done_cbs()
453 		if (&rsclp->head == rsclp->tails[i])  in rcu_segcblist_insert_done_cbs()
454 			WRITE_ONCE(rsclp->tails[i], rclp->tail);  in rcu_segcblist_insert_done_cbs()
465 void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,  in rcu_segcblist_insert_pend_cbs()  argument
471 	rcu_segcblist_add_seglen(rsclp, RCU_NEXT_TAIL, rclp->len);  in rcu_segcblist_insert_pend_cbs()
472 	WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head);  in rcu_segcblist_insert_pend_cbs()
473 	WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail);  in rcu_segcblist_insert_pend_cbs()
480 void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq)  in rcu_segcblist_advance()  argument
484 	WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp));  in rcu_segcblist_advance()
485 	if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL))  in rcu_segcblist_advance()
493 		if (ULONG_CMP_LT(seq, rsclp->gp_seq[i]))  in rcu_segcblist_advance()
495 		WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]);  in rcu_segcblist_advance()
496 		rcu_segcblist_move_seglen(rsclp, i, RCU_DONE_TAIL);  in rcu_segcblist_advance()
505 		WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]);  in rcu_segcblist_advance()
514 		if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL])  in rcu_segcblist_advance()
516 		WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]);  in rcu_segcblist_advance()
517 		rcu_segcblist_move_seglen(rsclp, i, j);  in rcu_segcblist_advance()
518 		rsclp->gp_seq[j] = rsclp->gp_seq[i];  in rcu_segcblist_advance()
537 bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq)  in rcu_segcblist_accelerate()  argument
541 	WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp));  in rcu_segcblist_accelerate()
542 	if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL))  in rcu_segcblist_accelerate()
554 		if (rsclp->tails[i] != rsclp->tails[i - 1] &&  in rcu_segcblist_accelerate()
555 		    ULONG_CMP_LT(rsclp->gp_seq[i], seq))  in rcu_segcblist_accelerate()
579 	if (rcu_segcblist_restempty(rsclp, i) || ++i >= RCU_NEXT_TAIL)  in rcu_segcblist_accelerate()
584 		rcu_segcblist_move_seglen(rsclp, j, i);  in rcu_segcblist_accelerate()
594 		WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL]);  in rcu_segcblist_accelerate()
595 		rsclp->gp_seq[i] = seq;  in rcu_segcblist_accelerate()