311 #define DESCS_COUNT(desc_ring)		_DESCS_COUNT((desc_ring)->count_bits)  argument
312 #define DESCS_COUNT_MASK(desc_ring)	(DESCS_COUNT(desc_ring) - 1)  argument
318 #define DESC_INDEX(desc_ring, n)	((n) & DESCS_COUNT_MASK(desc_ring))  argument
333 #define DESC_ID_PREV_WRAP(desc_ring, id) \  argument
334 DESC_ID((id) - DESCS_COUNT(desc_ring))
355 static struct prb_desc *to_desc(struct prb_desc_ring *desc_ring, u64 n)  in to_desc()  argument
357 	return &desc_ring->descs[DESC_INDEX(desc_ring, n)];  in to_desc()
364 static struct printk_info *to_info(struct prb_desc_ring *desc_ring, u64 n)  in to_info()  argument
366 	return &desc_ring->infos[DESC_INDEX(desc_ring, n)];  in to_info()
432 static enum desc_state desc_read(struct prb_desc_ring *desc_ring,  in desc_read()  argument
436 	struct printk_info *info = to_info(desc_ring, id);  in desc_read()
437 	struct prb_desc *desc = to_desc(desc_ring, id);  in desc_read()
543 static void desc_make_reusable(struct prb_desc_ring *desc_ring,  in desc_make_reusable()  argument
548 	struct prb_desc *desc = to_desc(desc_ring, id);  in desc_make_reusable()
571 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in data_make_reusable()  local
592 		d_state = desc_read(desc_ring, id, &desc,  in data_make_reusable()
607 			desc_make_reusable(desc_ring, id);  in data_make_reusable()
770 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in desc_push_tail()  local
774 	d_state = desc_read(desc_ring, tail_id, &desc, NULL, NULL);  in desc_push_tail()
784 		    DESC_ID_PREV_WRAP(desc_ring, tail_id)) {  in desc_push_tail()
799 		desc_make_reusable(desc_ring, tail_id);  in desc_push_tail()
824 	d_state = desc_read(desc_ring, DESC_ID(tail_id + 1), &desc,  in desc_push_tail()
835 		atomic_long_cmpxchg(&desc_ring->tail_id, tail_id,  in desc_push_tail()
867 		if (atomic_long_read(&desc_ring->tail_id) == tail_id) /* LMM(desc_push_tail:D) */  in desc_push_tail()
877 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in desc_reserve()  local
884 	head_id = atomic_long_read(&desc_ring->head_id); /* LMM(desc_reserve:A) */  in desc_reserve()
888 		id_prev_wrap = DESC_ID_PREV_WRAP(desc_ring, id);  in desc_reserve()
914 		if (id_prev_wrap == atomic_long_read(&desc_ring->tail_id  in desc_reserve()
965 	} while (!atomic_long_try_cmpxchg(&desc_ring->head_id, &head_id,  in desc_reserve()
968 	desc = to_desc(desc_ring, id);  in desc_reserve()
1259 static struct prb_desc *desc_reopen_last(struct prb_desc_ring *desc_ring,  in desc_reopen_last()  argument
1269 	id = atomic_long_read(&desc_ring->head_id);  in desc_reopen_last()
1275 	d_state = desc_read(desc_ring, id, &desc, NULL, &cid);  in desc_reopen_last()
1279 	d = to_desc(desc_ring, id);  in desc_reopen_last()
1353 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in prb_reserve_in_last()  local
1362 	d = desc_reopen_last(desc_ring, caller_id, &id);  in prb_reserve_in_last()
1370 	info = to_info(desc_ring, id);  in prb_reserve_in_last()
1448 static void desc_make_final(struct prb_desc_ring *desc_ring, unsigned long id)  in desc_make_final()  argument
1451 	struct prb_desc *d = to_desc(desc_ring, id);  in desc_make_final()
1457 	atomic_long_set(&desc_ring->last_finalized_id, id);  in desc_make_final()
1488 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in prb_reserve()  local
1512 	d = to_desc(desc_ring, id);  in prb_reserve()
1513 	info = to_info(desc_ring, id);  in prb_reserve()
1541 	if (seq == 0 && DESC_INDEX(desc_ring, id) != 0)  in prb_reserve()
1542 		info->seq = DESC_INDEX(desc_ring, id);  in prb_reserve()
1544 		info->seq = seq + DESCS_COUNT(desc_ring);  in prb_reserve()
1553 		desc_make_final(desc_ring, DESC_ID(id - 1));  in prb_reserve()
1578 	struct prb_desc_ring *desc_ring = &e->rb->desc_ring;  in _prb_commit()  local
1579 	struct prb_desc *d = to_desc(desc_ring, e->id);  in _prb_commit()
1634 	struct prb_desc_ring *desc_ring = &e->rb->desc_ring;  in prb_commit()  local
1644 	head_id = atomic_long_read(&desc_ring->head_id); /* LMM(prb_commit:A) */  in prb_commit()
1646 		desc_make_final(desc_ring, e->id);  in prb_commit()
1666 	struct prb_desc_ring *desc_ring = &e->rb->desc_ring;  in prb_final_commit()  local
1671 	atomic_long_set(&desc_ring->last_finalized_id, e->id);  in prb_final_commit()
1755 static int desc_read_finalized_seq(struct prb_desc_ring *desc_ring,  in desc_read_finalized_seq()  argument
1763 	d_state = desc_read(desc_ring, id, desc_out, &s, NULL);  in desc_read_finalized_seq()
1799 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in prb_read()  local
1800 	struct printk_info *info = to_info(desc_ring, seq);  in prb_read()
1801 	struct prb_desc *rdesc = to_desc(desc_ring, seq);  in prb_read()
1811 	err = desc_read_finalized_seq(desc_ring, id, seq, &desc);  in prb_read()
1831 	return desc_read_finalized_seq(desc_ring, id, seq, &desc);  in prb_read()
1837 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in prb_first_seq()  local
1844 		id = atomic_long_read(&rb->desc_ring.tail_id); /* LMM(prb_first_seq:A) */  in prb_first_seq()
1846 		d_state = desc_read(desc_ring, id, &desc, &seq, NULL); /* LMM(prb_first_seq:B) */  in prb_first_seq()
2019 	struct prb_desc_ring *desc_ring = &rb->desc_ring;  in prb_next_seq()  local
2025 	id = atomic_long_read(&desc_ring->last_finalized_id);  in prb_next_seq()
2026 	d_state = desc_read(desc_ring, id, NULL, &seq, NULL);  in prb_next_seq()
2083 	rb->desc_ring.count_bits = descbits;  in prb_init()
2084 	rb->desc_ring.descs = descs;  in prb_init()
2085 	rb->desc_ring.infos = infos;  in prb_init()
2086 	atomic_long_set(&rb->desc_ring.head_id, DESC0_ID(descbits));  in prb_init()
2087 	atomic_long_set(&rb->desc_ring.tail_id, DESC0_ID(descbits));  in prb_init()
2088 	atomic_long_set(&rb->desc_ring.last_finalized_id, DESC0_ID(descbits));  in prb_init()