574 struct topa {  struct
588 	((PAGE_SIZE - sizeof(struct topa)) / sizeof(struct topa_entry))  argument
597 	struct topa		topa;  member
600 static inline struct topa_page *topa_to_page(struct topa *topa)  in topa_to_page()  argument
602 	return container_of(topa, struct topa_page, topa);  in topa_to_page()
610 static inline phys_addr_t topa_pfn(struct topa *topa)  in topa_pfn()  argument
612 	return PFN_DOWN(virt_to_phys(topa_to_page(topa)));  in topa_pfn()
657 static struct topa *topa_alloc(int cpu, gfp_t gfp)  in topa_alloc()
668 	tp->topa.last = 0;  in topa_alloc()
675 		TOPA_ENTRY(&tp->topa, 1)->base = page_to_phys(p) >> TOPA_SHIFT;  in topa_alloc()
676 		TOPA_ENTRY(&tp->topa, 1)->end = 1;  in topa_alloc()
679 	return &tp->topa;  in topa_alloc()
686 static void topa_free(struct topa *topa)  in topa_free()  argument
688 	free_page((unsigned long)topa);  in topa_free()
700 static void topa_insert_table(struct pt_buffer *buf, struct topa *topa)  in topa_insert_table()  argument
702 	struct topa *last = buf->last;  in topa_insert_table()
704 	list_add_tail(&topa->list, &buf->tables);  in topa_insert_table()
707 		buf->first = buf->last = buf->cur = topa;  in topa_insert_table()
711 	topa->offset = last->offset + last->size;  in topa_insert_table()
712 	buf->last = topa;  in topa_insert_table()
719 	TOPA_ENTRY(last, -1)->base = topa_pfn(topa);  in topa_insert_table()
727 static bool topa_table_full(struct topa *topa)  in topa_table_full()  argument
731 		return !!topa->last;  in topa_table_full()
733 	return topa->last == TENTS_PER_PAGE - 1;  in topa_table_full()
748 	struct topa *topa = buf->last;  in topa_insert_pages()  local
756 	if (topa_table_full(topa)) {  in topa_insert_pages()
757 		topa = topa_alloc(cpu, gfp);  in topa_insert_pages()
758 		if (!topa)  in topa_insert_pages()
761 		topa_insert_table(buf, topa);  in topa_insert_pages()
764 	if (topa->z_count == topa->last - 1) {  in topa_insert_pages()
765 		if (order == TOPA_ENTRY(topa, topa->last - 1)->size)  in topa_insert_pages()
766 			topa->z_count++;  in topa_insert_pages()
769 	TOPA_ENTRY(topa, -1)->base = page_to_phys(p) >> TOPA_SHIFT;  in topa_insert_pages()
770 	TOPA_ENTRY(topa, -1)->size = order;  in topa_insert_pages()
773 		TOPA_ENTRY(topa, -1)->intr = 1;  in topa_insert_pages()
774 		TOPA_ENTRY(topa, -1)->stop = 1;  in topa_insert_pages()
777 	topa->last++;  in topa_insert_pages()
778 	topa->size += sizes(order);  in topa_insert_pages()
791 	struct topa *topa;  in pt_topa_dump()  local
793 	list_for_each_entry(topa, &buf->tables, list) {  in pt_topa_dump()
794 		struct topa_page *tp = topa_to_page(topa);  in pt_topa_dump()
798 			 topa->offset, topa->size);  in pt_topa_dump()
812 			if (!i && topa->z_count)  in pt_topa_dump()
813 				i += topa->z_count;  in pt_topa_dump()
833 			buf->cur = list_entry(buf->cur->list.next, struct topa,  in pt_buffer_advance()
963 		buf->cur = &tp->topa;  in pt_read_offset()
978 	struct topa *topa;  in pt_topa_entry_for_page()  local
991 	list_for_each_entry(topa, &buf->tables, list) {  in pt_topa_entry_for_page()
992 		if (topa->offset + topa->size > pg << PAGE_SHIFT)  in pt_topa_entry_for_page()
1008 	if (WARN_ON_ONCE(topa->last == -1))  in pt_topa_entry_for_page()
1011 	tp = topa_to_page(topa);  in pt_topa_entry_for_page()
1012 	cur_pg = PFN_DOWN(topa->offset);  in pt_topa_entry_for_page()
1013 	if (topa->z_count) {  in pt_topa_entry_for_page()
1014 		z_pg = TOPA_ENTRY_PAGES(topa, 0) * (topa->z_count + 1);  in pt_topa_entry_for_page()
1015 		start_idx = topa->z_count + 1;  in pt_topa_entry_for_page()
1023 		idx = (pg - cur_pg) / TOPA_ENTRY_PAGES(topa, 0);  in pt_topa_entry_for_page()
1030 	for (idx = start_idx, cur_pg += z_pg; idx < topa->last; idx++) {  in pt_topa_entry_for_page()
1031 		if (cur_pg + TOPA_ENTRY_PAGES(topa, idx) > pg)  in pt_topa_entry_for_page()
1034 		cur_pg += TOPA_ENTRY_PAGES(topa, idx);  in pt_topa_entry_for_page()
1050 	struct topa *topa;  in pt_topa_prev_entry()  local
1056 	topa = &tp->topa;  in pt_topa_prev_entry()
1057 	if (topa == buf->first)  in pt_topa_prev_entry()
1058 		topa = buf->last;  in pt_topa_prev_entry()
1060 		topa = list_prev_entry(topa, list);  in pt_topa_prev_entry()
1062 	tp = topa_to_page(topa);  in pt_topa_prev_entry()
1064 	return &tp->table[topa->last - 1];  in pt_topa_prev_entry()
1176 		buf->cur = &cur_tp->topa;  in pt_buffer_reset_offsets()
1193 	struct topa *topa, *iter;  in pt_buffer_fini_topa()  local
1198 	list_for_each_entry_safe(topa, iter, &buf->tables, list) {  in pt_buffer_fini_topa()
1203 		topa_free(topa);  in pt_buffer_fini_topa()
1216 	struct topa *topa;  in pt_buffer_init_topa()  local
1219 	topa = topa_alloc(cpu, gfp);  in pt_buffer_init_topa()
1220 	if (!topa)  in pt_buffer_init_topa()
1223 	topa_insert_table(buf, topa);  in pt_buffer_init_topa()
1759 	BUILD_BUG_ON(sizeof(struct topa) > PAGE_SIZE);  in pt_init()