118 static int thread_stack__grow(struct thread_stack *ts)  in thread_stack__grow()  argument
123 	new_sz = ts->sz + STACK_GROWTH;  in thread_stack__grow()
126 	new_stack = realloc(ts->stack, sz);  in thread_stack__grow()
130 	ts->stack = new_stack;  in thread_stack__grow()
131 	ts->sz = new_sz;  in thread_stack__grow()
136 static int thread_stack__init(struct thread_stack *ts, struct thread *thread,  in thread_stack__init()  argument
143 		err = thread_stack__grow(ts);  in thread_stack__init()
152 		ts->br_stack_rb = zalloc(sz);  in thread_stack__init()
153 		if (!ts->br_stack_rb)  in thread_stack__init()
155 		ts->br_stack_sz = br_stack_sz;  in thread_stack__init()
162 		ts->kernel_start = machine__kernel_start(machine);  in thread_stack__init()
164 			ts->rstate = X86_RETPOLINE_POSSIBLE;  in thread_stack__init()
166 		ts->kernel_start = 1ULL << 63;  in thread_stack__init()
168 	ts->crp = crp;  in thread_stack__init()
178 	struct thread_stack *ts = thread->ts, *new_ts;  in thread_stack__new()  local
179 	unsigned int old_sz = ts ? ts->arr_sz : 0;  in thread_stack__new()
185 	if (!ts || new_sz > old_sz) {  in thread_stack__new()
186 		new_ts = calloc(new_sz, sizeof(*ts));  in thread_stack__new()
189 		if (ts)  in thread_stack__new()
190 			memcpy(new_ts, ts, old_sz * sizeof(*ts));  in thread_stack__new()
192 		zfree(&thread->ts);  in thread_stack__new()
193 		thread->ts = new_ts;  in thread_stack__new()
194 		ts = new_ts;  in thread_stack__new()
198 	    (unsigned int)cpu < ts->arr_sz)  in thread_stack__new()
199 		ts += cpu;  in thread_stack__new()
201 	if (!ts->stack &&  in thread_stack__new()
202 	    thread_stack__init(ts, thread, crp, callstack, br_stack_sz))  in thread_stack__new()
205 	return ts;  in thread_stack__new()
210 	struct thread_stack *ts = thread->ts;  in thread__cpu_stack()  local
215 	if (!ts || (unsigned int)cpu >= ts->arr_sz)  in thread__cpu_stack()
218 	ts += cpu;  in thread__cpu_stack()
220 	if (!ts->stack)  in thread__cpu_stack()
223 	return ts;  in thread__cpu_stack()
235 	return thread->ts;  in thread__stack()
238 static int thread_stack__push(struct thread_stack *ts, u64 ret_addr,  in thread_stack__push()  argument
243 	if (ts->cnt == ts->sz) {  in thread_stack__push()
244 		err = thread_stack__grow(ts);  in thread_stack__push()
247 			ts->cnt = 0;  in thread_stack__push()
251 	ts->stack[ts->cnt].trace_end = trace_end;  in thread_stack__push()
252 	ts->stack[ts->cnt++].ret_addr = ret_addr;  in thread_stack__push()
257 static void thread_stack__pop(struct thread_stack *ts, u64 ret_addr)  in thread_stack__pop()  argument
270 	for (i = ts->cnt; i; ) {  in thread_stack__pop()
271 		if (ts->stack[--i].ret_addr == ret_addr) {  in thread_stack__pop()
272 			ts->cnt = i;  in thread_stack__pop()
278 static void thread_stack__pop_trace_end(struct thread_stack *ts)  in thread_stack__pop_trace_end()  argument
282 	for (i = ts->cnt; i; ) {  in thread_stack__pop_trace_end()
283 		if (ts->stack[--i].trace_end)  in thread_stack__pop_trace_end()
284 			ts->cnt = i;  in thread_stack__pop_trace_end()
290 static bool thread_stack__in_kernel(struct thread_stack *ts)  in thread_stack__in_kernel()  argument
292 	if (!ts->cnt)  in thread_stack__in_kernel()
295 	return ts->stack[ts->cnt - 1].cp->in_kernel;  in thread_stack__in_kernel()
299 				     struct thread_stack *ts, size_t idx,  in thread_stack__call_return()  argument
302 	struct call_return_processor *crp = ts->crp;  in thread_stack__call_return()
306 		.comm = ts->comm,  in thread_stack__call_return()
311 	tse = &ts->stack[idx];  in thread_stack__call_return()
315 	cr.branch_count = ts->branch_count - tse->branch_count;  in thread_stack__call_return()
316 	cr.insn_count = ts->insn_count - tse->insn_count;  in thread_stack__call_return()
317 	cr.cyc_count = ts->cyc_count - tse->cyc_count;  in thread_stack__call_return()
338 static int __thread_stack__flush(struct thread *thread, struct thread_stack *ts)  in __thread_stack__flush()  argument
340 	struct call_return_processor *crp = ts->crp;  in __thread_stack__flush()
344 		ts->cnt = 0;  in __thread_stack__flush()
345 		ts->br_stack_pos = 0;  in __thread_stack__flush()
346 		if (ts->br_stack_rb)  in __thread_stack__flush()
347 			ts->br_stack_rb->nr = 0;  in __thread_stack__flush()
351 	while (ts->cnt) {  in __thread_stack__flush()
352 		err = thread_stack__call_return(thread, ts, --ts->cnt,  in __thread_stack__flush()
353 						ts->last_time, 0, true);  in __thread_stack__flush()
356 			ts->cnt = 0;  in __thread_stack__flush()
366 	struct thread_stack *ts = thread->ts;  in thread_stack__flush()  local
370 	if (ts) {  in thread_stack__flush()
371 		for (pos = 0; pos < ts->arr_sz; pos++) {  in thread_stack__flush()
372 			int ret = __thread_stack__flush(thread, ts + pos);  in thread_stack__flush()
382 static void thread_stack__update_br_stack(struct thread_stack *ts, u32 flags,  in thread_stack__update_br_stack()  argument
385 	struct branch_stack *bs = ts->br_stack_rb;  in thread_stack__update_br_stack()
388 	if (!ts->br_stack_pos)  in thread_stack__update_br_stack()
389 		ts->br_stack_pos = ts->br_stack_sz;  in thread_stack__update_br_stack()
391 	ts->br_stack_pos -= 1;  in thread_stack__update_br_stack()
393 	be              = &bs->entries[ts->br_stack_pos];  in thread_stack__update_br_stack()
400 	be->flags.mispred = ts->mispred_all;  in thread_stack__update_br_stack()
402 	if (bs->nr < ts->br_stack_sz)  in thread_stack__update_br_stack()
410 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__event()  local
415 	if (!ts) {  in thread_stack__event()
416 		ts = thread_stack__new(thread, cpu, NULL, callstack, br_stack_sz);  in thread_stack__event()
417 		if (!ts) {  in thread_stack__event()
421 		ts->trace_nr = trace_nr;  in thread_stack__event()
422 		ts->mispred_all = mispred_all;  in thread_stack__event()
430 	if (trace_nr != ts->trace_nr) {  in thread_stack__event()
431 		if (ts->trace_nr)  in thread_stack__event()
432 			__thread_stack__flush(thread, ts);  in thread_stack__event()
433 		ts->trace_nr = trace_nr;  in thread_stack__event()
437 		thread_stack__update_br_stack(ts, flags, from_ip, to_ip);  in thread_stack__event()
443 	if (ts->crp || !callstack)  in thread_stack__event()
454 		return thread_stack__push(ts, ret_addr,  in thread_stack__event()
464 		thread_stack__pop(ts, to_ip);  in thread_stack__event()
465 		thread_stack__pop_trace_end(ts);  in thread_stack__event()
467 		thread_stack__pop(ts, to_ip);  in thread_stack__event()
475 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__set_trace_nr()  local
477 	if (!ts)  in thread_stack__set_trace_nr()
480 	if (trace_nr != ts->trace_nr) {  in thread_stack__set_trace_nr()
481 		if (ts->trace_nr)  in thread_stack__set_trace_nr()
482 			__thread_stack__flush(thread, ts);  in thread_stack__set_trace_nr()
483 		ts->trace_nr = trace_nr;  in thread_stack__set_trace_nr()
487 static void __thread_stack__free(struct thread *thread, struct thread_stack *ts)  in __thread_stack__free()  argument
489 	__thread_stack__flush(thread, ts);  in __thread_stack__free()
490 	zfree(&ts->stack);  in __thread_stack__free()
491 	zfree(&ts->br_stack_rb);  in __thread_stack__free()
494 static void thread_stack__reset(struct thread *thread, struct thread_stack *ts)  in thread_stack__reset()  argument
496 	unsigned int arr_sz = ts->arr_sz;  in thread_stack__reset()
498 	__thread_stack__free(thread, ts);  in thread_stack__reset()
499 	memset(ts, 0, sizeof(*ts));  in thread_stack__reset()
500 	ts->arr_sz = arr_sz;  in thread_stack__reset()
505 	struct thread_stack *ts = thread->ts;  in thread_stack__free()  local
508 	if (ts) {  in thread_stack__free()
509 		for (pos = 0; pos < ts->arr_sz; pos++)  in thread_stack__free()
510 			__thread_stack__free(thread, ts + pos);  in thread_stack__free()
511 		zfree(&thread->ts);  in thread_stack__free()
524 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__sample()  local
537 	if (!ts) {  in thread_stack__sample()
544 	for (i = 2, j = 1; i < sz && j <= ts->cnt; i++, j++) {  in thread_stack__sample()
545 		ip = ts->stack[ts->cnt - j].ret_addr;  in thread_stack__sample()
567 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__sample_late()  local
577 	if (!ts)  in thread_stack__sample_late()
585 	for (j = 1; j <= ts->cnt; j++) {  in thread_stack__sample_late()
586 		ip = ts->stack[ts->cnt - j].ret_addr;  in thread_stack__sample_late()
595 	for (; nr < sz && j <= ts->cnt; nr++, j++) {  in thread_stack__sample_late()
596 		ip = ts->stack[ts->cnt - j].ret_addr;  in thread_stack__sample_late()
619 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__br_sample()  local
627 	if (!ts)  in thread_stack__br_sample()
630 	src = ts->br_stack_rb;  in thread_stack__br_sample()
637 	nr = min(ts->br_stack_sz - ts->br_stack_pos, (unsigned int)dst->nr);  in thread_stack__br_sample()
638 	memcpy(be, &src->entries[ts->br_stack_pos], bsz * nr);  in thread_stack__br_sample()
640 	if (src->nr >= ts->br_stack_sz) {  in thread_stack__br_sample()
643 		nr = min(ts->br_stack_pos, sz);  in thread_stack__br_sample()
644 		memcpy(be, &src->entries[0], bsz * ts->br_stack_pos);  in thread_stack__br_sample()
681 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__br_sample_late()  local
689 	if (!ts)  in thread_stack__br_sample_late()
692 	src = ts->br_stack_rb;  in thread_stack__br_sample_late()
696 	spos = &src->entries[ts->br_stack_pos];  in thread_stack__br_sample_late()
697 	ssz  = &src->entries[ts->br_stack_sz];  in thread_stack__br_sample_late()
714 		if (src->nr >= ts->br_stack_sz) {  in thread_stack__br_sample_late()
738 		if (src->nr >= ts->br_stack_sz) {  in thread_stack__br_sample_late()
781 static int thread_stack__push_cp(struct thread_stack *ts, u64 ret_addr,  in thread_stack__push_cp()  argument
791 	if (ts->cnt == ts->sz) {  in thread_stack__push_cp()
792 		err = thread_stack__grow(ts);  in thread_stack__push_cp()
797 	tse = &ts->stack[ts->cnt++];  in thread_stack__push_cp()
801 	tse->branch_count = ts->branch_count;  in thread_stack__push_cp()
802 	tse->insn_count = ts->insn_count;  in thread_stack__push_cp()
803 	tse->cyc_count = ts->cyc_count;  in thread_stack__push_cp()
813 static int thread_stack__pop_cp(struct thread *thread, struct thread_stack *ts,  in thread_stack__pop_cp()  argument
819 	if (!ts->cnt)  in thread_stack__pop_cp()
822 	if (ts->cnt == 1) {  in thread_stack__pop_cp()
823 		struct thread_stack_entry *tse = &ts->stack[0];  in thread_stack__pop_cp()
826 			return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
830 	if (ts->stack[ts->cnt - 1].ret_addr == ret_addr &&  in thread_stack__pop_cp()
831 	    !ts->stack[ts->cnt - 1].non_call) {  in thread_stack__pop_cp()
832 		return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
835 		size_t i = ts->cnt - 1;  in thread_stack__pop_cp()
838 			if (ts->stack[i].ret_addr != ret_addr ||  in thread_stack__pop_cp()
839 			    ts->stack[i].non_call)  in thread_stack__pop_cp()
842 			while (ts->cnt > i) {  in thread_stack__pop_cp()
843 				err = thread_stack__call_return(thread, ts,  in thread_stack__pop_cp()
844 								--ts->cnt,  in thread_stack__pop_cp()
850 			return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
858 static int thread_stack__bottom(struct thread_stack *ts,  in thread_stack__bottom()  argument
863 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__bottom()
879 				ts->kernel_start);  in thread_stack__bottom()
881 	return thread_stack__push_cp(ts, ip, sample->time, ref, cp,  in thread_stack__bottom()
885 static int thread_stack__pop_ks(struct thread *thread, struct thread_stack *ts,  in thread_stack__pop_ks()  argument
892 	while (thread_stack__in_kernel(ts)) {  in thread_stack__pop_ks()
893 		err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_ks()
903 					struct thread_stack *ts,  in thread_stack__no_call_return()  argument
908 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__no_call_return()
913 	u64 ks = ts->kernel_start;  in thread_stack__no_call_return()
921 		err = thread_stack__pop_ks(thread, ts, sample, ref);  in thread_stack__no_call_return()
926 		if (!ts->cnt) {  in thread_stack__no_call_return()
928 			return thread_stack__push_cp(ts, 0, tm, ref, cp, true,  in thread_stack__no_call_return()
931 	} else if (thread_stack__in_kernel(ts) && ip < ks) {  in thread_stack__no_call_return()
933 		err = thread_stack__pop_ks(thread, ts, sample, ref);  in thread_stack__no_call_return()
938 	if (ts->cnt)  in thread_stack__no_call_return()
939 		parent = ts->stack[ts->cnt - 1].cp;  in thread_stack__no_call_return()
949 		if (ts->cnt == 1) {  in thread_stack__no_call_return()
950 			err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__no_call_return()
956 		if (!ts->cnt) {  in thread_stack__no_call_return()
959 			return thread_stack__push_cp(ts, addr, tm, ref, cp,  in thread_stack__no_call_return()
969 		err = thread_stack__push_cp(ts, 0, tm, ref, cp, true, false);  in thread_stack__no_call_return()
971 			ts->stack[ts->cnt - 1].non_call = true;  in thread_stack__no_call_return()
983 	err = thread_stack__push_cp(ts, addr, tm, ref, cp, true, false);  in thread_stack__no_call_return()
989 	err = thread_stack__push_cp(ts, ip, tm, ref, cp, true, false);  in thread_stack__no_call_return()
993 	return thread_stack__call_return(thread, ts, --ts->cnt, tm, ref, false);  in thread_stack__no_call_return()
997 				     struct thread_stack *ts, u64 timestamp,  in thread_stack__trace_begin()  argument
1003 	if (!ts->cnt)  in thread_stack__trace_begin()
1007 	tse = &ts->stack[ts->cnt - 1];  in thread_stack__trace_begin()
1009 		err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__trace_begin()
1018 static int thread_stack__trace_end(struct thread_stack *ts,  in thread_stack__trace_end()  argument
1021 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__trace_end()
1026 	if (!ts->cnt || (ts->cnt == 1 && ts->stack[0].ref == ref))  in thread_stack__trace_end()
1029 	cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp, NULL, 0,  in thread_stack__trace_end()
1030 				ts->kernel_start);  in thread_stack__trace_end()
1034 	return thread_stack__push_cp(ts, ret_addr, sample->time, ref, cp,  in thread_stack__trace_end()
1050 static int thread_stack__x86_retpoline(struct thread_stack *ts,  in thread_stack__x86_retpoline()  argument
1054 	struct thread_stack_entry *tse = &ts->stack[ts->cnt - 1];  in thread_stack__x86_retpoline()
1055 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__x86_retpoline()
1071 		ts->cnt -= 1;  in thread_stack__x86_retpoline()
1072 		sym = ts->stack[ts->cnt - 2].cp->sym;  in thread_stack__x86_retpoline()
1079 			ts->cnt -= 1;  in thread_stack__x86_retpoline()
1087 		ts->cnt -= 1;  in thread_stack__x86_retpoline()
1091 	cp = call_path__findnew(cpr, ts->stack[ts->cnt - 2].cp, tsym,  in thread_stack__x86_retpoline()
1092 				sample->addr, ts->kernel_start);  in thread_stack__x86_retpoline()
1097 	ts->stack[ts->cnt - 1].cp = cp;  in thread_stack__x86_retpoline()
1108 	struct thread_stack *ts = thread__stack(thread, sample->cpu);  in thread_stack__process()  local
1112 	if (ts && !ts->crp) {  in thread_stack__process()
1114 		thread_stack__reset(thread, ts);  in thread_stack__process()
1115 		ts = NULL;  in thread_stack__process()
1118 	if (!ts) {  in thread_stack__process()
1119 		ts = thread_stack__new(thread, sample->cpu, crp, true, 0);  in thread_stack__process()
1120 		if (!ts)  in thread_stack__process()
1122 		ts->comm = comm;  in thread_stack__process()
1125 	rstate = ts->rstate;  in thread_stack__process()
1127 		ts->rstate = X86_RETPOLINE_POSSIBLE;  in thread_stack__process()
1130 	if (ts->comm != comm && thread->pid_ == thread->tid) {  in thread_stack__process()
1131 		err = __thread_stack__flush(thread, ts);  in thread_stack__process()
1134 		ts->comm = comm;  in thread_stack__process()
1138 	if (!ts->cnt) {  in thread_stack__process()
1139 		err = thread_stack__bottom(ts, sample, from_al, to_al, ref);  in thread_stack__process()
1144 	ts->branch_count += 1;  in thread_stack__process()
1145 	ts->insn_count += sample->insn_cnt;  in thread_stack__process()
1146 	ts->cyc_count += sample->cyc_cnt;  in thread_stack__process()
1147 	ts->last_time = sample->time;  in thread_stack__process()
1151 		struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__process()
1162 		cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,  in thread_stack__process()
1164 					ts->kernel_start);  in thread_stack__process()
1165 		err = thread_stack__push_cp(ts, ret_addr, sample->time, ref,  in thread_stack__process()
1175 			ts->rstate = X86_RETPOLINE_DETECTED;  in thread_stack__process()
1186 			return thread_stack__pop_ks(thread, ts, sample, ref);  in thread_stack__process()
1193 		if (rstate == X86_RETPOLINE_DETECTED && ts->cnt > 2 &&  in thread_stack__process()
1194 		    ts->stack[ts->cnt - 1].ret_addr != sample->addr)  in thread_stack__process()
1195 			return thread_stack__x86_retpoline(ts, sample, to_al);  in thread_stack__process()
1197 		err = thread_stack__pop_cp(thread, ts, sample->addr,  in thread_stack__process()
1202 			err = thread_stack__no_call_return(thread, ts, sample,  in thread_stack__process()
1206 		err = thread_stack__trace_begin(thread, ts, sample->time, ref);  in thread_stack__process()
1208 		err = thread_stack__trace_end(ts, sample, ref);  in thread_stack__process()
1212 		struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__process()
1221 		cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,  in thread_stack__process()
1223 					ts->kernel_start);  in thread_stack__process()
1224 		err = thread_stack__push_cp(ts, 0, sample->time, ref, cp, false,  in thread_stack__process()
1227 			ts->stack[ts->cnt - 1].non_call = true;  in thread_stack__process()
1235 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__depth()  local
1237 	if (!ts)  in thread_stack__depth()
1239 	return ts->cnt;  in thread_stack__depth()