16 #define reserved_asid(info, cpu) *per_cpu_ptr((info)->reserved, cpu)  argument
18 #define ASID_MASK(info)			(~GENMASK((info)->bits - 1, 0))  argument
19 #define ASID_FIRST_VERSION(info)	(1UL << ((info)->bits))  argument
21 #define asid2idx(info, asid)		(((asid) & ~ASID_MASK(info)) >> (info)->ctxt_shift)  argument
22 #define idx2asid(info, idx)		(((idx) << (info)->ctxt_shift) & ~ASID_MASK(info))  argument
24 static void flush_context(struct asid_info *info)  in flush_context()  argument
30 	bitmap_zero(info->map, NUM_CTXT_ASIDS(info));  in flush_context()
33 		asid = atomic64_xchg_relaxed(&active_asid(info, i), 0);  in flush_context()
42 			asid = reserved_asid(info, i);  in flush_context()
43 		__set_bit(asid2idx(info, asid), info->map);  in flush_context()
44 		reserved_asid(info, i) = asid;  in flush_context()
51 	cpumask_setall(&info->flush_pending);  in flush_context()
54 static bool check_update_reserved_asid(struct asid_info *info, u64 asid,  in check_update_reserved_asid()  argument
70 		if (reserved_asid(info, cpu) == asid) {  in check_update_reserved_asid()
72 			reserved_asid(info, cpu) = newasid;  in check_update_reserved_asid()
79 static u64 new_context(struct asid_info *info, atomic64_t *pasid,  in new_context()  argument
84 	u64 generation = atomic64_read(&info->generation);  in new_context()
87 		u64 newasid = generation | (asid & ~ASID_MASK(info));  in new_context()
93 		if (check_update_reserved_asid(info, asid, newasid))  in new_context()
100 		if (!__test_and_set_bit(asid2idx(info, asid), info->map))  in new_context()
111 	asid = find_next_zero_bit(info->map, NUM_CTXT_ASIDS(info), cur_idx);  in new_context()
112 	if (asid != NUM_CTXT_ASIDS(info))  in new_context()
116 	generation = atomic64_add_return_relaxed(ASID_FIRST_VERSION(info),  in new_context()
117 						 &info->generation);  in new_context()
118 	flush_context(info);  in new_context()
121 	asid = find_next_zero_bit(info->map, NUM_CTXT_ASIDS(info), 1);  in new_context()
124 	__set_bit(asid, info->map);  in new_context()
127 	return idx2asid(info, asid) | generation;  in new_context()
137 void asid_new_context(struct asid_info *info, atomic64_t *pasid,  in asid_new_context()  argument
143 	raw_spin_lock_irqsave(&info->lock, flags);  in asid_new_context()
146 	if ((asid ^ atomic64_read(&info->generation)) >> info->bits) {  in asid_new_context()
147 		asid = new_context(info, pasid, mm);  in asid_new_context()
151 	if (cpumask_test_and_clear_cpu(cpu, &info->flush_pending))  in asid_new_context()
152 		info->flush_cpu_ctxt_cb();  in asid_new_context()
154 	atomic64_set(&active_asid(info, cpu), asid);  in asid_new_context()
156 	raw_spin_unlock_irqrestore(&info->lock, flags);  in asid_new_context()
168 int asid_allocator_init(struct asid_info *info,  in asid_allocator_init()  argument
172 	info->bits = bits;  in asid_allocator_init()
173 	info->ctxt_shift = ilog2(asid_per_ctxt);  in asid_allocator_init()
174 	info->flush_cpu_ctxt_cb = flush_cpu_ctxt_cb;  in asid_allocator_init()
179 	WARN_ON(NUM_CTXT_ASIDS(info) - 1 <= num_possible_cpus());  in asid_allocator_init()
180 	atomic64_set(&info->generation, ASID_FIRST_VERSION(info));  in asid_allocator_init()
181 	info->map = bitmap_zalloc(NUM_CTXT_ASIDS(info), GFP_KERNEL);  in asid_allocator_init()
182 	if (!info->map)  in asid_allocator_init()
185 	raw_spin_lock_init(&info->lock);  in asid_allocator_init()