35 void (*_cache_line_loop_ic_fn)(phys_addr_t paddr, unsigned long vaddr,
189 void __cache_line_loop_v3(phys_addr_t paddr, unsigned long vaddr,  in __cache_line_loop_v3()  argument
210 		sz += paddr & ~CACHE_LINE_MASK;  in __cache_line_loop_v3()
211 		paddr &= CACHE_LINE_MASK;  in __cache_line_loop_v3()
221 		write_aux_reg(aux_tag, paddr);  in __cache_line_loop_v3()
231 		write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32);  in __cache_line_loop_v3()
235 			write_aux_reg(aux_tag, paddr);  in __cache_line_loop_v3()
236 			paddr += L1_CACHE_BYTES;  in __cache_line_loop_v3()
249 void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr,  in __cache_line_loop_v4()  argument
269 		sz += paddr & ~CACHE_LINE_MASK;  in __cache_line_loop_v4()
270 		paddr &= CACHE_LINE_MASK;  in __cache_line_loop_v4()
286 			write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32);  in __cache_line_loop_v4()
288 			write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32);  in __cache_line_loop_v4()
292 		write_aux_reg(aux_cmd, paddr);  in __cache_line_loop_v4()
293 		paddr += L1_CACHE_BYTES;  in __cache_line_loop_v4()
303 void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr,  in __cache_line_loop_v4()  argument
319 		sz += paddr & ~CACHE_LINE_MASK;  in __cache_line_loop_v4()
320 		paddr &= CACHE_LINE_MASK;  in __cache_line_loop_v4()
332 			write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32);  in __cache_line_loop_v4()
334 			write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32);  in __cache_line_loop_v4()
338 	write_aux_reg(e, paddr + sz);	/* ENDR is exclusive */  in __cache_line_loop_v4()
339 	write_aux_reg(s, paddr);  in __cache_line_loop_v4()
461 static inline void __dc_line_op(phys_addr_t paddr, unsigned long vaddr,  in __dc_line_op()  argument
471 	__cache_line_loop(paddr, vaddr, sz, op, full_page);  in __dc_line_op()
483 #define __dc_line_op(paddr, vaddr, sz, op)  argument
484 #define __dc_line_op_k(paddr, sz, op)  argument
497 __ic_line_inv_vaddr_local(phys_addr_t paddr, unsigned long vaddr,  in __ic_line_inv_vaddr_local()  argument
504 	(*_cache_line_loop_ic_fn)(paddr, vaddr, sz, OP_INV_IC, full_page);  in __ic_line_inv_vaddr_local()
515 	phys_addr_t paddr, vaddr;  member
523         __ic_line_inv_vaddr_local(ic_inv->paddr, ic_inv->vaddr, ic_inv->sz);  in __ic_line_inv_vaddr_helper()
526 static void __ic_line_inv_vaddr(phys_addr_t paddr, unsigned long vaddr,  in __ic_line_inv_vaddr()  argument
530 		.paddr = paddr,  in __ic_line_inv_vaddr()
547 static noinline void slc_op_rgn(phys_addr_t paddr, unsigned long sz, const int op)  in slc_op_rgn()  argument
590 	end = paddr + sz + l2_line_sz - 1;  in slc_op_rgn()
597 		write_aux_reg(ARC_REG_SLC_RGN_START1, upper_32_bits(paddr));  in slc_op_rgn()
599 	write_aux_reg(ARC_REG_SLC_RGN_START, lower_32_bits(paddr));  in slc_op_rgn()
610 static __maybe_unused noinline void slc_op_line(phys_addr_t paddr, unsigned long sz, const int op)  in slc_op_line()  argument
641 	sz += paddr & ~SLC_LINE_MASK;  in slc_op_line()
642 	paddr &= SLC_LINE_MASK;  in slc_op_line()
647 		write_aux_reg(cmd, paddr);  in slc_op_line()
648 		paddr += l2_line_sz;  in slc_op_line()
660 #define slc_op(paddr, sz, op)	slc_op_rgn(paddr, sz, op)  argument
740 		phys_addr_t paddr = (unsigned long)folio_address(folio);  in flush_dcache_folio()  local
747 		if (addr_not_cache_congruent(paddr, vaddr))  in flush_dcache_folio()
748 			__flush_dcache_pages(paddr, vaddr,  in flush_dcache_folio()
890 void __sync_icache_dcache(phys_addr_t paddr, unsigned long vaddr, int len)  in __sync_icache_dcache()  argument
892 	__dc_line_op(paddr, vaddr, len, OP_FLUSH_N_INV);  in __sync_icache_dcache()
893 	__ic_line_inv_vaddr(paddr, vaddr, len);  in __sync_icache_dcache()
897 void __inv_icache_pages(phys_addr_t paddr, unsigned long vaddr, unsigned nr)  in __inv_icache_pages()  argument
899 	__ic_line_inv_vaddr(paddr, vaddr, nr * PAGE_SIZE);  in __inv_icache_pages()
906 void __flush_dcache_pages(phys_addr_t paddr, unsigned long vaddr, unsigned nr)  in __flush_dcache_pages()  argument
908 	__dc_line_op(paddr, vaddr & PAGE_MASK, nr * PAGE_SIZE, OP_FLUSH_N_INV);  in __flush_dcache_pages()
934 	phys_addr_t paddr = pfn << PAGE_SHIFT;  in flush_cache_page()  local
938 	__flush_dcache_pages(paddr, u_vaddr, 1);  in flush_cache_page()
941 		__inv_icache_pages(paddr, u_vaddr, 1);  in flush_cache_page()