18 #define cache_op(op,addr)						\  argument
26 	: "i" (op), "m" (*(unsigned char *)(addr)))
28 static inline void flush_icache_line_indexed(unsigned long addr)  in flush_icache_line_indexed()  argument
30 	cache_op(Index_Invalidate_I, addr);  in flush_icache_line_indexed()
33 static inline void flush_dcache_line_indexed(unsigned long addr)  in flush_dcache_line_indexed()  argument
35 	cache_op(Index_Writeback_Inv_D, addr);  in flush_dcache_line_indexed()
38 static inline void flush_scache_line_indexed(unsigned long addr)  in flush_scache_line_indexed()  argument
40 	cache_op(Index_Writeback_Inv_SD, addr);  in flush_scache_line_indexed()
43 static inline void flush_icache_line(unsigned long addr)  in flush_icache_line()  argument
45 	cache_op(Hit_Invalidate_I, addr);  in flush_icache_line()
48 static inline void flush_dcache_line(unsigned long addr)  in flush_dcache_line()  argument
50 	cache_op(Hit_Writeback_Inv_D, addr);  in flush_dcache_line()
53 static inline void invalidate_dcache_line(unsigned long addr)  in invalidate_dcache_line()  argument
55 	cache_op(Hit_Invalidate_D, addr);  in invalidate_dcache_line()
58 static inline void invalidate_scache_line(unsigned long addr)  in invalidate_scache_line()  argument
60 	cache_op(Hit_Invalidate_SD, addr);  in invalidate_scache_line()
63 static inline void flush_scache_line(unsigned long addr)  in flush_scache_line()  argument
65 	cache_op(Hit_Writeback_Inv_SD, addr);  in flush_scache_line()
71 static inline void protected_flush_icache_line(unsigned long addr)  in protected_flush_icache_line()  argument
83 		: "i" (Hit_Invalidate_I), "r" (addr));  in protected_flush_icache_line()
92 static inline void protected_writeback_dcache_line(unsigned long addr)  in protected_writeback_dcache_line()  argument
104 		: "i" (Hit_Writeback_Inv_D), "r" (addr));  in protected_writeback_dcache_line()
110 static inline void invalidate_tcache_page(unsigned long addr)  in invalidate_tcache_page()  argument
112 	cache_op(Page_Invalidate_T, addr);  in invalidate_tcache_page()
148 	unsigned long ws, addr;  in blast_dcache16()  local
151 		for (addr = start; addr < end; addr += 0x200)  in blast_dcache16()
152 			cache16_unroll32(addr|ws,Index_Writeback_Inv_D);  in blast_dcache16()
173 	unsigned long ws, addr;  in blast_dcache16_page_indexed()  local
176 		for (addr = start; addr < end; addr += 0x200)   in blast_dcache16_page_indexed()
177 			cache16_unroll32(addr|ws,Index_Writeback_Inv_D);  in blast_dcache16_page_indexed()
187 	unsigned long ws, addr;  in blast_icache16()  local
190 		for (addr = start; addr < end; addr += 0x200)   in blast_icache16()
191 			cache16_unroll32(addr|ws,Index_Invalidate_I);  in blast_icache16()
212 	unsigned long ws, addr;  in blast_icache16_page_indexed()  local
215 		for (addr = start; addr < end; addr += 0x200)   in blast_icache16_page_indexed()
216 			cache16_unroll32(addr|ws,Index_Invalidate_I);  in blast_icache16_page_indexed()
226 	unsigned long ws, addr;  in blast_scache16()  local
229 		for (addr = start; addr < end; addr += 0x200)  in blast_scache16()
230 			cache16_unroll32(addr|ws,Index_Writeback_Inv_SD);  in blast_scache16()
251 	unsigned long ws, addr;  in blast_scache16_page_indexed()  local
254 		for (addr = start; addr < end; addr += 0x200)   in blast_scache16_page_indexed()
255 			cache16_unroll32(addr|ws,Index_Writeback_Inv_SD);  in blast_scache16_page_indexed()
291 	unsigned long ws, addr;  in blast_dcache32()  local
294 		for (addr = start; addr < end; addr += 0x400)   in blast_dcache32()
295 			cache32_unroll32(addr|ws,Index_Writeback_Inv_D);  in blast_dcache32()
316 	unsigned long ws, addr;  in blast_dcache32_page_indexed()  local
319 		for (addr = start; addr < end; addr += 0x400)   in blast_dcache32_page_indexed()
320 			cache32_unroll32(addr|ws,Index_Writeback_Inv_D);  in blast_dcache32_page_indexed()
330 	unsigned long ws, addr;  in blast_icache32()  local
333 		for (addr = start; addr < end; addr += 0x400)   in blast_icache32()
334 			cache32_unroll32(addr|ws,Index_Invalidate_I);  in blast_icache32()
355 	unsigned long ws, addr;  in blast_icache32_page_indexed()  local
358 		for (addr = start; addr < end; addr += 0x400)   in blast_icache32_page_indexed()
359 			cache32_unroll32(addr|ws,Index_Invalidate_I);  in blast_icache32_page_indexed()
369 	unsigned long ws, addr;  in blast_scache32()  local
372 		for (addr = start; addr < end; addr += 0x400)  in blast_scache32()
373 			cache32_unroll32(addr|ws,Index_Writeback_Inv_SD);  in blast_scache32()
394 	unsigned long ws, addr;  in blast_scache32_page_indexed()  local
397 		for (addr = start; addr < end; addr += 0x400)   in blast_scache32_page_indexed()
398 			cache32_unroll32(addr|ws,Index_Writeback_Inv_SD);  in blast_scache32_page_indexed()
434 	unsigned long ws, addr;  in blast_icache64()  local
437 		for (addr = start; addr < end; addr += 0x800)   in blast_icache64()
438 			cache64_unroll32(addr|ws,Index_Invalidate_I);  in blast_icache64()
459 	unsigned long ws, addr;  in blast_icache64_page_indexed()  local
462 		for (addr = start; addr < end; addr += 0x800)   in blast_icache64_page_indexed()
463 			cache64_unroll32(addr|ws,Index_Invalidate_I);  in blast_icache64_page_indexed()
473 	unsigned long ws, addr;  in blast_scache64()  local
476 		for (addr = start; addr < end; addr += 0x800)  in blast_scache64()
477 			cache64_unroll32(addr|ws,Index_Writeback_Inv_SD);  in blast_scache64()
498 	unsigned long ws, addr;  in blast_scache64_page_indexed()  local
501 		for (addr = start; addr < end; addr += 0x800)   in blast_scache64_page_indexed()
502 			cache64_unroll32(addr|ws,Index_Writeback_Inv_SD);  in blast_scache64_page_indexed()
538 	unsigned long ws, addr;  in blast_scache128()  local
541 		for (addr = start; addr < end; addr += 0x1000)  in blast_scache128()
542 			cache128_unroll32(addr|ws,Index_Writeback_Inv_SD);  in blast_scache128()
563 	unsigned long ws, addr;  in blast_scache128_page_indexed()  local
566 		for (addr = start; addr < end; addr += 0x1000)   in blast_scache128_page_indexed()
567 			cache128_unroll32(addr|ws,Index_Writeback_Inv_SD);  in blast_scache128_page_indexed()