100 	unsigned long end, line_size;  in cpu_icache_inval_all()  local
103 	end =  in cpu_icache_inval_all()
107 		end -= line_size;  in cpu_icache_inval_all()
108 		__asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));  in cpu_icache_inval_all()
109 		end -= line_size;  in cpu_icache_inval_all()
110 		__asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));  in cpu_icache_inval_all()
111 		end -= line_size;  in cpu_icache_inval_all()
112 		__asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));  in cpu_icache_inval_all()
113 		end -= line_size;  in cpu_icache_inval_all()
114 		__asm__ volatile ("\n\tcctl %0, L1I_IX_INVAL"::"r" (end));  in cpu_icache_inval_all()
115 	} while (end > 0);  in cpu_icache_inval_all()
174 	unsigned long line_size, end;  in cpu_icache_inval_page()  local
177 	end = start + PAGE_SIZE;  in cpu_icache_inval_page()
180 		end -= line_size;  in cpu_icache_inval_page()
181 		__asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));  in cpu_icache_inval_page()
182 		end -= line_size;  in cpu_icache_inval_page()
183 		__asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));  in cpu_icache_inval_page()
184 		end -= line_size;  in cpu_icache_inval_page()
185 		__asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));  in cpu_icache_inval_page()
186 		end -= line_size;  in cpu_icache_inval_page()
187 		__asm__ volatile ("\n\tcctl %0, L1I_VA_INVAL"::"r" (end));  in cpu_icache_inval_page()
188 	} while (end != start);  in cpu_icache_inval_page()
194 	unsigned long line_size, end;  in cpu_dcache_inval_page()  local
197 	end = start + PAGE_SIZE;  in cpu_dcache_inval_page()
200 		end -= line_size;  in cpu_dcache_inval_page()
201 		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));  in cpu_dcache_inval_page()
202 		end -= line_size;  in cpu_dcache_inval_page()
203 		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));  in cpu_dcache_inval_page()
204 		end -= line_size;  in cpu_dcache_inval_page()
205 		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));  in cpu_dcache_inval_page()
206 		end -= line_size;  in cpu_dcache_inval_page()
207 		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));  in cpu_dcache_inval_page()
208 	} while (end != start);  in cpu_dcache_inval_page()
214 	unsigned long line_size, end;  in cpu_dcache_wb_page()  local
217 	end = start + PAGE_SIZE;  in cpu_dcache_wb_page()
220 		end -= line_size;  in cpu_dcache_wb_page()
221 		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));  in cpu_dcache_wb_page()
222 		end -= line_size;  in cpu_dcache_wb_page()
223 		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));  in cpu_dcache_wb_page()
224 		end -= line_size;  in cpu_dcache_wb_page()
225 		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));  in cpu_dcache_wb_page()
226 		end -= line_size;  in cpu_dcache_wb_page()
227 		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));  in cpu_dcache_wb_page()
228 	} while (end != start);  in cpu_dcache_wb_page()
235 	unsigned long line_size, end;  in cpu_dcache_wbinval_page()  local
238 	end = start + PAGE_SIZE;  in cpu_dcache_wbinval_page()
241 		end -= line_size;  in cpu_dcache_wbinval_page()
243 		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));  in cpu_dcache_wbinval_page()
245 		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));  in cpu_dcache_wbinval_page()
246 		end -= line_size;  in cpu_dcache_wbinval_page()
248 		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));  in cpu_dcache_wbinval_page()
250 		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));  in cpu_dcache_wbinval_page()
251 		end -= line_size;  in cpu_dcache_wbinval_page()
253 		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));  in cpu_dcache_wbinval_page()
255 		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));  in cpu_dcache_wbinval_page()
256 		end -= line_size;  in cpu_dcache_wbinval_page()
258 		__asm__ volatile ("\n\tcctl %0, L1D_VA_WB"::"r" (end));  in cpu_dcache_wbinval_page()
260 		__asm__ volatile ("\n\tcctl %0, L1D_VA_INVAL"::"r" (end));  in cpu_dcache_wbinval_page()
261 	} while (end != start);  in cpu_dcache_wbinval_page()
275 void cpu_icache_inval_range(unsigned long start, unsigned long end)  in cpu_icache_inval_range()  argument
281 	while (end > start) {  in cpu_icache_inval_range()
288 void cpu_dcache_inval_range(unsigned long start, unsigned long end)  in cpu_dcache_inval_range()  argument
294 	while (end > start) {  in cpu_dcache_inval_range()
300 void cpu_dcache_wb_range(unsigned long start, unsigned long end)  in cpu_dcache_wb_range()  argument
307 	while (end > start) {  in cpu_dcache_wb_range()
315 void cpu_dcache_wbinval_range(unsigned long start, unsigned long end)  in cpu_dcache_wbinval_range()  argument
321 	while (end > start) {  in cpu_dcache_wbinval_range()
331 void cpu_cache_wbinval_range(unsigned long start, unsigned long end, int flushi)  in cpu_cache_wbinval_range()  argument
337 	align_end = (end + line_size - 1) & ~(line_size - 1);  in cpu_cache_wbinval_range()
343 		align_end = (end + line_size - 1) & ~(line_size - 1);  in cpu_cache_wbinval_range()
349 				   unsigned long start, unsigned long end,  in cpu_cache_wbinval_range_check()  argument
358 	end = (end + line_size - 1) & ~(line_size - 1);  in cpu_cache_wbinval_range_check()
360 	if ((end - start) > (8 * PAGE_SIZE)) {  in cpu_cache_wbinval_range_check()
369 	t_end = ((end - 1) & PAGE_MASK);  in cpu_cache_wbinval_range_check()
374 				cpu_dcache_wbinval_range(start, end);  in cpu_cache_wbinval_range_check()
376 				cpu_icache_inval_range(start, end);  in cpu_cache_wbinval_range_check()
388 	if (va_present(vma->vm_mm, end - 1)) {  in cpu_cache_wbinval_range_check()
390 			cpu_dcache_wbinval_range(t_end, end);  in cpu_cache_wbinval_range_check()
392 			cpu_icache_inval_range(t_end, end);  in cpu_cache_wbinval_range_check()
407 static inline void cpu_l2cache_op(unsigned long start, unsigned long end, unsigned long op)  in cpu_l2cache_op()  argument
411 		unsigned long p_end = __pa(end);  in cpu_l2cache_op()
434 #define cpu_l2cache_op(start,end,op) do { } while (0)  argument
439 void cpu_dma_wb_range(unsigned long start, unsigned long end)  in cpu_dma_wb_range()  argument
445 	end = (end + line_size - 1) & (~(line_size - 1));  in cpu_dma_wb_range()
446 	if (unlikely(start == end))  in cpu_dma_wb_range()
450 	cpu_dcache_wb_range(start, end);  in cpu_dma_wb_range()
451 	cpu_l2cache_op(start, end, CCTL_CMD_L2_PA_WB);  in cpu_dma_wb_range()
456 void cpu_dma_inval_range(unsigned long start, unsigned long end)  in cpu_dma_inval_range()  argument
460 	unsigned long old_end = end;  in cpu_dma_inval_range()
464 	end = (end + line_size - 1) & (~(line_size - 1));  in cpu_dma_inval_range()
465 	if (unlikely(start == end))  in cpu_dma_inval_range()
472 	if (end != old_end) {  in cpu_dma_inval_range()
473 		cpu_dcache_wbinval_range(end - line_size, end);  in cpu_dma_inval_range()
474 		cpu_l2cache_op(end - line_size, end, CCTL_CMD_L2_PA_WBINVAL);  in cpu_dma_inval_range()
476 	cpu_dcache_inval_range(start, end);  in cpu_dma_inval_range()
477 	cpu_l2cache_op(start, end, CCTL_CMD_L2_PA_INVAL);  in cpu_dma_inval_range()
483 void cpu_dma_wbinval_range(unsigned long start, unsigned long end)  in cpu_dma_wbinval_range()  argument
489 	end = (end + line_size - 1) & (~(line_size - 1));  in cpu_dma_wbinval_range()
490 	if (unlikely(start == end))  in cpu_dma_wbinval_range()
494 	cpu_dcache_wbinval_range(start, end);  in cpu_dma_wbinval_range()
495 	cpu_l2cache_op(start, end, CCTL_CMD_L2_PA_WBINVAL);  in cpu_dma_wbinval_range()