1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * hypersparc.S: High speed Hypersparc mmu/cache operations.
4 *
5 * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu)
6 */
7
8#include <asm/ptrace.h>
9#include <asm/psr.h>
10#include <asm/asm-offsets.h>
11#include <asm/asi.h>
12#include <asm/page.h>
13#include <asm/pgtsrmmu.h>
14#include <linux/init.h>
15
16	.text
17	.align	4
18
19	.globl	hypersparc_flush_cache_all, hypersparc_flush_cache_mm
20	.globl	hypersparc_flush_cache_range, hypersparc_flush_cache_page
21	.globl	hypersparc_flush_page_to_ram
22	.globl	hypersparc_flush_page_for_dma, hypersparc_flush_sig_insns
23	.globl	hypersparc_flush_tlb_all, hypersparc_flush_tlb_mm
24	.globl	hypersparc_flush_tlb_range, hypersparc_flush_tlb_page
25
26hypersparc_flush_cache_all:
27	WINDOW_FLUSH(%g4, %g5)
28	sethi	%hi(vac_cache_size), %g4
29	ld	[%g4 + %lo(vac_cache_size)], %g5
30	sethi	%hi(vac_line_size), %g1
31	ld	[%g1 + %lo(vac_line_size)], %g2
321:
33	subcc	%g5, %g2, %g5			! hyper_flush_unconditional_combined
34	bne	1b
35	 sta	%g0, [%g5] ASI_M_FLUSH_CTX
36	retl
37	 sta	%g0, [%g0] ASI_M_FLUSH_IWHOLE	! hyper_flush_whole_icache
38
39	/* We expand the window flush to get maximum performance. */
40hypersparc_flush_cache_mm:
41#ifndef CONFIG_SMP
42	ld	[%o0 + AOFF_mm_context], %g1
43	cmp	%g1, -1
44	be	hypersparc_flush_cache_mm_out
45#endif
46	WINDOW_FLUSH(%g4, %g5)
47
48	sethi	%hi(vac_line_size), %g1
49	ld	[%g1 + %lo(vac_line_size)], %o1
50	sethi	%hi(vac_cache_size), %g2
51	ld	[%g2 + %lo(vac_cache_size)], %o0
52	add	%o1, %o1, %g1
53	add	%o1, %g1, %g2
54	add	%o1, %g2, %g3
55	add	%o1, %g3, %g4
56	add	%o1, %g4, %g5
57	add	%o1, %g5, %o4
58	add	%o1, %o4, %o5
59
60	/* BLAMMO! */
611:
62	subcc	%o0, %o5, %o0				! hyper_flush_cache_user
63	sta	%g0, [%o0 + %g0] ASI_M_FLUSH_USER
64	sta	%g0, [%o0 + %o1] ASI_M_FLUSH_USER
65	sta	%g0, [%o0 + %g1] ASI_M_FLUSH_USER
66	sta	%g0, [%o0 + %g2] ASI_M_FLUSH_USER
67	sta	%g0, [%o0 + %g3] ASI_M_FLUSH_USER
68	sta	%g0, [%o0 + %g4] ASI_M_FLUSH_USER
69	sta	%g0, [%o0 + %g5] ASI_M_FLUSH_USER
70	bne	1b
71	 sta	%g0, [%o0 + %o4] ASI_M_FLUSH_USER
72hypersparc_flush_cache_mm_out:
73	retl
74	 nop
75
76	/* The things we do for performance... */
77hypersparc_flush_cache_range:
78	ld	[%o0 + VMA_VM_MM], %o0
79#ifndef CONFIG_SMP
80	ld	[%o0 + AOFF_mm_context], %g1
81	cmp	%g1, -1
82	be	hypersparc_flush_cache_range_out
83#endif
84	WINDOW_FLUSH(%g4, %g5)
85
86	sethi	%hi(vac_line_size), %g1
87	ld	[%g1 + %lo(vac_line_size)], %o4
88	sethi	%hi(vac_cache_size), %g2
89	ld	[%g2 + %lo(vac_cache_size)], %o3
90
91	/* Here comes the fun part... */
92	add	%o2, (PAGE_SIZE - 1), %o2
93	andn	%o1, (PAGE_SIZE - 1), %o1
94	add	%o4, %o4, %o5
95	andn	%o2, (PAGE_SIZE - 1), %o2
96	add	%o4, %o5, %g1
97	sub	%o2, %o1, %g4
98	add	%o4, %g1, %g2
99	sll	%o3, 2, %g5
100	add	%o4, %g2, %g3
101	cmp	%g4, %g5
102	add	%o4, %g3, %g4
103	blu	0f
104	 add	%o4, %g4, %g5
105	add	%o4, %g5, %g7
106
107	/* Flush entire user space, believe it or not this is quicker
108	 * than page at a time flushings for range > (cache_size<<2).
109	 */
1101:
111	subcc	%o3, %g7, %o3
112	sta	%g0, [%o3 + %g0] ASI_M_FLUSH_USER
113	sta	%g0, [%o3 + %o4] ASI_M_FLUSH_USER
114	sta	%g0, [%o3 + %o5] ASI_M_FLUSH_USER
115	sta	%g0, [%o3 + %g1] ASI_M_FLUSH_USER
116	sta	%g0, [%o3 + %g2] ASI_M_FLUSH_USER
117	sta	%g0, [%o3 + %g3] ASI_M_FLUSH_USER
118	sta	%g0, [%o3 + %g4] ASI_M_FLUSH_USER
119	bne	1b
120	 sta	%g0, [%o3 + %g5] ASI_M_FLUSH_USER
121	retl
122	 nop
123
124	/* Below our threshold, flush one page at a time. */
1250:
126	ld	[%o0 + AOFF_mm_context], %o0
127	mov	SRMMU_CTX_REG, %g7
128	lda	[%g7] ASI_M_MMUREGS, %o3
129	sta	%o0, [%g7] ASI_M_MMUREGS
130	add	%o2, -PAGE_SIZE, %o0
1311:
132	or	%o0, 0x400, %g7
133	lda	[%g7] ASI_M_FLUSH_PROBE, %g7
134	orcc	%g7, 0, %g0
135	be,a	3f
136	 mov	%o0, %o2
137	add	%o4, %g5, %g7
1382:
139	sub	%o2, %g7, %o2
140	sta	%g0, [%o2 + %g0] ASI_M_FLUSH_PAGE
141	sta	%g0, [%o2 + %o4] ASI_M_FLUSH_PAGE
142	sta	%g0, [%o2 + %o5] ASI_M_FLUSH_PAGE
143	sta	%g0, [%o2 + %g1] ASI_M_FLUSH_PAGE
144	sta	%g0, [%o2 + %g2] ASI_M_FLUSH_PAGE
145	sta	%g0, [%o2 + %g3] ASI_M_FLUSH_PAGE
146	andcc	%o2, 0xffc, %g0
147	sta	%g0, [%o2 + %g4] ASI_M_FLUSH_PAGE
148	bne	2b
149	 sta	%g0, [%o2 + %g5] ASI_M_FLUSH_PAGE
1503:
151	cmp	%o2, %o1
152	bne	1b
153	 add	%o2, -PAGE_SIZE, %o0
154	mov	SRMMU_FAULT_STATUS, %g5
155	lda	[%g5] ASI_M_MMUREGS, %g0
156	mov	SRMMU_CTX_REG, %g7
157	sta	%o3, [%g7] ASI_M_MMUREGS
158hypersparc_flush_cache_range_out:
159	retl
160	 nop
161
162	/* HyperSparc requires a valid mapping where we are about to flush
163	 * in order to check for a physical tag match during the flush.
164	 */
165	/* Verified, my ass... */
166hypersparc_flush_cache_page:
167	ld	[%o0 + VMA_VM_MM], %o0
168	ld	[%o0 + AOFF_mm_context], %g2
169#ifndef CONFIG_SMP
170	cmp	%g2, -1
171	be	hypersparc_flush_cache_page_out
172#endif
173	WINDOW_FLUSH(%g4, %g5)
174
175	sethi	%hi(vac_line_size), %g1
176	ld	[%g1 + %lo(vac_line_size)], %o4
177	mov	SRMMU_CTX_REG, %o3
178	andn	%o1, (PAGE_SIZE - 1), %o1
179	lda	[%o3] ASI_M_MMUREGS, %o2
180	sta	%g2, [%o3] ASI_M_MMUREGS
181	or	%o1, 0x400, %o5
182	lda	[%o5] ASI_M_FLUSH_PROBE, %g1
183	orcc	%g0, %g1, %g0
184	be	2f
185	 add	%o4, %o4, %o5
186	sub	%o1, -PAGE_SIZE, %o1
187	add	%o4, %o5, %g1
188	add	%o4, %g1, %g2
189	add	%o4, %g2, %g3
190	add	%o4, %g3, %g4
191	add	%o4, %g4, %g5
192	add	%o4, %g5, %g7
193
194	/* BLAMMO! */
1951:
196	sub	%o1, %g7, %o1
197	sta	%g0, [%o1 + %g0] ASI_M_FLUSH_PAGE
198	sta	%g0, [%o1 + %o4] ASI_M_FLUSH_PAGE
199	sta	%g0, [%o1 + %o5] ASI_M_FLUSH_PAGE
200	sta	%g0, [%o1 + %g1] ASI_M_FLUSH_PAGE
201	sta	%g0, [%o1 + %g2] ASI_M_FLUSH_PAGE
202	sta	%g0, [%o1 + %g3] ASI_M_FLUSH_PAGE
203	andcc	%o1, 0xffc, %g0
204	sta	%g0, [%o1 + %g4] ASI_M_FLUSH_PAGE
205	bne	1b
206	 sta	%g0, [%o1 + %g5] ASI_M_FLUSH_PAGE
2072:
208	mov	SRMMU_FAULT_STATUS, %g7
209	mov	SRMMU_CTX_REG, %g4
210	lda	[%g7] ASI_M_MMUREGS, %g0
211	sta	%o2, [%g4] ASI_M_MMUREGS
212hypersparc_flush_cache_page_out:
213	retl
214	 nop
215
216hypersparc_flush_sig_insns:
217	flush	%o1
218	retl
219	 flush	%o1 + 4
220
221	/* HyperSparc is copy-back. */
222hypersparc_flush_page_to_ram:
223	sethi	%hi(vac_line_size), %g1
224	ld	[%g1 + %lo(vac_line_size)], %o4
225	andn	%o0, (PAGE_SIZE - 1), %o0
226	add	%o4, %o4, %o5
227	or	%o0, 0x400, %g7
228	lda	[%g7] ASI_M_FLUSH_PROBE, %g5
229	add	%o4, %o5, %g1
230	orcc	%g5, 0, %g0
231	be	2f
232	 add	%o4, %g1, %g2
233	add	%o4, %g2, %g3
234	sub	%o0, -PAGE_SIZE, %o0
235	add	%o4, %g3, %g4
236	add	%o4, %g4, %g5
237	add	%o4, %g5, %g7
238
239	/* BLAMMO! */
2401:
241	sub	%o0, %g7, %o0
242	sta	%g0, [%o0 + %g0] ASI_M_FLUSH_PAGE
243	sta	%g0, [%o0 + %o4] ASI_M_FLUSH_PAGE
244	sta	%g0, [%o0 + %o5] ASI_M_FLUSH_PAGE
245	sta	%g0, [%o0 + %g1] ASI_M_FLUSH_PAGE
246	sta	%g0, [%o0 + %g2] ASI_M_FLUSH_PAGE
247	sta	%g0, [%o0 + %g3] ASI_M_FLUSH_PAGE
248	andcc	%o0, 0xffc, %g0
249	sta	%g0, [%o0 + %g4] ASI_M_FLUSH_PAGE
250	bne	1b
251	 sta	%g0, [%o0 + %g5] ASI_M_FLUSH_PAGE
2522:
253	mov	SRMMU_FAULT_STATUS, %g1
254	retl
255	 lda	[%g1] ASI_M_MMUREGS, %g0
256
257	/* HyperSparc is IO cache coherent. */
258hypersparc_flush_page_for_dma:
259	retl
260	 nop
261
262	/* It was noted that at boot time a TLB flush all in a delay slot
263	 * can deliver an illegal instruction to the processor if the timing
264	 * is just right...
265	 */
266hypersparc_flush_tlb_all:
267	mov	0x400, %g1
268	sta	%g0, [%g1] ASI_M_FLUSH_PROBE
269	retl
270	 nop
271
272hypersparc_flush_tlb_mm:
273	mov	SRMMU_CTX_REG, %g1
274	ld	[%o0 + AOFF_mm_context], %o1
275	lda	[%g1] ASI_M_MMUREGS, %g5
276#ifndef CONFIG_SMP
277	cmp	%o1, -1
278	be	hypersparc_flush_tlb_mm_out
279#endif
280	 mov	0x300, %g2
281	sta	%o1, [%g1] ASI_M_MMUREGS
282	sta	%g0, [%g2] ASI_M_FLUSH_PROBE
283hypersparc_flush_tlb_mm_out:
284	retl
285	 sta	%g5, [%g1] ASI_M_MMUREGS
286
287hypersparc_flush_tlb_range:
288	ld	[%o0 + VMA_VM_MM], %o0
289	mov	SRMMU_CTX_REG, %g1
290	ld	[%o0 + AOFF_mm_context], %o3
291	lda	[%g1] ASI_M_MMUREGS, %g5
292#ifndef CONFIG_SMP
293	cmp	%o3, -1
294	be	hypersparc_flush_tlb_range_out
295#endif
296	 sethi	%hi(~((1 << SRMMU_PGDIR_SHIFT) - 1)), %o4
297	sta	%o3, [%g1] ASI_M_MMUREGS
298	and	%o1, %o4, %o1
299	add	%o1, 0x200, %o1
300	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
3011:
302	sub	%o1, %o4, %o1
303	cmp	%o1, %o2
304	blu,a	1b
305	 sta	%g0, [%o1] ASI_M_FLUSH_PROBE
306hypersparc_flush_tlb_range_out:
307	retl
308	 sta	%g5, [%g1] ASI_M_MMUREGS
309
310hypersparc_flush_tlb_page:
311	ld	[%o0 + VMA_VM_MM], %o0
312	mov	SRMMU_CTX_REG, %g1
313	ld	[%o0 + AOFF_mm_context], %o3
314	andn	%o1, (PAGE_SIZE - 1), %o1
315#ifndef CONFIG_SMP
316	cmp	%o3, -1
317	be	hypersparc_flush_tlb_page_out
318#endif
319	 lda	[%g1] ASI_M_MMUREGS, %g5
320	sta	%o3, [%g1] ASI_M_MMUREGS
321	sta	%g0, [%o1] ASI_M_FLUSH_PROBE
322hypersparc_flush_tlb_page_out:
323	retl
324	 sta	%g5, [%g1] ASI_M_MMUREGS
325
326	__INIT
327
328	/* High speed page clear/copy. */
329hypersparc_bzero_1page:
330/* NOTE: This routine has to be shorter than 40insns --jj */
331	clr	%g1
332	mov	32, %g2
333	mov	64, %g3
334	mov	96, %g4
335	mov	128, %g5
336	mov	160, %g7
337	mov	192, %o2
338	mov	224, %o3
339	mov	16, %o1
3401:
341	stda	%g0, [%o0 + %g0] ASI_M_BFILL
342	stda	%g0, [%o0 + %g2] ASI_M_BFILL
343	stda	%g0, [%o0 + %g3] ASI_M_BFILL
344	stda	%g0, [%o0 + %g4] ASI_M_BFILL
345	stda	%g0, [%o0 + %g5] ASI_M_BFILL
346	stda	%g0, [%o0 + %g7] ASI_M_BFILL
347	stda	%g0, [%o0 + %o2] ASI_M_BFILL
348	stda	%g0, [%o0 + %o3] ASI_M_BFILL
349	subcc	%o1, 1, %o1
350	bne	1b
351	 add	%o0, 256, %o0
352
353	retl
354	 nop
355
356hypersparc_copy_1page:
357/* NOTE: This routine has to be shorter than 70insns --jj */
358	sub	%o1, %o0, %o2		! difference
359	mov	16, %g1
3601:
361	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
362	add	%o0, 32, %o0
363	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
364	add	%o0, 32, %o0
365	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
366	add	%o0, 32, %o0
367	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
368	add	%o0, 32, %o0
369	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
370	add	%o0, 32, %o0
371	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
372	add	%o0, 32, %o0
373	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
374	add	%o0, 32, %o0
375	sta	%o0, [%o0 + %o2] ASI_M_BCOPY
376	subcc	%g1, 1, %g1
377	bne	1b
378	 add	%o0, 32, %o0
379
380	retl
381	 nop
382
383	.globl	hypersparc_setup_blockops
384hypersparc_setup_blockops:
385	sethi	%hi(bzero_1page), %o0
386	or	%o0, %lo(bzero_1page), %o0
387	sethi	%hi(hypersparc_bzero_1page), %o1
388	or	%o1, %lo(hypersparc_bzero_1page), %o1
389	sethi	%hi(hypersparc_copy_1page), %o2
390	or	%o2, %lo(hypersparc_copy_1page), %o2
391	ld	[%o1], %o4
3921:
393	add	%o1, 4, %o1
394	st	%o4, [%o0]
395	add	%o0, 4, %o0
396	cmp	%o1, %o2
397	bne	1b
398	 ld	[%o1], %o4
399	sethi	%hi(__copy_1page), %o0
400	or	%o0, %lo(__copy_1page), %o0
401	sethi	%hi(hypersparc_setup_blockops), %o2
402	or	%o2, %lo(hypersparc_setup_blockops), %o2
403	ld	[%o1], %o4
4041:
405	add	%o1, 4, %o1
406	st	%o4, [%o0]
407	add	%o0, 4, %o0
408	cmp	%o1, %o2
409	bne	1b
410	 ld	[%o1], %o4
411	sta	%g0, [%g0] ASI_M_FLUSH_IWHOLE
412	retl
413	 nop
414