1! SPDX-License-Identifier: GPL-2.0
2!
3! Fast SH memcpy
4!
5! by Toshiyasu Morita (tm@netcom.com)
6! hacked by J"orn Rernnecke (joern.rennecke@superh.com) ("o for o-umlaut)
7! SH5 code Copyright 2002 SuperH Ltd.
8!
9! Entry: ARG0: destination pointer
10!        ARG1: source pointer
11!        ARG2: byte count
12!
13! Exit:  RESULT: destination pointer
14!        any other registers in the range r0-r7: trashed
15!
16! Notes: Usually one wants to do small reads and write a longword, but
17!        unfortunately it is difficult in some cases to concatanate bytes
18!        into a longword on the SH, so this does a longword read and small
19!        writes.
20!
21! This implementation makes two assumptions about how it is called:
22!
23! 1.: If the byte count is nonzero, the address of the last byte to be
24!     copied is unsigned greater than the address of the first byte to
25!     be copied.  This could be easily swapped for a signed comparison,
26!     but the algorithm used needs some comparison.
27!
28! 2.: When there are two or three bytes in the last word of an 11-or-more
29!     bytes memory chunk to b copied, the rest of the word can be read
30!     without side effects.
31!     This could be easily changed by increasing the minimum size of
32!     a fast memcpy and the amount subtracted from r7 before L_2l_loop be 2,
33!     however, this would cost a few extra cyles on average.
34!     For SHmedia, the assumption is that any quadword can be read in its
35!     enirety if at least one byte is included in the copy.
36
37/* Imported into Linux kernel by Richard Curnow.  This is used to implement the
38   __copy_user function in the general case, so it has to be a distinct
39   function from intra-kernel memcpy to allow for exception fix-ups in the
40   event that the user pointer is bad somewhere in the copy (e.g. due to
41   running off the end of the vma).
42
43   Note, this algorithm will be slightly wasteful in the case where the source
44   and destination pointers are equally aligned, because the stlo/sthi pairs
45   could then be merged back into single stores.  If there are a lot of cache
46   misses, this is probably offset by the stall lengths on the preloads.
47
48*/
49
50/* NOTE : Prefetches removed and allocos guarded by synco to avoid TAKum03020
51 * erratum.  The first two prefetches are nop-ed out to avoid upsetting the
52 * instruction counts used in the jump address calculation.
53 * */
54
55	.section .text..SHmedia32,"ax"
56	.little
57	.balign 32
58	.global copy_user_memcpy
59	.global copy_user_memcpy_end
60copy_user_memcpy:
61
62#define LDUAQ(P,O,D0,D1) ldlo.q P,O,D0; ldhi.q P,O+7,D1
63#define STUAQ(P,O,D0,D1) stlo.q P,O,D0; sthi.q P,O+7,D1
64#define LDUAL(P,O,D0,D1) ldlo.l P,O,D0; ldhi.l P,O+3,D1
65#define STUAL(P,O,D0,D1) stlo.l P,O,D0; sthi.l P,O+3,D1
66
67	nop ! ld.b r3,0,r63 ! TAKum03020
68	pta/l Large,tr0
69	movi 25,r0
70	bgeu/u r4,r0,tr0
71	nsb r4,r0
72	shlli r0,5,r0
73	movi (L1-L0+63*32 + 1) & 0xffff,r1
74	sub r1, r0, r0
75L0:	ptrel r0,tr0
76	add r2,r4,r5
77	ptabs r18,tr1
78	add r3,r4,r6
79	blink tr0,r63
80
81/* Rearranged to make cut2 safe */
82	.balign 8
83L4_7:	/* 4..7 byte memcpy cntd. */
84	stlo.l r2, 0, r0
85	or r6, r7, r6
86	sthi.l r5, -1, r6
87	stlo.l r5, -4, r6
88	blink tr1,r63
89
90	.balign 8
91L1:	/* 0 byte memcpy */
92	nop
93	blink tr1,r63
94	nop
95	nop
96	nop
97	nop
98
99L2_3:	/* 2 or 3 byte memcpy cntd. */
100	st.b r5,-1,r6
101	blink tr1,r63
102
103	/* 1 byte memcpy */
104	ld.b r3,0,r0
105	st.b r2,0,r0
106	blink tr1,r63
107
108L8_15:	/* 8..15 byte memcpy cntd. */
109	stlo.q r2, 0, r0
110	or r6, r7, r6
111	sthi.q r5, -1, r6
112	stlo.q r5, -8, r6
113	blink tr1,r63
114
115	/* 2 or 3 byte memcpy */
116	ld.b r3,0,r0
117	nop ! ld.b r2,0,r63 ! TAKum03020
118	ld.b r3,1,r1
119	st.b r2,0,r0
120	pta/l L2_3,tr0
121	ld.b r6,-1,r6
122	st.b r2,1,r1
123	blink tr0, r63
124
125	/* 4 .. 7 byte memcpy */
126	LDUAL (r3, 0, r0, r1)
127	pta L4_7, tr0
128	ldlo.l r6, -4, r7
129	or r0, r1, r0
130	sthi.l r2, 3, r0
131	ldhi.l r6, -1, r6
132	blink tr0, r63
133
134	/* 8 .. 15 byte memcpy */
135	LDUAQ (r3, 0, r0, r1)
136	pta L8_15, tr0
137	ldlo.q r6, -8, r7
138	or r0, r1, r0
139	sthi.q r2, 7, r0
140	ldhi.q r6, -1, r6
141	blink tr0, r63
142
143	/* 16 .. 24 byte memcpy */
144	LDUAQ (r3, 0, r0, r1)
145	LDUAQ (r3, 8, r8, r9)
146	or r0, r1, r0
147	sthi.q r2, 7, r0
148	or r8, r9, r8
149	sthi.q r2, 15, r8
150	ldlo.q r6, -8, r7
151	ldhi.q r6, -1, r6
152	stlo.q r2, 8, r8
153	stlo.q r2, 0, r0
154	or r6, r7, r6
155	sthi.q r5, -1, r6
156	stlo.q r5, -8, r6
157	blink tr1,r63
158
159Large:
160	! ld.b r2, 0, r63 ! TAKum03020
161	pta/l  Loop_ua, tr1
162	ori r3, -8, r7
163	sub r2, r7, r22
164	sub r3, r2, r6
165	add r2, r4, r5
166	ldlo.q r3, 0, r0
167	addi r5, -16, r5
168	movi 64+8, r27 ! could subtract r7 from that.
169	stlo.q r2, 0, r0
170	sthi.q r2, 7, r0
171	ldx.q r22, r6, r0
172	bgtu/l r27, r4, tr1
173
174	addi r5, -48, r27
175	pta/l Loop_line, tr0
176	addi r6, 64, r36
177	addi r6, -24, r19
178	addi r6, -16, r20
179	addi r6, -8, r21
180
181Loop_line:
182	! ldx.q r22, r36, r63 ! TAKum03020
183	alloco r22, 32
184	synco
185	addi r22, 32, r22
186	ldx.q r22, r19, r23
187	sthi.q r22, -25, r0
188	ldx.q r22, r20, r24
189	ldx.q r22, r21, r25
190	stlo.q r22, -32, r0
191	ldx.q r22, r6,  r0
192	sthi.q r22, -17, r23
193	sthi.q r22,  -9, r24
194	sthi.q r22,  -1, r25
195	stlo.q r22, -24, r23
196	stlo.q r22, -16, r24
197	stlo.q r22,  -8, r25
198	bgeu r27, r22, tr0
199
200Loop_ua:
201	addi r22, 8, r22
202	sthi.q r22, -1, r0
203	stlo.q r22, -8, r0
204	ldx.q r22, r6, r0
205	bgtu/l r5, r22, tr1
206
207	add r3, r4, r7
208	ldlo.q r7, -8, r1
209	sthi.q r22, 7, r0
210	ldhi.q r7, -1, r7
211	ptabs r18,tr1
212	stlo.q r22, 0, r0
213	or r1, r7, r1
214	sthi.q r5, 15, r1
215	stlo.q r5, 8, r1
216	blink tr1, r63
217copy_user_memcpy_end:
218	nop
219