!
! Fast SH memcpy
!
! by Toshiyasu Morita (tm@netcom.com)
! hacked by J"orn Rernnecke (joern.rennecke@superh.com) ("o for o-umlaut)
/* SH5 code Copyright 2002 SuperH Ltd. */
!
! Entry: ARG0: destination pointer
!        ARG1: source pointer
!        ARG3: byte count
!
! Exit:  RESULT: destination pointer
!        any other registers in the range r0-r7: trashed
!
! Notes: Usually one wants to do small reads and write a longword, but
!        unfortunately it is difficult in some cases to concatanate bytes
!        into a longword on the SH, so this does a longword read and small
!        writes.
!
! This implementation makes two assumptions about how it is called:
!
! 1.: If the byte count is nonzero, the address of the last byte to be
!     copied is unsigned greater than the address of the first byte to
!     be copied.  This could be easily swapped for a signed comparison,
!     but the algorithm used needs some comparison.
!
! 2.: When there are two or three bytes in the last word of an 11-or-more
!     bytes memory chunk to b copied, the rest of the word can be read
!     without side effects.
!     This could be easily changed by increasing the minumum size of
!     a fast memcpy and the amount subtracted from r7 before L_2l_loop be 2,
!     however, this would cost a few extra cyles on average.
!     For SHmedia, the assumption is that any quadword can be read in its
!     enirety if at least one byte is included in the copy.
!

#include "asm.h"

ENTRY(memcpy)

#if __SHMEDIA__

#define LDUAQ(P,O,D0,D1) ldlo.q P,O,D0; ldhi.q P,O+7,D1
#define STUAQ(P,O,D0,D1) stlo.q P,O,D0; sthi.q P,O+7,D1
#define LDUAL(P,O,D0,D1) ldlo.l P,O,D0; ldhi.l P,O+3,D1
#define STUAL(P,O,D0,D1) stlo.l P,O,D0; sthi.l P,O+3,D1

	ld.b r3,0,r63
	pta/l Large,tr0
	movi 25,r0
	bgeu/u r4,r0,tr0
	nsb r4,r0
	shlli r0,5,r0
	movi (L1-L0+63*32 + 1) & 0xffff,r1
	sub r1, r0, r0
L0:	ptrel r0,tr0
	add r2,r4,r5
	ptabs r18,tr1
	add r3,r4,r6
	blink tr0,r63

	.balign 8
L1:
	/* 0 byte memcpy */
	blink tr1,r63

L4_7:	/* 4..7 byte memcpy cntd. */
	stlo.l r2, 0, r0
	or r6, r7, r6
	sthi.l r5, -1, r6
	stlo.l r5, -4, r6
	blink tr1,r63

L2_3:	/* 2 or 3 byte memcpy cntd. */
	st.b r5,-1,r6
	blink tr1,r63

	/* 1 byte memcpy */
	ld.b r3,0,r0
	st.b r2,0,r0
	blink tr1,r63

L8_15:	/* 8..15 byte memcpy cntd. */
	stlo.q r2, 0, r0
	or r6, r7, r6
	sthi.q r5, -1, r6
	stlo.q r5, -8, r6
	blink tr1,r63
	
	/* 2 or 3 byte memcpy */
	ld.b r3,0,r0
	ld.b r2,0,r63
	ld.b r3,1,r1
	st.b r2,0,r0
	pta/l L2_3,tr0
	ld.b r6,-1,r6
	st.b r2,1,r1
	blink tr0, r63

	/* 4 .. 7 byte memcpy */
	LDUAL (r3, 0, r0, r1)
	pta L4_7, tr0
	ldlo.l r6, -4, r7
	or r0, r1, r0
	sthi.l r2, 3, r0
	ldhi.l r6, -1, r6
	blink tr0, r63

	/* 8 .. 15 byte memcpy */
	LDUAQ (r3, 0, r0, r1)
	pta L8_15, tr0
	ldlo.q r6, -8, r7
	or r0, r1, r0
	sthi.q r2, 7, r0
	ldhi.q r6, -1, r6
	blink tr0, r63

	/* 16 .. 24 byte memcpy */
	LDUAQ (r3, 0, r0, r1)
	LDUAQ (r3, 8, r8, r9)
	or r0, r1, r0
	sthi.q r2, 7, r0
	or r8, r9, r8
	sthi.q r2, 15, r8
	ldlo.q r6, -8, r7
	ldhi.q r6, -1, r6
	stlo.q r2, 8, r8
	stlo.q r2, 0, r0
	or r6, r7, r6
	sthi.q r5, -1, r6
	stlo.q r5, -8, r6
	blink tr1,r63

Large:
	ld.b r2, 0, r63
	pta/l  Loop_ua, tr1
	ori r3, -8, r7
	sub r2, r7, r22
	sub r3, r2, r6
	add r2, r4, r5
	ldlo.q r3, 0, r0
	addi r5, -16, r5
	movi 64+8, r27 // could subtract r7 from that.
	stlo.q r2, 0, r0
	sthi.q r2, 7, r0
	ldx.q r22, r6, r0
	bgtu/l r27, r4, tr1

	addi r5, -48, r27
	pta/l Loop_line, tr0
	addi r6, 64, r36
	addi r6, -24, r19
	addi r6, -16, r20
	addi r6, -8, r21

Loop_line:
	ldx.q r22, r36, r63
	alloco r22, 32
	addi r22, 32, r22
	ldx.q r22, r19, r23
	sthi.q r22, -25, r0
	ldx.q r22, r20, r24
	ldx.q r22, r21, r25
	stlo.q r22, -32, r0
	ldx.q r22, r6,  r0
	sthi.q r22, -17, r23
	sthi.q r22,  -9, r24
	sthi.q r22,  -1, r25
	stlo.q r22, -24, r23
	stlo.q r22, -16, r24
	stlo.q r22,  -8, r25
	bgeu r27, r22, tr0

Loop_ua:
	addi r22, 8, r22
	sthi.q r22, -1, r0
	stlo.q r22, -8, r0
	ldx.q r22, r6, r0
	bgtu/l r5, r22, tr1

	add r3, r4, r7
	ldlo.q r7, -8, r1
	sthi.q r22, 7, r0
	ldhi.q r7, -1, r7
	ptabs r18,tr1
	stlo.q r22, 0, r0
	or r1, r7, r1
	sthi.q r5, 15, r1
	stlo.q r5, 8, r1
	blink tr1, r63

#else /* ! SHMEDIA, i.e. SH1 .. SH4 / SHcompact */

#ifdef __SH5__
#define DST r2
#define SRC r3
#define COUNT r4
#define TMP0 r5
#define TMP1 r6
#define RESULT r2
#else
#define DST r4
#define SRC r5
#define COUNT r6
#define TMP0 r2
#define TMP1 r3
#define RESULT r0
#endif

#ifdef __LITTLE_ENDIAN__
	! Little endian version copies with increasing addresses.
	mov DST,TMP1	! Save return value
	mov #11,r0	! Check if small number of bytes
	cmp/hs r0,COUNT
			! COUNT becomes src end address
	SL(bf, L_small, add SRC,COUNT)
	mov #1,r1
	tst r1,SRC	! check if source even
	SL(bt, L_even, mov COUNT,r7)
	mov.b @SRC+,r0	! no, make it even.
	mov.b r0,@DST
	add #1,DST
L_even:	tst r1,DST	! check if destination is even
	add #-3,r7
	SL(bf, L_odddst, mov #2,r1)
	tst r1,DST	! check if destination is 4-byte aligned
	mov DST,r0
	SL(bt, L_al4dst, sub SRC,r0)
	mov.w @SRC+,TMP0
	mov.w TMP0,@DST
	! add #2,DST  DST is dead here.
L_al4dst:
	tst r1,SRC
	bt L_al4both
	mov.w @SRC+,r1
	swap.w r1,r1
	add #-6,r0
	add #-6,r7	! r7 := src end address minus 9.
	.align 2
L_2l_loop:
	mov.l @SRC+,TMP0 ! Read & write two longwords per iteration
	xtrct TMP0,r1
	mov.l r1,@(r0,SRC)
	cmp/hs r7,SRC
	mov.l @SRC+,r1
	xtrct r1,TMP0
	mov.l TMP0,@(r0,SRC)
	bf L_2l_loop
	add #-2,SRC
	bra  L_cleanup
	add #5,r0
L_al4both:
	add #-4,r0
	.align 2
L_al4both_loop:
	mov.l @SRC+,DST   ! Read longword, write longword per iteration
	cmp/hs r7,SRC
	SL(bf, L_al4both_loop, mov.l DST,@(r0,SRC))

	bra L_cleanup
	add #3,r0

L_odddst:
	tst r1,SRC
	SL(bt, L_al4src, add #-1,DST)
	mov.w @SRC+,r0
	mov.b r0,@(1,DST)
	shlr8 r0
	mov.b r0,@(2,DST)
	add #2,DST
L_al4src:
	.align 2
L_odd_loop:
	mov.l @SRC+,r0   ! Read longword, write byte, word, byte per iteration
	cmp/hs r7,SRC
	mov.b r0,@(1,DST)
	shlr8 r0
	mov.w r0,@(2,DST)
	shlr16 r0
	mov.b r0,@(4,DST)
	SL(bf, L_odd_loop, add #4,DST)
	.align 2 ! avoid nop in more frequently executed code.
L_cleanup2:
	mov	DST,r0
	sub	SRC,r0
L_cleanup:
	cmp/eq	COUNT,SRC
	bt	L_ready
	.align 2
L_cleanup_loop:
	mov.b	@SRC+,r1
	cmp/eq	COUNT,SRC
	mov.b	r1,@(r0,SRC)
	bf	L_cleanup_loop
L_ready:
	rts
	mov	TMP1,RESULT
L_small:
	bra L_cleanup2
	add #-1,DST
#else /* ! __LITTLE_ENDIAN__ */
	! Big endian version copies with decreasing addresses.
	mov DST,r0
	add COUNT,r0
	sub DST,SRC
	mov #11,r1
	cmp/hs r1,COUNT
	SL(bf, L_small, add #-1,SRC)
	mov SRC,TMP1
	add r0,TMP1
	shlr TMP1
	SL(bt, L_even,
	mov DST,r7)
	mov.b @(r0,SRC),TMP0
	add #-1,TMP1
	mov.b TMP0,@-r0
L_even:
	tst #1,r0
	add #-1,SRC
	SL(bf, L_odddst, add #8,r7)
	tst #2,r0
	bt L_al4dst
	add #-1,TMP1
	mov.w @(r0,SRC),r1
	mov.w r1,@-r0
L_al4dst:
	shlr TMP1
	bt L_al4both
	mov.w @(r0,SRC),r1
	swap.w r1,r1
	add #4,r7
	add #-4,SRC
	.align 2
L_2l_loop:
	mov.l @(r0,SRC),TMP0
	xtrct TMP0,r1
	mov.l r1,@-r0
	cmp/hs r7,r0
	mov.l @(r0,SRC),r1
	xtrct r1,TMP0
	mov.l TMP0,@-r0
	bt L_2l_loop
	bra L_cleanup
	add #5,SRC

	nop ! avoid nop in executed code.
L_al4both:
	add #-2,SRC
	.align 2
L_al4both_loop:
	mov.l @(r0,SRC),r1
	cmp/hs r7,r0
	SL(bt, L_al4both_loop,
	mov.l r1,@-r0)
	bra L_cleanup
	add #3,SRC

	nop ! avoid nop in executed code.
L_odddst:
	shlr TMP1
	bt L_al4src
	mov.w @(r0,SRC),r1
	mov.b r1,@-r0
	shlr8 r1
	mov.b r1,@-r0
L_al4src:
	add #-2,SRC
	.align 2
L_odd_loop:
	mov.l @(r0,SRC),TMP0
	cmp/hs r7,r0
	mov.b TMP0,@-r0
	shlr8 TMP0
	mov.w TMP0,@-r0
	shlr16 TMP0
	mov.b TMP0,@-r0
	bt L_odd_loop

	add #3,SRC
L_cleanup:
L_small:
	cmp/eq DST,r0
	bt L_ready
	add #1,DST
	.align 2
L_cleanup_loop:
	mov.b @(r0,SRC),TMP0
	cmp/eq DST,r0
	mov.b TMP0,@-r0
	bf L_cleanup_loop
L_ready:
	rts
	mov r0,RESULT
#endif /* ! __LITTLE_ENDIAN__ */
#endif /* ! SHMEDIA */