1/* 2 * Copyright (c) 2013 3 * MIPS Technologies, Inc., California. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. Neither the name of the MIPS Technologies, Inc., nor the names of its 14 * contributors may be used to endorse or promote products derived from 15 * this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE MIPS TECHNOLOGIES, INC. ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE MIPS TECHNOLOGIES, INC. BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30#ifdef ANDROID_CHANGES 31# include "machine/asm.h" 32# include "machine/regdef.h" 33# define PREFETCH_STORE_HINT PREFETCH_HINT_PREPAREFORSTORE 34#elif _LIBC 35# include "machine/asm.h" 36# include "machine/regdef.h" 37# define PREFETCH_STORE_HINT PREFETCH_HINT_PREPAREFORSTORE 38#else 39# include <regdef.h> 40# include <sys/asm.h> 41#endif 42 43/* Check to see if the MIPS architecture we are compiling for supports 44 prefetching. */ 45 46#if (__mips == 4) || (__mips == 5) || (__mips == 32) || (__mips == 64) 47# ifndef DISABLE_PREFETCH 48# define USE_PREFETCH 49# endif 50#endif 51 52#if defined(_MIPS_SIM) && ((_MIPS_SIM == _ABI64) || (_MIPS_SIM == _ABIN32)) 53# ifndef DISABLE_DOUBLE 54# define USE_DOUBLE 55# endif 56#endif 57 58#ifndef USE_DOUBLE 59# ifndef DISABLE_DOUBLE_ALIGN 60# define DOUBLE_ALIGN 61# endif 62#endif 63 64/* Some asm.h files do not have the L macro definition. */ 65#ifndef L 66# if _MIPS_SIM == _ABIO32 67# define L(label) $L ## label 68# else 69# define L(label) .L ## label 70# endif 71#endif 72 73/* Some asm.h files do not have the PTR_ADDIU macro definition. */ 74#ifndef PTR_ADDIU 75# ifdef USE_DOUBLE 76# define PTR_ADDIU daddiu 77# else 78# define PTR_ADDIU addiu 79# endif 80#endif 81 82/* New R6 instructions that may not be in asm.h. */ 83#ifndef PTR_LSA 84# if _MIPS_SIM == _ABI64 85# define PTR_LSA dlsa 86# else 87# define PTR_LSA lsa 88# endif 89#endif 90 91/* Using PREFETCH_HINT_PREPAREFORSTORE instead of PREFETCH_STORE 92 or PREFETCH_STORE_STREAMED offers a large performance advantage 93 but PREPAREFORSTORE has some special restrictions to consider. 94 95 Prefetch with the 'prepare for store' hint does not copy a memory 96 location into the cache, it just allocates a cache line and zeros 97 it out. This means that if you do not write to the entire cache 98 line before writing it out to memory some data will get zero'ed out 99 when the cache line is written back to memory and data will be lost. 100 101 There are ifdef'ed sections of this memcpy to make sure that it does not 102 do prefetches on cache lines that are not going to be completely written. 103 This code is only needed and only used when PREFETCH_STORE_HINT is set to 104 PREFETCH_HINT_PREPAREFORSTORE. This code assumes that cache lines are 105 less than MAX_PREFETCH_SIZE bytes and if the cache line is larger it will 106 not work correctly. */ 107 108#ifdef USE_PREFETCH 109# define PREFETCH_HINT_STORE 1 110# define PREFETCH_HINT_STORE_STREAMED 5 111# define PREFETCH_HINT_STORE_RETAINED 7 112# define PREFETCH_HINT_PREPAREFORSTORE 30 113 114/* If we have not picked out what hints to use at this point use the 115 standard load and store prefetch hints. */ 116# ifndef PREFETCH_STORE_HINT 117# define PREFETCH_STORE_HINT PREFETCH_HINT_STORE 118# endif 119 120/* We double everything when USE_DOUBLE is true so we do 2 prefetches to 121 get 64 bytes in that case. The assumption is that each individual 122 prefetch brings in 32 bytes. */ 123# ifdef USE_DOUBLE 124# define PREFETCH_CHUNK 64 125# define PREFETCH_FOR_STORE(chunk, reg) \ 126 pref PREFETCH_STORE_HINT, (chunk)*64(reg); \ 127 pref PREFETCH_STORE_HINT, ((chunk)*64)+32(reg) 128# else 129# define PREFETCH_CHUNK 32 130# define PREFETCH_FOR_STORE(chunk, reg) \ 131 pref PREFETCH_STORE_HINT, (chunk)*32(reg) 132# endif 133 134/* MAX_PREFETCH_SIZE is the maximum size of a prefetch, it must not be less 135 than PREFETCH_CHUNK, the assumed size of each prefetch. If the real size 136 of a prefetch is greater than MAX_PREFETCH_SIZE and the PREPAREFORSTORE 137 hint is used, the code will not work correctly. If PREPAREFORSTORE is not 138 used than MAX_PREFETCH_SIZE does not matter. */ 139# define MAX_PREFETCH_SIZE 128 140/* PREFETCH_LIMIT is set based on the fact that we never use an offset greater 141 than 5 on a STORE prefetch and that a single prefetch can never be larger 142 than MAX_PREFETCH_SIZE. We add the extra 32 when USE_DOUBLE is set because 143 we actually do two prefetches in that case, one 32 bytes after the other. */ 144# ifdef USE_DOUBLE 145# define PREFETCH_LIMIT (5 * PREFETCH_CHUNK) + 32 + MAX_PREFETCH_SIZE 146# else 147# define PREFETCH_LIMIT (5 * PREFETCH_CHUNK) + MAX_PREFETCH_SIZE 148# endif 149 150# if (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE) \ 151 && ((PREFETCH_CHUNK * 4) < MAX_PREFETCH_SIZE) 152/* We cannot handle this because the initial prefetches may fetch bytes that 153 are before the buffer being copied. We start copies with an offset 154 of 4 so avoid this situation when using PREPAREFORSTORE. */ 155# error "PREFETCH_CHUNK is too large and/or MAX_PREFETCH_SIZE is too small." 156# endif 157#else /* USE_PREFETCH not defined */ 158# define PREFETCH_FOR_STORE(offset, reg) 159#endif 160 161#if __mips_isa_rev > 5 162# if (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE) 163# undef PREFETCH_STORE_HINT 164# define PREFETCH_STORE_HINT PREFETCH_HINT_STORE_STREAMED 165# endif 166# define R6_CODE 167#endif 168 169/* Allow the routine to be named something else if desired. */ 170#ifndef MEMSET_NAME 171# define MEMSET_NAME memset 172#endif 173 174/* We load/store 64 bits at a time when USE_DOUBLE is true. 175 The C_ prefix stands for CHUNK and is used to avoid macro name 176 conflicts with system header files. */ 177 178#ifdef USE_DOUBLE 179# define C_ST sd 180# if __MIPSEB 181# define C_STHI sdl /* high part is left in big-endian */ 182# else 183# define C_STHI sdr /* high part is right in little-endian */ 184# endif 185#else 186# define C_ST sw 187# if __MIPSEB 188# define C_STHI swl /* high part is left in big-endian */ 189# else 190# define C_STHI swr /* high part is right in little-endian */ 191# endif 192#endif 193 194/* Bookkeeping values for 32 vs. 64 bit mode. */ 195#ifdef USE_DOUBLE 196# define NSIZE 8 197# define NSIZEMASK 0x3f 198# define NSIZEDMASK 0x7f 199#else 200# define NSIZE 4 201# define NSIZEMASK 0x1f 202# define NSIZEDMASK 0x3f 203#endif 204#define UNIT(unit) ((unit)*NSIZE) 205#define UNITM1(unit) (((unit)*NSIZE)-1) 206 207#ifdef ANDROID_CHANGES 208LEAF(MEMSET_NAME,0) 209#else 210LEAF(MEMSET_NAME) 211#endif 212 213 .set nomips16 214 .set noreorder 215/* If the size is less than 2*NSIZE (8 or 16), go to L(lastb). Regardless of 216 size, copy dst pointer to v0 for the return value. */ 217 slti t2,a2,(2 * NSIZE) 218 bne t2,zero,L(lastb) 219 move v0,a0 220 221/* If memset value is not zero, we copy it to all the bytes in a 32 or 64 222 bit word. */ 223 beq a1,zero,L(set0) /* If memset value is zero no smear */ 224 PTR_SUBU a3,zero,a0 225 nop 226 227 /* smear byte into 32 or 64 bit word */ 228#if ((__mips == 64) || (__mips == 32)) && (__mips_isa_rev >= 2) 229# ifdef USE_DOUBLE 230 dins a1, a1, 8, 8 /* Replicate fill byte into half-word. */ 231 dins a1, a1, 16, 16 /* Replicate fill byte into word. */ 232 dins a1, a1, 32, 32 /* Replicate fill byte into dbl word. */ 233# else 234 ins a1, a1, 8, 8 /* Replicate fill byte into half-word. */ 235 ins a1, a1, 16, 16 /* Replicate fill byte into word. */ 236# endif 237#else 238# ifdef USE_DOUBLE 239 and a1,0xff 240 dsll t2,a1,8 241 or a1,t2 242 dsll t2,a1,16 243 or a1,t2 244 dsll t2,a1,32 245 or a1,t2 246# else 247 and a1,0xff 248 sll t2,a1,8 249 or a1,t2 250 sll t2,a1,16 251 or a1,t2 252# endif 253#endif 254 255/* If the destination address is not aligned do a partial store to get it 256 aligned. If it is already aligned just jump to L(aligned). */ 257L(set0): 258#ifndef R6_CODE 259 andi t2,a3,(NSIZE-1) /* word-unaligned address? */ 260 beq t2,zero,L(aligned) /* t2 is the unalignment count */ 261 PTR_SUBU a2,a2,t2 262 C_STHI a1,0(a0) 263 PTR_ADDU a0,a0,t2 264#else /* R6_CODE */ 265 andi t2,a0,(NSIZE-1) 266 lapc t9,L(atable) 267 PTR_LSA t9,t2,t9,2 268 jrc t9 269L(atable): 270 bc L(aligned) 271# ifdef USE_DOUBLE 272 bc L(lb7) 273 bc L(lb6) 274 bc L(lb5) 275 bc L(lb4) 276# endif 277 bc L(lb3) 278 bc L(lb2) 279 bc L(lb1) 280L(lb7): 281 sb a1,6(a0) 282L(lb6): 283 sb a1,5(a0) 284L(lb5): 285 sb a1,4(a0) 286L(lb4): 287 sb a1,3(a0) 288L(lb3): 289 sb a1,2(a0) 290L(lb2): 291 sb a1,1(a0) 292L(lb1): 293 sb a1,0(a0) 294 295 li t9,NSIZE 296 subu t2,t9,t2 297 PTR_SUBU a2,a2,t2 298 PTR_ADDU a0,a0,t2 299#endif /* R6_CODE */ 300 301L(aligned): 302/* If USE_DOUBLE is not set we may still want to align the data on a 16 303 byte boundry instead of an 8 byte boundry to maximize the opportunity 304 of proAptiv chips to do memory bonding (combining two sequential 4 305 byte stores into one 8 byte store). We know there are at least 4 bytes 306 left to store or we would have jumped to L(lastb) earlier in the code. */ 307#ifdef DOUBLE_ALIGN 308 andi t2,a3,4 309 beq t2,zero,L(double_aligned) 310 PTR_SUBU a2,a2,t2 311 sw a1,0(a0) 312 PTR_ADDU a0,a0,t2 313L(double_aligned): 314#endif 315 316/* Now the destination is aligned to (word or double word) aligned address 317 Set a2 to count how many bytes we have to copy after all the 64/128 byte 318 chunks are copied and a3 to the dest pointer after all the 64/128 byte 319 chunks have been copied. We will loop, incrementing a0 until it equals 320 a3. */ 321 andi t8,a2,NSIZEDMASK /* any whole 64-byte/128-byte chunks? */ 322 beq a2,t8,L(chkw) /* if a2==t8, no 64-byte/128-byte chunks */ 323 PTR_SUBU a3,a2,t8 /* subtract from a2 the reminder */ 324 PTR_ADDU a3,a0,a3 /* Now a3 is the final dst after loop */ 325 326/* When in the loop we may prefetch with the 'prepare to store' hint, 327 in this case the a0+x should not be past the "t0-32" address. This 328 means: for x=128 the last "safe" a0 address is "t0-160". Alternatively, 329 for x=64 the last "safe" a0 address is "t0-96" In the current version we 330 will use "prefetch hint,128(a0)", so "t0-160" is the limit. */ 331#if defined(USE_PREFETCH) \ 332 && (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE) 333 PTR_ADDU t0,a0,a2 /* t0 is the "past the end" address */ 334 PTR_SUBU t9,t0,PREFETCH_LIMIT /* t9 is the "last safe pref" address */ 335#endif 336#if defined(USE_PREFETCH) \ 337 && (PREFETCH_STORE_HINT != PREFETCH_HINT_PREPAREFORSTORE) 338 PREFETCH_FOR_STORE (1, a0) 339 PREFETCH_FOR_STORE (2, a0) 340 PREFETCH_FOR_STORE (3, a0) 341#endif 342 343L(loop16w): 344#if defined(USE_PREFETCH) \ 345 && (PREFETCH_STORE_HINT == PREFETCH_HINT_PREPAREFORSTORE) 346 sltu v1,t9,a0 /* If a0 > t9 don't use next prefetch */ 347 bgtz v1,L(skip_pref) 348 nop 349#endif 350#ifndef R6_CODE 351 PREFETCH_FOR_STORE (4, a0) 352 PREFETCH_FOR_STORE (5, a0) 353#else 354 PREFETCH_FOR_STORE (2, a0) 355#endif 356L(skip_pref): 357 C_ST a1,UNIT(0)(a0) 358 C_ST a1,UNIT(1)(a0) 359 C_ST a1,UNIT(2)(a0) 360 C_ST a1,UNIT(3)(a0) 361 C_ST a1,UNIT(4)(a0) 362 C_ST a1,UNIT(5)(a0) 363 C_ST a1,UNIT(6)(a0) 364 C_ST a1,UNIT(7)(a0) 365 C_ST a1,UNIT(8)(a0) 366 C_ST a1,UNIT(9)(a0) 367 C_ST a1,UNIT(10)(a0) 368 C_ST a1,UNIT(11)(a0) 369 C_ST a1,UNIT(12)(a0) 370 C_ST a1,UNIT(13)(a0) 371 C_ST a1,UNIT(14)(a0) 372 C_ST a1,UNIT(15)(a0) 373 PTR_ADDIU a0,a0,UNIT(16) /* adding 64/128 to dest */ 374 bne a0,a3,L(loop16w) 375 nop 376 move a2,t8 377 378/* Here we have dest word-aligned but less than 64-bytes or 128 bytes to go. 379 Check for a 32(64) byte chunk and copy if if there is one. Otherwise 380 jump down to L(chk1w) to handle the tail end of the copy. */ 381L(chkw): 382 andi t8,a2,NSIZEMASK /* is there a 32-byte/64-byte chunk. */ 383 /* the t8 is the reminder count past 32-bytes */ 384 beq a2,t8,L(chk1w)/* when a2==t8, no 32-byte chunk */ 385 nop 386 C_ST a1,UNIT(0)(a0) 387 C_ST a1,UNIT(1)(a0) 388 C_ST a1,UNIT(2)(a0) 389 C_ST a1,UNIT(3)(a0) 390 C_ST a1,UNIT(4)(a0) 391 C_ST a1,UNIT(5)(a0) 392 C_ST a1,UNIT(6)(a0) 393 C_ST a1,UNIT(7)(a0) 394 PTR_ADDIU a0,a0,UNIT(8) 395 396/* Here we have less than 32(64) bytes to set. Set up for a loop to 397 copy one word (or double word) at a time. Set a2 to count how many 398 bytes we have to copy after all the word (or double word) chunks are 399 copied and a3 to the dest pointer after all the (d)word chunks have 400 been copied. We will loop, incrementing a0 until a0 equals a3. */ 401L(chk1w): 402 andi a2,t8,(NSIZE-1) /* a2 is the reminder past one (d)word chunks */ 403 beq a2,t8,L(lastb) 404 PTR_SUBU a3,t8,a2 /* a3 is count of bytes in one (d)word chunks */ 405 PTR_ADDU a3,a0,a3 /* a3 is the dst address after loop */ 406 407/* copying in words (4-byte or 8 byte chunks) */ 408L(wordCopy_loop): 409 PTR_ADDIU a0,a0,UNIT(1) 410 bne a0,a3,L(wordCopy_loop) 411 C_ST a1,UNIT(-1)(a0) 412 413/* Copy the last 8 (or 16) bytes */ 414L(lastb): 415 blez a2,L(leave) 416 PTR_ADDU a3,a0,a2 /* a3 is the last dst address */ 417L(lastbloop): 418 PTR_ADDIU a0,a0,1 419 bne a0,a3,L(lastbloop) 420 sb a1,-1(a0) 421L(leave): 422 j ra 423 nop 424 425 .set at 426 .set reorder 427END(MEMSET_NAME) 428
