53 		"1: prefetch (%0)\n"		/* This set is 28 bytes */  in _mmx_memcpy()
54 		"   prefetch 64(%0)\n"  in _mmx_memcpy()
55 		"   prefetch 128(%0)\n"  in _mmx_memcpy()
56 		"   prefetch 192(%0)\n"  in _mmx_memcpy()
57 		"   prefetch 256(%0)\n"  in _mmx_memcpy()
58 		"2:  \n"  in _mmx_memcpy()
59 		".section .fixup, \"ax\"\n"  in _mmx_memcpy()
60 		"3: movw $0x1AEB, 1b\n"	/* jmp on 26 bytes */  in _mmx_memcpy()
61 		"   jmp 2b\n"  in _mmx_memcpy()
62 		".previous\n"  in _mmx_memcpy()
68 		"1:  prefetch 320(%0)\n"  in _mmx_memcpy()
69 		"2:  movq (%0), %%mm0\n"  in _mmx_memcpy()
70 		"  movq 8(%0), %%mm1\n"  in _mmx_memcpy()
71 		"  movq 16(%0), %%mm2\n"  in _mmx_memcpy()
72 		"  movq 24(%0), %%mm3\n"  in _mmx_memcpy()
73 		"  movq %%mm0, (%1)\n"  in _mmx_memcpy()
74 		"  movq %%mm1, 8(%1)\n"  in _mmx_memcpy()
75 		"  movq %%mm2, 16(%1)\n"  in _mmx_memcpy()
76 		"  movq %%mm3, 24(%1)\n"  in _mmx_memcpy()
77 		"  movq 32(%0), %%mm0\n"  in _mmx_memcpy()
78 		"  movq 40(%0), %%mm1\n"  in _mmx_memcpy()
79 		"  movq 48(%0), %%mm2\n"  in _mmx_memcpy()
80 		"  movq 56(%0), %%mm3\n"  in _mmx_memcpy()
81 		"  movq %%mm0, 32(%1)\n"  in _mmx_memcpy()
82 		"  movq %%mm1, 40(%1)\n"  in _mmx_memcpy()
83 		"  movq %%mm2, 48(%1)\n"  in _mmx_memcpy()
84 		"  movq %%mm3, 56(%1)\n"  in _mmx_memcpy()
85 		".section .fixup, \"ax\"\n"  in _mmx_memcpy()
86 		"3: movw $0x05EB, 1b\n"	/* jmp on 5 bytes */  in _mmx_memcpy()
87 		"   jmp 2b\n"  in _mmx_memcpy()
88 		".previous\n"  in _mmx_memcpy()
98 		"  movq (%0), %%mm0\n"  in _mmx_memcpy()
99 		"  movq 8(%0), %%mm1\n"  in _mmx_memcpy()
100 		"  movq 16(%0), %%mm2\n"  in _mmx_memcpy()
101 		"  movq 24(%0), %%mm3\n"  in _mmx_memcpy()
102 		"  movq %%mm0, (%1)\n"  in _mmx_memcpy()
103 		"  movq %%mm1, 8(%1)\n"  in _mmx_memcpy()
104 		"  movq %%mm2, 16(%1)\n"  in _mmx_memcpy()
105 		"  movq %%mm3, 24(%1)\n"  in _mmx_memcpy()
106 		"  movq 32(%0), %%mm0\n"  in _mmx_memcpy()
107 		"  movq 40(%0), %%mm1\n"  in _mmx_memcpy()
108 		"  movq 48(%0), %%mm2\n"  in _mmx_memcpy()
109 		"  movq 56(%0), %%mm3\n"  in _mmx_memcpy()
110 		"  movq %%mm0, 32(%1)\n"  in _mmx_memcpy()
111 		"  movq %%mm1, 40(%1)\n"  in _mmx_memcpy()
112 		"  movq %%mm2, 48(%1)\n"  in _mmx_memcpy()
113 		"  movq %%mm3, 56(%1)\n"  in _mmx_memcpy()
143 		"  pxor %%mm0, %%mm0\n" : :  in fast_clear_page()
148 		"  movntq %%mm0, (%0)\n"  in fast_clear_page()
149 		"  movntq %%mm0, 8(%0)\n"  in fast_clear_page()
150 		"  movntq %%mm0, 16(%0)\n"  in fast_clear_page()
151 		"  movntq %%mm0, 24(%0)\n"  in fast_clear_page()
152 		"  movntq %%mm0, 32(%0)\n"  in fast_clear_page()
153 		"  movntq %%mm0, 40(%0)\n"  in fast_clear_page()
154 		"  movntq %%mm0, 48(%0)\n"  in fast_clear_page()
155 		"  movntq %%mm0, 56(%0)\n"  in fast_clear_page()
164 	__asm__ __volatile__("sfence\n"::);  in fast_clear_page()
180 		"1: prefetch (%0)\n"  in fast_copy_page()
181 		"   prefetch 64(%0)\n"  in fast_copy_page()
182 		"   prefetch 128(%0)\n"  in fast_copy_page()
183 		"   prefetch 192(%0)\n"  in fast_copy_page()
184 		"   prefetch 256(%0)\n"  in fast_copy_page()
185 		"2:  \n"  in fast_copy_page()
186 		".section .fixup, \"ax\"\n"  in fast_copy_page()
187 		"3: movw $0x1AEB, 1b\n"	/* jmp on 26 bytes */  in fast_copy_page()
188 		"   jmp 2b\n"  in fast_copy_page()
189 		".previous\n"  in fast_copy_page()
194 		"1: prefetch 320(%0)\n"  in fast_copy_page()
195 		"2: movq (%0), %%mm0\n"  in fast_copy_page()
196 		"   movntq %%mm0, (%1)\n"  in fast_copy_page()
197 		"   movq 8(%0), %%mm1\n"  in fast_copy_page()
198 		"   movntq %%mm1, 8(%1)\n"  in fast_copy_page()
199 		"   movq 16(%0), %%mm2\n"  in fast_copy_page()
200 		"   movntq %%mm2, 16(%1)\n"  in fast_copy_page()
201 		"   movq 24(%0), %%mm3\n"  in fast_copy_page()
202 		"   movntq %%mm3, 24(%1)\n"  in fast_copy_page()
203 		"   movq 32(%0), %%mm4\n"  in fast_copy_page()
204 		"   movntq %%mm4, 32(%1)\n"  in fast_copy_page()
205 		"   movq 40(%0), %%mm5\n"  in fast_copy_page()
206 		"   movntq %%mm5, 40(%1)\n"  in fast_copy_page()
207 		"   movq 48(%0), %%mm6\n"  in fast_copy_page()
208 		"   movntq %%mm6, 48(%1)\n"  in fast_copy_page()
209 		"   movq 56(%0), %%mm7\n"  in fast_copy_page()
210 		"   movntq %%mm7, 56(%1)\n"  in fast_copy_page()
211 		".section .fixup, \"ax\"\n"  in fast_copy_page()
212 		"3: movw $0x05EB, 1b\n"	/* jmp on 5 bytes */  in fast_copy_page()
213 		"   jmp 2b\n"  in fast_copy_page()
214 		".previous\n"  in fast_copy_page()
223 		"2: movq (%0), %%mm0\n"  in fast_copy_page()
224 		"   movntq %%mm0, (%1)\n"  in fast_copy_page()
225 		"   movq 8(%0), %%mm1\n"  in fast_copy_page()
226 		"   movntq %%mm1, 8(%1)\n"  in fast_copy_page()
227 		"   movq 16(%0), %%mm2\n"  in fast_copy_page()
228 		"   movntq %%mm2, 16(%1)\n"  in fast_copy_page()
229 		"   movq 24(%0), %%mm3\n"  in fast_copy_page()
230 		"   movntq %%mm3, 24(%1)\n"  in fast_copy_page()
231 		"   movq 32(%0), %%mm4\n"  in fast_copy_page()
232 		"   movntq %%mm4, 32(%1)\n"  in fast_copy_page()
233 		"   movq 40(%0), %%mm5\n"  in fast_copy_page()
234 		"   movntq %%mm5, 40(%1)\n"  in fast_copy_page()
235 		"   movq 48(%0), %%mm6\n"  in fast_copy_page()
236 		"   movntq %%mm6, 48(%1)\n"  in fast_copy_page()
237 		"   movq 56(%0), %%mm7\n"  in fast_copy_page()
238 		"   movntq %%mm7, 56(%1)\n"  in fast_copy_page()
247 	__asm__ __volatile__("sfence \n"::);  in fast_copy_page()
263 		"  pxor %%mm0, %%mm0\n" : :  in fast_clear_page()
268 		"  movq %%mm0, (%0)\n"  in fast_clear_page()
269 		"  movq %%mm0, 8(%0)\n"  in fast_clear_page()
270 		"  movq %%mm0, 16(%0)\n"  in fast_clear_page()
271 		"  movq %%mm0, 24(%0)\n"  in fast_clear_page()
272 		"  movq %%mm0, 32(%0)\n"  in fast_clear_page()
273 		"  movq %%mm0, 40(%0)\n"  in fast_clear_page()
274 		"  movq %%mm0, 48(%0)\n"  in fast_clear_page()
275 		"  movq %%mm0, 56(%0)\n"  in fast_clear_page()
276 		"  movq %%mm0, 64(%0)\n"  in fast_clear_page()
277 		"  movq %%mm0, 72(%0)\n"  in fast_clear_page()
278 		"  movq %%mm0, 80(%0)\n"  in fast_clear_page()
279 		"  movq %%mm0, 88(%0)\n"  in fast_clear_page()
280 		"  movq %%mm0, 96(%0)\n"  in fast_clear_page()
281 		"  movq %%mm0, 104(%0)\n"  in fast_clear_page()
282 		"  movq %%mm0, 112(%0)\n"  in fast_clear_page()
283 		"  movq %%mm0, 120(%0)\n"  in fast_clear_page()
298 		"1: prefetch (%0)\n"  in fast_copy_page()
299 		"   prefetch 64(%0)\n"  in fast_copy_page()
300 		"   prefetch 128(%0)\n"  in fast_copy_page()
301 		"   prefetch 192(%0)\n"  in fast_copy_page()
302 		"   prefetch 256(%0)\n"  in fast_copy_page()
303 		"2:  \n"  in fast_copy_page()
304 		".section .fixup, \"ax\"\n"  in fast_copy_page()
305 		"3: movw $0x1AEB, 1b\n"	/* jmp on 26 bytes */  in fast_copy_page()
306 		"   jmp 2b\n"  in fast_copy_page()
307 		".previous\n"  in fast_copy_page()
312 		"1: prefetch 320(%0)\n"  in fast_copy_page()
313 		"2: movq (%0), %%mm0\n"  in fast_copy_page()
314 		"   movq 8(%0), %%mm1\n"  in fast_copy_page()
315 		"   movq 16(%0), %%mm2\n"  in fast_copy_page()
316 		"   movq 24(%0), %%mm3\n"  in fast_copy_page()
317 		"   movq %%mm0, (%1)\n"  in fast_copy_page()
318 		"   movq %%mm1, 8(%1)\n"  in fast_copy_page()
319 		"   movq %%mm2, 16(%1)\n"  in fast_copy_page()
320 		"   movq %%mm3, 24(%1)\n"  in fast_copy_page()
321 		"   movq 32(%0), %%mm0\n"  in fast_copy_page()
322 		"   movq 40(%0), %%mm1\n"  in fast_copy_page()
323 		"   movq 48(%0), %%mm2\n"  in fast_copy_page()
324 		"   movq 56(%0), %%mm3\n"  in fast_copy_page()
325 		"   movq %%mm0, 32(%1)\n"  in fast_copy_page()
326 		"   movq %%mm1, 40(%1)\n"  in fast_copy_page()
327 		"   movq %%mm2, 48(%1)\n"  in fast_copy_page()
328 		"   movq %%mm3, 56(%1)\n"  in fast_copy_page()
329 		".section .fixup, \"ax\"\n"  in fast_copy_page()
330 		"3: movw $0x05EB, 1b\n"	/* jmp on 5 bytes */  in fast_copy_page()
331 		"   jmp 2b\n"  in fast_copy_page()
332 		".previous\n"  in fast_copy_page()
352 		"cld\n\t"  in slow_zero_page()
374 		"cld\n\t"  in slow_copy_page()