1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Accelerated CRC32(C) using AArch64 CRC instructions
4 *
5 * Copyright (C) 2016 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
6 */
7
8#include <linux/linkage.h>
9#include <asm/alternative.h>
10#include <asm/assembler.h>
11
12	.cpu		generic+crc
13
14	.macro		__crc32, c
15	cmp		x2, #16
16	b.lt		8f			// less than 16 bytes
17
18	and		x7, x2, #0x1f
19	and		x2, x2, #~0x1f
20	cbz		x7, 32f			// multiple of 32 bytes
21
22	and		x8, x7, #0xf
23	ldp		x3, x4, [x1]
24	add		x8, x8, x1
25	add		x1, x1, x7
26	ldp		x5, x6, [x8]
27CPU_BE(	rev		x3, x3		)
28CPU_BE(	rev		x4, x4		)
29CPU_BE(	rev		x5, x5		)
30CPU_BE(	rev		x6, x6		)
31
32	tst		x7, #8
33	crc32\c\()x	w8, w0, x3
34	csel		x3, x3, x4, eq
35	csel		w0, w0, w8, eq
36	tst		x7, #4
37	lsr		x4, x3, #32
38	crc32\c\()w	w8, w0, w3
39	csel		x3, x3, x4, eq
40	csel		w0, w0, w8, eq
41	tst		x7, #2
42	lsr		w4, w3, #16
43	crc32\c\()h	w8, w0, w3
44	csel		w3, w3, w4, eq
45	csel		w0, w0, w8, eq
46	tst		x7, #1
47	crc32\c\()b	w8, w0, w3
48	csel		w0, w0, w8, eq
49	tst		x7, #16
50	crc32\c\()x	w8, w0, x5
51	crc32\c\()x	w8, w8, x6
52	csel		w0, w0, w8, eq
53	cbz		x2, 0f
54
5532:	ldp		x3, x4, [x1], #32
56	sub		x2, x2, #32
57	ldp		x5, x6, [x1, #-16]
58CPU_BE(	rev		x3, x3		)
59CPU_BE(	rev		x4, x4		)
60CPU_BE(	rev		x5, x5		)
61CPU_BE(	rev		x6, x6		)
62	crc32\c\()x	w0, w0, x3
63	crc32\c\()x	w0, w0, x4
64	crc32\c\()x	w0, w0, x5
65	crc32\c\()x	w0, w0, x6
66	cbnz		x2, 32b
670:	ret
68
698:	tbz		x2, #3, 4f
70	ldr		x3, [x1], #8
71CPU_BE(	rev		x3, x3		)
72	crc32\c\()x	w0, w0, x3
734:	tbz		x2, #2, 2f
74	ldr		w3, [x1], #4
75CPU_BE(	rev		w3, w3		)
76	crc32\c\()w	w0, w0, w3
772:	tbz		x2, #1, 1f
78	ldrh		w3, [x1], #2
79CPU_BE(	rev16		w3, w3		)
80	crc32\c\()h	w0, w0, w3
811:	tbz		x2, #0, 0f
82	ldrb		w3, [x1]
83	crc32\c\()b	w0, w0, w3
840:	ret
85	.endm
86
87	.align		5
88ENTRY(crc32_le)
89alternative_if_not ARM64_HAS_CRC32
90	b		crc32_le_base
91alternative_else_nop_endif
92	__crc32
93ENDPROC(crc32_le)
94
95	.align		5
96ENTRY(__crc32c_le)
97alternative_if_not ARM64_HAS_CRC32
98	b		__crc32c_le_base
99alternative_else_nop_endif
100	__crc32		c
101ENDPROC(__crc32c_le)
102