1/* SPDX-License-Identifier: GPL-2.0-only */
2/* bpf_jit.S: Packet/header access helper functions
3 * for PPC64 BPF compiler.
4 *
5 * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation
6 */
7
8#include <asm/ppc_asm.h>
9#include <asm/asm-compat.h>
10#include "bpf_jit32.h"
11
12/*
13 * All of these routines are called directly from generated code,
14 * whose register usage is:
15 *
16 * r3		skb
17 * r4,r5	A,X
18 * r6		*** address parameter to helper ***
19 * r7-r10	scratch
20 * r14		skb->data
21 * r15		skb headlen
22 * r16-31	M[]
23 */
24
25/*
26 * To consider: These helpers are so small it could be better to just
27 * generate them inline.  Inline code can do the simple headlen check
28 * then branch directly to slow_path_XXX if required.  (In fact, could
29 * load a spare GPR with the address of slow_path_generic and pass size
30 * as an argument, making the call site a mtlr, li and bllr.)
31 */
32	.globl	sk_load_word
33sk_load_word:
34	PPC_LCMPI	r_addr, 0
35	blt	bpf_slow_path_word_neg
36	.globl	sk_load_word_positive_offset
37sk_load_word_positive_offset:
38	/* Are we accessing past headlen? */
39	subi	r_scratch1, r_HL, 4
40	PPC_LCMP	r_scratch1, r_addr
41	blt	bpf_slow_path_word
42	/* Nope, just hitting the header.  cr0 here is eq or gt! */
43#ifdef __LITTLE_ENDIAN__
44	lwbrx	r_A, r_D, r_addr
45#else
46	lwzx	r_A, r_D, r_addr
47#endif
48	blr	/* Return success, cr0 != LT */
49
50	.globl	sk_load_half
51sk_load_half:
52	PPC_LCMPI	r_addr, 0
53	blt	bpf_slow_path_half_neg
54	.globl	sk_load_half_positive_offset
55sk_load_half_positive_offset:
56	subi	r_scratch1, r_HL, 2
57	PPC_LCMP	r_scratch1, r_addr
58	blt	bpf_slow_path_half
59#ifdef __LITTLE_ENDIAN__
60	lhbrx	r_A, r_D, r_addr
61#else
62	lhzx	r_A, r_D, r_addr
63#endif
64	blr
65
66	.globl	sk_load_byte
67sk_load_byte:
68	PPC_LCMPI	r_addr, 0
69	blt	bpf_slow_path_byte_neg
70	.globl	sk_load_byte_positive_offset
71sk_load_byte_positive_offset:
72	PPC_LCMP	r_HL, r_addr
73	ble	bpf_slow_path_byte
74	lbzx	r_A, r_D, r_addr
75	blr
76
77/*
78 * BPF_LDX | BPF_B | BPF_MSH: ldxb  4*([offset]&0xf)
79 * r_addr is the offset value
80 */
81	.globl sk_load_byte_msh
82sk_load_byte_msh:
83	PPC_LCMPI	r_addr, 0
84	blt	bpf_slow_path_byte_msh_neg
85	.globl sk_load_byte_msh_positive_offset
86sk_load_byte_msh_positive_offset:
87	PPC_LCMP	r_HL, r_addr
88	ble	bpf_slow_path_byte_msh
89	lbzx	r_X, r_D, r_addr
90	rlwinm	r_X, r_X, 2, 32-4-2, 31-2
91	blr
92
93/* Call out to skb_copy_bits:
94 * We'll need to back up our volatile regs first; we have
95 * local variable space at r1+(BPF_PPC_STACK_BASIC).
96 * Allocate a new stack frame here to remain ABI-compliant in
97 * stashing LR.
98 */
99#define bpf_slow_path_common(SIZE)				\
100	mflr	r0;						\
101	PPC_STL	r0, PPC_LR_STKOFF(r1);					\
102	/* R3 goes in parameter space of caller's frame */	\
103	PPC_STL	r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);		\
104	PPC_STL	r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);		\
105	PPC_STL	r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);		\
106	addi	r5, r1, BPF_PPC_STACK_BASIC+(2*REG_SZ);		\
107	PPC_STLU	r1, -BPF_PPC_SLOWPATH_FRAME(r1);		\
108	/* R3 = r_skb, as passed */				\
109	mr	r4, r_addr;					\
110	li	r6, SIZE;					\
111	bl	skb_copy_bits;					\
112	nop;							\
113	/* R3 = 0 on success */					\
114	addi	r1, r1, BPF_PPC_SLOWPATH_FRAME;			\
115	PPC_LL	r0, PPC_LR_STKOFF(r1);					\
116	PPC_LL	r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);		\
117	PPC_LL	r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);		\
118	mtlr	r0;						\
119	PPC_LCMPI	r3, 0;						\
120	blt	bpf_error;	/* cr0 = LT */			\
121	PPC_LL	r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);		\
122	/* Great success! */
123
124bpf_slow_path_word:
125	bpf_slow_path_common(4)
126	/* Data value is on stack, and cr0 != LT */
127	lwz	r_A, BPF_PPC_STACK_BASIC+(2*REG_SZ)(r1)
128	blr
129
130bpf_slow_path_half:
131	bpf_slow_path_common(2)
132	lhz	r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
133	blr
134
135bpf_slow_path_byte:
136	bpf_slow_path_common(1)
137	lbz	r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
138	blr
139
140bpf_slow_path_byte_msh:
141	bpf_slow_path_common(1)
142	lbz	r_X, BPF_PPC_STACK_BASIC+(2*8)(r1)
143	rlwinm	r_X, r_X, 2, 32-4-2, 31-2
144	blr
145
146/* Call out to bpf_internal_load_pointer_neg_helper:
147 * We'll need to back up our volatile regs first; we have
148 * local variable space at r1+(BPF_PPC_STACK_BASIC).
149 * Allocate a new stack frame here to remain ABI-compliant in
150 * stashing LR.
151 */
152#define sk_negative_common(SIZE)				\
153	mflr	r0;						\
154	PPC_STL	r0, PPC_LR_STKOFF(r1);					\
155	/* R3 goes in parameter space of caller's frame */	\
156	PPC_STL	r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);		\
157	PPC_STL	r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);		\
158	PPC_STL	r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);		\
159	PPC_STLU	r1, -BPF_PPC_SLOWPATH_FRAME(r1);		\
160	/* R3 = r_skb, as passed */				\
161	mr	r4, r_addr;					\
162	li	r5, SIZE;					\
163	bl	bpf_internal_load_pointer_neg_helper;		\
164	nop;							\
165	/* R3 != 0 on success */				\
166	addi	r1, r1, BPF_PPC_SLOWPATH_FRAME;			\
167	PPC_LL	r0, PPC_LR_STKOFF(r1);					\
168	PPC_LL	r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1);		\
169	PPC_LL	r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1);		\
170	mtlr	r0;						\
171	PPC_LCMPLI	r3, 0;						\
172	beq	bpf_error_slow;	/* cr0 = EQ */			\
173	mr	r_addr, r3;					\
174	PPC_LL	r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1);		\
175	/* Great success! */
176
177bpf_slow_path_word_neg:
178	lis     r_scratch1,-32	/* SKF_LL_OFF */
179	PPC_LCMP	r_addr, r_scratch1	/* addr < SKF_* */
180	blt	bpf_error	/* cr0 = LT */
181	.globl	sk_load_word_negative_offset
182sk_load_word_negative_offset:
183	sk_negative_common(4)
184	lwz	r_A, 0(r_addr)
185	blr
186
187bpf_slow_path_half_neg:
188	lis     r_scratch1,-32	/* SKF_LL_OFF */
189	PPC_LCMP	r_addr, r_scratch1	/* addr < SKF_* */
190	blt	bpf_error	/* cr0 = LT */
191	.globl	sk_load_half_negative_offset
192sk_load_half_negative_offset:
193	sk_negative_common(2)
194	lhz	r_A, 0(r_addr)
195	blr
196
197bpf_slow_path_byte_neg:
198	lis     r_scratch1,-32	/* SKF_LL_OFF */
199	PPC_LCMP	r_addr, r_scratch1	/* addr < SKF_* */
200	blt	bpf_error	/* cr0 = LT */
201	.globl	sk_load_byte_negative_offset
202sk_load_byte_negative_offset:
203	sk_negative_common(1)
204	lbz	r_A, 0(r_addr)
205	blr
206
207bpf_slow_path_byte_msh_neg:
208	lis     r_scratch1,-32	/* SKF_LL_OFF */
209	PPC_LCMP	r_addr, r_scratch1	/* addr < SKF_* */
210	blt	bpf_error	/* cr0 = LT */
211	.globl	sk_load_byte_msh_negative_offset
212sk_load_byte_msh_negative_offset:
213	sk_negative_common(1)
214	lbz	r_X, 0(r_addr)
215	rlwinm	r_X, r_X, 2, 32-4-2, 31-2
216	blr
217
218bpf_error_slow:
219	/* fabricate a cr0 = lt */
220	li	r_scratch1, -1
221	PPC_LCMPI	r_scratch1, 0
222bpf_error:
223	/* Entered with cr0 = lt */
224	li	r3, 0
225	/* Generated code will 'blt epilogue', returning 0. */
226	blr
227