1 /*
2 * Copyright (C) 2015-2017 Netronome Systems, Inc.
3 *
4 * This software is dual licensed under the GNU General License Version 2,
5 * June 1991 as shown in the file COPYING in the top-level directory of this
6 * source tree or the BSD 2-Clause License provided below. You have the
7 * option to license this software under the complete terms of either license.
8 *
9 * The BSD 2-Clause License:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * 1. Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * 2. Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #include <linux/delay.h>
35 #include <linux/device.h>
36 #include <linux/jiffies.h>
37 #include <linux/types.h>
38 #include <linux/slab.h>
39 #include <linux/wait.h>
40
41 #include "nfp_cpp.h"
42 #include "nfp6000/nfp6000.h"
43
44 struct nfp_cpp_mutex {
45 struct nfp_cpp *cpp;
46 int target;
47 u16 depth;
48 unsigned long long address;
49 u32 key;
50 };
51
nfp_mutex_locked(u16 interface)52 static u32 nfp_mutex_locked(u16 interface)
53 {
54 return (u32)interface << 16 | 0x000f;
55 }
56
nfp_mutex_unlocked(u16 interface)57 static u32 nfp_mutex_unlocked(u16 interface)
58 {
59 return (u32)interface << 16 | 0x0000;
60 }
61
nfp_mutex_owner(u32 val)62 static u32 nfp_mutex_owner(u32 val)
63 {
64 return val >> 16;
65 }
66
nfp_mutex_is_locked(u32 val)67 static bool nfp_mutex_is_locked(u32 val)
68 {
69 return (val & 0xffff) == 0x000f;
70 }
71
nfp_mutex_is_unlocked(u32 val)72 static bool nfp_mutex_is_unlocked(u32 val)
73 {
74 return (val & 0xffff) == 0000;
75 }
76
77 /* If you need more than 65536 recursive locks, please rethink your code. */
78 #define NFP_MUTEX_DEPTH_MAX 0xffff
79
80 static int
nfp_cpp_mutex_validate(u16 interface,int * target,unsigned long long address)81 nfp_cpp_mutex_validate(u16 interface, int *target, unsigned long long address)
82 {
83 /* Not permitted on invalid interfaces */
84 if (NFP_CPP_INTERFACE_TYPE_of(interface) ==
85 NFP_CPP_INTERFACE_TYPE_INVALID)
86 return -EINVAL;
87
88 /* Address must be 64-bit aligned */
89 if (address & 7)
90 return -EINVAL;
91
92 if (*target != NFP_CPP_TARGET_MU)
93 return -EINVAL;
94
95 return 0;
96 }
97
98 /**
99 * nfp_cpp_mutex_init() - Initialize a mutex location
100 * @cpp: NFP CPP handle
101 * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
102 * @address: Offset into the address space of the NFP CPP target ID
103 * @key: Unique 32-bit value for this mutex
104 *
105 * The CPP target:address must point to a 64-bit aligned location, and
106 * will initialize 64 bits of data at the location.
107 *
108 * This creates the initial mutex state, as locked by this
109 * nfp_cpp_interface().
110 *
111 * This function should only be called when setting up
112 * the initial lock state upon boot-up of the system.
113 *
114 * Return: 0 on success, or -errno on failure
115 */
nfp_cpp_mutex_init(struct nfp_cpp * cpp,int target,unsigned long long address,u32 key)116 int nfp_cpp_mutex_init(struct nfp_cpp *cpp,
117 int target, unsigned long long address, u32 key)
118 {
119 const u32 muw = NFP_CPP_ID(target, 4, 0); /* atomic_write */
120 u16 interface = nfp_cpp_interface(cpp);
121 int err;
122
123 err = nfp_cpp_mutex_validate(interface, &target, address);
124 if (err)
125 return err;
126
127 err = nfp_cpp_writel(cpp, muw, address + 4, key);
128 if (err)
129 return err;
130
131 err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_locked(interface));
132 if (err)
133 return err;
134
135 return 0;
136 }
137
138 /**
139 * nfp_cpp_mutex_alloc() - Create a mutex handle
140 * @cpp: NFP CPP handle
141 * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
142 * @address: Offset into the address space of the NFP CPP target ID
143 * @key: 32-bit unique key (must match the key at this location)
144 *
145 * The CPP target:address must point to a 64-bit aligned location, and
146 * reserve 64 bits of data at the location for use by the handle.
147 *
148 * Only target/address pairs that point to entities that support the
149 * MU Atomic Engine's CmpAndSwap32 command are supported.
150 *
151 * Return: A non-NULL struct nfp_cpp_mutex * on success, NULL on failure.
152 */
nfp_cpp_mutex_alloc(struct nfp_cpp * cpp,int target,unsigned long long address,u32 key)153 struct nfp_cpp_mutex *nfp_cpp_mutex_alloc(struct nfp_cpp *cpp, int target,
154 unsigned long long address, u32 key)
155 {
156 const u32 mur = NFP_CPP_ID(target, 3, 0); /* atomic_read */
157 u16 interface = nfp_cpp_interface(cpp);
158 struct nfp_cpp_mutex *mutex;
159 int err;
160 u32 tmp;
161
162 err = nfp_cpp_mutex_validate(interface, &target, address);
163 if (err)
164 return NULL;
165
166 err = nfp_cpp_readl(cpp, mur, address + 4, &tmp);
167 if (err < 0)
168 return NULL;
169
170 if (tmp != key)
171 return NULL;
172
173 mutex = kzalloc(sizeof(*mutex), GFP_KERNEL);
174 if (!mutex)
175 return NULL;
176
177 mutex->cpp = cpp;
178 mutex->target = target;
179 mutex->address = address;
180 mutex->key = key;
181 mutex->depth = 0;
182
183 return mutex;
184 }
185
186 /**
187 * nfp_cpp_mutex_free() - Free a mutex handle - does not alter the lock state
188 * @mutex: NFP CPP Mutex handle
189 */
nfp_cpp_mutex_free(struct nfp_cpp_mutex * mutex)190 void nfp_cpp_mutex_free(struct nfp_cpp_mutex *mutex)
191 {
192 kfree(mutex);
193 }
194
195 /**
196 * nfp_cpp_mutex_lock() - Lock a mutex handle, using the NFP MU Atomic Engine
197 * @mutex: NFP CPP Mutex handle
198 *
199 * Return: 0 on success, or -errno on failure
200 */
nfp_cpp_mutex_lock(struct nfp_cpp_mutex * mutex)201 int nfp_cpp_mutex_lock(struct nfp_cpp_mutex *mutex)
202 {
203 unsigned long warn_at = jiffies + NFP_MUTEX_WAIT_FIRST_WARN * HZ;
204 unsigned long err_at = jiffies + NFP_MUTEX_WAIT_ERROR * HZ;
205 unsigned int timeout_ms = 1;
206 int err;
207
208 /* We can't use a waitqueue here, because the unlocker
209 * might be on a separate CPU.
210 *
211 * So just wait for now.
212 */
213 for (;;) {
214 err = nfp_cpp_mutex_trylock(mutex);
215 if (err != -EBUSY)
216 break;
217
218 err = msleep_interruptible(timeout_ms);
219 if (err != 0) {
220 nfp_info(mutex->cpp,
221 "interrupted waiting for NFP mutex\n");
222 return -ERESTARTSYS;
223 }
224
225 if (time_is_before_eq_jiffies(warn_at)) {
226 warn_at = jiffies + NFP_MUTEX_WAIT_NEXT_WARN * HZ;
227 nfp_warn(mutex->cpp,
228 "Warning: waiting for NFP mutex [depth:%hd target:%d addr:%llx key:%08x]\n",
229 mutex->depth,
230 mutex->target, mutex->address, mutex->key);
231 }
232 if (time_is_before_eq_jiffies(err_at)) {
233 nfp_err(mutex->cpp, "Error: mutex wait timed out\n");
234 return -EBUSY;
235 }
236 }
237
238 return err;
239 }
240
241 /**
242 * nfp_cpp_mutex_unlock() - Unlock a mutex handle, using the MU Atomic Engine
243 * @mutex: NFP CPP Mutex handle
244 *
245 * Return: 0 on success, or -errno on failure
246 */
nfp_cpp_mutex_unlock(struct nfp_cpp_mutex * mutex)247 int nfp_cpp_mutex_unlock(struct nfp_cpp_mutex *mutex)
248 {
249 const u32 muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */
250 const u32 mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */
251 struct nfp_cpp *cpp = mutex->cpp;
252 u32 key, value;
253 u16 interface;
254 int err;
255
256 interface = nfp_cpp_interface(cpp);
257
258 if (mutex->depth > 1) {
259 mutex->depth--;
260 return 0;
261 }
262
263 err = nfp_cpp_readl(mutex->cpp, mur, mutex->address + 4, &key);
264 if (err < 0)
265 return err;
266
267 if (key != mutex->key)
268 return -EPERM;
269
270 err = nfp_cpp_readl(mutex->cpp, mur, mutex->address, &value);
271 if (err < 0)
272 return err;
273
274 if (value != nfp_mutex_locked(interface))
275 return -EACCES;
276
277 err = nfp_cpp_writel(cpp, muw, mutex->address,
278 nfp_mutex_unlocked(interface));
279 if (err < 0)
280 return err;
281
282 mutex->depth = 0;
283 return 0;
284 }
285
286 /**
287 * nfp_cpp_mutex_trylock() - Attempt to lock a mutex handle
288 * @mutex: NFP CPP Mutex handle
289 *
290 * Return: 0 if the lock succeeded, -errno on failure
291 */
nfp_cpp_mutex_trylock(struct nfp_cpp_mutex * mutex)292 int nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex)
293 {
294 const u32 muw = NFP_CPP_ID(mutex->target, 4, 0); /* atomic_write */
295 const u32 mus = NFP_CPP_ID(mutex->target, 5, 3); /* test_set_imm */
296 const u32 mur = NFP_CPP_ID(mutex->target, 3, 0); /* atomic_read */
297 struct nfp_cpp *cpp = mutex->cpp;
298 u32 key, value, tmp;
299 int err;
300
301 if (mutex->depth > 0) {
302 if (mutex->depth == NFP_MUTEX_DEPTH_MAX)
303 return -E2BIG;
304 mutex->depth++;
305 return 0;
306 }
307
308 /* Verify that the lock marker is not damaged */
309 err = nfp_cpp_readl(cpp, mur, mutex->address + 4, &key);
310 if (err < 0)
311 return err;
312
313 if (key != mutex->key)
314 return -EPERM;
315
316 /* Compare against the unlocked state, and if true,
317 * write the interface id into the top 16 bits, and
318 * mark as locked.
319 */
320 value = nfp_mutex_locked(nfp_cpp_interface(cpp));
321
322 /* We use test_set_imm here, as it implies a read
323 * of the current state, and sets the bits in the
324 * bytemask of the command to 1s. Since the mutex
325 * is guaranteed to be 64-bit aligned, the bytemask
326 * of this 32-bit command is ensured to be 8'b00001111,
327 * which implies that the lower 4 bits will be set to
328 * ones regardless of the initial state.
329 *
330 * Since this is a 'Readback' operation, with no Pull
331 * data, we can treat this as a normal Push (read)
332 * atomic, which returns the original value.
333 */
334 err = nfp_cpp_readl(cpp, mus, mutex->address, &tmp);
335 if (err < 0)
336 return err;
337
338 /* Was it unlocked? */
339 if (nfp_mutex_is_unlocked(tmp)) {
340 /* The read value can only be 0x....0000 in the unlocked state.
341 * If there was another contending for this lock, then
342 * the lock state would be 0x....000f
343 */
344
345 /* Write our owner ID into the lock
346 * While not strictly necessary, this helps with
347 * debug and bookkeeping.
348 */
349 err = nfp_cpp_writel(cpp, muw, mutex->address, value);
350 if (err < 0)
351 return err;
352
353 mutex->depth = 1;
354 return 0;
355 }
356
357 return nfp_mutex_is_locked(tmp) ? -EBUSY : -EINVAL;
358 }
359
360 /**
361 * nfp_cpp_mutex_reclaim() - Unlock mutex if held by local endpoint
362 * @cpp: NFP CPP handle
363 * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU)
364 * @address: Offset into the address space of the NFP CPP target ID
365 *
366 * Release lock if held by local system. Extreme care is advised, call only
367 * when no local lock users can exist.
368 *
369 * Return: 0 if the lock was OK, 1 if locked by us, -errno on invalid mutex
370 */
nfp_cpp_mutex_reclaim(struct nfp_cpp * cpp,int target,unsigned long long address)371 int nfp_cpp_mutex_reclaim(struct nfp_cpp *cpp, int target,
372 unsigned long long address)
373 {
374 const u32 mur = NFP_CPP_ID(target, 3, 0); /* atomic_read */
375 const u32 muw = NFP_CPP_ID(target, 4, 0); /* atomic_write */
376 u16 interface = nfp_cpp_interface(cpp);
377 int err;
378 u32 tmp;
379
380 err = nfp_cpp_mutex_validate(interface, &target, address);
381 if (err)
382 return err;
383
384 /* Check lock */
385 err = nfp_cpp_readl(cpp, mur, address, &tmp);
386 if (err < 0)
387 return err;
388
389 if (nfp_mutex_is_unlocked(tmp) || nfp_mutex_owner(tmp) != interface)
390 return 0;
391
392 /* Bust the lock */
393 err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_unlocked(interface));
394 if (err < 0)
395 return err;
396
397 return 1;
398 }
399
