1 /*
2 * Copyright 2016-17 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10 #define pr_fmt(fmt) "vas: " fmt
11
12 #include <linux/types.h>
13 #include <linux/mutex.h>
14 #include <linux/slab.h>
15 #include <linux/io.h>
16 #include <linux/log2.h>
17 #include <linux/rcupdate.h>
18 #include <linux/cred.h>
19 #include <asm/switch_to.h>
20 #include <asm/ppc-opcode.h>
21 #include "vas.h"
22 #include "copy-paste.h"
23
24 #define CREATE_TRACE_POINTS
25 #include "vas-trace.h"
26
27 /*
28 * Compute the paste address region for the window @window using the
29 * ->paste_base_addr and ->paste_win_id_shift we got from device tree.
30 */
compute_paste_address(struct vas_window * window,u64 * addr,int * len)31 static void compute_paste_address(struct vas_window *window, u64 *addr, int *len)
32 {
33 int winid;
34 u64 base, shift;
35
36 base = window->vinst->paste_base_addr;
37 shift = window->vinst->paste_win_id_shift;
38 winid = window->winid;
39
40 *addr = base + (winid << shift);
41 if (len)
42 *len = PAGE_SIZE;
43
44 pr_debug("Txwin #%d: Paste addr 0x%llx\n", winid, *addr);
45 }
46
vas_win_paste_addr(struct vas_window * win)47 u64 vas_win_paste_addr(struct vas_window *win)
48 {
49 u64 addr;
50
51 compute_paste_address(win, &addr, NULL);
52
53 return addr;
54 }
55 EXPORT_SYMBOL(vas_win_paste_addr);
56
get_hvwc_mmio_bar(struct vas_window * window,u64 * start,int * len)57 static inline void get_hvwc_mmio_bar(struct vas_window *window,
58 u64 *start, int *len)
59 {
60 u64 pbaddr;
61
62 pbaddr = window->vinst->hvwc_bar_start;
63 *start = pbaddr + window->winid * VAS_HVWC_SIZE;
64 *len = VAS_HVWC_SIZE;
65 }
66
get_uwc_mmio_bar(struct vas_window * window,u64 * start,int * len)67 static inline void get_uwc_mmio_bar(struct vas_window *window,
68 u64 *start, int *len)
69 {
70 u64 pbaddr;
71
72 pbaddr = window->vinst->uwc_bar_start;
73 *start = pbaddr + window->winid * VAS_UWC_SIZE;
74 *len = VAS_UWC_SIZE;
75 }
76
77 /*
78 * Map the paste bus address of the given send window into kernel address
79 * space. Unlike MMIO regions (map_mmio_region() below), paste region must
80 * be mapped cache-able and is only applicable to send windows.
81 */
map_paste_region(struct vas_window * txwin)82 static void *map_paste_region(struct vas_window *txwin)
83 {
84 int len;
85 void *map;
86 char *name;
87 u64 start;
88
89 name = kasprintf(GFP_KERNEL, "window-v%d-w%d", txwin->vinst->vas_id,
90 txwin->winid);
91 if (!name)
92 goto free_name;
93
94 txwin->paste_addr_name = name;
95 compute_paste_address(txwin, &start, &len);
96
97 if (!request_mem_region(start, len, name)) {
98 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
99 __func__, start, len);
100 goto free_name;
101 }
102
103 map = ioremap_cache(start, len);
104 if (!map) {
105 pr_devel("%s(): ioremap_cache(0x%llx, %d) failed\n", __func__,
106 start, len);
107 goto free_name;
108 }
109
110 pr_devel("Mapped paste addr 0x%llx to kaddr 0x%p\n", start, map);
111 return map;
112
113 free_name:
114 kfree(name);
115 return ERR_PTR(-ENOMEM);
116 }
117
map_mmio_region(char * name,u64 start,int len)118 static void *map_mmio_region(char *name, u64 start, int len)
119 {
120 void *map;
121
122 if (!request_mem_region(start, len, name)) {
123 pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
124 __func__, start, len);
125 return NULL;
126 }
127
128 map = ioremap(start, len);
129 if (!map) {
130 pr_devel("%s(): ioremap(0x%llx, %d) failed\n", __func__, start,
131 len);
132 return NULL;
133 }
134
135 return map;
136 }
137
unmap_region(void * addr,u64 start,int len)138 static void unmap_region(void *addr, u64 start, int len)
139 {
140 iounmap(addr);
141 release_mem_region((phys_addr_t)start, len);
142 }
143
144 /*
145 * Unmap the paste address region for a window.
146 */
unmap_paste_region(struct vas_window * window)147 static void unmap_paste_region(struct vas_window *window)
148 {
149 int len;
150 u64 busaddr_start;
151
152 if (window->paste_kaddr) {
153 compute_paste_address(window, &busaddr_start, &len);
154 unmap_region(window->paste_kaddr, busaddr_start, len);
155 window->paste_kaddr = NULL;
156 kfree(window->paste_addr_name);
157 window->paste_addr_name = NULL;
158 }
159 }
160
161 /*
162 * Unmap the MMIO regions for a window. Hold the vas_mutex so we don't
163 * unmap when the window's debugfs dir is in use. This serializes close
164 * of a window even on another VAS instance but since its not a critical
165 * path, just minimize the time we hold the mutex for now. We can add
166 * a per-instance mutex later if necessary.
167 */
unmap_winctx_mmio_bars(struct vas_window * window)168 static void unmap_winctx_mmio_bars(struct vas_window *window)
169 {
170 int len;
171 void *uwc_map;
172 void *hvwc_map;
173 u64 busaddr_start;
174
175 mutex_lock(&vas_mutex);
176
177 hvwc_map = window->hvwc_map;
178 window->hvwc_map = NULL;
179
180 uwc_map = window->uwc_map;
181 window->uwc_map = NULL;
182
183 mutex_unlock(&vas_mutex);
184
185 if (hvwc_map) {
186 get_hvwc_mmio_bar(window, &busaddr_start, &len);
187 unmap_region(hvwc_map, busaddr_start, len);
188 }
189
190 if (uwc_map) {
191 get_uwc_mmio_bar(window, &busaddr_start, &len);
192 unmap_region(uwc_map, busaddr_start, len);
193 }
194 }
195
196 /*
197 * Find the Hypervisor Window Context (HVWC) MMIO Base Address Region and the
198 * OS/User Window Context (UWC) MMIO Base Address Region for the given window.
199 * Map these bus addresses and save the mapped kernel addresses in @window.
200 */
map_winctx_mmio_bars(struct vas_window * window)201 int map_winctx_mmio_bars(struct vas_window *window)
202 {
203 int len;
204 u64 start;
205
206 get_hvwc_mmio_bar(window, &start, &len);
207 window->hvwc_map = map_mmio_region("HVWCM_Window", start, len);
208
209 get_uwc_mmio_bar(window, &start, &len);
210 window->uwc_map = map_mmio_region("UWCM_Window", start, len);
211
212 if (!window->hvwc_map || !window->uwc_map) {
213 unmap_winctx_mmio_bars(window);
214 return -1;
215 }
216
217 return 0;
218 }
219
220 /*
221 * Reset all valid registers in the HV and OS/User Window Contexts for
222 * the window identified by @window.
223 *
224 * NOTE: We cannot really use a for loop to reset window context. Not all
225 * offsets in a window context are valid registers and the valid
226 * registers are not sequential. And, we can only write to offsets
227 * with valid registers.
228 */
reset_window_regs(struct vas_window * window)229 void reset_window_regs(struct vas_window *window)
230 {
231 write_hvwc_reg(window, VREG(LPID), 0ULL);
232 write_hvwc_reg(window, VREG(PID), 0ULL);
233 write_hvwc_reg(window, VREG(XLATE_MSR), 0ULL);
234 write_hvwc_reg(window, VREG(XLATE_LPCR), 0ULL);
235 write_hvwc_reg(window, VREG(XLATE_CTL), 0ULL);
236 write_hvwc_reg(window, VREG(AMR), 0ULL);
237 write_hvwc_reg(window, VREG(SEIDR), 0ULL);
238 write_hvwc_reg(window, VREG(FAULT_TX_WIN), 0ULL);
239 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
240 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), 0ULL);
241 write_hvwc_reg(window, VREG(PSWID), 0ULL);
242 write_hvwc_reg(window, VREG(LFIFO_BAR), 0ULL);
243 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), 0ULL);
244 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), 0ULL);
245 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
246 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
247 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
248 write_hvwc_reg(window, VREG(LRX_WCRED), 0ULL);
249 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
250 write_hvwc_reg(window, VREG(TX_WCRED), 0ULL);
251 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
252 write_hvwc_reg(window, VREG(LFIFO_SIZE), 0ULL);
253 write_hvwc_reg(window, VREG(WINCTL), 0ULL);
254 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
255 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), 0ULL);
256 write_hvwc_reg(window, VREG(TX_RSVD_BUF_COUNT), 0ULL);
257 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), 0ULL);
258 write_hvwc_reg(window, VREG(LNOTIFY_CTL), 0ULL);
259 write_hvwc_reg(window, VREG(LNOTIFY_PID), 0ULL);
260 write_hvwc_reg(window, VREG(LNOTIFY_LPID), 0ULL);
261 write_hvwc_reg(window, VREG(LNOTIFY_TID), 0ULL);
262 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), 0ULL);
263 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
264
265 /* Skip read-only registers: NX_UTIL and NX_UTIL_SE */
266
267 /*
268 * The send and receive window credit adder registers are also
269 * accessible from HVWC and have been initialized above. We don't
270 * need to initialize from the OS/User Window Context, so skip
271 * following calls:
272 *
273 * write_uwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
274 * write_uwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
275 */
276 }
277
278 /*
279 * Initialize window context registers related to Address Translation.
280 * These registers are common to send/receive windows although they
281 * differ for user/kernel windows. As we resolve the TODOs we may
282 * want to add fields to vas_winctx and move the initialization to
283 * init_vas_winctx_regs().
284 */
init_xlate_regs(struct vas_window * window,bool user_win)285 static void init_xlate_regs(struct vas_window *window, bool user_win)
286 {
287 u64 lpcr, val;
288
289 /*
290 * MSR_TA, MSR_US are false for both kernel and user.
291 * MSR_DR and MSR_PR are false for kernel.
292 */
293 val = 0ULL;
294 val = SET_FIELD(VAS_XLATE_MSR_HV, val, 1);
295 val = SET_FIELD(VAS_XLATE_MSR_SF, val, 1);
296 if (user_win) {
297 val = SET_FIELD(VAS_XLATE_MSR_DR, val, 1);
298 val = SET_FIELD(VAS_XLATE_MSR_PR, val, 1);
299 }
300 write_hvwc_reg(window, VREG(XLATE_MSR), val);
301
302 lpcr = mfspr(SPRN_LPCR);
303 val = 0ULL;
304 /*
305 * NOTE: From Section 5.7.8.1 Segment Lookaside Buffer of the
306 * Power ISA, v3.0B, Page size encoding is 0 = 4KB, 5 = 64KB.
307 *
308 * NOTE: From Section 1.3.1, Address Translation Context of the
309 * Nest MMU Workbook, LPCR_SC should be 0 for Power9.
310 */
311 val = SET_FIELD(VAS_XLATE_LPCR_PAGE_SIZE, val, 5);
312 val = SET_FIELD(VAS_XLATE_LPCR_ISL, val, lpcr & LPCR_ISL);
313 val = SET_FIELD(VAS_XLATE_LPCR_TC, val, lpcr & LPCR_TC);
314 val = SET_FIELD(VAS_XLATE_LPCR_SC, val, 0);
315 write_hvwc_reg(window, VREG(XLATE_LPCR), val);
316
317 /*
318 * Section 1.3.1 (Address translation Context) of NMMU workbook.
319 * 0b00 Hashed Page Table mode
320 * 0b01 Reserved
321 * 0b10 Radix on HPT
322 * 0b11 Radix on Radix
323 */
324 val = 0ULL;
325 val = SET_FIELD(VAS_XLATE_MODE, val, radix_enabled() ? 3 : 2);
326 write_hvwc_reg(window, VREG(XLATE_CTL), val);
327
328 /*
329 * TODO: Can we mfspr(AMR) even for user windows?
330 */
331 val = 0ULL;
332 val = SET_FIELD(VAS_AMR, val, mfspr(SPRN_AMR));
333 write_hvwc_reg(window, VREG(AMR), val);
334
335 val = 0ULL;
336 val = SET_FIELD(VAS_SEIDR, val, 0);
337 write_hvwc_reg(window, VREG(SEIDR), val);
338 }
339
340 /*
341 * Initialize Reserved Send Buffer Count for the send window. It involves
342 * writing to the register, reading it back to confirm that the hardware
343 * has enough buffers to reserve. See section 1.3.1.2.1 of VAS workbook.
344 *
345 * Since we can only make a best-effort attempt to fulfill the request,
346 * we don't return any errors if we cannot.
347 *
348 * TODO: Reserved (aka dedicated) send buffers are not supported yet.
349 */
init_rsvd_tx_buf_count(struct vas_window * txwin,struct vas_winctx * winctx)350 static void init_rsvd_tx_buf_count(struct vas_window *txwin,
351 struct vas_winctx *winctx)
352 {
353 write_hvwc_reg(txwin, VREG(TX_RSVD_BUF_COUNT), 0ULL);
354 }
355
356 /*
357 * init_winctx_regs()
358 * Initialize window context registers for a receive window.
359 * Except for caching control and marking window open, the registers
360 * are initialized in the order listed in Section 3.1.4 (Window Context
361 * Cache Register Details) of the VAS workbook although they don't need
362 * to be.
363 *
364 * Design note: For NX receive windows, NX allocates the FIFO buffer in OPAL
365 * (so that it can get a large contiguous area) and passes that buffer
366 * to kernel via device tree. We now write that buffer address to the
367 * FIFO BAR. Would it make sense to do this all in OPAL? i.e have OPAL
368 * write the per-chip RX FIFO addresses to the windows during boot-up
369 * as a one-time task? That could work for NX but what about other
370 * receivers? Let the receivers tell us the rx-fifo buffers for now.
371 */
init_winctx_regs(struct vas_window * window,struct vas_winctx * winctx)372 int init_winctx_regs(struct vas_window *window, struct vas_winctx *winctx)
373 {
374 u64 val;
375 int fifo_size;
376
377 reset_window_regs(window);
378
379 val = 0ULL;
380 val = SET_FIELD(VAS_LPID, val, winctx->lpid);
381 write_hvwc_reg(window, VREG(LPID), val);
382
383 val = 0ULL;
384 val = SET_FIELD(VAS_PID_ID, val, winctx->pidr);
385 write_hvwc_reg(window, VREG(PID), val);
386
387 init_xlate_regs(window, winctx->user_win);
388
389 val = 0ULL;
390 val = SET_FIELD(VAS_FAULT_TX_WIN, val, 0);
391 write_hvwc_reg(window, VREG(FAULT_TX_WIN), val);
392
393 /* In PowerNV, interrupts go to HV. */
394 write_hvwc_reg(window, VREG(OSU_INTR_SRC_RA), 0ULL);
395
396 val = 0ULL;
397 val = SET_FIELD(VAS_HV_INTR_SRC_RA, val, winctx->irq_port);
398 write_hvwc_reg(window, VREG(HV_INTR_SRC_RA), val);
399
400 val = 0ULL;
401 val = SET_FIELD(VAS_PSWID_EA_HANDLE, val, winctx->pswid);
402 write_hvwc_reg(window, VREG(PSWID), val);
403
404 write_hvwc_reg(window, VREG(SPARE1), 0ULL);
405 write_hvwc_reg(window, VREG(SPARE2), 0ULL);
406 write_hvwc_reg(window, VREG(SPARE3), 0ULL);
407
408 /*
409 * NOTE: VAS expects the FIFO address to be copied into the LFIFO_BAR
410 * register as is - do NOT shift the address into VAS_LFIFO_BAR
411 * bit fields! Ok to set the page migration select fields -
412 * VAS ignores the lower 10+ bits in the address anyway, because
413 * the minimum FIFO size is 1K?
414 *
415 * See also: Design note in function header.
416 */
417 val = __pa(winctx->rx_fifo);
418 val = SET_FIELD(VAS_PAGE_MIGRATION_SELECT, val, 0);
419 write_hvwc_reg(window, VREG(LFIFO_BAR), val);
420
421 val = 0ULL;
422 val = SET_FIELD(VAS_LDATA_STAMP, val, winctx->data_stamp);
423 write_hvwc_reg(window, VREG(LDATA_STAMP_CTL), val);
424
425 val = 0ULL;
426 val = SET_FIELD(VAS_LDMA_TYPE, val, winctx->dma_type);
427 val = SET_FIELD(VAS_LDMA_FIFO_DISABLE, val, winctx->fifo_disable);
428 write_hvwc_reg(window, VREG(LDMA_CACHE_CTL), val);
429
430 write_hvwc_reg(window, VREG(LRFIFO_PUSH), 0ULL);
431 write_hvwc_reg(window, VREG(CURR_MSG_COUNT), 0ULL);
432 write_hvwc_reg(window, VREG(LNOTIFY_AFTER_COUNT), 0ULL);
433
434 val = 0ULL;
435 val = SET_FIELD(VAS_LRX_WCRED, val, winctx->wcreds_max);
436 write_hvwc_reg(window, VREG(LRX_WCRED), val);
437
438 val = 0ULL;
439 val = SET_FIELD(VAS_TX_WCRED, val, winctx->wcreds_max);
440 write_hvwc_reg(window, VREG(TX_WCRED), val);
441
442 write_hvwc_reg(window, VREG(LRX_WCRED_ADDER), 0ULL);
443 write_hvwc_reg(window, VREG(TX_WCRED_ADDER), 0ULL);
444
445 fifo_size = winctx->rx_fifo_size / 1024;
446
447 val = 0ULL;
448 val = SET_FIELD(VAS_LFIFO_SIZE, val, ilog2(fifo_size));
449 write_hvwc_reg(window, VREG(LFIFO_SIZE), val);
450
451 /* Update window control and caching control registers last so
452 * we mark the window open only after fully initializing it and
453 * pushing context to cache.
454 */
455
456 write_hvwc_reg(window, VREG(WIN_STATUS), 0ULL);
457
458 init_rsvd_tx_buf_count(window, winctx);
459
460 /* for a send window, point to the matching receive window */
461 val = 0ULL;
462 val = SET_FIELD(VAS_LRX_WIN_ID, val, winctx->rx_win_id);
463 write_hvwc_reg(window, VREG(LRFIFO_WIN_PTR), val);
464
465 write_hvwc_reg(window, VREG(SPARE4), 0ULL);
466
467 val = 0ULL;
468 val = SET_FIELD(VAS_NOTIFY_DISABLE, val, winctx->notify_disable);
469 val = SET_FIELD(VAS_INTR_DISABLE, val, winctx->intr_disable);
470 val = SET_FIELD(VAS_NOTIFY_EARLY, val, winctx->notify_early);
471 val = SET_FIELD(VAS_NOTIFY_OSU_INTR, val, winctx->notify_os_intr_reg);
472 write_hvwc_reg(window, VREG(LNOTIFY_CTL), val);
473
474 val = 0ULL;
475 val = SET_FIELD(VAS_LNOTIFY_PID, val, winctx->lnotify_pid);
476 write_hvwc_reg(window, VREG(LNOTIFY_PID), val);
477
478 val = 0ULL;
479 val = SET_FIELD(VAS_LNOTIFY_LPID, val, winctx->lnotify_lpid);
480 write_hvwc_reg(window, VREG(LNOTIFY_LPID), val);
481
482 val = 0ULL;
483 val = SET_FIELD(VAS_LNOTIFY_TID, val, winctx->lnotify_tid);
484 write_hvwc_reg(window, VREG(LNOTIFY_TID), val);
485
486 val = 0ULL;
487 val = SET_FIELD(VAS_LNOTIFY_MIN_SCOPE, val, winctx->min_scope);
488 val = SET_FIELD(VAS_LNOTIFY_MAX_SCOPE, val, winctx->max_scope);
489 write_hvwc_reg(window, VREG(LNOTIFY_SCOPE), val);
490
491 /* Skip read-only registers NX_UTIL and NX_UTIL_SE */
492
493 write_hvwc_reg(window, VREG(SPARE5), 0ULL);
494 write_hvwc_reg(window, VREG(NX_UTIL_ADDER), 0ULL);
495 write_hvwc_reg(window, VREG(SPARE6), 0ULL);
496
497 /* Finally, push window context to memory and... */
498 val = 0ULL;
499 val = SET_FIELD(VAS_PUSH_TO_MEM, val, 1);
500 write_hvwc_reg(window, VREG(WIN_CTX_CACHING_CTL), val);
501
502 /* ... mark the window open for business */
503 val = 0ULL;
504 val = SET_FIELD(VAS_WINCTL_REJ_NO_CREDIT, val, winctx->rej_no_credit);
505 val = SET_FIELD(VAS_WINCTL_PIN, val, winctx->pin_win);
506 val = SET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val, winctx->tx_wcred_mode);
507 val = SET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val, winctx->rx_wcred_mode);
508 val = SET_FIELD(VAS_WINCTL_TX_WORD_MODE, val, winctx->tx_word_mode);
509 val = SET_FIELD(VAS_WINCTL_RX_WORD_MODE, val, winctx->rx_word_mode);
510 val = SET_FIELD(VAS_WINCTL_FAULT_WIN, val, winctx->fault_win);
511 val = SET_FIELD(VAS_WINCTL_NX_WIN, val, winctx->nx_win);
512 val = SET_FIELD(VAS_WINCTL_OPEN, val, 1);
513 write_hvwc_reg(window, VREG(WINCTL), val);
514
515 return 0;
516 }
517
vas_release_window_id(struct ida * ida,int winid)518 static void vas_release_window_id(struct ida *ida, int winid)
519 {
520 ida_free(ida, winid);
521 }
522
vas_assign_window_id(struct ida * ida)523 static int vas_assign_window_id(struct ida *ida)
524 {
525 int winid = ida_alloc_max(ida, VAS_WINDOWS_PER_CHIP - 1, GFP_KERNEL);
526
527 if (winid == -ENOSPC) {
528 pr_err("Too many (%d) open windows\n", VAS_WINDOWS_PER_CHIP);
529 return -EAGAIN;
530 }
531
532 return winid;
533 }
534
vas_window_free(struct vas_window * window)535 static void vas_window_free(struct vas_window *window)
536 {
537 int winid = window->winid;
538 struct vas_instance *vinst = window->vinst;
539
540 unmap_winctx_mmio_bars(window);
541
542 vas_window_free_dbgdir(window);
543
544 kfree(window);
545
546 vas_release_window_id(&vinst->ida, winid);
547 }
548
vas_window_alloc(struct vas_instance * vinst)549 static struct vas_window *vas_window_alloc(struct vas_instance *vinst)
550 {
551 int winid;
552 struct vas_window *window;
553
554 winid = vas_assign_window_id(&vinst->ida);
555 if (winid < 0)
556 return ERR_PTR(winid);
557
558 window = kzalloc(sizeof(*window), GFP_KERNEL);
559 if (!window)
560 goto out_free;
561
562 window->vinst = vinst;
563 window->winid = winid;
564
565 if (map_winctx_mmio_bars(window))
566 goto out_free;
567
568 vas_window_init_dbgdir(window);
569
570 return window;
571
572 out_free:
573 kfree(window);
574 vas_release_window_id(&vinst->ida, winid);
575 return ERR_PTR(-ENOMEM);
576 }
577
put_rx_win(struct vas_window * rxwin)578 static void put_rx_win(struct vas_window *rxwin)
579 {
580 /* Better not be a send window! */
581 WARN_ON_ONCE(rxwin->tx_win);
582
583 atomic_dec(&rxwin->num_txwins);
584 }
585
586 /*
587 * Find the user space receive window given the @pswid.
588 * - We must have a valid vasid and it must belong to this instance.
589 * (so both send and receive windows are on the same VAS instance)
590 * - The window must refer to an OPEN, FTW, RECEIVE window.
591 *
592 * NOTE: We access ->windows[] table and assume that vinst->mutex is held.
593 */
get_user_rxwin(struct vas_instance * vinst,u32 pswid)594 static struct vas_window *get_user_rxwin(struct vas_instance *vinst, u32 pswid)
595 {
596 int vasid, winid;
597 struct vas_window *rxwin;
598
599 decode_pswid(pswid, &vasid, &winid);
600
601 if (vinst->vas_id != vasid)
602 return ERR_PTR(-EINVAL);
603
604 rxwin = vinst->windows[winid];
605
606 if (!rxwin || rxwin->tx_win || rxwin->cop != VAS_COP_TYPE_FTW)
607 return ERR_PTR(-EINVAL);
608
609 return rxwin;
610 }
611
612 /*
613 * Get the VAS receive window associated with NX engine identified
614 * by @cop and if applicable, @pswid.
615 *
616 * See also function header of set_vinst_win().
617 */
get_vinst_rxwin(struct vas_instance * vinst,enum vas_cop_type cop,u32 pswid)618 static struct vas_window *get_vinst_rxwin(struct vas_instance *vinst,
619 enum vas_cop_type cop, u32 pswid)
620 {
621 struct vas_window *rxwin;
622
623 mutex_lock(&vinst->mutex);
624
625 if (cop == VAS_COP_TYPE_FTW)
626 rxwin = get_user_rxwin(vinst, pswid);
627 else
628 rxwin = vinst->rxwin[cop] ?: ERR_PTR(-EINVAL);
629
630 if (!IS_ERR(rxwin))
631 atomic_inc(&rxwin->num_txwins);
632
633 mutex_unlock(&vinst->mutex);
634
635 return rxwin;
636 }
637
638 /*
639 * We have two tables of windows in a VAS instance. The first one,
640 * ->windows[], contains all the windows in the instance and allows
641 * looking up a window by its id. It is used to look up send windows
642 * during fault handling and receive windows when pairing user space
643 * send/receive windows.
644 *
645 * The second table, ->rxwin[], contains receive windows that are
646 * associated with NX engines. This table has VAS_COP_TYPE_MAX
647 * entries and is used to look up a receive window by its
648 * coprocessor type.
649 *
650 * Here, we save @window in the ->windows[] table. If it is a receive
651 * window, we also save the window in the ->rxwin[] table.
652 */
set_vinst_win(struct vas_instance * vinst,struct vas_window * window)653 static void set_vinst_win(struct vas_instance *vinst,
654 struct vas_window *window)
655 {
656 int id = window->winid;
657
658 mutex_lock(&vinst->mutex);
659
660 /*
661 * There should only be one receive window for a coprocessor type
662 * unless its a user (FTW) window.
663 */
664 if (!window->user_win && !window->tx_win) {
665 WARN_ON_ONCE(vinst->rxwin[window->cop]);
666 vinst->rxwin[window->cop] = window;
667 }
668
669 WARN_ON_ONCE(vinst->windows[id] != NULL);
670 vinst->windows[id] = window;
671
672 mutex_unlock(&vinst->mutex);
673 }
674
675 /*
676 * Clear this window from the table(s) of windows for this VAS instance.
677 * See also function header of set_vinst_win().
678 */
clear_vinst_win(struct vas_window * window)679 static void clear_vinst_win(struct vas_window *window)
680 {
681 int id = window->winid;
682 struct vas_instance *vinst = window->vinst;
683
684 mutex_lock(&vinst->mutex);
685
686 if (!window->user_win && !window->tx_win) {
687 WARN_ON_ONCE(!vinst->rxwin[window->cop]);
688 vinst->rxwin[window->cop] = NULL;
689 }
690
691 WARN_ON_ONCE(vinst->windows[id] != window);
692 vinst->windows[id] = NULL;
693
694 mutex_unlock(&vinst->mutex);
695 }
696
init_winctx_for_rxwin(struct vas_window * rxwin,struct vas_rx_win_attr * rxattr,struct vas_winctx * winctx)697 static void init_winctx_for_rxwin(struct vas_window *rxwin,
698 struct vas_rx_win_attr *rxattr,
699 struct vas_winctx *winctx)
700 {
701 /*
702 * We first zero (memset()) all fields and only set non-zero fields.
703 * Following fields are 0/false but maybe deserve a comment:
704 *
705 * ->notify_os_intr_reg In powerNV, send intrs to HV
706 * ->notify_disable False for NX windows
707 * ->intr_disable False for Fault Windows
708 * ->xtra_write False for NX windows
709 * ->notify_early NA for NX windows
710 * ->rsvd_txbuf_count NA for Rx windows
711 * ->lpid, ->pid, ->tid NA for Rx windows
712 */
713
714 memset(winctx, 0, sizeof(struct vas_winctx));
715
716 winctx->rx_fifo = rxattr->rx_fifo;
717 winctx->rx_fifo_size = rxattr->rx_fifo_size;
718 winctx->wcreds_max = rxwin->wcreds_max;
719 winctx->pin_win = rxattr->pin_win;
720
721 winctx->nx_win = rxattr->nx_win;
722 winctx->fault_win = rxattr->fault_win;
723 winctx->user_win = rxattr->user_win;
724 winctx->rej_no_credit = rxattr->rej_no_credit;
725 winctx->rx_word_mode = rxattr->rx_win_ord_mode;
726 winctx->tx_word_mode = rxattr->tx_win_ord_mode;
727 winctx->rx_wcred_mode = rxattr->rx_wcred_mode;
728 winctx->tx_wcred_mode = rxattr->tx_wcred_mode;
729 winctx->notify_early = rxattr->notify_early;
730
731 if (winctx->nx_win) {
732 winctx->data_stamp = true;
733 winctx->intr_disable = true;
734 winctx->pin_win = true;
735
736 WARN_ON_ONCE(winctx->fault_win);
737 WARN_ON_ONCE(!winctx->rx_word_mode);
738 WARN_ON_ONCE(!winctx->tx_word_mode);
739 WARN_ON_ONCE(winctx->notify_after_count);
740 } else if (winctx->fault_win) {
741 winctx->notify_disable = true;
742 } else if (winctx->user_win) {
743 /*
744 * Section 1.8.1 Low Latency Core-Core Wake up of
745 * the VAS workbook:
746 *
747 * - disable credit checks ([tr]x_wcred_mode = false)
748 * - disable FIFO writes
749 * - enable ASB_Notify, disable interrupt
750 */
751 winctx->fifo_disable = true;
752 winctx->intr_disable = true;
753 winctx->rx_fifo = NULL;
754 }
755
756 winctx->lnotify_lpid = rxattr->lnotify_lpid;
757 winctx->lnotify_pid = rxattr->lnotify_pid;
758 winctx->lnotify_tid = rxattr->lnotify_tid;
759 winctx->pswid = rxattr->pswid;
760 winctx->dma_type = VAS_DMA_TYPE_INJECT;
761 winctx->tc_mode = rxattr->tc_mode;
762
763 winctx->min_scope = VAS_SCOPE_LOCAL;
764 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
765 }
766
rx_win_args_valid(enum vas_cop_type cop,struct vas_rx_win_attr * attr)767 static bool rx_win_args_valid(enum vas_cop_type cop,
768 struct vas_rx_win_attr *attr)
769 {
770 pr_debug("Rxattr: fault %d, notify %d, intr %d, early %d, fifo %d\n",
771 attr->fault_win, attr->notify_disable,
772 attr->intr_disable, attr->notify_early,
773 attr->rx_fifo_size);
774
775 if (cop >= VAS_COP_TYPE_MAX)
776 return false;
777
778 if (cop != VAS_COP_TYPE_FTW &&
779 attr->rx_fifo_size < VAS_RX_FIFO_SIZE_MIN)
780 return false;
781
782 if (attr->rx_fifo_size > VAS_RX_FIFO_SIZE_MAX)
783 return false;
784
785 if (attr->wcreds_max > VAS_RX_WCREDS_MAX)
786 return false;
787
788 if (attr->nx_win) {
789 /* cannot be fault or user window if it is nx */
790 if (attr->fault_win || attr->user_win)
791 return false;
792 /*
793 * Section 3.1.4.32: NX Windows must not disable notification,
794 * and must not enable interrupts or early notification.
795 */
796 if (attr->notify_disable || !attr->intr_disable ||
797 attr->notify_early)
798 return false;
799 } else if (attr->fault_win) {
800 /* cannot be both fault and user window */
801 if (attr->user_win)
802 return false;
803
804 /*
805 * Section 3.1.4.32: Fault windows must disable notification
806 * but not interrupts.
807 */
808 if (!attr->notify_disable || attr->intr_disable)
809 return false;
810
811 } else if (attr->user_win) {
812 /*
813 * User receive windows are only for fast-thread-wakeup
814 * (FTW). They don't need a FIFO and must disable interrupts
815 */
816 if (attr->rx_fifo || attr->rx_fifo_size || !attr->intr_disable)
817 return false;
818 } else {
819 /* Rx window must be one of NX or Fault or User window. */
820 return false;
821 }
822
823 return true;
824 }
825
vas_init_rx_win_attr(struct vas_rx_win_attr * rxattr,enum vas_cop_type cop)826 void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop)
827 {
828 memset(rxattr, 0, sizeof(*rxattr));
829
830 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) {
831 rxattr->pin_win = true;
832 rxattr->nx_win = true;
833 rxattr->fault_win = false;
834 rxattr->intr_disable = true;
835 rxattr->rx_wcred_mode = true;
836 rxattr->tx_wcred_mode = true;
837 rxattr->rx_win_ord_mode = true;
838 rxattr->tx_win_ord_mode = true;
839 } else if (cop == VAS_COP_TYPE_FAULT) {
840 rxattr->pin_win = true;
841 rxattr->fault_win = true;
842 rxattr->notify_disable = true;
843 rxattr->rx_wcred_mode = true;
844 rxattr->tx_wcred_mode = true;
845 rxattr->rx_win_ord_mode = true;
846 rxattr->tx_win_ord_mode = true;
847 } else if (cop == VAS_COP_TYPE_FTW) {
848 rxattr->user_win = true;
849 rxattr->intr_disable = true;
850
851 /*
852 * As noted in the VAS Workbook we disable credit checks.
853 * If we enable credit checks in the future, we must also
854 * implement a mechanism to return the user credits or new
855 * paste operations will fail.
856 */
857 }
858 }
859 EXPORT_SYMBOL_GPL(vas_init_rx_win_attr);
860
vas_rx_win_open(int vasid,enum vas_cop_type cop,struct vas_rx_win_attr * rxattr)861 struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop,
862 struct vas_rx_win_attr *rxattr)
863 {
864 struct vas_window *rxwin;
865 struct vas_winctx winctx;
866 struct vas_instance *vinst;
867
868 trace_vas_rx_win_open(current, vasid, cop, rxattr);
869
870 if (!rx_win_args_valid(cop, rxattr))
871 return ERR_PTR(-EINVAL);
872
873 vinst = find_vas_instance(vasid);
874 if (!vinst) {
875 pr_devel("vasid %d not found!\n", vasid);
876 return ERR_PTR(-EINVAL);
877 }
878 pr_devel("Found instance %d\n", vasid);
879
880 rxwin = vas_window_alloc(vinst);
881 if (IS_ERR(rxwin)) {
882 pr_devel("Unable to allocate memory for Rx window\n");
883 return rxwin;
884 }
885
886 rxwin->tx_win = false;
887 rxwin->nx_win = rxattr->nx_win;
888 rxwin->user_win = rxattr->user_win;
889 rxwin->cop = cop;
890 rxwin->wcreds_max = rxattr->wcreds_max ?: VAS_WCREDS_DEFAULT;
891 if (rxattr->user_win)
892 rxwin->pid = task_pid_vnr(current);
893
894 init_winctx_for_rxwin(rxwin, rxattr, &winctx);
895 init_winctx_regs(rxwin, &winctx);
896
897 set_vinst_win(vinst, rxwin);
898
899 return rxwin;
900 }
901 EXPORT_SYMBOL_GPL(vas_rx_win_open);
902
vas_init_tx_win_attr(struct vas_tx_win_attr * txattr,enum vas_cop_type cop)903 void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop)
904 {
905 memset(txattr, 0, sizeof(*txattr));
906
907 if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) {
908 txattr->rej_no_credit = false;
909 txattr->rx_wcred_mode = true;
910 txattr->tx_wcred_mode = true;
911 txattr->rx_win_ord_mode = true;
912 txattr->tx_win_ord_mode = true;
913 } else if (cop == VAS_COP_TYPE_FTW) {
914 txattr->user_win = true;
915 }
916 }
917 EXPORT_SYMBOL_GPL(vas_init_tx_win_attr);
918
init_winctx_for_txwin(struct vas_window * txwin,struct vas_tx_win_attr * txattr,struct vas_winctx * winctx)919 static void init_winctx_for_txwin(struct vas_window *txwin,
920 struct vas_tx_win_attr *txattr,
921 struct vas_winctx *winctx)
922 {
923 /*
924 * We first zero all fields and only set non-zero ones. Following
925 * are some fields set to 0/false for the stated reason:
926 *
927 * ->notify_os_intr_reg In powernv, send intrs to HV
928 * ->rsvd_txbuf_count Not supported yet.
929 * ->notify_disable False for NX windows
930 * ->xtra_write False for NX windows
931 * ->notify_early NA for NX windows
932 * ->lnotify_lpid NA for Tx windows
933 * ->lnotify_pid NA for Tx windows
934 * ->lnotify_tid NA for Tx windows
935 * ->tx_win_cred_mode Ignore for now for NX windows
936 * ->rx_win_cred_mode Ignore for now for NX windows
937 */
938 memset(winctx, 0, sizeof(struct vas_winctx));
939
940 winctx->wcreds_max = txwin->wcreds_max;
941
942 winctx->user_win = txattr->user_win;
943 winctx->nx_win = txwin->rxwin->nx_win;
944 winctx->pin_win = txattr->pin_win;
945 winctx->rej_no_credit = txattr->rej_no_credit;
946 winctx->rsvd_txbuf_enable = txattr->rsvd_txbuf_enable;
947
948 winctx->rx_wcred_mode = txattr->rx_wcred_mode;
949 winctx->tx_wcred_mode = txattr->tx_wcred_mode;
950 winctx->rx_word_mode = txattr->rx_win_ord_mode;
951 winctx->tx_word_mode = txattr->tx_win_ord_mode;
952 winctx->rsvd_txbuf_count = txattr->rsvd_txbuf_count;
953
954 winctx->intr_disable = true;
955 if (winctx->nx_win)
956 winctx->data_stamp = true;
957
958 winctx->lpid = txattr->lpid;
959 winctx->pidr = txattr->pidr;
960 winctx->rx_win_id = txwin->rxwin->winid;
961
962 winctx->dma_type = VAS_DMA_TYPE_INJECT;
963 winctx->tc_mode = txattr->tc_mode;
964 winctx->min_scope = VAS_SCOPE_LOCAL;
965 winctx->max_scope = VAS_SCOPE_VECTORED_GROUP;
966
967 winctx->pswid = 0;
968 }
969
tx_win_args_valid(enum vas_cop_type cop,struct vas_tx_win_attr * attr)970 static bool tx_win_args_valid(enum vas_cop_type cop,
971 struct vas_tx_win_attr *attr)
972 {
973 if (attr->tc_mode != VAS_THRESH_DISABLED)
974 return false;
975
976 if (cop > VAS_COP_TYPE_MAX)
977 return false;
978
979 if (attr->wcreds_max > VAS_TX_WCREDS_MAX)
980 return false;
981
982 if (attr->user_win &&
983 (cop != VAS_COP_TYPE_FTW || attr->rsvd_txbuf_count))
984 return false;
985
986 return true;
987 }
988
vas_tx_win_open(int vasid,enum vas_cop_type cop,struct vas_tx_win_attr * attr)989 struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
990 struct vas_tx_win_attr *attr)
991 {
992 int rc;
993 struct vas_window *txwin;
994 struct vas_window *rxwin;
995 struct vas_winctx winctx;
996 struct vas_instance *vinst;
997
998 trace_vas_tx_win_open(current, vasid, cop, attr);
999
1000 if (!tx_win_args_valid(cop, attr))
1001 return ERR_PTR(-EINVAL);
1002
1003 /*
1004 * If caller did not specify a vasid but specified the PSWID of a
1005 * receive window (applicable only to FTW windows), use the vasid
1006 * from that receive window.
1007 */
1008 if (vasid == -1 && attr->pswid)
1009 decode_pswid(attr->pswid, &vasid, NULL);
1010
1011 vinst = find_vas_instance(vasid);
1012 if (!vinst) {
1013 pr_devel("vasid %d not found!\n", vasid);
1014 return ERR_PTR(-EINVAL);
1015 }
1016
1017 rxwin = get_vinst_rxwin(vinst, cop, attr->pswid);
1018 if (IS_ERR(rxwin)) {
1019 pr_devel("No RxWin for vasid %d, cop %d\n", vasid, cop);
1020 return rxwin;
1021 }
1022
1023 txwin = vas_window_alloc(vinst);
1024 if (IS_ERR(txwin)) {
1025 rc = PTR_ERR(txwin);
1026 goto put_rxwin;
1027 }
1028
1029 txwin->cop = cop;
1030 txwin->tx_win = 1;
1031 txwin->rxwin = rxwin;
1032 txwin->nx_win = txwin->rxwin->nx_win;
1033 txwin->pid = attr->pid;
1034 txwin->user_win = attr->user_win;
1035 txwin->wcreds_max = attr->wcreds_max ?: VAS_WCREDS_DEFAULT;
1036
1037 init_winctx_for_txwin(txwin, attr, &winctx);
1038
1039 init_winctx_regs(txwin, &winctx);
1040
1041 /*
1042 * If its a kernel send window, map the window address into the
1043 * kernel's address space. For user windows, user must issue an
1044 * mmap() to map the window into their address space.
1045 *
1046 * NOTE: If kernel ever resubmits a user CRB after handling a page
1047 * fault, we will need to map this into kernel as well.
1048 */
1049 if (!txwin->user_win) {
1050 txwin->paste_kaddr = map_paste_region(txwin);
1051 if (IS_ERR(txwin->paste_kaddr)) {
1052 rc = PTR_ERR(txwin->paste_kaddr);
1053 goto free_window;
1054 }
1055 } else {
1056 /*
1057 * A user mapping must ensure that context switch issues
1058 * CP_ABORT for this thread.
1059 */
1060 rc = set_thread_uses_vas();
1061 if (rc)
1062 goto free_window;
1063 }
1064
1065 set_vinst_win(vinst, txwin);
1066
1067 return txwin;
1068
1069 free_window:
1070 vas_window_free(txwin);
1071
1072 put_rxwin:
1073 put_rx_win(rxwin);
1074 return ERR_PTR(rc);
1075
1076 }
1077 EXPORT_SYMBOL_GPL(vas_tx_win_open);
1078
vas_copy_crb(void * crb,int offset)1079 int vas_copy_crb(void *crb, int offset)
1080 {
1081 return vas_copy(crb, offset);
1082 }
1083 EXPORT_SYMBOL_GPL(vas_copy_crb);
1084
1085 #define RMA_LSMP_REPORT_ENABLE PPC_BIT(53)
vas_paste_crb(struct vas_window * txwin,int offset,bool re)1086 int vas_paste_crb(struct vas_window *txwin, int offset, bool re)
1087 {
1088 int rc;
1089 void *addr;
1090 uint64_t val;
1091
1092 trace_vas_paste_crb(current, txwin);
1093
1094 /*
1095 * Only NX windows are supported for now and hardware assumes
1096 * report-enable flag is set for NX windows. Ensure software
1097 * complies too.
1098 */
1099 WARN_ON_ONCE(txwin->nx_win && !re);
1100
1101 addr = txwin->paste_kaddr;
1102 if (re) {
1103 /*
1104 * Set the REPORT_ENABLE bit (equivalent to writing
1105 * to 1K offset of the paste address)
1106 */
1107 val = SET_FIELD(RMA_LSMP_REPORT_ENABLE, 0ULL, 1);
1108 addr += val;
1109 }
1110
1111 /*
1112 * Map the raw CR value from vas_paste() to an error code (there
1113 * is just pass or fail for now though).
1114 */
1115 rc = vas_paste(addr, offset);
1116 if (rc == 2)
1117 rc = 0;
1118 else
1119 rc = -EINVAL;
1120
1121 pr_debug("Txwin #%d: Msg count %llu\n", txwin->winid,
1122 read_hvwc_reg(txwin, VREG(LRFIFO_PUSH)));
1123
1124 return rc;
1125 }
1126 EXPORT_SYMBOL_GPL(vas_paste_crb);
1127
1128 /*
1129 * If credit checking is enabled for this window, poll for the return
1130 * of window credits (i.e for NX engines to process any outstanding CRBs).
1131 * Since NX-842 waits for the CRBs to be processed before closing the
1132 * window, we should not have to wait for too long.
1133 *
1134 * TODO: We retry in 10ms intervals now. We could/should probably peek at
1135 * the VAS_LRFIFO_PUSH_OFFSET register to get an estimate of pending
1136 * CRBs on the FIFO and compute the delay dynamically on each retry.
1137 * But that is not really needed until we support NX-GZIP access from
1138 * user space. (NX-842 driver waits for CSB and Fast thread-wakeup
1139 * doesn't use credit checking).
1140 */
poll_window_credits(struct vas_window * window)1141 static void poll_window_credits(struct vas_window *window)
1142 {
1143 u64 val;
1144 int creds, mode;
1145
1146 val = read_hvwc_reg(window, VREG(WINCTL));
1147 if (window->tx_win)
1148 mode = GET_FIELD(VAS_WINCTL_TX_WCRED_MODE, val);
1149 else
1150 mode = GET_FIELD(VAS_WINCTL_RX_WCRED_MODE, val);
1151
1152 if (!mode)
1153 return;
1154 retry:
1155 if (window->tx_win) {
1156 val = read_hvwc_reg(window, VREG(TX_WCRED));
1157 creds = GET_FIELD(VAS_TX_WCRED, val);
1158 } else {
1159 val = read_hvwc_reg(window, VREG(LRX_WCRED));
1160 creds = GET_FIELD(VAS_LRX_WCRED, val);
1161 }
1162
1163 if (creds < window->wcreds_max) {
1164 val = 0;
1165 set_current_state(TASK_UNINTERRUPTIBLE);
1166 schedule_timeout(msecs_to_jiffies(10));
1167 goto retry;
1168 }
1169 }
1170
1171 /*
1172 * Wait for the window to go to "not-busy" state. It should only take a
1173 * short time to queue a CRB, so window should not be busy for too long.
1174 * Trying 5ms intervals.
1175 */
poll_window_busy_state(struct vas_window * window)1176 static void poll_window_busy_state(struct vas_window *window)
1177 {
1178 int busy;
1179 u64 val;
1180
1181 retry:
1182 val = read_hvwc_reg(window, VREG(WIN_STATUS));
1183 busy = GET_FIELD(VAS_WIN_BUSY, val);
1184 if (busy) {
1185 val = 0;
1186 set_current_state(TASK_UNINTERRUPTIBLE);
1187 schedule_timeout(msecs_to_jiffies(5));
1188 goto retry;
1189 }
1190 }
1191
1192 /*
1193 * Have the hardware cast a window out of cache and wait for it to
1194 * be completed.
1195 *
1196 * NOTE: It can take a relatively long time to cast the window context
1197 * out of the cache. It is not strictly necessary to cast out if:
1198 *
1199 * - we clear the "Pin Window" bit (so hardware is free to evict)
1200 *
1201 * - we re-initialize the window context when it is reassigned.
1202 *
1203 * We do the former in vas_win_close() and latter in vas_win_open().
1204 * So, ignoring the cast-out for now. We can add it as needed. If
1205 * casting out becomes necessary we should consider offloading the
1206 * job to a worker thread, so the window close can proceed quickly.
1207 */
poll_window_castout(struct vas_window * window)1208 static void poll_window_castout(struct vas_window *window)
1209 {
1210 /* stub for now */
1211 }
1212
1213 /*
1214 * Unpin and close a window so no new requests are accepted and the
1215 * hardware can evict this window from cache if necessary.
1216 */
unpin_close_window(struct vas_window * window)1217 static void unpin_close_window(struct vas_window *window)
1218 {
1219 u64 val;
1220
1221 val = read_hvwc_reg(window, VREG(WINCTL));
1222 val = SET_FIELD(VAS_WINCTL_PIN, val, 0);
1223 val = SET_FIELD(VAS_WINCTL_OPEN, val, 0);
1224 write_hvwc_reg(window, VREG(WINCTL), val);
1225 }
1226
1227 /*
1228 * Close a window.
1229 *
1230 * See Section 1.12.1 of VAS workbook v1.05 for details on closing window:
1231 * - Disable new paste operations (unmap paste address)
1232 * - Poll for the "Window Busy" bit to be cleared
1233 * - Clear the Open/Enable bit for the Window.
1234 * - Poll for return of window Credits (implies FIFO empty for Rx win?)
1235 * - Unpin and cast window context out of cache
1236 *
1237 * Besides the hardware, kernel has some bookkeeping of course.
1238 */
vas_win_close(struct vas_window * window)1239 int vas_win_close(struct vas_window *window)
1240 {
1241 if (!window)
1242 return 0;
1243
1244 if (!window->tx_win && atomic_read(&window->num_txwins) != 0) {
1245 pr_devel("Attempting to close an active Rx window!\n");
1246 WARN_ON_ONCE(1);
1247 return -EBUSY;
1248 }
1249
1250 unmap_paste_region(window);
1251
1252 clear_vinst_win(window);
1253
1254 poll_window_busy_state(window);
1255
1256 unpin_close_window(window);
1257
1258 poll_window_credits(window);
1259
1260 poll_window_castout(window);
1261
1262 /* if send window, drop reference to matching receive window */
1263 if (window->tx_win)
1264 put_rx_win(window->rxwin);
1265
1266 vas_window_free(window);
1267
1268 return 0;
1269 }
1270 EXPORT_SYMBOL_GPL(vas_win_close);
1271
1272 /*
1273 * Return a system-wide unique window id for the window @win.
1274 */
vas_win_id(struct vas_window * win)1275 u32 vas_win_id(struct vas_window *win)
1276 {
1277 return encode_pswid(win->vinst->vas_id, win->winid);
1278 }
1279 EXPORT_SYMBOL_GPL(vas_win_id);
1280
