1 /*
2 * Copyright 2017 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22 #include "vmm.h"
23
24 #include <core/client.h>
25 #include <subdev/fb.h>
26 #include <subdev/ltc.h>
27 #include <subdev/timer.h>
28 #include <engine/gr.h>
29
30 #include <nvif/ifc00d.h>
31 #include <nvif/unpack.h>
32
33 static void
gp100_vmm_pfn_unmap(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes)34 gp100_vmm_pfn_unmap(struct nvkm_vmm *vmm,
35 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
36 {
37 struct device *dev = vmm->mmu->subdev.device->dev;
38 dma_addr_t addr;
39
40 nvkm_kmap(pt->memory);
41 while (ptes--) {
42 u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0);
43 u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4);
44 u64 data = (u64)datahi << 32 | datalo;
45 if ((data & (3ULL << 1)) != 0) {
46 addr = (data >> 8) << 12;
47 dma_unmap_page(dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
48 }
49 ptei++;
50 }
51 nvkm_done(pt->memory);
52 }
53
54 static bool
gp100_vmm_pfn_clear(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes)55 gp100_vmm_pfn_clear(struct nvkm_vmm *vmm,
56 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
57 {
58 bool dma = false;
59 nvkm_kmap(pt->memory);
60 while (ptes--) {
61 u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0);
62 u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4);
63 u64 data = (u64)datahi << 32 | datalo;
64 if ((data & BIT_ULL(0)) && (data & (3ULL << 1)) != 0) {
65 VMM_WO064(pt, vmm, ptei * 8, data & ~BIT_ULL(0));
66 dma = true;
67 }
68 ptei++;
69 }
70 nvkm_done(pt->memory);
71 return dma;
72 }
73
74 static void
gp100_vmm_pgt_pfn(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map)75 gp100_vmm_pgt_pfn(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
76 u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
77 {
78 struct device *dev = vmm->mmu->subdev.device->dev;
79 dma_addr_t addr;
80
81 nvkm_kmap(pt->memory);
82 while (ptes--) {
83 u64 data = 0;
84 if (!(*map->pfn & NVKM_VMM_PFN_W))
85 data |= BIT_ULL(6); /* RO. */
86
87 if (!(*map->pfn & NVKM_VMM_PFN_VRAM)) {
88 addr = *map->pfn >> NVKM_VMM_PFN_ADDR_SHIFT;
89 addr = dma_map_page(dev, pfn_to_page(addr), 0,
90 PAGE_SIZE, DMA_BIDIRECTIONAL);
91 if (!WARN_ON(dma_mapping_error(dev, addr))) {
92 data |= addr >> 4;
93 data |= 2ULL << 1; /* SYSTEM_COHERENT_MEMORY. */
94 data |= BIT_ULL(3); /* VOL. */
95 data |= BIT_ULL(0); /* VALID. */
96 }
97 } else {
98 data |= (*map->pfn & NVKM_VMM_PFN_ADDR) >> 4;
99 data |= BIT_ULL(0); /* VALID. */
100 }
101
102 VMM_WO064(pt, vmm, ptei++ * 8, data);
103 map->pfn++;
104 }
105 nvkm_done(pt->memory);
106 }
107
108 static inline void
gp100_vmm_pgt_pte(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map,u64 addr)109 gp100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
110 u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
111 {
112 u64 data = (addr >> 4) | map->type;
113
114 map->type += ptes * map->ctag;
115
116 while (ptes--) {
117 VMM_WO064(pt, vmm, ptei++ * 8, data);
118 data += map->next;
119 }
120 }
121
122 static void
gp100_vmm_pgt_sgl(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map)123 gp100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
124 u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
125 {
126 VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte);
127 }
128
129 static void
gp100_vmm_pgt_dma(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map)130 gp100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
131 u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
132 {
133 if (map->page->shift == PAGE_SHIFT) {
134 VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes);
135 nvkm_kmap(pt->memory);
136 while (ptes--) {
137 const u64 data = (*map->dma++ >> 4) | map->type;
138 VMM_WO064(pt, vmm, ptei++ * 8, data);
139 map->type += map->ctag;
140 }
141 nvkm_done(pt->memory);
142 return;
143 }
144
145 VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte);
146 }
147
148 static void
gp100_vmm_pgt_mem(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map)149 gp100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
150 u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
151 {
152 VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gp100_vmm_pgt_pte);
153 }
154
155 static void
gp100_vmm_pgt_sparse(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes)156 gp100_vmm_pgt_sparse(struct nvkm_vmm *vmm,
157 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
158 {
159 /* VALID_FALSE + VOL tells the MMU to treat the PTE as sparse. */
160 VMM_FO064(pt, vmm, ptei * 8, BIT_ULL(3) /* VOL. */, ptes);
161 }
162
163 static const struct nvkm_vmm_desc_func
164 gp100_vmm_desc_spt = {
165 .unmap = gf100_vmm_pgt_unmap,
166 .sparse = gp100_vmm_pgt_sparse,
167 .mem = gp100_vmm_pgt_mem,
168 .dma = gp100_vmm_pgt_dma,
169 .sgl = gp100_vmm_pgt_sgl,
170 .pfn = gp100_vmm_pgt_pfn,
171 .pfn_clear = gp100_vmm_pfn_clear,
172 .pfn_unmap = gp100_vmm_pfn_unmap,
173 };
174
175 static void
gp100_vmm_lpt_invalid(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes)176 gp100_vmm_lpt_invalid(struct nvkm_vmm *vmm,
177 struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
178 {
179 /* VALID_FALSE + PRIV tells the MMU to ignore corresponding SPTEs. */
180 VMM_FO064(pt, vmm, ptei * 8, BIT_ULL(5) /* PRIV. */, ptes);
181 }
182
183 static const struct nvkm_vmm_desc_func
184 gp100_vmm_desc_lpt = {
185 .invalid = gp100_vmm_lpt_invalid,
186 .unmap = gf100_vmm_pgt_unmap,
187 .sparse = gp100_vmm_pgt_sparse,
188 .mem = gp100_vmm_pgt_mem,
189 };
190
191 static inline void
gp100_vmm_pd0_pte(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map,u64 addr)192 gp100_vmm_pd0_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
193 u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
194 {
195 u64 data = (addr >> 4) | map->type;
196
197 map->type += ptes * map->ctag;
198
199 while (ptes--) {
200 VMM_WO128(pt, vmm, ptei++ * 0x10, data, 0ULL);
201 data += map->next;
202 }
203 }
204
205 static void
gp100_vmm_pd0_mem(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map)206 gp100_vmm_pd0_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
207 u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
208 {
209 VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gp100_vmm_pd0_pte);
210 }
211
212 static inline bool
gp100_vmm_pde(struct nvkm_mmu_pt * pt,u64 * data)213 gp100_vmm_pde(struct nvkm_mmu_pt *pt, u64 *data)
214 {
215 switch (nvkm_memory_target(pt->memory)) {
216 case NVKM_MEM_TARGET_VRAM: *data |= 1ULL << 1; break;
217 case NVKM_MEM_TARGET_HOST: *data |= 2ULL << 1;
218 *data |= BIT_ULL(3); /* VOL. */
219 break;
220 case NVKM_MEM_TARGET_NCOH: *data |= 3ULL << 1; break;
221 default:
222 WARN_ON(1);
223 return false;
224 }
225 *data |= pt->addr >> 4;
226 return true;
227 }
228
229 static void
gp100_vmm_pd0_pde(struct nvkm_vmm * vmm,struct nvkm_vmm_pt * pgd,u32 pdei)230 gp100_vmm_pd0_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)
231 {
232 struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
233 struct nvkm_mmu_pt *pd = pgd->pt[0];
234 u64 data[2] = {};
235
236 if (pgt->pt[0] && !gp100_vmm_pde(pgt->pt[0], &data[0]))
237 return;
238 if (pgt->pt[1] && !gp100_vmm_pde(pgt->pt[1], &data[1]))
239 return;
240
241 nvkm_kmap(pd->memory);
242 VMM_WO128(pd, vmm, pdei * 0x10, data[0], data[1]);
243 nvkm_done(pd->memory);
244 }
245
246 static void
gp100_vmm_pd0_sparse(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 pdei,u32 pdes)247 gp100_vmm_pd0_sparse(struct nvkm_vmm *vmm,
248 struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes)
249 {
250 /* VALID_FALSE + VOL_BIG tells the MMU to treat the PDE as sparse. */
251 VMM_FO128(pt, vmm, pdei * 0x10, BIT_ULL(3) /* VOL_BIG. */, 0ULL, pdes);
252 }
253
254 static void
gp100_vmm_pd0_unmap(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 pdei,u32 pdes)255 gp100_vmm_pd0_unmap(struct nvkm_vmm *vmm,
256 struct nvkm_mmu_pt *pt, u32 pdei, u32 pdes)
257 {
258 VMM_FO128(pt, vmm, pdei * 0x10, 0ULL, 0ULL, pdes);
259 }
260
261 static const struct nvkm_vmm_desc_func
262 gp100_vmm_desc_pd0 = {
263 .unmap = gp100_vmm_pd0_unmap,
264 .sparse = gp100_vmm_pd0_sparse,
265 .pde = gp100_vmm_pd0_pde,
266 .mem = gp100_vmm_pd0_mem,
267 };
268
269 static void
gp100_vmm_pd1_pde(struct nvkm_vmm * vmm,struct nvkm_vmm_pt * pgd,u32 pdei)270 gp100_vmm_pd1_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)
271 {
272 struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
273 struct nvkm_mmu_pt *pd = pgd->pt[0];
274 u64 data = 0;
275
276 if (!gp100_vmm_pde(pgt->pt[0], &data))
277 return;
278
279 nvkm_kmap(pd->memory);
280 VMM_WO064(pd, vmm, pdei * 8, data);
281 nvkm_done(pd->memory);
282 }
283
284 static const struct nvkm_vmm_desc_func
285 gp100_vmm_desc_pd1 = {
286 .unmap = gf100_vmm_pgt_unmap,
287 .sparse = gp100_vmm_pgt_sparse,
288 .pde = gp100_vmm_pd1_pde,
289 };
290
291 const struct nvkm_vmm_desc
292 gp100_vmm_desc_16[] = {
293 { LPT, 5, 8, 0x0100, &gp100_vmm_desc_lpt },
294 { PGD, 8, 16, 0x1000, &gp100_vmm_desc_pd0 },
295 { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 },
296 { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 },
297 { PGD, 2, 8, 0x1000, &gp100_vmm_desc_pd1 },
298 {}
299 };
300
301 const struct nvkm_vmm_desc
302 gp100_vmm_desc_12[] = {
303 { SPT, 9, 8, 0x1000, &gp100_vmm_desc_spt },
304 { PGD, 8, 16, 0x1000, &gp100_vmm_desc_pd0 },
305 { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 },
306 { PGD, 9, 8, 0x1000, &gp100_vmm_desc_pd1 },
307 { PGD, 2, 8, 0x1000, &gp100_vmm_desc_pd1 },
308 {}
309 };
310
311 int
gp100_vmm_valid(struct nvkm_vmm * vmm,void * argv,u32 argc,struct nvkm_vmm_map * map)312 gp100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc,
313 struct nvkm_vmm_map *map)
314 {
315 const enum nvkm_memory_target target = nvkm_memory_target(map->memory);
316 const struct nvkm_vmm_page *page = map->page;
317 union {
318 struct gp100_vmm_map_vn vn;
319 struct gp100_vmm_map_v0 v0;
320 } *args = argv;
321 struct nvkm_device *device = vmm->mmu->subdev.device;
322 struct nvkm_memory *memory = map->memory;
323 u8 kind, priv, ro, vol;
324 int kindn, aper, ret = -ENOSYS;
325 const u8 *kindm;
326
327 map->next = (1ULL << page->shift) >> 4;
328 map->type = 0;
329
330 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
331 vol = !!args->v0.vol;
332 ro = !!args->v0.ro;
333 priv = !!args->v0.priv;
334 kind = args->v0.kind;
335 } else
336 if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
337 vol = target == NVKM_MEM_TARGET_HOST;
338 ro = 0;
339 priv = 0;
340 kind = 0x00;
341 } else {
342 VMM_DEBUG(vmm, "args");
343 return ret;
344 }
345
346 aper = vmm->func->aper(target);
347 if (WARN_ON(aper < 0))
348 return aper;
349
350 kindm = vmm->mmu->func->kind(vmm->mmu, &kindn);
351 if (kind >= kindn || kindm[kind] == 0xff) {
352 VMM_DEBUG(vmm, "kind %02x", kind);
353 return -EINVAL;
354 }
355
356 if (kindm[kind] != kind) {
357 u64 tags = nvkm_memory_size(memory) >> 16;
358 if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) {
359 VMM_DEBUG(vmm, "comp %d %02x", aper, page->type);
360 return -EINVAL;
361 }
362
363 ret = nvkm_memory_tags_get(memory, device, tags,
364 nvkm_ltc_tags_clear,
365 &map->tags);
366 if (ret) {
367 VMM_DEBUG(vmm, "comp %d", ret);
368 return ret;
369 }
370
371 if (map->tags->mn) {
372 tags = map->tags->mn->offset + (map->offset >> 16);
373 map->ctag |= ((1ULL << page->shift) >> 16) << 36;
374 map->type |= tags << 36;
375 map->next |= map->ctag;
376 } else {
377 kind = kindm[kind];
378 }
379 }
380
381 map->type |= BIT(0);
382 map->type |= (u64)aper << 1;
383 map->type |= (u64) vol << 3;
384 map->type |= (u64)priv << 5;
385 map->type |= (u64) ro << 6;
386 map->type |= (u64)kind << 56;
387 return 0;
388 }
389
390 static int
gp100_vmm_fault_cancel(struct nvkm_vmm * vmm,void * argv,u32 argc)391 gp100_vmm_fault_cancel(struct nvkm_vmm *vmm, void *argv, u32 argc)
392 {
393 struct nvkm_device *device = vmm->mmu->subdev.device;
394 union {
395 struct gp100_vmm_fault_cancel_v0 v0;
396 } *args = argv;
397 int ret = -ENOSYS;
398 u32 inst, aper;
399
400 if ((ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false)))
401 return ret;
402
403 /* Translate MaxwellFaultBufferA instance pointer to the same
404 * format as the NV_GR_FECS_CURRENT_CTX register.
405 */
406 aper = (args->v0.inst >> 8) & 3;
407 args->v0.inst >>= 12;
408 args->v0.inst |= aper << 28;
409 args->v0.inst |= 0x80000000;
410
411 if (!WARN_ON(nvkm_gr_ctxsw_pause(device))) {
412 if ((inst = nvkm_gr_ctxsw_inst(device)) == args->v0.inst) {
413 gf100_vmm_invalidate(vmm, 0x0000001b
414 /* CANCEL_TARGETED. */ |
415 (args->v0.hub << 20) |
416 (args->v0.gpc << 15) |
417 (args->v0.client << 9));
418 }
419 WARN_ON(nvkm_gr_ctxsw_resume(device));
420 }
421
422 return 0;
423 }
424
425 static int
gp100_vmm_fault_replay(struct nvkm_vmm * vmm,void * argv,u32 argc)426 gp100_vmm_fault_replay(struct nvkm_vmm *vmm, void *argv, u32 argc)
427 {
428 union {
429 struct gp100_vmm_fault_replay_vn vn;
430 } *args = argv;
431 int ret = -ENOSYS;
432
433 if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
434 gf100_vmm_invalidate(vmm, 0x0000000b); /* REPLAY_GLOBAL. */
435 }
436
437 return ret;
438 }
439
440 int
gp100_vmm_mthd(struct nvkm_vmm * vmm,struct nvkm_client * client,u32 mthd,void * argv,u32 argc)441 gp100_vmm_mthd(struct nvkm_vmm *vmm,
442 struct nvkm_client *client, u32 mthd, void *argv, u32 argc)
443 {
444 if (client->super) {
445 switch (mthd) {
446 case GP100_VMM_VN_FAULT_REPLAY:
447 return gp100_vmm_fault_replay(vmm, argv, argc);
448 case GP100_VMM_VN_FAULT_CANCEL:
449 return gp100_vmm_fault_cancel(vmm, argv, argc);
450 default:
451 break;
452 }
453 }
454 return -EINVAL;
455 }
456
457 void
gp100_vmm_invalidate_pdb(struct nvkm_vmm * vmm,u64 addr)458 gp100_vmm_invalidate_pdb(struct nvkm_vmm *vmm, u64 addr)
459 {
460 struct nvkm_device *device = vmm->mmu->subdev.device;
461 nvkm_wr32(device, 0x100cb8, lower_32_bits(addr));
462 nvkm_wr32(device, 0x100cec, upper_32_bits(addr));
463 }
464
465 void
gp100_vmm_flush(struct nvkm_vmm * vmm,int depth)466 gp100_vmm_flush(struct nvkm_vmm *vmm, int depth)
467 {
468 u32 type = (5 /* CACHE_LEVEL_UP_TO_PDE3 */ - depth) << 24;
469 type = 0; /*XXX: need to confirm stuff works with depth enabled... */
470 if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR]))
471 type |= 0x00000004; /* HUB_ONLY */
472 type |= 0x00000001; /* PAGE_ALL */
473 gf100_vmm_invalidate(vmm, type);
474 }
475
476 int
gp100_vmm_join(struct nvkm_vmm * vmm,struct nvkm_memory * inst)477 gp100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
478 {
479 u64 base = BIT_ULL(10) /* VER2 */ | BIT_ULL(11) /* 64KiB */;
480 if (vmm->replay) {
481 base |= BIT_ULL(4); /* FAULT_REPLAY_TEX */
482 base |= BIT_ULL(5); /* FAULT_REPLAY_GCC */
483 }
484 return gf100_vmm_join_(vmm, inst, base);
485 }
486
487 static const struct nvkm_vmm_func
488 gp100_vmm = {
489 .join = gp100_vmm_join,
490 .part = gf100_vmm_part,
491 .aper = gf100_vmm_aper,
492 .valid = gp100_vmm_valid,
493 .flush = gp100_vmm_flush,
494 .mthd = gp100_vmm_mthd,
495 .invalidate_pdb = gp100_vmm_invalidate_pdb,
496 .page = {
497 { 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx },
498 { 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx },
499 { 29, &gp100_vmm_desc_16[2], NVKM_VMM_PAGE_Sxxx },
500 { 21, &gp100_vmm_desc_16[1], NVKM_VMM_PAGE_SVxC },
501 { 16, &gp100_vmm_desc_16[0], NVKM_VMM_PAGE_SVxC },
502 { 12, &gp100_vmm_desc_12[0], NVKM_VMM_PAGE_SVHx },
503 {}
504 }
505 };
506
507 int
gp100_vmm_new_(const struct nvkm_vmm_func * func,struct nvkm_mmu * mmu,bool managed,u64 addr,u64 size,void * argv,u32 argc,struct lock_class_key * key,const char * name,struct nvkm_vmm ** pvmm)508 gp100_vmm_new_(const struct nvkm_vmm_func *func,
509 struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,
510 void *argv, u32 argc, struct lock_class_key *key,
511 const char *name, struct nvkm_vmm **pvmm)
512 {
513 union {
514 struct gp100_vmm_vn vn;
515 struct gp100_vmm_v0 v0;
516 } *args = argv;
517 int ret = -ENOSYS;
518 bool replay;
519
520 if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
521 replay = args->v0.fault_replay != 0;
522 } else
523 if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
524 replay = false;
525 } else
526 return ret;
527
528 ret = nvkm_vmm_new_(func, mmu, 0, managed, addr, size, key, name, pvmm);
529 if (ret)
530 return ret;
531
532 (*pvmm)->replay = replay;
533 return 0;
534 }
535
536 int
gp100_vmm_new(struct nvkm_mmu * mmu,bool managed,u64 addr,u64 size,void * argv,u32 argc,struct lock_class_key * key,const char * name,struct nvkm_vmm ** pvmm)537 gp100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,
538 void *argv, u32 argc, struct lock_class_key *key,
539 const char *name, struct nvkm_vmm **pvmm)
540 {
541 return gp100_vmm_new_(&gp100_vmm, mmu, managed, addr, size,
542 argv, argc, key, name, pvmm);
543 }
544
