1 /*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26
27 #include <linux/io-64-nonatomic-lo-hi.h>
28
29 #include "amdgpu.h"
30
31 /**
32 * amdgpu_gmc_get_pde_for_bo - get the PDE for a BO
33 *
34 * @bo: the BO to get the PDE for
35 * @level: the level in the PD hirarchy
36 * @addr: resulting addr
37 * @flags: resulting flags
38 *
39 * Get the address and flags to be used for a PDE (Page Directory Entry).
40 */
amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo * bo,int level,uint64_t * addr,uint64_t * flags)41 void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo *bo, int level,
42 uint64_t *addr, uint64_t *flags)
43 {
44 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
45 struct ttm_dma_tt *ttm;
46
47 switch (bo->tbo.mem.mem_type) {
48 case TTM_PL_TT:
49 ttm = container_of(bo->tbo.ttm, struct ttm_dma_tt, ttm);
50 *addr = ttm->dma_address[0];
51 break;
52 case TTM_PL_VRAM:
53 *addr = amdgpu_bo_gpu_offset(bo);
54 break;
55 default:
56 *addr = 0;
57 break;
58 }
59 *flags = amdgpu_ttm_tt_pde_flags(bo->tbo.ttm, &bo->tbo.mem);
60 amdgpu_gmc_get_vm_pde(adev, level, addr, flags);
61 }
62
63 /**
64 * amdgpu_gmc_pd_addr - return the address of the root directory
65 *
66 */
amdgpu_gmc_pd_addr(struct amdgpu_bo * bo)67 uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo)
68 {
69 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
70 uint64_t pd_addr;
71
72 /* TODO: move that into ASIC specific code */
73 if (adev->asic_type >= CHIP_VEGA10) {
74 uint64_t flags = AMDGPU_PTE_VALID;
75
76 amdgpu_gmc_get_pde_for_bo(bo, -1, &pd_addr, &flags);
77 pd_addr |= flags;
78 } else {
79 pd_addr = amdgpu_bo_gpu_offset(bo);
80 }
81 return pd_addr;
82 }
83
84 /**
85 * amdgpu_gmc_set_pte_pde - update the page tables using CPU
86 *
87 * @adev: amdgpu_device pointer
88 * @cpu_pt_addr: cpu address of the page table
89 * @gpu_page_idx: entry in the page table to update
90 * @addr: dst addr to write into pte/pde
91 * @flags: access flags
92 *
93 * Update the page tables using CPU.
94 */
amdgpu_gmc_set_pte_pde(struct amdgpu_device * adev,void * cpu_pt_addr,uint32_t gpu_page_idx,uint64_t addr,uint64_t flags)95 int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
96 uint32_t gpu_page_idx, uint64_t addr,
97 uint64_t flags)
98 {
99 void __iomem *ptr = (void *)cpu_pt_addr;
100 uint64_t value;
101
102 /*
103 * The following is for PTE only. GART does not have PDEs.
104 */
105 value = addr & 0x0000FFFFFFFFF000ULL;
106 value |= flags;
107 writeq(value, ptr + (gpu_page_idx * 8));
108 return 0;
109 }
110
111 /**
112 * amdgpu_gmc_agp_addr - return the address in the AGP address space
113 *
114 * @tbo: TTM BO which needs the address, must be in GTT domain
115 *
116 * Tries to figure out how to access the BO through the AGP aperture. Returns
117 * AMDGPU_BO_INVALID_OFFSET if that is not possible.
118 */
amdgpu_gmc_agp_addr(struct ttm_buffer_object * bo)119 uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo)
120 {
121 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
122 struct ttm_dma_tt *ttm;
123
124 if (bo->num_pages != 1 || bo->ttm->caching_state == tt_cached)
125 return AMDGPU_BO_INVALID_OFFSET;
126
127 ttm = container_of(bo->ttm, struct ttm_dma_tt, ttm);
128 if (ttm->dma_address[0] + PAGE_SIZE >= adev->gmc.agp_size)
129 return AMDGPU_BO_INVALID_OFFSET;
130
131 return adev->gmc.agp_start + ttm->dma_address[0];
132 }
133
134 /**
135 * amdgpu_gmc_vram_location - try to find VRAM location
136 *
137 * @adev: amdgpu device structure holding all necessary informations
138 * @mc: memory controller structure holding memory informations
139 * @base: base address at which to put VRAM
140 *
141 * Function will try to place VRAM at base address provided
142 * as parameter.
143 */
amdgpu_gmc_vram_location(struct amdgpu_device * adev,struct amdgpu_gmc * mc,u64 base)144 void amdgpu_gmc_vram_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc,
145 u64 base)
146 {
147 uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
148
149 mc->vram_start = base;
150 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
151 if (limit && limit < mc->real_vram_size)
152 mc->real_vram_size = limit;
153
154 if (mc->xgmi.num_physical_nodes == 0) {
155 mc->fb_start = mc->vram_start;
156 mc->fb_end = mc->vram_end;
157 }
158 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
159 mc->mc_vram_size >> 20, mc->vram_start,
160 mc->vram_end, mc->real_vram_size >> 20);
161 }
162
163 /**
164 * amdgpu_gmc_gart_location - try to find GART location
165 *
166 * @adev: amdgpu device structure holding all necessary informations
167 * @mc: memory controller structure holding memory informations
168 *
169 * Function will place try to place GART before or after VRAM.
170 *
171 * If GART size is bigger than space left then we ajust GART size.
172 * Thus function will never fails.
173 */
amdgpu_gmc_gart_location(struct amdgpu_device * adev,struct amdgpu_gmc * mc)174 void amdgpu_gmc_gart_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc)
175 {
176 const uint64_t four_gb = 0x100000000ULL;
177 u64 size_af, size_bf;
178 /*To avoid the hole, limit the max mc address to AMDGPU_GMC_HOLE_START*/
179 u64 max_mc_address = min(adev->gmc.mc_mask, AMDGPU_GMC_HOLE_START - 1);
180
181 mc->gart_size += adev->pm.smu_prv_buffer_size;
182
183 /* VCE doesn't like it when BOs cross a 4GB segment, so align
184 * the GART base on a 4GB boundary as well.
185 */
186 size_bf = mc->fb_start;
187 size_af = max_mc_address + 1 - ALIGN(mc->fb_end + 1, four_gb);
188
189 if (mc->gart_size > max(size_bf, size_af)) {
190 dev_warn(adev->dev, "limiting GART\n");
191 mc->gart_size = max(size_bf, size_af);
192 }
193
194 if ((size_bf >= mc->gart_size && size_bf < size_af) ||
195 (size_af < mc->gart_size))
196 mc->gart_start = 0;
197 else
198 mc->gart_start = max_mc_address - mc->gart_size + 1;
199
200 mc->gart_start &= ~(four_gb - 1);
201 mc->gart_end = mc->gart_start + mc->gart_size - 1;
202 dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n",
203 mc->gart_size >> 20, mc->gart_start, mc->gart_end);
204 }
205
206 /**
207 * amdgpu_gmc_agp_location - try to find AGP location
208 * @adev: amdgpu device structure holding all necessary informations
209 * @mc: memory controller structure holding memory informations
210 *
211 * Function will place try to find a place for the AGP BAR in the MC address
212 * space.
213 *
214 * AGP BAR will be assigned the largest available hole in the address space.
215 * Should be called after VRAM and GART locations are setup.
216 */
amdgpu_gmc_agp_location(struct amdgpu_device * adev,struct amdgpu_gmc * mc)217 void amdgpu_gmc_agp_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc)
218 {
219 const uint64_t sixteen_gb = 1ULL << 34;
220 const uint64_t sixteen_gb_mask = ~(sixteen_gb - 1);
221 u64 size_af, size_bf;
222
223 if (amdgpu_sriov_vf(adev)) {
224 mc->agp_start = 0xffffffff;
225 mc->agp_end = 0x0;
226 mc->agp_size = 0;
227
228 return;
229 }
230
231 if (mc->fb_start > mc->gart_start) {
232 size_bf = (mc->fb_start & sixteen_gb_mask) -
233 ALIGN(mc->gart_end + 1, sixteen_gb);
234 size_af = mc->mc_mask + 1 - ALIGN(mc->fb_end + 1, sixteen_gb);
235 } else {
236 size_bf = mc->fb_start & sixteen_gb_mask;
237 size_af = (mc->gart_start & sixteen_gb_mask) -
238 ALIGN(mc->fb_end + 1, sixteen_gb);
239 }
240
241 if (size_bf > size_af) {
242 mc->agp_start = (mc->fb_start - size_bf) & sixteen_gb_mask;
243 mc->agp_size = size_bf;
244 } else {
245 mc->agp_start = ALIGN(mc->fb_end + 1, sixteen_gb);
246 mc->agp_size = size_af;
247 }
248
249 mc->agp_end = mc->agp_start + mc->agp_size - 1;
250 dev_info(adev->dev, "AGP: %lluM 0x%016llX - 0x%016llX\n",
251 mc->agp_size >> 20, mc->agp_start, mc->agp_end);
252 }
253
254 /**
255 * amdgpu_gmc_filter_faults - filter VM faults
256 *
257 * @adev: amdgpu device structure
258 * @addr: address of the VM fault
259 * @pasid: PASID of the process causing the fault
260 * @timestamp: timestamp of the fault
261 *
262 * Returns:
263 * True if the fault was filtered and should not be processed further.
264 * False if the fault is a new one and needs to be handled.
265 */
amdgpu_gmc_filter_faults(struct amdgpu_device * adev,uint64_t addr,uint16_t pasid,uint64_t timestamp)266 bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr,
267 uint16_t pasid, uint64_t timestamp)
268 {
269 struct amdgpu_gmc *gmc = &adev->gmc;
270
271 uint64_t stamp, key = addr << 4 | pasid;
272 struct amdgpu_gmc_fault *fault;
273 uint32_t hash;
274
275 /* If we don't have space left in the ring buffer return immediately */
276 stamp = max(timestamp, AMDGPU_GMC_FAULT_TIMEOUT + 1) -
277 AMDGPU_GMC_FAULT_TIMEOUT;
278 if (gmc->fault_ring[gmc->last_fault].timestamp >= stamp)
279 return true;
280
281 /* Try to find the fault in the hash */
282 hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER);
283 fault = &gmc->fault_ring[gmc->fault_hash[hash].idx];
284 while (fault->timestamp >= stamp) {
285 uint64_t tmp;
286
287 if (fault->key == key)
288 return true;
289
290 tmp = fault->timestamp;
291 fault = &gmc->fault_ring[fault->next];
292
293 /* Check if the entry was reused */
294 if (fault->timestamp >= tmp)
295 break;
296 }
297
298 /* Add the fault to the ring */
299 fault = &gmc->fault_ring[gmc->last_fault];
300 fault->key = key;
301 fault->timestamp = timestamp;
302
303 /* And update the hash */
304 fault->next = gmc->fault_hash[hash].idx;
305 gmc->fault_hash[hash].idx = gmc->last_fault++;
306 return false;
307 }
308
