1 /*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 *
23 */
24 #include <linux/list.h>
25 #include "amdgpu.h"
26 #include "amdgpu_xgmi.h"
27 #include "amdgpu_smu.h"
28 #include "df/df_3_6_offset.h"
29
30 static DEFINE_MUTEX(xgmi_mutex);
31
32 #define AMDGPU_MAX_XGMI_HIVE 8
33 #define AMDGPU_MAX_XGMI_DEVICE_PER_HIVE 4
34
35 static struct amdgpu_hive_info xgmi_hives[AMDGPU_MAX_XGMI_HIVE];
36 static unsigned hive_count = 0;
37
amdgpu_xgmi_hive_try_lock(struct amdgpu_hive_info * hive)38 void *amdgpu_xgmi_hive_try_lock(struct amdgpu_hive_info *hive)
39 {
40 return &hive->device_list;
41 }
42
43 /**
44 * DOC: AMDGPU XGMI Support
45 *
46 * XGMI is a high speed interconnect that joins multiple GPU cards
47 * into a homogeneous memory space that is organized by a collective
48 * hive ID and individual node IDs, both of which are 64-bit numbers.
49 *
50 * The file xgmi_device_id contains the unique per GPU device ID and
51 * is stored in the /sys/class/drm/card${cardno}/device/ directory.
52 *
53 * Inside the device directory a sub-directory 'xgmi_hive_info' is
54 * created which contains the hive ID and the list of nodes.
55 *
56 * The hive ID is stored in:
57 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/xgmi_hive_id
58 *
59 * The node information is stored in numbered directories:
60 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/node${nodeno}/xgmi_device_id
61 *
62 * Each device has their own xgmi_hive_info direction with a mirror
63 * set of node sub-directories.
64 *
65 * The XGMI memory space is built by contiguously adding the power of
66 * two padded VRAM space from each node to each other.
67 *
68 */
69
70
amdgpu_xgmi_show_hive_id(struct device * dev,struct device_attribute * attr,char * buf)71 static ssize_t amdgpu_xgmi_show_hive_id(struct device *dev,
72 struct device_attribute *attr, char *buf)
73 {
74 struct amdgpu_hive_info *hive =
75 container_of(attr, struct amdgpu_hive_info, dev_attr);
76
77 return snprintf(buf, PAGE_SIZE, "%llu\n", hive->hive_id);
78 }
79
amdgpu_xgmi_sysfs_create(struct amdgpu_device * adev,struct amdgpu_hive_info * hive)80 static int amdgpu_xgmi_sysfs_create(struct amdgpu_device *adev,
81 struct amdgpu_hive_info *hive)
82 {
83 int ret = 0;
84
85 if (WARN_ON(hive->kobj))
86 return -EINVAL;
87
88 hive->kobj = kobject_create_and_add("xgmi_hive_info", &adev->dev->kobj);
89 if (!hive->kobj) {
90 dev_err(adev->dev, "XGMI: Failed to allocate sysfs entry!\n");
91 return -EINVAL;
92 }
93
94 hive->dev_attr = (struct device_attribute) {
95 .attr = {
96 .name = "xgmi_hive_id",
97 .mode = S_IRUGO,
98
99 },
100 .show = amdgpu_xgmi_show_hive_id,
101 };
102
103 ret = sysfs_create_file(hive->kobj, &hive->dev_attr.attr);
104 if (ret) {
105 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_hive_id\n");
106 kobject_del(hive->kobj);
107 kobject_put(hive->kobj);
108 hive->kobj = NULL;
109 }
110
111 return ret;
112 }
113
amdgpu_xgmi_sysfs_destroy(struct amdgpu_device * adev,struct amdgpu_hive_info * hive)114 static void amdgpu_xgmi_sysfs_destroy(struct amdgpu_device *adev,
115 struct amdgpu_hive_info *hive)
116 {
117 sysfs_remove_file(hive->kobj, &hive->dev_attr.attr);
118 kobject_del(hive->kobj);
119 kobject_put(hive->kobj);
120 hive->kobj = NULL;
121 }
122
amdgpu_xgmi_show_device_id(struct device * dev,struct device_attribute * attr,char * buf)123 static ssize_t amdgpu_xgmi_show_device_id(struct device *dev,
124 struct device_attribute *attr,
125 char *buf)
126 {
127 struct drm_device *ddev = dev_get_drvdata(dev);
128 struct amdgpu_device *adev = ddev->dev_private;
129
130 return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.xgmi.node_id);
131
132 }
133
134 #define AMDGPU_XGMI_SET_FICAA(o) ((o) | 0x456801)
amdgpu_xgmi_show_error(struct device * dev,struct device_attribute * attr,char * buf)135 static ssize_t amdgpu_xgmi_show_error(struct device *dev,
136 struct device_attribute *attr,
137 char *buf)
138 {
139 struct drm_device *ddev = dev_get_drvdata(dev);
140 struct amdgpu_device *adev = ddev->dev_private;
141 uint32_t ficaa_pie_ctl_in, ficaa_pie_status_in;
142 uint64_t fica_out;
143 unsigned int error_count = 0;
144
145 ficaa_pie_ctl_in = AMDGPU_XGMI_SET_FICAA(0x200);
146 ficaa_pie_status_in = AMDGPU_XGMI_SET_FICAA(0x208);
147
148 fica_out = adev->df_funcs->get_fica(adev, ficaa_pie_ctl_in);
149 if (fica_out != 0x1f)
150 pr_err("xGMI error counters not enabled!\n");
151
152 fica_out = adev->df_funcs->get_fica(adev, ficaa_pie_status_in);
153
154 if ((fica_out & 0xffff) == 2)
155 error_count = ((fica_out >> 62) & 0x1) + (fica_out >> 63);
156
157 adev->df_funcs->set_fica(adev, ficaa_pie_status_in, 0, 0);
158
159 return snprintf(buf, PAGE_SIZE, "%d\n", error_count);
160 }
161
162
163 static DEVICE_ATTR(xgmi_device_id, S_IRUGO, amdgpu_xgmi_show_device_id, NULL);
164 static DEVICE_ATTR(xgmi_error, S_IRUGO, amdgpu_xgmi_show_error, NULL);
165
amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device * adev,struct amdgpu_hive_info * hive)166 static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,
167 struct amdgpu_hive_info *hive)
168 {
169 int ret = 0;
170 char node[10] = { 0 };
171
172 /* Create xgmi device id file */
173 ret = device_create_file(adev->dev, &dev_attr_xgmi_device_id);
174 if (ret) {
175 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_device_id\n");
176 return ret;
177 }
178
179 /* Create xgmi error file */
180 ret = device_create_file(adev->dev, &dev_attr_xgmi_error);
181 if (ret)
182 pr_err("failed to create xgmi_error\n");
183
184
185 /* Create sysfs link to hive info folder on the first device */
186 if (adev != hive->adev) {
187 ret = sysfs_create_link(&adev->dev->kobj, hive->kobj,
188 "xgmi_hive_info");
189 if (ret) {
190 dev_err(adev->dev, "XGMI: Failed to create link to hive info");
191 goto remove_file;
192 }
193 }
194
195 sprintf(node, "node%d", hive->number_devices);
196 /* Create sysfs link form the hive folder to yourself */
197 ret = sysfs_create_link(hive->kobj, &adev->dev->kobj, node);
198 if (ret) {
199 dev_err(adev->dev, "XGMI: Failed to create link from hive info");
200 goto remove_link;
201 }
202
203 goto success;
204
205
206 remove_link:
207 sysfs_remove_link(&adev->dev->kobj, adev->ddev->unique);
208
209 remove_file:
210 device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
211
212 success:
213 return ret;
214 }
215
amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device * adev,struct amdgpu_hive_info * hive)216 static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev,
217 struct amdgpu_hive_info *hive)
218 {
219 device_remove_file(adev->dev, &dev_attr_xgmi_device_id);
220 sysfs_remove_link(&adev->dev->kobj, adev->ddev->unique);
221 sysfs_remove_link(hive->kobj, adev->ddev->unique);
222 }
223
224
225
amdgpu_get_xgmi_hive(struct amdgpu_device * adev,int lock)226 struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lock)
227 {
228 int i;
229 struct amdgpu_hive_info *tmp;
230
231 if (!adev->gmc.xgmi.hive_id)
232 return NULL;
233
234 mutex_lock(&xgmi_mutex);
235
236 for (i = 0 ; i < hive_count; ++i) {
237 tmp = &xgmi_hives[i];
238 if (tmp->hive_id == adev->gmc.xgmi.hive_id) {
239 if (lock)
240 mutex_lock(&tmp->hive_lock);
241 mutex_unlock(&xgmi_mutex);
242 return tmp;
243 }
244 }
245 if (i >= AMDGPU_MAX_XGMI_HIVE) {
246 mutex_unlock(&xgmi_mutex);
247 return NULL;
248 }
249
250 /* initialize new hive if not exist */
251 tmp = &xgmi_hives[hive_count++];
252
253 if (amdgpu_xgmi_sysfs_create(adev, tmp)) {
254 mutex_unlock(&xgmi_mutex);
255 return NULL;
256 }
257
258 tmp->adev = adev;
259 tmp->hive_id = adev->gmc.xgmi.hive_id;
260 INIT_LIST_HEAD(&tmp->device_list);
261 mutex_init(&tmp->hive_lock);
262 mutex_init(&tmp->reset_lock);
263
264 if (lock)
265 mutex_lock(&tmp->hive_lock);
266 tmp->pstate = -1;
267 mutex_unlock(&xgmi_mutex);
268
269 return tmp;
270 }
271
amdgpu_xgmi_set_pstate(struct amdgpu_device * adev,int pstate)272 int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate)
273 {
274 int ret = 0;
275 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev, 0);
276
277 if (!hive)
278 return 0;
279
280 if (hive->pstate == pstate)
281 return 0;
282
283 dev_dbg(adev->dev, "Set xgmi pstate %d.\n", pstate);
284
285 if (is_support_sw_smu_xgmi(adev))
286 ret = smu_set_xgmi_pstate(&adev->smu, pstate);
287 if (ret)
288 dev_err(adev->dev,
289 "XGMI: Set pstate failure on device %llx, hive %llx, ret %d",
290 adev->gmc.xgmi.node_id,
291 adev->gmc.xgmi.hive_id, ret);
292
293 return ret;
294 }
295
amdgpu_xgmi_update_topology(struct amdgpu_hive_info * hive,struct amdgpu_device * adev)296 int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev)
297 {
298 int ret = -EINVAL;
299
300 /* Each psp need to set the latest topology */
301 ret = psp_xgmi_set_topology_info(&adev->psp,
302 hive->number_devices,
303 &adev->psp.xgmi_context.top_info);
304 if (ret)
305 dev_err(adev->dev,
306 "XGMI: Set topology failure on device %llx, hive %llx, ret %d",
307 adev->gmc.xgmi.node_id,
308 adev->gmc.xgmi.hive_id, ret);
309
310 return ret;
311 }
312
313
amdgpu_xgmi_get_hops_count(struct amdgpu_device * adev,struct amdgpu_device * peer_adev)314 int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
315 struct amdgpu_device *peer_adev)
316 {
317 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
318 int i;
319
320 for (i = 0 ; i < top->num_nodes; ++i)
321 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
322 return top->nodes[i].num_hops;
323 return -EINVAL;
324 }
325
amdgpu_xgmi_add_device(struct amdgpu_device * adev)326 int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
327 {
328 struct psp_xgmi_topology_info *top_info;
329 struct amdgpu_hive_info *hive;
330 struct amdgpu_xgmi *entry;
331 struct amdgpu_device *tmp_adev = NULL;
332
333 int count = 0, ret = 0;
334
335 if (!adev->gmc.xgmi.supported)
336 return 0;
337
338 if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
339 ret = psp_xgmi_get_hive_id(&adev->psp, &adev->gmc.xgmi.hive_id);
340 if (ret) {
341 dev_err(adev->dev,
342 "XGMI: Failed to get hive id\n");
343 return ret;
344 }
345
346 ret = psp_xgmi_get_node_id(&adev->psp, &adev->gmc.xgmi.node_id);
347 if (ret) {
348 dev_err(adev->dev,
349 "XGMI: Failed to get node id\n");
350 return ret;
351 }
352 } else {
353 adev->gmc.xgmi.hive_id = 16;
354 adev->gmc.xgmi.node_id = adev->gmc.xgmi.physical_node_id + 16;
355 }
356
357 hive = amdgpu_get_xgmi_hive(adev, 1);
358 if (!hive) {
359 ret = -EINVAL;
360 dev_err(adev->dev,
361 "XGMI: node 0x%llx, can not match hive 0x%llx in the hive list.\n",
362 adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id);
363 goto exit;
364 }
365
366 top_info = &adev->psp.xgmi_context.top_info;
367
368 list_add_tail(&adev->gmc.xgmi.head, &hive->device_list);
369 list_for_each_entry(entry, &hive->device_list, head)
370 top_info->nodes[count++].node_id = entry->node_id;
371 top_info->num_nodes = count;
372 hive->number_devices = count;
373
374 if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {
375 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
376 /* update node list for other device in the hive */
377 if (tmp_adev != adev) {
378 top_info = &tmp_adev->psp.xgmi_context.top_info;
379 top_info->nodes[count - 1].node_id =
380 adev->gmc.xgmi.node_id;
381 top_info->num_nodes = count;
382 }
383 ret = amdgpu_xgmi_update_topology(hive, tmp_adev);
384 if (ret)
385 goto exit;
386 }
387
388 /* get latest topology info for each device from psp */
389 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
390 ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,
391 &tmp_adev->psp.xgmi_context.top_info);
392 if (ret) {
393 dev_err(tmp_adev->dev,
394 "XGMI: Get topology failure on device %llx, hive %llx, ret %d",
395 tmp_adev->gmc.xgmi.node_id,
396 tmp_adev->gmc.xgmi.hive_id, ret);
397 /* To do : continue with some node failed or disable the whole hive */
398 goto exit;
399 }
400 }
401 }
402
403 if (!ret)
404 ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive);
405
406
407 mutex_unlock(&hive->hive_lock);
408 exit:
409 if (!ret)
410 dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n",
411 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id);
412 else
413 dev_err(adev->dev, "XGMI: Failed to add node %d, hive 0x%llx ret: %d\n",
414 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id,
415 ret);
416
417 return ret;
418 }
419
amdgpu_xgmi_remove_device(struct amdgpu_device * adev)420 void amdgpu_xgmi_remove_device(struct amdgpu_device *adev)
421 {
422 struct amdgpu_hive_info *hive;
423
424 if (!adev->gmc.xgmi.supported)
425 return;
426
427 hive = amdgpu_get_xgmi_hive(adev, 1);
428 if (!hive)
429 return;
430
431 if (!(hive->number_devices--)) {
432 amdgpu_xgmi_sysfs_destroy(adev, hive);
433 mutex_destroy(&hive->hive_lock);
434 mutex_destroy(&hive->reset_lock);
435 } else {
436 amdgpu_xgmi_sysfs_rem_dev_info(adev, hive);
437 mutex_unlock(&hive->hive_lock);
438 }
439 }
440
