1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Intel MIC Platform Software Stack (MPSS)
4 *
5 * Copyright(c) 2013 Intel Corporation.
6 *
7 * Intel MIC Host driver.
8 */
9 #include <linux/fs.h>
10 #include <linux/pci.h>
11 #include <linux/sched.h>
12 #include <linux/firmware.h>
13 #include <linux/delay.h>
14
15 #include "../common/mic_dev.h"
16 #include "mic_device.h"
17 #include "mic_x100.h"
18 #include "mic_smpt.h"
19
20 /**
21 * mic_x100_write_spad - write to the scratchpad register
22 * @mdev: pointer to mic_device instance
23 * @idx: index to the scratchpad register, 0 based
24 * @val: the data value to put into the register
25 *
26 * This function allows writing of a 32bit value to the indexed scratchpad
27 * register.
28 *
29 * RETURNS: none.
30 */
31 static void
mic_x100_write_spad(struct mic_device * mdev,unsigned int idx,u32 val)32 mic_x100_write_spad(struct mic_device *mdev, unsigned int idx, u32 val)
33 {
34 dev_dbg(&mdev->pdev->dev, "Writing 0x%x to scratch pad index %d\n",
35 val, idx);
36 mic_mmio_write(&mdev->mmio, val,
37 MIC_X100_SBOX_BASE_ADDRESS +
38 MIC_X100_SBOX_SPAD0 + idx * 4);
39 }
40
41 /**
42 * mic_x100_read_spad - read from the scratchpad register
43 * @mdev: pointer to mic_device instance
44 * @idx: index to scratchpad register, 0 based
45 *
46 * This function allows reading of the 32bit scratchpad register.
47 *
48 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
49 */
50 static u32
mic_x100_read_spad(struct mic_device * mdev,unsigned int idx)51 mic_x100_read_spad(struct mic_device *mdev, unsigned int idx)
52 {
53 u32 val = mic_mmio_read(&mdev->mmio,
54 MIC_X100_SBOX_BASE_ADDRESS +
55 MIC_X100_SBOX_SPAD0 + idx * 4);
56
57 dev_dbg(&mdev->pdev->dev,
58 "Reading 0x%x from scratch pad index %d\n", val, idx);
59 return val;
60 }
61
62 /**
63 * mic_x100_enable_interrupts - Enable interrupts.
64 * @mdev: pointer to mic_device instance
65 */
mic_x100_enable_interrupts(struct mic_device * mdev)66 static void mic_x100_enable_interrupts(struct mic_device *mdev)
67 {
68 u32 reg;
69 struct mic_mw *mw = &mdev->mmio;
70 u32 sice0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICE0;
71 u32 siac0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SIAC0;
72
73 reg = mic_mmio_read(mw, sice0);
74 reg |= MIC_X100_SBOX_DBR_BITS(0xf) | MIC_X100_SBOX_DMA_BITS(0xff);
75 mic_mmio_write(mw, reg, sice0);
76
77 /*
78 * Enable auto-clear when enabling interrupts. Applicable only for
79 * MSI-x. Legacy and MSI mode cannot have auto-clear enabled.
80 */
81 if (mdev->irq_info.num_vectors > 1) {
82 reg = mic_mmio_read(mw, siac0);
83 reg |= MIC_X100_SBOX_DBR_BITS(0xf) |
84 MIC_X100_SBOX_DMA_BITS(0xff);
85 mic_mmio_write(mw, reg, siac0);
86 }
87 }
88
89 /**
90 * mic_x100_disable_interrupts - Disable interrupts.
91 * @mdev: pointer to mic_device instance
92 */
mic_x100_disable_interrupts(struct mic_device * mdev)93 static void mic_x100_disable_interrupts(struct mic_device *mdev)
94 {
95 u32 reg;
96 struct mic_mw *mw = &mdev->mmio;
97 u32 sice0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICE0;
98 u32 siac0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SIAC0;
99 u32 sicc0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICC0;
100
101 reg = mic_mmio_read(mw, sice0);
102 mic_mmio_write(mw, reg, sicc0);
103
104 if (mdev->irq_info.num_vectors > 1) {
105 reg = mic_mmio_read(mw, siac0);
106 reg &= ~(MIC_X100_SBOX_DBR_BITS(0xf) |
107 MIC_X100_SBOX_DMA_BITS(0xff));
108 mic_mmio_write(mw, reg, siac0);
109 }
110 }
111
112 /**
113 * mic_x100_send_sbox_intr - Send an MIC_X100_SBOX interrupt to MIC.
114 * @mdev: pointer to mic_device instance
115 */
mic_x100_send_sbox_intr(struct mic_device * mdev,int doorbell)116 static void mic_x100_send_sbox_intr(struct mic_device *mdev,
117 int doorbell)
118 {
119 struct mic_mw *mw = &mdev->mmio;
120 u64 apic_icr_offset = MIC_X100_SBOX_APICICR0 + doorbell * 8;
121 u32 apicicr_low = mic_mmio_read(mw, MIC_X100_SBOX_BASE_ADDRESS +
122 apic_icr_offset);
123
124 /* for MIC we need to make sure we "hit" the send_icr bit (13) */
125 apicicr_low = (apicicr_low | (1 << 13));
126
127 /* Ensure that the interrupt is ordered w.r.t. previous stores. */
128 wmb();
129 mic_mmio_write(mw, apicicr_low,
130 MIC_X100_SBOX_BASE_ADDRESS + apic_icr_offset);
131 }
132
133 /**
134 * mic_x100_send_rdmasr_intr - Send an RDMASR interrupt to MIC.
135 * @mdev: pointer to mic_device instance
136 */
mic_x100_send_rdmasr_intr(struct mic_device * mdev,int doorbell)137 static void mic_x100_send_rdmasr_intr(struct mic_device *mdev,
138 int doorbell)
139 {
140 int rdmasr_offset = MIC_X100_SBOX_RDMASR0 + (doorbell << 2);
141 /* Ensure that the interrupt is ordered w.r.t. previous stores. */
142 wmb();
143 mic_mmio_write(&mdev->mmio, 0,
144 MIC_X100_SBOX_BASE_ADDRESS + rdmasr_offset);
145 }
146
147 /**
148 * __mic_x100_send_intr - Send interrupt to MIC.
149 * @mdev: pointer to mic_device instance
150 * @doorbell: doorbell number.
151 */
mic_x100_send_intr(struct mic_device * mdev,int doorbell)152 static void mic_x100_send_intr(struct mic_device *mdev, int doorbell)
153 {
154 int rdmasr_db;
155 if (doorbell < MIC_X100_NUM_SBOX_IRQ) {
156 mic_x100_send_sbox_intr(mdev, doorbell);
157 } else {
158 rdmasr_db = doorbell - MIC_X100_NUM_SBOX_IRQ;
159 mic_x100_send_rdmasr_intr(mdev, rdmasr_db);
160 }
161 }
162
163 /**
164 * mic_x100_ack_interrupt - Read the interrupt sources register and
165 * clear it. This function will be called in the MSI/INTx case.
166 * @mdev: Pointer to mic_device instance.
167 *
168 * Returns: bitmask of interrupt sources triggered.
169 */
mic_x100_ack_interrupt(struct mic_device * mdev)170 static u32 mic_x100_ack_interrupt(struct mic_device *mdev)
171 {
172 u32 sicr0 = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_SICR0;
173 u32 reg = mic_mmio_read(&mdev->mmio, sicr0);
174 mic_mmio_write(&mdev->mmio, reg, sicr0);
175 return reg;
176 }
177
178 /**
179 * mic_x100_intr_workarounds - These hardware specific workarounds are
180 * to be invoked everytime an interrupt is handled.
181 * @mdev: Pointer to mic_device instance.
182 *
183 * Returns: none
184 */
mic_x100_intr_workarounds(struct mic_device * mdev)185 static void mic_x100_intr_workarounds(struct mic_device *mdev)
186 {
187 struct mic_mw *mw = &mdev->mmio;
188
189 /* Clear pending bit array. */
190 if (MIC_A0_STEP == mdev->stepping)
191 mic_mmio_write(mw, 1, MIC_X100_SBOX_BASE_ADDRESS +
192 MIC_X100_SBOX_MSIXPBACR);
193
194 if (mdev->stepping >= MIC_B0_STEP)
195 mdev->intr_ops->enable_interrupts(mdev);
196 }
197
198 /**
199 * mic_x100_hw_intr_init - Initialize h/w specific interrupt
200 * information.
201 * @mdev: pointer to mic_device instance
202 */
mic_x100_hw_intr_init(struct mic_device * mdev)203 static void mic_x100_hw_intr_init(struct mic_device *mdev)
204 {
205 mdev->intr_info = (struct mic_intr_info *)mic_x100_intr_init;
206 }
207
208 /**
209 * mic_x100_read_msi_to_src_map - read from the MSI mapping registers
210 * @mdev: pointer to mic_device instance
211 * @idx: index to the mapping register, 0 based
212 *
213 * This function allows reading of the 32bit MSI mapping register.
214 *
215 * RETURNS: The value in the register.
216 */
217 static u32
mic_x100_read_msi_to_src_map(struct mic_device * mdev,int idx)218 mic_x100_read_msi_to_src_map(struct mic_device *mdev, int idx)
219 {
220 return mic_mmio_read(&mdev->mmio,
221 MIC_X100_SBOX_BASE_ADDRESS +
222 MIC_X100_SBOX_MXAR0 + idx * 4);
223 }
224
225 /**
226 * mic_x100_program_msi_to_src_map - program the MSI mapping registers
227 * @mdev: pointer to mic_device instance
228 * @idx: index to the mapping register, 0 based
229 * @offset: The bit offset in the register that needs to be updated.
230 * @set: boolean specifying if the bit in the specified offset needs
231 * to be set or cleared.
232 *
233 * RETURNS: None.
234 */
235 static void
mic_x100_program_msi_to_src_map(struct mic_device * mdev,int idx,int offset,bool set)236 mic_x100_program_msi_to_src_map(struct mic_device *mdev,
237 int idx, int offset, bool set)
238 {
239 unsigned long reg;
240 struct mic_mw *mw = &mdev->mmio;
241 u32 mxar = MIC_X100_SBOX_BASE_ADDRESS +
242 MIC_X100_SBOX_MXAR0 + idx * 4;
243
244 reg = mic_mmio_read(mw, mxar);
245 if (set)
246 __set_bit(offset, ®);
247 else
248 __clear_bit(offset, ®);
249 mic_mmio_write(mw, reg, mxar);
250 }
251
252 /*
253 * mic_x100_reset_fw_ready - Reset Firmware ready status field.
254 * @mdev: pointer to mic_device instance
255 */
mic_x100_reset_fw_ready(struct mic_device * mdev)256 static void mic_x100_reset_fw_ready(struct mic_device *mdev)
257 {
258 mdev->ops->write_spad(mdev, MIC_X100_DOWNLOAD_INFO, 0);
259 }
260
261 /*
262 * mic_x100_is_fw_ready - Check if firmware is ready.
263 * @mdev: pointer to mic_device instance
264 */
mic_x100_is_fw_ready(struct mic_device * mdev)265 static bool mic_x100_is_fw_ready(struct mic_device *mdev)
266 {
267 u32 scratch2 = mdev->ops->read_spad(mdev, MIC_X100_DOWNLOAD_INFO);
268 return MIC_X100_SPAD2_DOWNLOAD_STATUS(scratch2) ? true : false;
269 }
270
271 /**
272 * mic_x100_get_apic_id - Get bootstrap APIC ID.
273 * @mdev: pointer to mic_device instance
274 */
mic_x100_get_apic_id(struct mic_device * mdev)275 static u32 mic_x100_get_apic_id(struct mic_device *mdev)
276 {
277 u32 scratch2 = 0;
278
279 scratch2 = mdev->ops->read_spad(mdev, MIC_X100_DOWNLOAD_INFO);
280 return MIC_X100_SPAD2_APIC_ID(scratch2);
281 }
282
283 /**
284 * mic_x100_send_firmware_intr - Send an interrupt to the firmware on MIC.
285 * @mdev: pointer to mic_device instance
286 */
mic_x100_send_firmware_intr(struct mic_device * mdev)287 static void mic_x100_send_firmware_intr(struct mic_device *mdev)
288 {
289 u32 apicicr_low;
290 u64 apic_icr_offset = MIC_X100_SBOX_APICICR7;
291 int vector = MIC_X100_BSP_INTERRUPT_VECTOR;
292 struct mic_mw *mw = &mdev->mmio;
293
294 /*
295 * For MIC we need to make sure we "hit"
296 * the send_icr bit (13).
297 */
298 apicicr_low = (vector | (1 << 13));
299
300 mic_mmio_write(mw, mic_x100_get_apic_id(mdev),
301 MIC_X100_SBOX_BASE_ADDRESS + apic_icr_offset + 4);
302
303 /* Ensure that the interrupt is ordered w.r.t. previous stores. */
304 wmb();
305 mic_mmio_write(mw, apicicr_low,
306 MIC_X100_SBOX_BASE_ADDRESS + apic_icr_offset);
307 }
308
309 /**
310 * mic_x100_hw_reset - Reset the MIC device.
311 * @mdev: pointer to mic_device instance
312 */
mic_x100_hw_reset(struct mic_device * mdev)313 static void mic_x100_hw_reset(struct mic_device *mdev)
314 {
315 u32 reset_reg;
316 u32 rgcr = MIC_X100_SBOX_BASE_ADDRESS + MIC_X100_SBOX_RGCR;
317 struct mic_mw *mw = &mdev->mmio;
318
319 /* Ensure that the reset is ordered w.r.t. previous loads and stores */
320 mb();
321 /* Trigger reset */
322 reset_reg = mic_mmio_read(mw, rgcr);
323 reset_reg |= 0x1;
324 mic_mmio_write(mw, reset_reg, rgcr);
325 /*
326 * It seems we really want to delay at least 1 second
327 * after touching reset to prevent a lot of problems.
328 */
329 msleep(1000);
330 }
331
332 /**
333 * mic_x100_load_command_line - Load command line to MIC.
334 * @mdev: pointer to mic_device instance
335 * @fw: the firmware image
336 *
337 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
338 */
339 static int
mic_x100_load_command_line(struct mic_device * mdev,const struct firmware * fw)340 mic_x100_load_command_line(struct mic_device *mdev, const struct firmware *fw)
341 {
342 u32 len = 0;
343 u32 boot_mem;
344 char *buf;
345 void __iomem *cmd_line_va = mdev->aper.va + mdev->bootaddr + fw->size;
346 #define CMDLINE_SIZE 2048
347
348 boot_mem = mdev->aper.len >> 20;
349 buf = kzalloc(CMDLINE_SIZE, GFP_KERNEL);
350 if (!buf)
351 return -ENOMEM;
352
353 len += snprintf(buf, CMDLINE_SIZE - len,
354 " mem=%dM", boot_mem);
355 if (mdev->cosm_dev->cmdline)
356 snprintf(buf + len, CMDLINE_SIZE - len, " %s",
357 mdev->cosm_dev->cmdline);
358 memcpy_toio(cmd_line_va, buf, strlen(buf) + 1);
359 kfree(buf);
360 return 0;
361 }
362
363 /**
364 * mic_x100_load_ramdisk - Load ramdisk to MIC.
365 * @mdev: pointer to mic_device instance
366 *
367 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
368 */
369 static int
mic_x100_load_ramdisk(struct mic_device * mdev)370 mic_x100_load_ramdisk(struct mic_device *mdev)
371 {
372 const struct firmware *fw;
373 int rc;
374 struct boot_params __iomem *bp = mdev->aper.va + mdev->bootaddr;
375
376 rc = request_firmware(&fw, mdev->cosm_dev->ramdisk, &mdev->pdev->dev);
377 if (rc < 0) {
378 dev_err(&mdev->pdev->dev,
379 "ramdisk request_firmware failed: %d %s\n",
380 rc, mdev->cosm_dev->ramdisk);
381 goto error;
382 }
383 /*
384 * Typically the bootaddr for card OS is 64M
385 * so copy over the ramdisk @ 128M.
386 */
387 memcpy_toio(mdev->aper.va + (mdev->bootaddr << 1), fw->data, fw->size);
388 iowrite32(mdev->bootaddr << 1, &bp->hdr.ramdisk_image);
389 iowrite32(fw->size, &bp->hdr.ramdisk_size);
390 release_firmware(fw);
391 error:
392 return rc;
393 }
394
395 /**
396 * mic_x100_get_boot_addr - Get MIC boot address.
397 * @mdev: pointer to mic_device instance
398 *
399 * This function is called during firmware load to determine
400 * the address at which the OS should be downloaded in card
401 * memory i.e. GDDR.
402 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
403 */
404 static int
mic_x100_get_boot_addr(struct mic_device * mdev)405 mic_x100_get_boot_addr(struct mic_device *mdev)
406 {
407 u32 scratch2, boot_addr;
408 int rc = 0;
409
410 scratch2 = mdev->ops->read_spad(mdev, MIC_X100_DOWNLOAD_INFO);
411 boot_addr = MIC_X100_SPAD2_DOWNLOAD_ADDR(scratch2);
412 dev_dbg(&mdev->pdev->dev, "%s %d boot_addr 0x%x\n",
413 __func__, __LINE__, boot_addr);
414 if (boot_addr > (1 << 31)) {
415 dev_err(&mdev->pdev->dev,
416 "incorrect bootaddr 0x%x\n",
417 boot_addr);
418 rc = -EINVAL;
419 goto error;
420 }
421 mdev->bootaddr = boot_addr;
422 error:
423 return rc;
424 }
425
426 /**
427 * mic_x100_load_firmware - Load firmware to MIC.
428 * @mdev: pointer to mic_device instance
429 * @buf: buffer containing boot string including firmware/ramdisk path.
430 *
431 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
432 */
433 static int
mic_x100_load_firmware(struct mic_device * mdev,const char * buf)434 mic_x100_load_firmware(struct mic_device *mdev, const char *buf)
435 {
436 int rc;
437 const struct firmware *fw;
438
439 rc = mic_x100_get_boot_addr(mdev);
440 if (rc)
441 return rc;
442 /* load OS */
443 rc = request_firmware(&fw, mdev->cosm_dev->firmware, &mdev->pdev->dev);
444 if (rc < 0) {
445 dev_err(&mdev->pdev->dev,
446 "ramdisk request_firmware failed: %d %s\n",
447 rc, mdev->cosm_dev->firmware);
448 return rc;
449 }
450 if (mdev->bootaddr > mdev->aper.len - fw->size) {
451 rc = -EINVAL;
452 dev_err(&mdev->pdev->dev, "%s %d rc %d bootaddr 0x%x\n",
453 __func__, __LINE__, rc, mdev->bootaddr);
454 goto error;
455 }
456 memcpy_toio(mdev->aper.va + mdev->bootaddr, fw->data, fw->size);
457 mdev->ops->write_spad(mdev, MIC_X100_FW_SIZE, fw->size);
458 if (!strcmp(mdev->cosm_dev->bootmode, "flash")) {
459 rc = -EINVAL;
460 dev_err(&mdev->pdev->dev, "%s %d rc %d\n",
461 __func__, __LINE__, rc);
462 goto error;
463 }
464 /* load command line */
465 rc = mic_x100_load_command_line(mdev, fw);
466 if (rc) {
467 dev_err(&mdev->pdev->dev, "%s %d rc %d\n",
468 __func__, __LINE__, rc);
469 goto error;
470 }
471 release_firmware(fw);
472 /* load ramdisk */
473 if (mdev->cosm_dev->ramdisk)
474 rc = mic_x100_load_ramdisk(mdev);
475
476 return rc;
477
478 error:
479 release_firmware(fw);
480 return rc;
481 }
482
483 /**
484 * mic_x100_get_postcode - Get postcode status from firmware.
485 * @mdev: pointer to mic_device instance
486 *
487 * RETURNS: postcode.
488 */
mic_x100_get_postcode(struct mic_device * mdev)489 static u32 mic_x100_get_postcode(struct mic_device *mdev)
490 {
491 return mic_mmio_read(&mdev->mmio, MIC_X100_POSTCODE);
492 }
493
494 /**
495 * mic_x100_smpt_set - Update an SMPT entry with a DMA address.
496 * @mdev: pointer to mic_device instance
497 *
498 * RETURNS: none.
499 */
500 static void
mic_x100_smpt_set(struct mic_device * mdev,dma_addr_t dma_addr,u8 index)501 mic_x100_smpt_set(struct mic_device *mdev, dma_addr_t dma_addr, u8 index)
502 {
503 #define SNOOP_ON (0 << 0)
504 #define SNOOP_OFF (1 << 0)
505 /*
506 * Sbox Smpt Reg Bits:
507 * Bits 31:2 Host address
508 * Bits 1 RSVD
509 * Bits 0 No snoop
510 */
511 #define BUILD_SMPT(NO_SNOOP, HOST_ADDR) \
512 (u32)(((HOST_ADDR) << 2) | ((NO_SNOOP) & 0x01))
513
514 uint32_t smpt_reg_val = BUILD_SMPT(SNOOP_ON,
515 dma_addr >> mdev->smpt->info.page_shift);
516 mic_mmio_write(&mdev->mmio, smpt_reg_val,
517 MIC_X100_SBOX_BASE_ADDRESS +
518 MIC_X100_SBOX_SMPT00 + (4 * index));
519 }
520
521 /**
522 * mic_x100_smpt_hw_init - Initialize SMPT X100 specific fields.
523 * @mdev: pointer to mic_device instance
524 *
525 * RETURNS: none.
526 */
mic_x100_smpt_hw_init(struct mic_device * mdev)527 static void mic_x100_smpt_hw_init(struct mic_device *mdev)
528 {
529 struct mic_smpt_hw_info *info = &mdev->smpt->info;
530
531 info->num_reg = 32;
532 info->page_shift = 34;
533 info->page_size = (1ULL << info->page_shift);
534 info->base = 0x8000000000ULL;
535 }
536
537 struct mic_smpt_ops mic_x100_smpt_ops = {
538 .init = mic_x100_smpt_hw_init,
539 .set = mic_x100_smpt_set,
540 };
541
mic_x100_dma_filter(struct dma_chan * chan,void * param)542 static bool mic_x100_dma_filter(struct dma_chan *chan, void *param)
543 {
544 if (chan->device->dev->parent == (struct device *)param)
545 return true;
546 return false;
547 }
548
549 struct mic_hw_ops mic_x100_ops = {
550 .aper_bar = MIC_X100_APER_BAR,
551 .mmio_bar = MIC_X100_MMIO_BAR,
552 .read_spad = mic_x100_read_spad,
553 .write_spad = mic_x100_write_spad,
554 .send_intr = mic_x100_send_intr,
555 .ack_interrupt = mic_x100_ack_interrupt,
556 .intr_workarounds = mic_x100_intr_workarounds,
557 .reset = mic_x100_hw_reset,
558 .reset_fw_ready = mic_x100_reset_fw_ready,
559 .is_fw_ready = mic_x100_is_fw_ready,
560 .send_firmware_intr = mic_x100_send_firmware_intr,
561 .load_mic_fw = mic_x100_load_firmware,
562 .get_postcode = mic_x100_get_postcode,
563 .dma_filter = mic_x100_dma_filter,
564 };
565
566 struct mic_hw_intr_ops mic_x100_intr_ops = {
567 .intr_init = mic_x100_hw_intr_init,
568 .enable_interrupts = mic_x100_enable_interrupts,
569 .disable_interrupts = mic_x100_disable_interrupts,
570 .program_msi_to_src_map = mic_x100_program_msi_to_src_map,
571 .read_msi_to_src_map = mic_x100_read_msi_to_src_map,
572 };
573
