Lines Matching +full:combined +full:- +full:power +full:- +full:req
2 * Copyright(c) 2015 - 2018 Intel Corporation.
24 * - Redistributions of source code must retain the above copyright
26 * - Redistributions in binary form must reproduce the above copyright
30 * - Neither the name of Intel Corporation nor the names of its
64 /* must be a power of 2 >= 64 <= 32768 */
254 * sdma_state_name() - return state string from enum
264 kref_get(&ss->kref); in sdma_get()
272 complete(&ss->comp); in sdma_complete()
277 kref_put(&ss->kref, sdma_complete); in sdma_put()
283 wait_for_completion(&ss->comp); in sdma_finalput()
291 write_kctxt_csr(sde->dd, sde->this_idx, offset0, value); in write_sde_csr()
298 return read_kctxt_csr(sde->dd, sde->this_idx, offset0); in read_sde_csr()
302 * sdma_wait_for_packet_egress() - wait for the VL FIFO occupancy for
308 u64 off = 8 * sde->this_idx; in sdma_wait_for_packet_egress()
309 struct hfi1_devdata *dd = sde->dd; in sdma_wait_for_packet_egress()
326 /* timed out - bounce the link */ in sdma_wait_for_packet_egress()
328 __func__, sde->this_idx, (u32)reg); in sdma_wait_for_packet_egress()
329 queue_work(dd->pport->link_wq, in sdma_wait_for_packet_egress()
330 &dd->pport->link_bounce_work); in sdma_wait_for_packet_egress()
338 * sdma_wait() - wait for packet egress to complete for all SDMA engines,
345 for (i = 0; i < dd->num_sdma; i++) { in sdma_wait()
346 struct sdma_engine *sde = &dd->per_sdma[i]; in sdma_wait()
356 if (!(sde->dd->flags & HFI1_HAS_SDMA_TIMEOUT)) in sdma_set_desc_cnt()
369 struct iowait *wait = tx->wait; in complete_tx()
370 callback_t complete = tx->complete; in complete_tx()
373 trace_hfi1_sdma_out_sn(sde, tx->sn); in complete_tx()
374 if (WARN_ON_ONCE(sde->head_sn != tx->sn)) in complete_tx()
375 dd_dev_err(sde->dd, "expected %llu got %llu\n", in complete_tx()
376 sde->head_sn, tx->sn); in complete_tx()
377 sde->head_sn++; in complete_tx()
379 __sdma_txclean(sde->dd, tx); in complete_tx()
390 * - in the descq ring
391 * - in the flush list
397 * - From a work queue item
398 * - Directly from the state machine just before setting the
413 spin_lock_irqsave(&sde->flushlist_lock, flags); in sdma_flush()
415 list_splice_init(&sde->flushlist, &flushlist); in sdma_flush()
416 spin_unlock_irqrestore(&sde->flushlist_lock, flags); in sdma_flush()
424 seq = read_seqbegin(&sde->waitlock); in sdma_flush()
425 if (!list_empty(&sde->dmawait)) { in sdma_flush()
426 write_seqlock(&sde->waitlock); in sdma_flush()
427 list_for_each_entry_safe(w, nw, &sde->dmawait, list) { in sdma_flush()
428 if (w->wakeup) { in sdma_flush()
429 w->wakeup(w, SDMA_AVAIL_REASON); in sdma_flush()
430 list_del_init(&w->list); in sdma_flush()
433 write_sequnlock(&sde->waitlock); in sdma_flush()
435 } while (read_seqretry(&sde->waitlock, seq)); in sdma_flush()
454 write_seqlock_irqsave(&sde->head_lock, flags); in sdma_field_flush()
457 write_sequnlock_irqrestore(&sde->head_lock, flags); in sdma_field_flush()
474 dd_dev_err(sde->dd, in sdma_err_halt_wait()
475 "SDMA engine %d - timeout waiting for engine to halt\n", in sdma_err_halt_wait()
476 sde->this_idx); in sdma_err_halt_wait()
491 if (!is_bx(sde->dd) && HFI1_CAP_IS_KSET(SDMA_AHG)) { in sdma_err_progress_check_schedule()
493 struct hfi1_devdata *dd = sde->dd; in sdma_err_progress_check_schedule()
495 for (index = 0; index < dd->num_sdma; index++) { in sdma_err_progress_check_schedule()
496 struct sdma_engine *curr_sdma = &dd->per_sdma[index]; in sdma_err_progress_check_schedule()
499 curr_sdma->progress_check_head = in sdma_err_progress_check_schedule()
500 curr_sdma->descq_head; in sdma_err_progress_check_schedule()
502 dd_dev_err(sde->dd, in sdma_err_progress_check_schedule()
503 "SDMA engine %d - check scheduled\n", in sdma_err_progress_check_schedule()
504 sde->this_idx); in sdma_err_progress_check_schedule()
505 mod_timer(&sde->err_progress_check_timer, jiffies + 10); in sdma_err_progress_check_schedule()
514 dd_dev_err(sde->dd, "SDE progress check event\n"); in sdma_err_progress_check()
515 for (index = 0; index < sde->dd->num_sdma; index++) { in sdma_err_progress_check()
516 struct sdma_engine *curr_sde = &sde->dd->per_sdma[index]; in sdma_err_progress_check()
523 * We must lock interrupts when acquiring sde->lock, in sdma_err_progress_check()
527 spin_lock_irqsave(&curr_sde->tail_lock, flags); in sdma_err_progress_check()
528 write_seqlock(&curr_sde->head_lock); in sdma_err_progress_check()
530 /* skip non-running queues */ in sdma_err_progress_check()
531 if (curr_sde->state.current_state != sdma_state_s99_running) { in sdma_err_progress_check()
532 write_sequnlock(&curr_sde->head_lock); in sdma_err_progress_check()
533 spin_unlock_irqrestore(&curr_sde->tail_lock, flags); in sdma_err_progress_check()
537 if ((curr_sde->descq_head != curr_sde->descq_tail) && in sdma_err_progress_check()
538 (curr_sde->descq_head == in sdma_err_progress_check()
539 curr_sde->progress_check_head)) in sdma_err_progress_check()
542 write_sequnlock(&curr_sde->head_lock); in sdma_err_progress_check()
543 spin_unlock_irqrestore(&curr_sde->tail_lock, flags); in sdma_err_progress_check()
545 schedule_work(&sde->err_halt_worker); in sdma_err_progress_check()
556 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", in sdma_hw_clean_up_task()
557 sde->this_idx, slashstrip(__FILE__), __LINE__, in sdma_hw_clean_up_task()
572 return sde->tx_ring[sde->tx_head & sde->sdma_mask]; in get_txhead()
589 head = sde->descq_head & sde->sdma_mask; in sdma_flush_descq()
590 tail = sde->descq_tail & sde->sdma_mask; in sdma_flush_descq()
593 head = ++sde->descq_head & sde->sdma_mask; in sdma_flush_descq()
595 if (txp && txp->next_descq_idx == head) { in sdma_flush_descq()
597 sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL; in sdma_flush_descq()
613 spin_lock_irqsave(&sde->tail_lock, flags); in sdma_sw_clean_up_task()
614 write_seqlock(&sde->head_lock); in sdma_sw_clean_up_task()
618 * - We are halted, so no more descriptors are getting retired. in sdma_sw_clean_up_task()
619 * - We are not running, so no one is submitting new work. in sdma_sw_clean_up_task()
620 * - Only we can send the e40_sw_cleaned, so we can't start in sdma_sw_clean_up_task()
632 * latest physical hardware head - we are not running so speed does in sdma_sw_clean_up_task()
644 sde->descq_tail = 0; in sdma_sw_clean_up_task()
645 sde->descq_head = 0; in sdma_sw_clean_up_task()
646 sde->desc_avail = sdma_descq_freecnt(sde); in sdma_sw_clean_up_task()
647 *sde->head_dma = 0; in sdma_sw_clean_up_task()
651 write_sequnlock(&sde->head_lock); in sdma_sw_clean_up_task()
652 spin_unlock_irqrestore(&sde->tail_lock, flags); in sdma_sw_clean_up_task()
657 struct sdma_state *ss = &sde->state; in sdma_sw_tear_down()
663 atomic_set(&sde->dd->sdma_unfreeze_count, -1); in sdma_sw_tear_down()
664 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); in sdma_sw_tear_down()
669 tasklet_hi_schedule(&sde->sdma_hw_clean_up_task); in sdma_start_hw_clean_up()
675 struct sdma_state *ss = &sde->state; in sdma_set_state()
681 sdma_state_names[ss->current_state], in sdma_set_state()
685 ss->previous_state = ss->current_state; in sdma_set_state()
686 ss->previous_op = ss->current_op; in sdma_set_state()
687 ss->current_state = next_state; in sdma_set_state()
689 if (ss->previous_state != sdma_state_s99_running && in sdma_set_state()
706 ss->go_s99_running = 0; in sdma_set_state()
709 ss->go_s99_running = 1; in sdma_set_state()
711 ss->current_op = op; in sdma_set_state()
712 sdma_sendctrl(sde, ss->current_op); in sdma_set_state()
716 * sdma_get_descq_cnt() - called when device probed
733 /* count must be a power of 2 greater than 64 and less than in sdma_get_descq_cnt()
744 * sdma_engine_get_vl() - return vl for a given sdma engine
752 struct hfi1_devdata *dd = sde->dd; in sdma_engine_get_vl()
756 if (sde->this_idx >= TXE_NUM_SDMA_ENGINES) in sdma_engine_get_vl()
757 return -EINVAL; in sdma_engine_get_vl()
760 m = rcu_dereference(dd->sdma_map); in sdma_engine_get_vl()
763 return -EINVAL; in sdma_engine_get_vl()
765 vl = m->engine_to_vl[sde->this_idx]; in sdma_engine_get_vl()
772 * sdma_select_engine_vl() - select sdma engine
790 /* NOTE This should only happen if SC->VL changed after the initial in sdma_select_engine_vl()
800 m = rcu_dereference(dd->sdma_map); in sdma_select_engine_vl()
803 return &dd->per_sdma[0]; in sdma_select_engine_vl()
805 e = m->map[vl & m->mask]; in sdma_select_engine_vl()
806 rval = e->sde[selector & e->mask]; in sdma_select_engine_vl()
810 rval = !rval ? &dd->per_sdma[0] : rval; in sdma_select_engine_vl()
811 trace_hfi1_sdma_engine_select(dd, selector, vl, rval->this_idx); in sdma_select_engine_vl()
816 * sdma_select_engine_sc() - select sdma engine
859 * sdma_select_user_engine() - select sdma engine based on user setup
880 if (current->nr_cpus_allowed != 1) in sdma_select_user_engine()
885 rht_node = rhashtable_lookup(dd->sdma_rht, &cpu_id, in sdma_select_user_engine()
888 if (rht_node && rht_node->map[vl]) { in sdma_select_user_engine()
889 struct sdma_rht_map_elem *map = rht_node->map[vl]; in sdma_select_user_engine()
891 sde = map->sde[selector & map->mask]; in sdma_select_user_engine()
906 for (i = 0; i < roundup_pow_of_two(map->ctr ? : 1) - map->ctr; i++) in sdma_populate_sde_map()
907 map->sde[map->ctr + i] = map->sde[i]; in sdma_populate_sde_map()
916 for (i = 0; i < map->ctr; i++) { in sdma_cleanup_sde_map()
917 if (map->sde[i] == sde) { in sdma_cleanup_sde_map()
918 memmove(&map->sde[i], &map->sde[i + 1], in sdma_cleanup_sde_map()
919 (map->ctr - i - 1) * sizeof(map->sde[0])); in sdma_cleanup_sde_map()
920 map->ctr--; in sdma_cleanup_sde_map()
921 pow = roundup_pow_of_two(map->ctr ? : 1); in sdma_cleanup_sde_map()
922 map->mask = pow - 1; in sdma_cleanup_sde_map()
937 struct hfi1_devdata *dd = sde->dd; in sdma_set_cpu_to_sde_map()
944 if (unlikely(vl < 0 || vl >= ARRAY_SIZE(rht_node->map))) in sdma_set_cpu_to_sde_map()
945 return -EINVAL; in sdma_set_cpu_to_sde_map()
949 return -ENOMEM; in sdma_set_cpu_to_sde_map()
954 return -ENOMEM; in sdma_set_cpu_to_sde_map()
961 dd_dev_warn(sde->dd, "Invalid CPU mask\n"); in sdma_set_cpu_to_sde_map()
962 ret = -EINVAL; in sdma_set_cpu_to_sde_map()
973 if (cpumask_test_cpu(cpu, &sde->cpu_mask)) { in sdma_set_cpu_to_sde_map()
978 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu, in sdma_set_cpu_to_sde_map()
983 ret = -ENOMEM; in sdma_set_cpu_to_sde_map()
987 rht_node->map[vl] = kzalloc(sz, GFP_KERNEL); in sdma_set_cpu_to_sde_map()
988 if (!rht_node->map[vl]) { in sdma_set_cpu_to_sde_map()
990 ret = -ENOMEM; in sdma_set_cpu_to_sde_map()
993 rht_node->cpu_id = cpu; in sdma_set_cpu_to_sde_map()
994 rht_node->map[vl]->mask = 0; in sdma_set_cpu_to_sde_map()
995 rht_node->map[vl]->ctr = 1; in sdma_set_cpu_to_sde_map()
996 rht_node->map[vl]->sde[0] = sde; in sdma_set_cpu_to_sde_map()
998 ret = rhashtable_insert_fast(dd->sdma_rht, in sdma_set_cpu_to_sde_map()
999 &rht_node->node, in sdma_set_cpu_to_sde_map()
1002 kfree(rht_node->map[vl]); in sdma_set_cpu_to_sde_map()
1004 dd_dev_err(sde->dd, "Failed to set process to sde affinity for cpu %lu\n", in sdma_set_cpu_to_sde_map()
1013 if (!rht_node->map[vl]) in sdma_set_cpu_to_sde_map()
1014 rht_node->map[vl] = kzalloc(sz, GFP_KERNEL); in sdma_set_cpu_to_sde_map()
1016 if (!rht_node->map[vl]) { in sdma_set_cpu_to_sde_map()
1017 ret = -ENOMEM; in sdma_set_cpu_to_sde_map()
1021 rht_node->map[vl]->ctr++; in sdma_set_cpu_to_sde_map()
1022 ctr = rht_node->map[vl]->ctr; in sdma_set_cpu_to_sde_map()
1023 rht_node->map[vl]->sde[ctr - 1] = sde; in sdma_set_cpu_to_sde_map()
1025 rht_node->map[vl]->mask = pow - 1; in sdma_set_cpu_to_sde_map()
1028 sdma_populate_sde_map(rht_node->map[vl]); in sdma_set_cpu_to_sde_map()
1041 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu, in sdma_set_cpu_to_sde_map()
1049 if (rht_node->map[i]) in sdma_set_cpu_to_sde_map()
1050 sdma_cleanup_sde_map(rht_node->map[i], in sdma_set_cpu_to_sde_map()
1055 if (!rht_node->map[i]) in sdma_set_cpu_to_sde_map()
1058 if (rht_node->map[i]->ctr) { in sdma_set_cpu_to_sde_map()
1065 ret = rhashtable_remove_fast(dd->sdma_rht, in sdma_set_cpu_to_sde_map()
1066 &rht_node->node, in sdma_set_cpu_to_sde_map()
1071 kfree(rht_node->map[i]); in sdma_set_cpu_to_sde_map()
1078 cpumask_copy(&sde->cpu_mask, new_mask); in sdma_set_cpu_to_sde_map()
1090 if (cpumask_empty(&sde->cpu_mask)) in sdma_get_cpu_to_sde_map()
1093 cpumap_print_to_pagebuf(true, buf, &sde->cpu_mask); in sdma_get_cpu_to_sde_map()
1104 kfree(rht_node->map[i]); in sdma_rht_free()
1110 * sdma_seqfile_dump_cpu_list() - debugfs dump the cpu to sdma mappings
1124 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpuid, in sdma_seqfile_dump_cpu_list()
1131 if (!rht_node->map[i] || !rht_node->map[i]->ctr) in sdma_seqfile_dump_cpu_list()
1136 for (j = 0; j < rht_node->map[i]->ctr; j++) { in sdma_seqfile_dump_cpu_list()
1137 if (!rht_node->map[i]->sde[j]) in sdma_seqfile_dump_cpu_list()
1144 rht_node->map[i]->sde[j]->this_idx); in sdma_seqfile_dump_cpu_list()
1159 for (i = 0; m && i < m->actual_vls; i++) in sdma_map_free()
1160 kfree(m->map[i]); in sdma_map_free()
1175 * sdma_map_init - called when # vls change
1183 * vl_engines is used to specify a non-uniform vl/engine loading. NULL
1192 * If either the num_vls or num_sdma are non-power of 2, the array sizes
1194 * up to the next highest power of 2 and the first entry is reused
1209 if (!(dd->flags & HFI1_HAS_SEND_DMA)) in sdma_map_init()
1214 sde_per_vl = dd->num_sdma / num_vls; in sdma_map_init()
1216 extra = dd->num_sdma % num_vls; in sdma_map_init()
1219 for (i = num_vls - 1; i >= 0; i--, extra--) in sdma_map_init()
1230 newmap->actual_vls = num_vls; in sdma_map_init()
1231 newmap->vls = roundup_pow_of_two(num_vls); in sdma_map_init()
1232 newmap->mask = (1 << ilog2(newmap->vls)) - 1; in sdma_map_init()
1233 /* initialize back-map */ in sdma_map_init()
1235 newmap->engine_to_vl[i] = -1; in sdma_map_init()
1236 for (i = 0; i < newmap->vls; i++) { in sdma_map_init()
1240 if (i < newmap->actual_vls) { in sdma_map_init()
1244 newmap->map[i] = kzalloc( in sdma_map_init()
1248 if (!newmap->map[i]) in sdma_map_init()
1250 newmap->map[i]->mask = (1 << ilog2(sz)) - 1; in sdma_map_init()
1253 newmap->map[i]->sde[j] = in sdma_map_init()
1254 &dd->per_sdma[engine]; in sdma_map_init()
1259 /* assign back-map */ in sdma_map_init()
1261 newmap->engine_to_vl[first_engine + j] = i; in sdma_map_init()
1263 /* just re-use entry without allocating */ in sdma_map_init()
1264 newmap->map[i] = newmap->map[i % num_vls]; in sdma_map_init()
1269 spin_lock_irq(&dd->sde_map_lock); in sdma_map_init()
1270 oldmap = rcu_dereference_protected(dd->sdma_map, in sdma_map_init()
1271 lockdep_is_held(&dd->sde_map_lock)); in sdma_map_init()
1274 rcu_assign_pointer(dd->sdma_map, newmap); in sdma_map_init()
1276 spin_unlock_irq(&dd->sde_map_lock); in sdma_map_init()
1279 call_rcu(&oldmap->list, sdma_map_rcu_callback); in sdma_map_init()
1284 return -ENOMEM; in sdma_map_init()
1300 if (dd->sdma_pad_dma) { in sdma_clean()
1301 dma_free_coherent(&dd->pcidev->dev, SDMA_PAD, in sdma_clean()
1302 (void *)dd->sdma_pad_dma, in sdma_clean()
1303 dd->sdma_pad_phys); in sdma_clean()
1304 dd->sdma_pad_dma = NULL; in sdma_clean()
1305 dd->sdma_pad_phys = 0; in sdma_clean()
1307 if (dd->sdma_heads_dma) { in sdma_clean()
1308 dma_free_coherent(&dd->pcidev->dev, dd->sdma_heads_size, in sdma_clean()
1309 (void *)dd->sdma_heads_dma, in sdma_clean()
1310 dd->sdma_heads_phys); in sdma_clean()
1311 dd->sdma_heads_dma = NULL; in sdma_clean()
1312 dd->sdma_heads_phys = 0; in sdma_clean()
1314 for (i = 0; dd->per_sdma && i < num_engines; ++i) { in sdma_clean()
1315 sde = &dd->per_sdma[i]; in sdma_clean()
1317 sde->head_dma = NULL; in sdma_clean()
1318 sde->head_phys = 0; in sdma_clean()
1320 if (sde->descq) { in sdma_clean()
1322 &dd->pcidev->dev, in sdma_clean()
1323 sde->descq_cnt * sizeof(u64[2]), in sdma_clean()
1324 sde->descq, in sdma_clean()
1325 sde->descq_phys in sdma_clean()
1327 sde->descq = NULL; in sdma_clean()
1328 sde->descq_phys = 0; in sdma_clean()
1330 kvfree(sde->tx_ring); in sdma_clean()
1331 sde->tx_ring = NULL; in sdma_clean()
1333 spin_lock_irq(&dd->sde_map_lock); in sdma_clean()
1334 sdma_map_free(rcu_access_pointer(dd->sdma_map)); in sdma_clean()
1335 RCU_INIT_POINTER(dd->sdma_map, NULL); in sdma_clean()
1336 spin_unlock_irq(&dd->sde_map_lock); in sdma_clean()
1338 kfree(dd->per_sdma); in sdma_clean()
1339 dd->per_sdma = NULL; in sdma_clean()
1341 if (dd->sdma_rht) { in sdma_clean()
1342 rhashtable_free_and_destroy(dd->sdma_rht, sdma_rht_free, NULL); in sdma_clean()
1343 kfree(dd->sdma_rht); in sdma_clean()
1344 dd->sdma_rht = NULL; in sdma_clean()
1349 * sdma_init() - called when device probed
1357 * 0 - success, -errno on failure
1366 struct hfi1_pportdata *ppd = dd->pport + port; in sdma_init()
1370 int ret = -ENOMEM; in sdma_init()
1392 init_waitqueue_head(&dd->sdma_unfreeze_wq); in sdma_init()
1393 atomic_set(&dd->sdma_unfreeze_count, 0); in sdma_init()
1400 dd->per_sdma = kcalloc_node(num_engines, sizeof(*dd->per_sdma), in sdma_init()
1401 GFP_KERNEL, dd->node); in sdma_init()
1402 if (!dd->per_sdma) in sdma_init()
1407 dd->default_desc1 = in sdma_init()
1410 dd->default_desc1 = in sdma_init()
1418 sde = &dd->per_sdma[this_idx]; in sdma_init()
1419 sde->dd = dd; in sdma_init()
1420 sde->ppd = ppd; in sdma_init()
1421 sde->this_idx = this_idx; in sdma_init()
1422 sde->descq_cnt = descq_cnt; in sdma_init()
1423 sde->desc_avail = sdma_descq_freecnt(sde); in sdma_init()
1424 sde->sdma_shift = ilog2(descq_cnt); in sdma_init()
1425 sde->sdma_mask = (1 << sde->sdma_shift) - 1; in sdma_init()
1428 sde->int_mask = (u64)1 << (0 * TXE_NUM_SDMA_ENGINES + in sdma_init()
1430 sde->progress_mask = (u64)1 << (1 * TXE_NUM_SDMA_ENGINES + in sdma_init()
1432 sde->idle_mask = (u64)1 << (2 * TXE_NUM_SDMA_ENGINES + in sdma_init()
1434 /* Create a combined mask to cover all 3 interrupt sources */ in sdma_init()
1435 sde->imask = sde->int_mask | sde->progress_mask | in sdma_init()
1436 sde->idle_mask; in sdma_init()
1438 spin_lock_init(&sde->tail_lock); in sdma_init()
1439 seqlock_init(&sde->head_lock); in sdma_init()
1440 spin_lock_init(&sde->senddmactrl_lock); in sdma_init()
1441 spin_lock_init(&sde->flushlist_lock); in sdma_init()
1442 seqlock_init(&sde->waitlock); in sdma_init()
1444 sde->ahg_bits = 0xfffffffe00000000ULL; in sdma_init()
1449 kref_init(&sde->state.kref); in sdma_init()
1450 init_completion(&sde->state.comp); in sdma_init()
1452 INIT_LIST_HEAD(&sde->flushlist); in sdma_init()
1453 INIT_LIST_HEAD(&sde->dmawait); in sdma_init()
1455 sde->tail_csr = in sdma_init()
1458 tasklet_setup(&sde->sdma_hw_clean_up_task, in sdma_init()
1460 tasklet_setup(&sde->sdma_sw_clean_up_task, in sdma_init()
1462 INIT_WORK(&sde->err_halt_worker, sdma_err_halt_wait); in sdma_init()
1463 INIT_WORK(&sde->flush_worker, sdma_field_flush); in sdma_init()
1465 sde->progress_check_head = 0; in sdma_init()
1467 timer_setup(&sde->err_progress_check_timer, in sdma_init()
1470 sde->descq = dma_alloc_coherent(&dd->pcidev->dev, in sdma_init()
1472 &sde->descq_phys, GFP_KERNEL); in sdma_init()
1473 if (!sde->descq) in sdma_init()
1475 sde->tx_ring = in sdma_init()
1478 GFP_KERNEL, dd->node); in sdma_init()
1479 if (!sde->tx_ring) in sdma_init()
1483 dd->sdma_heads_size = L1_CACHE_BYTES * num_engines; in sdma_init()
1485 dd->sdma_heads_dma = dma_alloc_coherent(&dd->pcidev->dev, in sdma_init()
1486 dd->sdma_heads_size, in sdma_init()
1487 &dd->sdma_heads_phys, in sdma_init()
1489 if (!dd->sdma_heads_dma) { in sdma_init()
1495 dd->sdma_pad_dma = dma_alloc_coherent(&dd->pcidev->dev, SDMA_PAD, in sdma_init()
1496 &dd->sdma_pad_phys, GFP_KERNEL); in sdma_init()
1497 if (!dd->sdma_pad_dma) { in sdma_init()
1503 curr_head = (void *)dd->sdma_heads_dma; in sdma_init()
1507 sde = &dd->per_sdma[this_idx]; in sdma_init()
1509 sde->head_dma = curr_head; in sdma_init()
1511 phys_offset = (unsigned long)sde->head_dma - in sdma_init()
1512 (unsigned long)dd->sdma_heads_dma; in sdma_init()
1513 sde->head_phys = dd->sdma_heads_phys + phys_offset; in sdma_init()
1516 dd->flags |= HFI1_HAS_SEND_DMA; in sdma_init()
1517 dd->flags |= idle_cnt ? HFI1_HAS_SDMA_TIMEOUT : 0; in sdma_init()
1518 dd->num_sdma = num_engines; in sdma_init()
1519 ret = sdma_map_init(dd, port, ppd->vls_operational, NULL); in sdma_init()
1525 ret = -ENOMEM; in sdma_init()
1535 dd->sdma_rht = tmp_sdma_rht; in sdma_init()
1537 dd_dev_info(dd, "SDMA num_sdma: %u\n", dd->num_sdma); in sdma_init()
1546 * sdma_all_running() - called when the link goes up
1557 for (i = 0; i < dd->num_sdma; ++i) { in sdma_all_running()
1558 sde = &dd->per_sdma[i]; in sdma_all_running()
1564 * sdma_all_idle() - called when the link goes down
1575 for (i = 0; i < dd->num_sdma; ++i) { in sdma_all_idle()
1576 sde = &dd->per_sdma[i]; in sdma_all_idle()
1582 * sdma_start() - called to kick off state processing for all engines
1595 for (i = 0; i < dd->num_sdma; ++i) { in sdma_start()
1596 sde = &dd->per_sdma[i]; in sdma_start()
1602 * sdma_exit() - used when module is removed
1610 for (this_idx = 0; dd->per_sdma && this_idx < dd->num_sdma; in sdma_exit()
1612 sde = &dd->per_sdma[this_idx]; in sdma_exit()
1613 if (!list_empty(&sde->dmawait)) in sdma_exit()
1615 sde->this_idx); in sdma_exit()
1618 del_timer_sync(&sde->err_progress_check_timer); in sdma_exit()
1625 sdma_finalput(&sde->state); in sdma_exit()
1639 &dd->pcidev->dev, in sdma_unmap_desc()
1646 &dd->pcidev->dev, in sdma_unmap_desc()
1660 return (tx->descp[0].qw[1] & SDMA_DESC1_HEADER_MODE_SMASK) in ahg_mode()
1665 * __sdma_txclean() - clean tx of mappings, descp *kmalloc's
1670 * by the ULP to toss an in-process tx build.
1681 if (tx->num_desc) { in __sdma_txclean()
1685 sdma_unmap_desc(dd, &tx->descp[0]); in __sdma_txclean()
1689 for (i = 1 + skip; i < tx->num_desc; i++) in __sdma_txclean()
1690 sdma_unmap_desc(dd, &tx->descp[i]); in __sdma_txclean()
1691 tx->num_desc = 0; in __sdma_txclean()
1693 kfree(tx->coalesce_buf); in __sdma_txclean()
1694 tx->coalesce_buf = NULL; in __sdma_txclean()
1696 if (unlikely(tx->desc_limit > ARRAY_SIZE(tx->descs))) { in __sdma_txclean()
1697 tx->desc_limit = ARRAY_SIZE(tx->descs); in __sdma_txclean()
1698 kfree(tx->descp); in __sdma_txclean()
1704 struct hfi1_devdata *dd = sde->dd; in sdma_gethead()
1709 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", in sdma_gethead()
1710 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); in sdma_gethead()
1715 (dd->flags & HFI1_HAS_SDMA_TIMEOUT); in sdma_gethead()
1717 (u16)le64_to_cpu(*sde->head_dma) : in sdma_gethead()
1726 swhead = sde->descq_head & sde->sdma_mask; in sdma_gethead()
1728 swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; in sdma_gethead()
1729 cnt = sde->descq_cnt; in sdma_gethead()
1744 sde->this_idx, in sdma_gethead()
1772 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, in sdma_desc_avail()
1774 dd_dev_err(sde->dd, "avail: %u\n", avail); in sdma_desc_avail()
1778 seq = read_seqbegin(&sde->waitlock); in sdma_desc_avail()
1779 if (!list_empty(&sde->dmawait)) { in sdma_desc_avail()
1781 write_seqlock(&sde->waitlock); in sdma_desc_avail()
1786 &sde->dmawait, in sdma_desc_avail()
1790 if (!wait->wakeup) in sdma_desc_avail()
1798 avail -= num_desc; in sdma_desc_avail()
1799 /* Find the top-priority wait memeber */ in sdma_desc_avail()
1808 list_del_init(&wait->list); in sdma_desc_avail()
1811 write_sequnlock(&sde->waitlock); in sdma_desc_avail()
1814 } while (read_seqretry(&sde->waitlock, seq)); in sdma_desc_avail()
1816 /* Schedule the top-priority entry first */ in sdma_desc_avail()
1818 waits[tidx]->wakeup(waits[tidx], SDMA_AVAIL_REASON); in sdma_desc_avail()
1822 waits[i]->wakeup(waits[i], SDMA_AVAIL_REASON); in sdma_desc_avail()
1843 swhead = sde->descq_head & sde->sdma_mask; in sdma_make_progress()
1847 swhead = ++sde->descq_head & sde->sdma_mask; in sdma_make_progress()
1850 if (txp && txp->next_descq_idx == swhead) { in sdma_make_progress()
1852 sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL; in sdma_make_progress()
1870 if ((status & sde->idle_mask) && !idle_check_done) { in sdma_make_progress()
1873 swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; in sdma_make_progress()
1881 sde->last_status = status; in sdma_make_progress()
1887 * sdma_engine_interrupt() - interrupt handler for engine
1898 write_seqlock(&sde->head_lock); in sdma_engine_interrupt()
1900 if (status & sde->idle_mask) in sdma_engine_interrupt()
1901 sde->idle_int_cnt++; in sdma_engine_interrupt()
1902 else if (status & sde->progress_mask) in sdma_engine_interrupt()
1903 sde->progress_int_cnt++; in sdma_engine_interrupt()
1904 else if (status & sde->int_mask) in sdma_engine_interrupt()
1905 sde->sdma_int_cnt++; in sdma_engine_interrupt()
1907 write_sequnlock(&sde->head_lock); in sdma_engine_interrupt()
1911 * sdma_engine_error() - error handler for engine
1920 dd_dev_err(sde->dd, "CONFIG SDMA(%u) error status 0x%llx state %s\n", in sdma_engine_error()
1921 sde->this_idx, in sdma_engine_error()
1923 sdma_state_names[sde->state.current_state]); in sdma_engine_error()
1925 spin_lock_irqsave(&sde->tail_lock, flags); in sdma_engine_error()
1926 write_seqlock(&sde->head_lock); in sdma_engine_error()
1930 dd_dev_err(sde->dd, in sdma_engine_error()
1932 sde->this_idx, in sdma_engine_error()
1934 sdma_state_names[sde->state.current_state]); in sdma_engine_error()
1937 write_sequnlock(&sde->head_lock); in sdma_engine_error()
1938 spin_unlock_irqrestore(&sde->tail_lock, flags); in sdma_engine_error()
1948 dd_dev_err(sde->dd, "CONFIG SDMA(%u) senddmactrl E=%d I=%d H=%d C=%d\n", in sdma_sendctrl()
1949 sde->this_idx, in sdma_sendctrl()
1971 spin_lock_irqsave(&sde->senddmactrl_lock, flags); in sdma_sendctrl()
1973 sde->p_senddmactrl |= set_senddmactrl; in sdma_sendctrl()
1974 sde->p_senddmactrl &= ~clr_senddmactrl; in sdma_sendctrl()
1978 sde->p_senddmactrl | in sdma_sendctrl()
1981 write_sde_csr(sde, SD(CTRL), sde->p_senddmactrl); in sdma_sendctrl()
1983 spin_unlock_irqrestore(&sde->senddmactrl_lock, flags); in sdma_sendctrl()
1993 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", in sdma_setlengen()
1994 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); in sdma_setlengen()
1998 * Set SendDmaLenGen and clear-then-set the MSB of the generation in sdma_setlengen()
2003 (sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT)); in sdma_setlengen()
2005 ((sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT)) | in sdma_setlengen()
2013 writeq(tail, sde->tail_csr); in sdma_update_tail()
2025 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", in sdma_hw_start_up()
2026 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); in sdma_hw_start_up()
2031 *sde->head_dma = 0; in sdma_hw_start_up()
2045 struct hfi1_devdata *dd = sde->dd; in set_sdma_integrity()
2058 struct hfi1_devdata *dd = sde->dd; in init_sdma_regs()
2061 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); in init_sdma_regs()
2064 write_sde_csr(sde, SD(BASE_ADDR), sde->descq_phys); in init_sdma_regs()
2069 write_sde_csr(sde, SD(HEAD_ADDR), sde->head_phys); in init_sdma_regs()
2072 ((u64)(credits * sde->this_idx) << in init_sdma_regs()
2086 csr = read_csr(sde->dd, reg); \
2087 dd_dev_err(sde->dd, "%36s 0x%016llx\n", #reg, csr); \
2092 dd_dev_err(sde->dd, "%36s[%02u] 0x%016llx\n", \
2093 #reg, sde->this_idx, csr); \
2097 csr = read_csr(sde->dd, reg + (8 * i)); \
2098 dd_dev_err(sde->dd, "%33s_%02u 0x%016llx\n", \
2152 head = sde->descq_head & sde->sdma_mask; in dump_sdma_state()
2153 tail = sde->descq_tail & sde->sdma_mask; in dump_sdma_state()
2156 dd_dev_err(sde->dd, in dump_sdma_state()
2158 sde->this_idx, head, tail, cnt, in dump_sdma_state()
2159 !list_empty(&sde->flushlist)); in dump_sdma_state()
2165 descqp = &sde->descq[head]; in dump_sdma_state()
2166 desc[0] = le64_to_cpu(descqp->qw[0]); in dump_sdma_state()
2167 desc[1] = le64_to_cpu(descqp->qw[1]); in dump_sdma_state()
2168 flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-'; in dump_sdma_state()
2170 'H' : '-'; in dump_sdma_state()
2171 flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-'; in dump_sdma_state()
2172 flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-'; in dump_sdma_state()
2179 dd_dev_err(sde->dd, in dump_sdma_state()
2182 dd_dev_err(sde->dd, in dump_sdma_state()
2186 dd_dev_err(sde->dd, in dump_sdma_state()
2198 head &= sde->sdma_mask; in dump_sdma_state()
2205 * sdma_seqfile_dump_sde() - debugfs dump of sde
2220 head = sde->descq_head & sde->sdma_mask; in sdma_seqfile_dump_sde()
2221 tail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; in sdma_seqfile_dump_sde()
2222 seq_printf(s, SDE_FMT, sde->this_idx, in sdma_seqfile_dump_sde()
2223 sde->cpu, in sdma_seqfile_dump_sde()
2224 sdma_state_name(sde->state.current_state), in sdma_seqfile_dump_sde()
2230 (unsigned long long)le64_to_cpu(*sde->head_dma), in sdma_seqfile_dump_sde()
2234 (unsigned long long)sde->last_status, in sdma_seqfile_dump_sde()
2235 (unsigned long long)sde->ahg_bits, in sdma_seqfile_dump_sde()
2236 sde->tx_tail, in sdma_seqfile_dump_sde()
2237 sde->tx_head, in sdma_seqfile_dump_sde()
2238 sde->descq_tail, in sdma_seqfile_dump_sde()
2239 sde->descq_head, in sdma_seqfile_dump_sde()
2240 !list_empty(&sde->flushlist), in sdma_seqfile_dump_sde()
2241 sde->descq_full_count, in sdma_seqfile_dump_sde()
2248 descqp = &sde->descq[head]; in sdma_seqfile_dump_sde()
2249 desc[0] = le64_to_cpu(descqp->qw[0]); in sdma_seqfile_dump_sde()
2250 desc[1] = le64_to_cpu(descqp->qw[1]); in sdma_seqfile_dump_sde()
2251 flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-'; in sdma_seqfile_dump_sde()
2253 'H' : '-'; in sdma_seqfile_dump_sde()
2254 flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-'; in sdma_seqfile_dump_sde()
2255 flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-'; in sdma_seqfile_dump_sde()
2273 head = (head + 1) & sde->sdma_mask; in sdma_seqfile_dump_sde()
2283 u8 generation = (sde->descq_tail >> sde->sdma_shift) & 3; in add_gen()
2311 struct sdma_desc *descp = tx->descp; in submit_tx()
2314 tail = sde->descq_tail & sde->sdma_mask; in submit_tx()
2315 sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]); in submit_tx()
2316 sde->descq[tail].qw[1] = cpu_to_le64(add_gen(sde, descp->qw[1])); in submit_tx()
2317 trace_hfi1_sdma_descriptor(sde, descp->qw[0], descp->qw[1], in submit_tx()
2318 tail, &sde->descq[tail]); in submit_tx()
2319 tail = ++sde->descq_tail & sde->sdma_mask; in submit_tx()
2323 for (i = 1; i < tx->num_desc; i++, descp++) { in submit_tx()
2326 sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]); in submit_tx()
2329 qw1 = descp->qw[1]; in submit_tx()
2330 skip--; in submit_tx()
2332 /* replace generation with real one for non-edits */ in submit_tx()
2333 qw1 = add_gen(sde, descp->qw[1]); in submit_tx()
2335 sde->descq[tail].qw[1] = cpu_to_le64(qw1); in submit_tx()
2336 trace_hfi1_sdma_descriptor(sde, descp->qw[0], qw1, in submit_tx()
2337 tail, &sde->descq[tail]); in submit_tx()
2338 tail = ++sde->descq_tail & sde->sdma_mask; in submit_tx()
2340 tx->next_descq_idx = tail; in submit_tx()
2342 tx->sn = sde->tail_sn++; in submit_tx()
2343 trace_hfi1_sdma_in_sn(sde, tx->sn); in submit_tx()
2344 WARN_ON_ONCE(sde->tx_ring[sde->tx_tail & sde->sdma_mask]); in submit_tx()
2346 sde->tx_ring[sde->tx_tail++ & sde->sdma_mask] = tx; in submit_tx()
2347 sde->desc_avail -= tx->num_desc; in submit_tx()
2362 sde->desc_avail = sdma_descq_freecnt(sde); in sdma_check_progress()
2363 if (tx->num_desc <= sde->desc_avail) in sdma_check_progress()
2364 return -EAGAIN; in sdma_check_progress()
2366 if (wait && iowait_ioww_to_iow(wait)->sleep) { in sdma_check_progress()
2370 (const seqcount_t *)&sde->head_lock.seqcount); in sdma_check_progress()
2371 ret = wait->iow->sleep(sde, wait, tx, seq, pkts_sent); in sdma_check_progress()
2372 if (ret == -EAGAIN) in sdma_check_progress()
2373 sde->desc_avail = sdma_descq_freecnt(sde); in sdma_check_progress()
2375 ret = -EBUSY; in sdma_check_progress()
2381 * sdma_send_txreq() - submit a tx req to ring
2387 * The call submits the tx into the ring. If a iowait structure is non-NULL
2391 * 0 - Success, -EINVAL - sdma_txreq incomplete, -EBUSY - no space in
2393 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
2405 if (unlikely(tx->tlen)) in sdma_send_txreq()
2406 return -EINVAL; in sdma_send_txreq()
2407 tx->wait = iowait_ioww_to_iow(wait); in sdma_send_txreq()
2408 spin_lock_irqsave(&sde->tail_lock, flags); in sdma_send_txreq()
2412 if (unlikely(tx->num_desc > sde->desc_avail)) in sdma_send_txreq()
2419 spin_unlock_irqrestore(&sde->tail_lock, flags); in sdma_send_txreq()
2424 tx->next_descq_idx = 0; in sdma_send_txreq()
2426 tx->sn = sde->tail_sn++; in sdma_send_txreq()
2427 trace_hfi1_sdma_in_sn(sde, tx->sn); in sdma_send_txreq()
2429 spin_lock(&sde->flushlist_lock); in sdma_send_txreq()
2430 list_add_tail(&tx->list, &sde->flushlist); in sdma_send_txreq()
2431 spin_unlock(&sde->flushlist_lock); in sdma_send_txreq()
2432 iowait_inc_wait_count(wait, tx->num_desc); in sdma_send_txreq()
2433 queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker); in sdma_send_txreq()
2434 ret = -ECOMM; in sdma_send_txreq()
2438 if (ret == -EAGAIN) { in sdma_send_txreq()
2442 sde->descq_full_count++; in sdma_send_txreq()
2447 * sdma_send_txlist() - submit a list of tx req to ring
2459 * If the iowait structure is non-NULL and not equal to the iowait list
2470 * 0 - Success,
2471 * -EINVAL - sdma_txreq incomplete, -EBUSY - no space in ring (wait == NULL)
2472 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
2483 spin_lock_irqsave(&sde->tail_lock, flags); in sdma_send_txlist()
2486 tx->wait = iowait_ioww_to_iow(wait); in sdma_send_txlist()
2489 if (unlikely(tx->num_desc > sde->desc_avail)) in sdma_send_txlist()
2491 if (unlikely(tx->tlen)) { in sdma_send_txlist()
2492 ret = -EINVAL; in sdma_send_txlist()
2495 list_del_init(&tx->list); in sdma_send_txlist()
2513 spin_unlock_irqrestore(&sde->tail_lock, flags); in sdma_send_txlist()
2517 spin_lock(&sde->flushlist_lock); in sdma_send_txlist()
2519 tx->wait = iowait_ioww_to_iow(wait); in sdma_send_txlist()
2520 list_del_init(&tx->list); in sdma_send_txlist()
2521 tx->next_descq_idx = 0; in sdma_send_txlist()
2523 tx->sn = sde->tail_sn++; in sdma_send_txlist()
2524 trace_hfi1_sdma_in_sn(sde, tx->sn); in sdma_send_txlist()
2526 list_add_tail(&tx->list, &sde->flushlist); in sdma_send_txlist()
2528 iowait_inc_wait_count(wait, tx->num_desc); in sdma_send_txlist()
2530 spin_unlock(&sde->flushlist_lock); in sdma_send_txlist()
2531 queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker); in sdma_send_txlist()
2532 ret = -ECOMM; in sdma_send_txlist()
2536 if (ret == -EAGAIN) { in sdma_send_txlist()
2540 sde->descq_full_count++; in sdma_send_txlist()
2548 spin_lock_irqsave(&sde->tail_lock, flags); in sdma_process_event()
2549 write_seqlock(&sde->head_lock); in sdma_process_event()
2553 if (sde->state.current_state == sdma_state_s99_running) in sdma_process_event()
2556 write_sequnlock(&sde->head_lock); in sdma_process_event()
2557 spin_unlock_irqrestore(&sde->tail_lock, flags); in sdma_process_event()
2563 struct sdma_state *ss = &sde->state; in __sdma_process_event()
2568 dd_dev_err(sde->dd, "CONFIG SDMA(%u) [%s] %s\n", sde->this_idx, in __sdma_process_event()
2569 sdma_state_names[ss->current_state], in __sdma_process_event()
2573 switch (ss->current_state) { in __sdma_process_event()
2586 ss->go_s99_running = 1; in __sdma_process_event()
2590 sdma_get(&sde->state); in __sdma_process_event()
2636 ss->go_s99_running = 1; in __sdma_process_event()
2643 schedule_work(&sde->err_halt_worker); in __sdma_process_event()
2646 ss->go_s99_running = 0; in __sdma_process_event()
2673 sdma_set_state(sde, ss->go_s99_running ? in __sdma_process_event()
2678 ss->go_s99_running = 1; in __sdma_process_event()
2687 ss->go_s99_running = 0; in __sdma_process_event()
2716 ss->go_s99_running = 1; in __sdma_process_event()
2724 schedule_work(&sde->err_halt_worker); in __sdma_process_event()
2731 atomic_dec(&sde->dd->sdma_unfreeze_count); in __sdma_process_event()
2732 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); in __sdma_process_event()
2755 ss->go_s99_running = 1; in __sdma_process_event()
2766 ss->go_s99_running = 0; in __sdma_process_event()
2775 ss->go_s99_running = 0; in __sdma_process_event()
2786 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2794 sdma_set_state(sde, ss->go_s99_running ? in __sdma_process_event()
2799 ss->go_s99_running = 1; in __sdma_process_event()
2808 ss->go_s99_running = 0; in __sdma_process_event()
2817 ss->go_s99_running = 0; in __sdma_process_event()
2828 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2834 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2839 ss->go_s99_running = 1; in __sdma_process_event()
2846 schedule_work(&sde->err_halt_worker); in __sdma_process_event()
2849 ss->go_s99_running = 0; in __sdma_process_event()
2858 ss->go_s99_running = 0; in __sdma_process_event()
2869 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2875 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2880 ss->go_s99_running = 1; in __sdma_process_event()
2887 schedule_work(&sde->err_halt_worker); in __sdma_process_event()
2890 ss->go_s99_running = 0; in __sdma_process_event()
2909 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2918 ss->go_s99_running = 1; in __sdma_process_event()
2927 ss->go_s99_running = 0; in __sdma_process_event()
2933 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2948 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2957 ss->go_s99_running = 1; in __sdma_process_event()
2961 atomic_dec(&sde->dd->sdma_unfreeze_count); in __sdma_process_event()
2962 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); in __sdma_process_event()
2969 ss->go_s99_running = 0; in __sdma_process_event()
2977 sdma_set_state(sde, ss->go_s99_running ? in __sdma_process_event()
2992 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
3016 schedule_work(&sde->err_halt_worker); in __sdma_process_event()
3022 ss->go_s99_running = 0; in __sdma_process_event()
3026 atomic_dec(&sde->dd->sdma_unfreeze_count); in __sdma_process_event()
3027 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); in __sdma_process_event()
3037 ss->last_event = event; in __sdma_process_event()
3043 * _extend_sdma_tx_descs() - helper to extend txreq
3060 if (unlikely((tx->num_desc == (MAX_DESC - 1)))) { in _extend_sdma_tx_descs()
3062 if (!tx->tlen) { in _extend_sdma_tx_descs()
3063 tx->desc_limit = MAX_DESC; in _extend_sdma_tx_descs()
3064 } else if (!tx->coalesce_buf) { in _extend_sdma_tx_descs()
3066 tx->coalesce_buf = kmalloc(tx->tlen + sizeof(u32), in _extend_sdma_tx_descs()
3068 if (!tx->coalesce_buf) in _extend_sdma_tx_descs()
3070 tx->coalesce_idx = 0; in _extend_sdma_tx_descs()
3075 if (unlikely(tx->num_desc == MAX_DESC)) in _extend_sdma_tx_descs()
3078 tx->descp = kmalloc_array( in _extend_sdma_tx_descs()
3082 if (!tx->descp) in _extend_sdma_tx_descs()
3086 tx->desc_limit = MAX_DESC - 1; in _extend_sdma_tx_descs()
3088 for (i = 0; i < tx->num_desc; i++) in _extend_sdma_tx_descs()
3089 tx->descp[i] = tx->descs[i]; in _extend_sdma_tx_descs()
3093 return -ENOMEM; in _extend_sdma_tx_descs()
3097 * ext_coal_sdma_tx_descs() - extend or coalesce sdma tx descriptors
3108 * <0 - error
3109 * 0 - coalescing, don't populate descriptor
3110 * 1 - continue with populating descriptor
3126 if (tx->coalesce_buf) { in ext_coal_sdma_tx_descs()
3129 return -EINVAL; in ext_coal_sdma_tx_descs()
3137 return -EINVAL; in ext_coal_sdma_tx_descs()
3140 memcpy(tx->coalesce_buf + tx->coalesce_idx, kvaddr, len); in ext_coal_sdma_tx_descs()
3141 tx->coalesce_idx += len; in ext_coal_sdma_tx_descs()
3146 if (tx->tlen - tx->coalesce_idx) in ext_coal_sdma_tx_descs()
3150 pad_len = tx->packet_len & (sizeof(u32) - 1); in ext_coal_sdma_tx_descs()
3152 pad_len = sizeof(u32) - pad_len; in ext_coal_sdma_tx_descs()
3153 memset(tx->coalesce_buf + tx->coalesce_idx, 0, pad_len); in ext_coal_sdma_tx_descs()
3155 tx->packet_len += pad_len; in ext_coal_sdma_tx_descs()
3156 tx->tlen += pad_len; in ext_coal_sdma_tx_descs()
3160 addr = dma_map_single(&dd->pcidev->dev, in ext_coal_sdma_tx_descs()
3161 tx->coalesce_buf, in ext_coal_sdma_tx_descs()
3162 tx->tlen, in ext_coal_sdma_tx_descs()
3165 if (unlikely(dma_mapping_error(&dd->pcidev->dev, addr))) { in ext_coal_sdma_tx_descs()
3167 return -ENOSPC; in ext_coal_sdma_tx_descs()
3171 tx->desc_limit = MAX_DESC; in ext_coal_sdma_tx_descs()
3173 addr, tx->tlen); in ext_coal_sdma_tx_descs()
3191 for (i = 0; i < dd->num_sdma; i++) { in sdma_update_lmc()
3194 sde = &dd->per_sdma[i]; in sdma_update_lmc()
3199 /* tx not dword sized - pad */
3204 tx->num_desc++; in _pad_sdma_tx_descs()
3205 if ((unlikely(tx->num_desc == tx->desc_limit))) { in _pad_sdma_tx_descs()
3216 dd->sdma_pad_phys, in _pad_sdma_tx_descs()
3217 sizeof(u32) - (tx->packet_len & (sizeof(u32) - 1))); in _pad_sdma_tx_descs()
3247 tx->num_desc++; in _sdma_txreq_ahgadd()
3251 tx->num_desc++; in _sdma_txreq_ahgadd()
3252 tx->descs[2].qw[0] = 0; in _sdma_txreq_ahgadd()
3253 tx->descs[2].qw[1] = 0; in _sdma_txreq_ahgadd()
3256 tx->num_desc++; in _sdma_txreq_ahgadd()
3257 tx->descs[1].qw[0] = 0; in _sdma_txreq_ahgadd()
3258 tx->descs[1].qw[1] = 0; in _sdma_txreq_ahgadd()
3262 tx->descs[0].qw[1] |= in _sdma_txreq_ahgadd()
3271 for (i = 0; i < (num_ahg - 1); i++) { in _sdma_txreq_ahgadd()
3274 tx->descs[desc].qw[!!(i & 2)] |= in _sdma_txreq_ahgadd()
3282 * sdma_ahg_alloc - allocate an AHG entry
3286 * 0-31 when successful, -EOPNOTSUPP if AHG is not enabled,
3287 * -ENOSPC if an entry is not available
3295 trace_hfi1_ahg_allocate(sde, -EINVAL); in sdma_ahg_alloc()
3296 return -EINVAL; in sdma_ahg_alloc()
3299 nr = ffz(READ_ONCE(sde->ahg_bits)); in sdma_ahg_alloc()
3301 trace_hfi1_ahg_allocate(sde, -ENOSPC); in sdma_ahg_alloc()
3302 return -ENOSPC; in sdma_ahg_alloc()
3304 oldbit = test_and_set_bit(nr, &sde->ahg_bits); in sdma_ahg_alloc()
3314 * sdma_ahg_free - free an AHG entry
3327 clear_bit(ahg_index, &sde->ahg_bits); in sdma_ahg_free()
3345 atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma); in sdma_freeze_notify()
3348 for (i = 0; i < dd->num_sdma; i++) in sdma_freeze_notify()
3349 sdma_process_event(&dd->per_sdma[i], event); in sdma_freeze_notify()
3367 ret = wait_event_interruptible(dd->sdma_unfreeze_wq, in sdma_freeze()
3368 atomic_read(&dd->sdma_unfreeze_count) <= in sdma_freeze()
3370 /* interrupted or count is negative, then unloading - just exit */ in sdma_freeze()
3371 if (ret || atomic_read(&dd->sdma_unfreeze_count) < 0) in sdma_freeze()
3375 atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma); in sdma_freeze()
3378 for (i = 0; i < dd->num_sdma; i++) in sdma_freeze()
3379 sdma_process_event(&dd->per_sdma[i], sdma_event_e81_hw_frozen); in sdma_freeze()
3386 (void)wait_event_interruptible(dd->sdma_unfreeze_wq, in sdma_freeze()
3387 atomic_read(&dd->sdma_unfreeze_count) <= 0); in sdma_freeze()
3388 /* no need to check results - done no matter what */ in sdma_freeze()
3394 * The SPC freeze acts like a SDMA halt and a hardware clean combined. All
3404 for (i = 0; i < dd->num_sdma; i++) in sdma_unfreeze()
3405 sdma_process_event(&dd->per_sdma[i], in sdma_unfreeze()
3410 * _sdma_engine_progress_schedule() - schedule progress on engine
3417 trace_hfi1_sdma_engine_progress(sde, sde->progress_mask); in _sdma_engine_progress_schedule()
3419 write_csr(sde->dd, in _sdma_engine_progress_schedule()
3421 sde->progress_mask); in _sdma_engine_progress_schedule()