Lines Matching +full:num +full:- +full:ss +full:- +full:bits
1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
3 * Copyright(c) 2015 - 2018 Intel Corporation.
212 * sdma_state_name() - return state string from enum
220 static void sdma_get(struct sdma_state *ss) in sdma_get() argument
222 kref_get(&ss->kref); in sdma_get()
227 struct sdma_state *ss = in sdma_complete() local
230 complete(&ss->comp); in sdma_complete()
233 static void sdma_put(struct sdma_state *ss) in sdma_put() argument
235 kref_put(&ss->kref, sdma_complete); in sdma_put()
238 static void sdma_finalput(struct sdma_state *ss) in sdma_finalput() argument
240 sdma_put(ss); in sdma_finalput()
241 wait_for_completion(&ss->comp); in sdma_finalput()
249 write_kctxt_csr(sde->dd, sde->this_idx, offset0, value); in write_sde_csr()
256 return read_kctxt_csr(sde->dd, sde->this_idx, offset0); in read_sde_csr()
260 * sdma_wait_for_packet_egress() - wait for the VL FIFO occupancy for
266 u64 off = 8 * sde->this_idx; in sdma_wait_for_packet_egress()
267 struct hfi1_devdata *dd = sde->dd; in sdma_wait_for_packet_egress()
284 /* timed out - bounce the link */ in sdma_wait_for_packet_egress()
286 __func__, sde->this_idx, (u32)reg); in sdma_wait_for_packet_egress()
287 queue_work(dd->pport->link_wq, in sdma_wait_for_packet_egress()
288 &dd->pport->link_bounce_work); in sdma_wait_for_packet_egress()
296 * sdma_wait() - wait for packet egress to complete for all SDMA engines,
303 for (i = 0; i < dd->num_sdma; i++) { in sdma_wait()
304 struct sdma_engine *sde = &dd->per_sdma[i]; in sdma_wait()
314 if (!(sde->dd->flags & HFI1_HAS_SDMA_TIMEOUT)) in sdma_set_desc_cnt()
327 struct iowait *wait = tx->wait; in complete_tx()
328 callback_t complete = tx->complete; in complete_tx()
331 trace_hfi1_sdma_out_sn(sde, tx->sn); in complete_tx()
332 if (WARN_ON_ONCE(sde->head_sn != tx->sn)) in complete_tx()
333 dd_dev_err(sde->dd, "expected %llu got %llu\n", in complete_tx()
334 sde->head_sn, tx->sn); in complete_tx()
335 sde->head_sn++; in complete_tx()
337 __sdma_txclean(sde->dd, tx); in complete_tx()
348 * - in the descq ring
349 * - in the flush list
355 * - From a work queue item
356 * - Directly from the state machine just before setting the
371 spin_lock_irqsave(&sde->flushlist_lock, flags); in sdma_flush()
373 list_splice_init(&sde->flushlist, &flushlist); in sdma_flush()
374 spin_unlock_irqrestore(&sde->flushlist_lock, flags); in sdma_flush()
382 seq = read_seqbegin(&sde->waitlock); in sdma_flush()
383 if (!list_empty(&sde->dmawait)) { in sdma_flush()
384 write_seqlock(&sde->waitlock); in sdma_flush()
385 list_for_each_entry_safe(w, nw, &sde->dmawait, list) { in sdma_flush()
386 if (w->wakeup) { in sdma_flush()
387 w->wakeup(w, SDMA_AVAIL_REASON); in sdma_flush()
388 list_del_init(&w->list); in sdma_flush()
391 write_sequnlock(&sde->waitlock); in sdma_flush()
393 } while (read_seqretry(&sde->waitlock, seq)); in sdma_flush()
412 write_seqlock_irqsave(&sde->head_lock, flags); in sdma_field_flush()
415 write_sequnlock_irqrestore(&sde->head_lock, flags); in sdma_field_flush()
432 dd_dev_err(sde->dd, in sdma_err_halt_wait()
433 "SDMA engine %d - timeout waiting for engine to halt\n", in sdma_err_halt_wait()
434 sde->this_idx); in sdma_err_halt_wait()
449 if (!is_bx(sde->dd) && HFI1_CAP_IS_KSET(SDMA_AHG)) { in sdma_err_progress_check_schedule()
451 struct hfi1_devdata *dd = sde->dd; in sdma_err_progress_check_schedule()
453 for (index = 0; index < dd->num_sdma; index++) { in sdma_err_progress_check_schedule()
454 struct sdma_engine *curr_sdma = &dd->per_sdma[index]; in sdma_err_progress_check_schedule()
457 curr_sdma->progress_check_head = in sdma_err_progress_check_schedule()
458 curr_sdma->descq_head; in sdma_err_progress_check_schedule()
460 dd_dev_err(sde->dd, in sdma_err_progress_check_schedule()
461 "SDMA engine %d - check scheduled\n", in sdma_err_progress_check_schedule()
462 sde->this_idx); in sdma_err_progress_check_schedule()
463 mod_timer(&sde->err_progress_check_timer, jiffies + 10); in sdma_err_progress_check_schedule()
472 dd_dev_err(sde->dd, "SDE progress check event\n"); in sdma_err_progress_check()
473 for (index = 0; index < sde->dd->num_sdma; index++) { in sdma_err_progress_check()
474 struct sdma_engine *curr_sde = &sde->dd->per_sdma[index]; in sdma_err_progress_check()
481 * We must lock interrupts when acquiring sde->lock, in sdma_err_progress_check()
485 spin_lock_irqsave(&curr_sde->tail_lock, flags); in sdma_err_progress_check()
486 write_seqlock(&curr_sde->head_lock); in sdma_err_progress_check()
488 /* skip non-running queues */ in sdma_err_progress_check()
489 if (curr_sde->state.current_state != sdma_state_s99_running) { in sdma_err_progress_check()
490 write_sequnlock(&curr_sde->head_lock); in sdma_err_progress_check()
491 spin_unlock_irqrestore(&curr_sde->tail_lock, flags); in sdma_err_progress_check()
495 if ((curr_sde->descq_head != curr_sde->descq_tail) && in sdma_err_progress_check()
496 (curr_sde->descq_head == in sdma_err_progress_check()
497 curr_sde->progress_check_head)) in sdma_err_progress_check()
500 write_sequnlock(&curr_sde->head_lock); in sdma_err_progress_check()
501 spin_unlock_irqrestore(&curr_sde->tail_lock, flags); in sdma_err_progress_check()
503 schedule_work(&sde->err_halt_worker); in sdma_err_progress_check()
514 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", in sdma_hw_clean_up_task()
515 sde->this_idx, slashstrip(__FILE__), __LINE__, in sdma_hw_clean_up_task()
530 return sde->tx_ring[sde->tx_head & sde->sdma_mask]; in get_txhead()
547 head = sde->descq_head & sde->sdma_mask; in sdma_flush_descq()
548 tail = sde->descq_tail & sde->sdma_mask; in sdma_flush_descq()
551 head = ++sde->descq_head & sde->sdma_mask; in sdma_flush_descq()
553 if (txp && txp->next_descq_idx == head) { in sdma_flush_descq()
555 sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL; in sdma_flush_descq()
571 spin_lock_irqsave(&sde->tail_lock, flags); in sdma_sw_clean_up_task()
572 write_seqlock(&sde->head_lock); in sdma_sw_clean_up_task()
576 * - We are halted, so no more descriptors are getting retired. in sdma_sw_clean_up_task()
577 * - We are not running, so no one is submitting new work. in sdma_sw_clean_up_task()
578 * - Only we can send the e40_sw_cleaned, so we can't start in sdma_sw_clean_up_task()
590 * latest physical hardware head - we are not running so speed does in sdma_sw_clean_up_task()
602 sde->descq_tail = 0; in sdma_sw_clean_up_task()
603 sde->descq_head = 0; in sdma_sw_clean_up_task()
604 sde->desc_avail = sdma_descq_freecnt(sde); in sdma_sw_clean_up_task()
605 *sde->head_dma = 0; in sdma_sw_clean_up_task()
609 write_sequnlock(&sde->head_lock); in sdma_sw_clean_up_task()
610 spin_unlock_irqrestore(&sde->tail_lock, flags); in sdma_sw_clean_up_task()
615 struct sdma_state *ss = &sde->state; in sdma_sw_tear_down() local
618 sdma_put(ss); in sdma_sw_tear_down()
621 atomic_set(&sde->dd->sdma_unfreeze_count, -1); in sdma_sw_tear_down()
622 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); in sdma_sw_tear_down()
627 tasklet_hi_schedule(&sde->sdma_hw_clean_up_task); in sdma_start_hw_clean_up()
633 struct sdma_state *ss = &sde->state; in sdma_set_state() local
639 sdma_state_names[ss->current_state], in sdma_set_state()
643 ss->previous_state = ss->current_state; in sdma_set_state()
644 ss->previous_op = ss->current_op; in sdma_set_state()
645 ss->current_state = next_state; in sdma_set_state()
647 if (ss->previous_state != sdma_state_s99_running && in sdma_set_state()
664 ss->go_s99_running = 0; in sdma_set_state()
667 ss->go_s99_running = 1; in sdma_set_state()
669 ss->current_op = op; in sdma_set_state()
670 sdma_sendctrl(sde, ss->current_op); in sdma_set_state()
674 * sdma_get_descq_cnt() - called when device probed
702 * sdma_engine_get_vl() - return vl for a given sdma engine
710 struct hfi1_devdata *dd = sde->dd; in sdma_engine_get_vl()
714 if (sde->this_idx >= TXE_NUM_SDMA_ENGINES) in sdma_engine_get_vl()
715 return -EINVAL; in sdma_engine_get_vl()
718 m = rcu_dereference(dd->sdma_map); in sdma_engine_get_vl()
721 return -EINVAL; in sdma_engine_get_vl()
723 vl = m->engine_to_vl[sde->this_idx]; in sdma_engine_get_vl()
730 * sdma_select_engine_vl() - select sdma engine
748 /* NOTE This should only happen if SC->VL changed after the initial in sdma_select_engine_vl()
758 m = rcu_dereference(dd->sdma_map); in sdma_select_engine_vl()
761 return &dd->per_sdma[0]; in sdma_select_engine_vl()
763 e = m->map[vl & m->mask]; in sdma_select_engine_vl()
764 rval = e->sde[selector & e->mask]; in sdma_select_engine_vl()
768 rval = !rval ? &dd->per_sdma[0] : rval; in sdma_select_engine_vl()
769 trace_hfi1_sdma_engine_select(dd, selector, vl, rval->this_idx); in sdma_select_engine_vl()
774 * sdma_select_engine_sc() - select sdma engine
817 * sdma_select_user_engine() - select sdma engine based on user setup
838 if (current->nr_cpus_allowed != 1) in sdma_select_user_engine()
843 rht_node = rhashtable_lookup(dd->sdma_rht, &cpu_id, in sdma_select_user_engine()
846 if (rht_node && rht_node->map[vl]) { in sdma_select_user_engine()
847 struct sdma_rht_map_elem *map = rht_node->map[vl]; in sdma_select_user_engine()
849 sde = map->sde[selector & map->mask]; in sdma_select_user_engine()
864 for (i = 0; i < roundup_pow_of_two(map->ctr ? : 1) - map->ctr; i++) in sdma_populate_sde_map()
865 map->sde[map->ctr + i] = map->sde[i]; in sdma_populate_sde_map()
874 for (i = 0; i < map->ctr; i++) { in sdma_cleanup_sde_map()
875 if (map->sde[i] == sde) { in sdma_cleanup_sde_map()
876 memmove(&map->sde[i], &map->sde[i + 1], in sdma_cleanup_sde_map()
877 (map->ctr - i - 1) * sizeof(map->sde[0])); in sdma_cleanup_sde_map()
878 map->ctr--; in sdma_cleanup_sde_map()
879 pow = roundup_pow_of_two(map->ctr ? : 1); in sdma_cleanup_sde_map()
880 map->mask = pow - 1; in sdma_cleanup_sde_map()
895 struct hfi1_devdata *dd = sde->dd; in sdma_set_cpu_to_sde_map()
902 if (unlikely(vl < 0 || vl >= ARRAY_SIZE(rht_node->map))) in sdma_set_cpu_to_sde_map()
903 return -EINVAL; in sdma_set_cpu_to_sde_map()
907 return -ENOMEM; in sdma_set_cpu_to_sde_map()
912 return -ENOMEM; in sdma_set_cpu_to_sde_map()
919 dd_dev_warn(sde->dd, "Invalid CPU mask\n"); in sdma_set_cpu_to_sde_map()
920 ret = -EINVAL; in sdma_set_cpu_to_sde_map()
931 if (cpumask_test_cpu(cpu, &sde->cpu_mask)) { in sdma_set_cpu_to_sde_map()
936 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu, in sdma_set_cpu_to_sde_map()
941 ret = -ENOMEM; in sdma_set_cpu_to_sde_map()
945 rht_node->map[vl] = kzalloc(sz, GFP_KERNEL); in sdma_set_cpu_to_sde_map()
946 if (!rht_node->map[vl]) { in sdma_set_cpu_to_sde_map()
948 ret = -ENOMEM; in sdma_set_cpu_to_sde_map()
951 rht_node->cpu_id = cpu; in sdma_set_cpu_to_sde_map()
952 rht_node->map[vl]->mask = 0; in sdma_set_cpu_to_sde_map()
953 rht_node->map[vl]->ctr = 1; in sdma_set_cpu_to_sde_map()
954 rht_node->map[vl]->sde[0] = sde; in sdma_set_cpu_to_sde_map()
956 ret = rhashtable_insert_fast(dd->sdma_rht, in sdma_set_cpu_to_sde_map()
957 &rht_node->node, in sdma_set_cpu_to_sde_map()
960 kfree(rht_node->map[vl]); in sdma_set_cpu_to_sde_map()
962 dd_dev_err(sde->dd, "Failed to set process to sde affinity for cpu %lu\n", in sdma_set_cpu_to_sde_map()
971 if (!rht_node->map[vl]) in sdma_set_cpu_to_sde_map()
972 rht_node->map[vl] = kzalloc(sz, GFP_KERNEL); in sdma_set_cpu_to_sde_map()
974 if (!rht_node->map[vl]) { in sdma_set_cpu_to_sde_map()
975 ret = -ENOMEM; in sdma_set_cpu_to_sde_map()
979 rht_node->map[vl]->ctr++; in sdma_set_cpu_to_sde_map()
980 ctr = rht_node->map[vl]->ctr; in sdma_set_cpu_to_sde_map()
981 rht_node->map[vl]->sde[ctr - 1] = sde; in sdma_set_cpu_to_sde_map()
983 rht_node->map[vl]->mask = pow - 1; in sdma_set_cpu_to_sde_map()
986 sdma_populate_sde_map(rht_node->map[vl]); in sdma_set_cpu_to_sde_map()
999 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu, in sdma_set_cpu_to_sde_map()
1007 if (rht_node->map[i]) in sdma_set_cpu_to_sde_map()
1008 sdma_cleanup_sde_map(rht_node->map[i], in sdma_set_cpu_to_sde_map()
1013 if (!rht_node->map[i]) in sdma_set_cpu_to_sde_map()
1016 if (rht_node->map[i]->ctr) { in sdma_set_cpu_to_sde_map()
1023 ret = rhashtable_remove_fast(dd->sdma_rht, in sdma_set_cpu_to_sde_map()
1024 &rht_node->node, in sdma_set_cpu_to_sde_map()
1029 kfree(rht_node->map[i]); in sdma_set_cpu_to_sde_map()
1036 cpumask_copy(&sde->cpu_mask, new_mask); in sdma_set_cpu_to_sde_map()
1048 if (cpumask_empty(&sde->cpu_mask)) in sdma_get_cpu_to_sde_map()
1051 cpumap_print_to_pagebuf(true, buf, &sde->cpu_mask); in sdma_get_cpu_to_sde_map()
1062 kfree(rht_node->map[i]); in sdma_rht_free()
1068 * sdma_seqfile_dump_cpu_list() - debugfs dump the cpu to sdma mappings
1082 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpuid, in sdma_seqfile_dump_cpu_list()
1089 if (!rht_node->map[i] || !rht_node->map[i]->ctr) in sdma_seqfile_dump_cpu_list()
1094 for (j = 0; j < rht_node->map[i]->ctr; j++) { in sdma_seqfile_dump_cpu_list()
1095 if (!rht_node->map[i]->sde[j]) in sdma_seqfile_dump_cpu_list()
1102 rht_node->map[i]->sde[j]->this_idx); in sdma_seqfile_dump_cpu_list()
1117 for (i = 0; m && i < m->actual_vls; i++) in sdma_map_free()
1118 kfree(m->map[i]); in sdma_map_free()
1133 * sdma_map_init - called when # vls change
1141 * vl_engines is used to specify a non-uniform vl/engine loading. NULL
1150 * If either the num_vls or num_sdma are non-power of 2, the array sizes
1167 if (!(dd->flags & HFI1_HAS_SEND_DMA)) in sdma_map_init()
1172 sde_per_vl = dd->num_sdma / num_vls; in sdma_map_init()
1174 extra = dd->num_sdma % num_vls; in sdma_map_init()
1177 for (i = num_vls - 1; i >= 0; i--, extra--) in sdma_map_init()
1188 newmap->actual_vls = num_vls; in sdma_map_init()
1189 newmap->vls = roundup_pow_of_two(num_vls); in sdma_map_init()
1190 newmap->mask = (1 << ilog2(newmap->vls)) - 1; in sdma_map_init()
1191 /* initialize back-map */ in sdma_map_init()
1193 newmap->engine_to_vl[i] = -1; in sdma_map_init()
1194 for (i = 0; i < newmap->vls; i++) { in sdma_map_init()
1198 if (i < newmap->actual_vls) { in sdma_map_init()
1202 newmap->map[i] = kzalloc( in sdma_map_init()
1206 if (!newmap->map[i]) in sdma_map_init()
1208 newmap->map[i]->mask = (1 << ilog2(sz)) - 1; in sdma_map_init()
1211 newmap->map[i]->sde[j] = in sdma_map_init()
1212 &dd->per_sdma[engine]; in sdma_map_init()
1217 /* assign back-map */ in sdma_map_init()
1219 newmap->engine_to_vl[first_engine + j] = i; in sdma_map_init()
1221 /* just re-use entry without allocating */ in sdma_map_init()
1222 newmap->map[i] = newmap->map[i % num_vls]; in sdma_map_init()
1227 spin_lock_irq(&dd->sde_map_lock); in sdma_map_init()
1228 oldmap = rcu_dereference_protected(dd->sdma_map, in sdma_map_init()
1229 lockdep_is_held(&dd->sde_map_lock)); in sdma_map_init()
1232 rcu_assign_pointer(dd->sdma_map, newmap); in sdma_map_init()
1234 spin_unlock_irq(&dd->sde_map_lock); in sdma_map_init()
1237 call_rcu(&oldmap->list, sdma_map_rcu_callback); in sdma_map_init()
1242 return -ENOMEM; in sdma_map_init()
1246 * sdma_clean - Clean up allocated memory
1248 * @num_engines: num sdma engines
1258 if (dd->sdma_pad_dma) { in sdma_clean()
1259 dma_free_coherent(&dd->pcidev->dev, SDMA_PAD, in sdma_clean()
1260 (void *)dd->sdma_pad_dma, in sdma_clean()
1261 dd->sdma_pad_phys); in sdma_clean()
1262 dd->sdma_pad_dma = NULL; in sdma_clean()
1263 dd->sdma_pad_phys = 0; in sdma_clean()
1265 if (dd->sdma_heads_dma) { in sdma_clean()
1266 dma_free_coherent(&dd->pcidev->dev, dd->sdma_heads_size, in sdma_clean()
1267 (void *)dd->sdma_heads_dma, in sdma_clean()
1268 dd->sdma_heads_phys); in sdma_clean()
1269 dd->sdma_heads_dma = NULL; in sdma_clean()
1270 dd->sdma_heads_phys = 0; in sdma_clean()
1272 for (i = 0; dd->per_sdma && i < num_engines; ++i) { in sdma_clean()
1273 sde = &dd->per_sdma[i]; in sdma_clean()
1275 sde->head_dma = NULL; in sdma_clean()
1276 sde->head_phys = 0; in sdma_clean()
1278 if (sde->descq) { in sdma_clean()
1280 &dd->pcidev->dev, in sdma_clean()
1281 sde->descq_cnt * sizeof(u64[2]), in sdma_clean()
1282 sde->descq, in sdma_clean()
1283 sde->descq_phys in sdma_clean()
1285 sde->descq = NULL; in sdma_clean()
1286 sde->descq_phys = 0; in sdma_clean()
1288 kvfree(sde->tx_ring); in sdma_clean()
1289 sde->tx_ring = NULL; in sdma_clean()
1291 if (rcu_access_pointer(dd->sdma_map)) { in sdma_clean()
1292 spin_lock_irq(&dd->sde_map_lock); in sdma_clean()
1293 sdma_map_free(rcu_access_pointer(dd->sdma_map)); in sdma_clean()
1294 RCU_INIT_POINTER(dd->sdma_map, NULL); in sdma_clean()
1295 spin_unlock_irq(&dd->sde_map_lock); in sdma_clean()
1298 kfree(dd->per_sdma); in sdma_clean()
1299 dd->per_sdma = NULL; in sdma_clean()
1301 if (dd->sdma_rht) { in sdma_clean()
1302 rhashtable_free_and_destroy(dd->sdma_rht, sdma_rht_free, NULL); in sdma_clean()
1303 kfree(dd->sdma_rht); in sdma_clean()
1304 dd->sdma_rht = NULL; in sdma_clean()
1309 * sdma_init() - called when device probed
1317 * 0 - success, -errno on failure
1326 struct hfi1_pportdata *ppd = dd->pport + port; in sdma_init()
1330 int ret = -ENOMEM; in sdma_init()
1352 init_waitqueue_head(&dd->sdma_unfreeze_wq); in sdma_init()
1353 atomic_set(&dd->sdma_unfreeze_count, 0); in sdma_init()
1360 dd->per_sdma = kcalloc_node(num_engines, sizeof(*dd->per_sdma), in sdma_init()
1361 GFP_KERNEL, dd->node); in sdma_init()
1362 if (!dd->per_sdma) in sdma_init()
1367 dd->default_desc1 = in sdma_init()
1370 dd->default_desc1 = in sdma_init()
1378 sde = &dd->per_sdma[this_idx]; in sdma_init()
1379 sde->dd = dd; in sdma_init()
1380 sde->ppd = ppd; in sdma_init()
1381 sde->this_idx = this_idx; in sdma_init()
1382 sde->descq_cnt = descq_cnt; in sdma_init()
1383 sde->desc_avail = sdma_descq_freecnt(sde); in sdma_init()
1384 sde->sdma_shift = ilog2(descq_cnt); in sdma_init()
1385 sde->sdma_mask = (1 << sde->sdma_shift) - 1; in sdma_init()
1388 sde->int_mask = (u64)1 << (0 * TXE_NUM_SDMA_ENGINES + in sdma_init()
1390 sde->progress_mask = (u64)1 << (1 * TXE_NUM_SDMA_ENGINES + in sdma_init()
1392 sde->idle_mask = (u64)1 << (2 * TXE_NUM_SDMA_ENGINES + in sdma_init()
1395 sde->imask = sde->int_mask | sde->progress_mask | in sdma_init()
1396 sde->idle_mask; in sdma_init()
1398 spin_lock_init(&sde->tail_lock); in sdma_init()
1399 seqlock_init(&sde->head_lock); in sdma_init()
1400 spin_lock_init(&sde->senddmactrl_lock); in sdma_init()
1401 spin_lock_init(&sde->flushlist_lock); in sdma_init()
1402 seqlock_init(&sde->waitlock); in sdma_init()
1404 sde->ahg_bits = 0xfffffffe00000000ULL; in sdma_init()
1409 kref_init(&sde->state.kref); in sdma_init()
1410 init_completion(&sde->state.comp); in sdma_init()
1412 INIT_LIST_HEAD(&sde->flushlist); in sdma_init()
1413 INIT_LIST_HEAD(&sde->dmawait); in sdma_init()
1415 sde->tail_csr = in sdma_init()
1418 tasklet_setup(&sde->sdma_hw_clean_up_task, in sdma_init()
1420 tasklet_setup(&sde->sdma_sw_clean_up_task, in sdma_init()
1422 INIT_WORK(&sde->err_halt_worker, sdma_err_halt_wait); in sdma_init()
1423 INIT_WORK(&sde->flush_worker, sdma_field_flush); in sdma_init()
1425 sde->progress_check_head = 0; in sdma_init()
1427 timer_setup(&sde->err_progress_check_timer, in sdma_init()
1430 sde->descq = dma_alloc_coherent(&dd->pcidev->dev, in sdma_init()
1432 &sde->descq_phys, GFP_KERNEL); in sdma_init()
1433 if (!sde->descq) in sdma_init()
1435 sde->tx_ring = in sdma_init()
1438 GFP_KERNEL, dd->node); in sdma_init()
1439 if (!sde->tx_ring) in sdma_init()
1443 dd->sdma_heads_size = L1_CACHE_BYTES * num_engines; in sdma_init()
1445 dd->sdma_heads_dma = dma_alloc_coherent(&dd->pcidev->dev, in sdma_init()
1446 dd->sdma_heads_size, in sdma_init()
1447 &dd->sdma_heads_phys, in sdma_init()
1449 if (!dd->sdma_heads_dma) { in sdma_init()
1455 dd->sdma_pad_dma = dma_alloc_coherent(&dd->pcidev->dev, SDMA_PAD, in sdma_init()
1456 &dd->sdma_pad_phys, GFP_KERNEL); in sdma_init()
1457 if (!dd->sdma_pad_dma) { in sdma_init()
1463 curr_head = (void *)dd->sdma_heads_dma; in sdma_init()
1467 sde = &dd->per_sdma[this_idx]; in sdma_init()
1469 sde->head_dma = curr_head; in sdma_init()
1471 phys_offset = (unsigned long)sde->head_dma - in sdma_init()
1472 (unsigned long)dd->sdma_heads_dma; in sdma_init()
1473 sde->head_phys = dd->sdma_heads_phys + phys_offset; in sdma_init()
1476 dd->flags |= HFI1_HAS_SEND_DMA; in sdma_init()
1477 dd->flags |= idle_cnt ? HFI1_HAS_SDMA_TIMEOUT : 0; in sdma_init()
1478 dd->num_sdma = num_engines; in sdma_init()
1479 ret = sdma_map_init(dd, port, ppd->vls_operational, NULL); in sdma_init()
1485 ret = -ENOMEM; in sdma_init()
1495 dd->sdma_rht = tmp_sdma_rht; in sdma_init()
1497 dd_dev_info(dd, "SDMA num_sdma: %u\n", dd->num_sdma); in sdma_init()
1506 * sdma_all_running() - called when the link goes up
1517 for (i = 0; i < dd->num_sdma; ++i) { in sdma_all_running()
1518 sde = &dd->per_sdma[i]; in sdma_all_running()
1524 * sdma_all_idle() - called when the link goes down
1535 for (i = 0; i < dd->num_sdma; ++i) { in sdma_all_idle()
1536 sde = &dd->per_sdma[i]; in sdma_all_idle()
1542 * sdma_start() - called to kick off state processing for all engines
1555 for (i = 0; i < dd->num_sdma; ++i) { in sdma_start()
1556 sde = &dd->per_sdma[i]; in sdma_start()
1562 * sdma_exit() - used when module is removed
1570 for (this_idx = 0; dd->per_sdma && this_idx < dd->num_sdma; in sdma_exit()
1572 sde = &dd->per_sdma[this_idx]; in sdma_exit()
1573 if (!list_empty(&sde->dmawait)) in sdma_exit()
1575 sde->this_idx); in sdma_exit()
1578 del_timer_sync(&sde->err_progress_check_timer); in sdma_exit()
1585 sdma_finalput(&sde->state); in sdma_exit()
1599 &dd->pcidev->dev, in sdma_unmap_desc()
1606 &dd->pcidev->dev, in sdma_unmap_desc()
1620 return (tx->descp[0].qw[1] & SDMA_DESC1_HEADER_MODE_SMASK) in ahg_mode()
1625 * __sdma_txclean() - clean tx of mappings, descp *kmalloc's
1630 * by the ULP to toss an in-process tx build.
1641 if (tx->num_desc) { in __sdma_txclean()
1645 sdma_unmap_desc(dd, &tx->descp[0]); in __sdma_txclean()
1649 for (i = 1 + skip; i < tx->num_desc; i++) in __sdma_txclean()
1650 sdma_unmap_desc(dd, &tx->descp[i]); in __sdma_txclean()
1651 tx->num_desc = 0; in __sdma_txclean()
1653 kfree(tx->coalesce_buf); in __sdma_txclean()
1654 tx->coalesce_buf = NULL; in __sdma_txclean()
1656 if (unlikely(tx->desc_limit > ARRAY_SIZE(tx->descs))) { in __sdma_txclean()
1657 tx->desc_limit = ARRAY_SIZE(tx->descs); in __sdma_txclean()
1658 kfree(tx->descp); in __sdma_txclean()
1664 struct hfi1_devdata *dd = sde->dd; in sdma_gethead()
1669 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", in sdma_gethead()
1670 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); in sdma_gethead()
1675 (dd->flags & HFI1_HAS_SDMA_TIMEOUT); in sdma_gethead()
1677 (u16)le64_to_cpu(*sde->head_dma) : in sdma_gethead()
1686 swhead = sde->descq_head & sde->sdma_mask; in sdma_gethead()
1688 swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; in sdma_gethead()
1689 cnt = sde->descq_cnt; in sdma_gethead()
1704 sde->this_idx, in sdma_gethead()
1732 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, in sdma_desc_avail()
1734 dd_dev_err(sde->dd, "avail: %u\n", avail); in sdma_desc_avail()
1738 seq = read_seqbegin(&sde->waitlock); in sdma_desc_avail()
1739 if (!list_empty(&sde->dmawait)) { in sdma_desc_avail()
1741 write_seqlock(&sde->waitlock); in sdma_desc_avail()
1746 &sde->dmawait, in sdma_desc_avail()
1750 if (!wait->wakeup) in sdma_desc_avail()
1758 avail -= num_desc; in sdma_desc_avail()
1759 /* Find the top-priority wait memeber */ in sdma_desc_avail()
1768 list_del_init(&wait->list); in sdma_desc_avail()
1771 write_sequnlock(&sde->waitlock); in sdma_desc_avail()
1774 } while (read_seqretry(&sde->waitlock, seq)); in sdma_desc_avail()
1776 /* Schedule the top-priority entry first */ in sdma_desc_avail()
1778 waits[tidx]->wakeup(waits[tidx], SDMA_AVAIL_REASON); in sdma_desc_avail()
1782 waits[i]->wakeup(waits[i], SDMA_AVAIL_REASON); in sdma_desc_avail()
1803 swhead = sde->descq_head & sde->sdma_mask; in sdma_make_progress()
1807 swhead = ++sde->descq_head & sde->sdma_mask; in sdma_make_progress()
1810 if (txp && txp->next_descq_idx == swhead) { in sdma_make_progress()
1812 sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL; in sdma_make_progress()
1830 if ((status & sde->idle_mask) && !idle_check_done) { in sdma_make_progress()
1833 swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; in sdma_make_progress()
1841 sde->last_status = status; in sdma_make_progress()
1847 * sdma_engine_interrupt() - interrupt handler for engine
1852 * contain bits _only_ for this SDMA engine. It will contain at least one
1858 write_seqlock(&sde->head_lock); in sdma_engine_interrupt()
1860 if (status & sde->idle_mask) in sdma_engine_interrupt()
1861 sde->idle_int_cnt++; in sdma_engine_interrupt()
1862 else if (status & sde->progress_mask) in sdma_engine_interrupt()
1863 sde->progress_int_cnt++; in sdma_engine_interrupt()
1864 else if (status & sde->int_mask) in sdma_engine_interrupt()
1865 sde->sdma_int_cnt++; in sdma_engine_interrupt()
1867 write_sequnlock(&sde->head_lock); in sdma_engine_interrupt()
1871 * sdma_engine_error() - error handler for engine
1880 dd_dev_err(sde->dd, "CONFIG SDMA(%u) error status 0x%llx state %s\n", in sdma_engine_error()
1881 sde->this_idx, in sdma_engine_error()
1883 sdma_state_names[sde->state.current_state]); in sdma_engine_error()
1885 spin_lock_irqsave(&sde->tail_lock, flags); in sdma_engine_error()
1886 write_seqlock(&sde->head_lock); in sdma_engine_error()
1890 dd_dev_err(sde->dd, in sdma_engine_error()
1892 sde->this_idx, in sdma_engine_error()
1894 sdma_state_names[sde->state.current_state]); in sdma_engine_error()
1897 write_sequnlock(&sde->head_lock); in sdma_engine_error()
1898 spin_unlock_irqrestore(&sde->tail_lock, flags); in sdma_engine_error()
1908 dd_dev_err(sde->dd, "CONFIG SDMA(%u) senddmactrl E=%d I=%d H=%d C=%d\n", in sdma_sendctrl()
1909 sde->this_idx, in sdma_sendctrl()
1931 spin_lock_irqsave(&sde->senddmactrl_lock, flags); in sdma_sendctrl()
1933 sde->p_senddmactrl |= set_senddmactrl; in sdma_sendctrl()
1934 sde->p_senddmactrl &= ~clr_senddmactrl; in sdma_sendctrl()
1938 sde->p_senddmactrl | in sdma_sendctrl()
1941 write_sde_csr(sde, SD(CTRL), sde->p_senddmactrl); in sdma_sendctrl()
1943 spin_unlock_irqrestore(&sde->senddmactrl_lock, flags); in sdma_sendctrl()
1953 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", in sdma_setlengen()
1954 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); in sdma_setlengen()
1958 * Set SendDmaLenGen and clear-then-set the MSB of the generation in sdma_setlengen()
1963 (sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT)); in sdma_setlengen()
1965 ((sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT)) | in sdma_setlengen()
1973 writeq(tail, sde->tail_csr); in sdma_update_tail()
1985 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", in sdma_hw_start_up()
1986 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); in sdma_hw_start_up()
1991 *sde->head_dma = 0; in sdma_hw_start_up()
2005 struct hfi1_devdata *dd = sde->dd; in set_sdma_integrity()
2018 struct hfi1_devdata *dd = sde->dd; in init_sdma_regs()
2021 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); in init_sdma_regs()
2024 write_sde_csr(sde, SD(BASE_ADDR), sde->descq_phys); in init_sdma_regs()
2029 write_sde_csr(sde, SD(HEAD_ADDR), sde->head_phys); in init_sdma_regs()
2032 ((u64)(credits * sde->this_idx) << in init_sdma_regs()
2046 csr = read_csr(sde->dd, reg); \
2047 dd_dev_err(sde->dd, "%36s 0x%016llx\n", #reg, csr); \
2052 dd_dev_err(sde->dd, "%36s[%02u] 0x%016llx\n", \
2053 #reg, sde->this_idx, csr); \
2057 csr = read_csr(sde->dd, reg + (8 * i)); \
2058 dd_dev_err(sde->dd, "%33s_%02u 0x%016llx\n", \
2112 head = sde->descq_head & sde->sdma_mask; in dump_sdma_state()
2113 tail = sde->descq_tail & sde->sdma_mask; in dump_sdma_state()
2116 dd_dev_err(sde->dd, in dump_sdma_state()
2118 sde->this_idx, head, tail, cnt, in dump_sdma_state()
2119 !list_empty(&sde->flushlist)); in dump_sdma_state()
2125 descqp = &sde->descq[head]; in dump_sdma_state()
2126 desc[0] = le64_to_cpu(descqp->qw[0]); in dump_sdma_state()
2127 desc[1] = le64_to_cpu(descqp->qw[1]); in dump_sdma_state()
2128 flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-'; in dump_sdma_state()
2130 'H' : '-'; in dump_sdma_state()
2131 flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-'; in dump_sdma_state()
2132 flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-'; in dump_sdma_state()
2139 dd_dev_err(sde->dd, in dump_sdma_state()
2142 dd_dev_err(sde->dd, in dump_sdma_state()
2146 dd_dev_err(sde->dd, in dump_sdma_state()
2158 head &= sde->sdma_mask; in dump_sdma_state()
2165 * sdma_seqfile_dump_sde() - debugfs dump of sde
2180 head = sde->descq_head & sde->sdma_mask; in sdma_seqfile_dump_sde()
2181 tail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; in sdma_seqfile_dump_sde()
2182 seq_printf(s, SDE_FMT, sde->this_idx, in sdma_seqfile_dump_sde()
2183 sde->cpu, in sdma_seqfile_dump_sde()
2184 sdma_state_name(sde->state.current_state), in sdma_seqfile_dump_sde()
2190 (unsigned long long)le64_to_cpu(*sde->head_dma), in sdma_seqfile_dump_sde()
2194 (unsigned long long)sde->last_status, in sdma_seqfile_dump_sde()
2195 (unsigned long long)sde->ahg_bits, in sdma_seqfile_dump_sde()
2196 sde->tx_tail, in sdma_seqfile_dump_sde()
2197 sde->tx_head, in sdma_seqfile_dump_sde()
2198 sde->descq_tail, in sdma_seqfile_dump_sde()
2199 sde->descq_head, in sdma_seqfile_dump_sde()
2200 !list_empty(&sde->flushlist), in sdma_seqfile_dump_sde()
2201 sde->descq_full_count, in sdma_seqfile_dump_sde()
2208 descqp = &sde->descq[head]; in sdma_seqfile_dump_sde()
2209 desc[0] = le64_to_cpu(descqp->qw[0]); in sdma_seqfile_dump_sde()
2210 desc[1] = le64_to_cpu(descqp->qw[1]); in sdma_seqfile_dump_sde()
2211 flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-'; in sdma_seqfile_dump_sde()
2213 'H' : '-'; in sdma_seqfile_dump_sde()
2214 flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-'; in sdma_seqfile_dump_sde()
2215 flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-'; in sdma_seqfile_dump_sde()
2233 head = (head + 1) & sde->sdma_mask; in sdma_seqfile_dump_sde()
2243 u8 generation = (sde->descq_tail >> sde->sdma_shift) & 3; in add_gen()
2271 struct sdma_desc *descp = tx->descp; in submit_tx()
2274 tail = sde->descq_tail & sde->sdma_mask; in submit_tx()
2275 sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]); in submit_tx()
2276 sde->descq[tail].qw[1] = cpu_to_le64(add_gen(sde, descp->qw[1])); in submit_tx()
2277 trace_hfi1_sdma_descriptor(sde, descp->qw[0], descp->qw[1], in submit_tx()
2278 tail, &sde->descq[tail]); in submit_tx()
2279 tail = ++sde->descq_tail & sde->sdma_mask; in submit_tx()
2283 for (i = 1; i < tx->num_desc; i++, descp++) { in submit_tx()
2286 sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]); in submit_tx()
2289 qw1 = descp->qw[1]; in submit_tx()
2290 skip--; in submit_tx()
2292 /* replace generation with real one for non-edits */ in submit_tx()
2293 qw1 = add_gen(sde, descp->qw[1]); in submit_tx()
2295 sde->descq[tail].qw[1] = cpu_to_le64(qw1); in submit_tx()
2296 trace_hfi1_sdma_descriptor(sde, descp->qw[0], qw1, in submit_tx()
2297 tail, &sde->descq[tail]); in submit_tx()
2298 tail = ++sde->descq_tail & sde->sdma_mask; in submit_tx()
2300 tx->next_descq_idx = tail; in submit_tx()
2302 tx->sn = sde->tail_sn++; in submit_tx()
2303 trace_hfi1_sdma_in_sn(sde, tx->sn); in submit_tx()
2304 WARN_ON_ONCE(sde->tx_ring[sde->tx_tail & sde->sdma_mask]); in submit_tx()
2306 sde->tx_ring[sde->tx_tail++ & sde->sdma_mask] = tx; in submit_tx()
2307 sde->desc_avail -= tx->num_desc; in submit_tx()
2322 sde->desc_avail = sdma_descq_freecnt(sde); in sdma_check_progress()
2323 if (tx->num_desc <= sde->desc_avail) in sdma_check_progress()
2324 return -EAGAIN; in sdma_check_progress()
2326 if (wait && iowait_ioww_to_iow(wait)->sleep) { in sdma_check_progress()
2330 (const seqcount_t *)&sde->head_lock.seqcount); in sdma_check_progress()
2331 ret = wait->iow->sleep(sde, wait, tx, seq, pkts_sent); in sdma_check_progress()
2332 if (ret == -EAGAIN) in sdma_check_progress()
2333 sde->desc_avail = sdma_descq_freecnt(sde); in sdma_check_progress()
2335 ret = -EBUSY; in sdma_check_progress()
2341 * sdma_send_txreq() - submit a tx req to ring
2347 * The call submits the tx into the ring. If a iowait structure is non-NULL
2351 * 0 - Success, -EINVAL - sdma_txreq incomplete, -EBUSY - no space in
2353 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
2365 if (unlikely(tx->tlen)) in sdma_send_txreq()
2366 return -EINVAL; in sdma_send_txreq()
2367 tx->wait = iowait_ioww_to_iow(wait); in sdma_send_txreq()
2368 spin_lock_irqsave(&sde->tail_lock, flags); in sdma_send_txreq()
2372 if (unlikely(tx->num_desc > sde->desc_avail)) in sdma_send_txreq()
2379 spin_unlock_irqrestore(&sde->tail_lock, flags); in sdma_send_txreq()
2384 tx->next_descq_idx = 0; in sdma_send_txreq()
2386 tx->sn = sde->tail_sn++; in sdma_send_txreq()
2387 trace_hfi1_sdma_in_sn(sde, tx->sn); in sdma_send_txreq()
2389 spin_lock(&sde->flushlist_lock); in sdma_send_txreq()
2390 list_add_tail(&tx->list, &sde->flushlist); in sdma_send_txreq()
2391 spin_unlock(&sde->flushlist_lock); in sdma_send_txreq()
2392 iowait_inc_wait_count(wait, tx->num_desc); in sdma_send_txreq()
2393 queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker); in sdma_send_txreq()
2394 ret = -ECOMM; in sdma_send_txreq()
2398 if (ret == -EAGAIN) { in sdma_send_txreq()
2402 sde->descq_full_count++; in sdma_send_txreq()
2407 * sdma_send_txlist() - submit a list of tx req to ring
2419 * If the iowait structure is non-NULL and not equal to the iowait list
2430 * 0 - Success,
2431 * -EINVAL - sdma_txreq incomplete, -EBUSY - no space in ring (wait == NULL)
2432 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state
2443 spin_lock_irqsave(&sde->tail_lock, flags); in sdma_send_txlist()
2446 tx->wait = iowait_ioww_to_iow(wait); in sdma_send_txlist()
2449 if (unlikely(tx->num_desc > sde->desc_avail)) in sdma_send_txlist()
2451 if (unlikely(tx->tlen)) { in sdma_send_txlist()
2452 ret = -EINVAL; in sdma_send_txlist()
2455 list_del_init(&tx->list); in sdma_send_txlist()
2473 spin_unlock_irqrestore(&sde->tail_lock, flags); in sdma_send_txlist()
2477 spin_lock(&sde->flushlist_lock); in sdma_send_txlist()
2479 tx->wait = iowait_ioww_to_iow(wait); in sdma_send_txlist()
2480 list_del_init(&tx->list); in sdma_send_txlist()
2481 tx->next_descq_idx = 0; in sdma_send_txlist()
2483 tx->sn = sde->tail_sn++; in sdma_send_txlist()
2484 trace_hfi1_sdma_in_sn(sde, tx->sn); in sdma_send_txlist()
2486 list_add_tail(&tx->list, &sde->flushlist); in sdma_send_txlist()
2488 iowait_inc_wait_count(wait, tx->num_desc); in sdma_send_txlist()
2490 spin_unlock(&sde->flushlist_lock); in sdma_send_txlist()
2491 queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker); in sdma_send_txlist()
2492 ret = -ECOMM; in sdma_send_txlist()
2496 if (ret == -EAGAIN) { in sdma_send_txlist()
2500 sde->descq_full_count++; in sdma_send_txlist()
2508 spin_lock_irqsave(&sde->tail_lock, flags); in sdma_process_event()
2509 write_seqlock(&sde->head_lock); in sdma_process_event()
2513 if (sde->state.current_state == sdma_state_s99_running) in sdma_process_event()
2516 write_sequnlock(&sde->head_lock); in sdma_process_event()
2517 spin_unlock_irqrestore(&sde->tail_lock, flags); in sdma_process_event()
2523 struct sdma_state *ss = &sde->state; in __sdma_process_event() local
2528 dd_dev_err(sde->dd, "CONFIG SDMA(%u) [%s] %s\n", sde->this_idx, in __sdma_process_event()
2529 sdma_state_names[ss->current_state], in __sdma_process_event()
2533 switch (ss->current_state) { in __sdma_process_event()
2546 ss->go_s99_running = 1; in __sdma_process_event()
2550 sdma_get(&sde->state); in __sdma_process_event()
2596 ss->go_s99_running = 1; in __sdma_process_event()
2603 schedule_work(&sde->err_halt_worker); in __sdma_process_event()
2606 ss->go_s99_running = 0; in __sdma_process_event()
2633 sdma_set_state(sde, ss->go_s99_running ? in __sdma_process_event()
2638 ss->go_s99_running = 1; in __sdma_process_event()
2647 ss->go_s99_running = 0; in __sdma_process_event()
2676 ss->go_s99_running = 1; in __sdma_process_event()
2684 schedule_work(&sde->err_halt_worker); in __sdma_process_event()
2691 atomic_dec(&sde->dd->sdma_unfreeze_count); in __sdma_process_event()
2692 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); in __sdma_process_event()
2715 ss->go_s99_running = 1; in __sdma_process_event()
2726 ss->go_s99_running = 0; in __sdma_process_event()
2735 ss->go_s99_running = 0; in __sdma_process_event()
2746 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2754 sdma_set_state(sde, ss->go_s99_running ? in __sdma_process_event()
2759 ss->go_s99_running = 1; in __sdma_process_event()
2768 ss->go_s99_running = 0; in __sdma_process_event()
2777 ss->go_s99_running = 0; in __sdma_process_event()
2788 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2794 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2799 ss->go_s99_running = 1; in __sdma_process_event()
2806 schedule_work(&sde->err_halt_worker); in __sdma_process_event()
2809 ss->go_s99_running = 0; in __sdma_process_event()
2818 ss->go_s99_running = 0; in __sdma_process_event()
2829 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2835 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2840 ss->go_s99_running = 1; in __sdma_process_event()
2847 schedule_work(&sde->err_halt_worker); in __sdma_process_event()
2850 ss->go_s99_running = 0; in __sdma_process_event()
2869 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2878 ss->go_s99_running = 1; in __sdma_process_event()
2887 ss->go_s99_running = 0; in __sdma_process_event()
2893 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2908 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2917 ss->go_s99_running = 1; in __sdma_process_event()
2921 atomic_dec(&sde->dd->sdma_unfreeze_count); in __sdma_process_event()
2922 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); in __sdma_process_event()
2929 ss->go_s99_running = 0; in __sdma_process_event()
2937 sdma_set_state(sde, ss->go_s99_running ? in __sdma_process_event()
2952 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); in __sdma_process_event()
2976 schedule_work(&sde->err_halt_worker); in __sdma_process_event()
2982 ss->go_s99_running = 0; in __sdma_process_event()
2986 atomic_dec(&sde->dd->sdma_unfreeze_count); in __sdma_process_event()
2987 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); in __sdma_process_event()
2997 ss->last_event = event; in __sdma_process_event()
3003 * _extend_sdma_tx_descs() - helper to extend txreq
3021 if (unlikely((tx->num_desc == (MAX_DESC - 1)))) { in _extend_sdma_tx_descs()
3023 if (!tx->tlen) { in _extend_sdma_tx_descs()
3024 tx->desc_limit = MAX_DESC; in _extend_sdma_tx_descs()
3025 } else if (!tx->coalesce_buf) { in _extend_sdma_tx_descs()
3027 tx->coalesce_buf = kmalloc(tx->tlen + sizeof(u32), in _extend_sdma_tx_descs()
3029 if (!tx->coalesce_buf) in _extend_sdma_tx_descs()
3031 tx->coalesce_idx = 0; in _extend_sdma_tx_descs()
3036 if (unlikely(tx->num_desc == MAX_DESC)) in _extend_sdma_tx_descs()
3042 tx->descp = descp; in _extend_sdma_tx_descs()
3045 tx->desc_limit = MAX_DESC - 1; in _extend_sdma_tx_descs()
3047 for (i = 0; i < tx->num_desc; i++) in _extend_sdma_tx_descs()
3048 tx->descp[i] = tx->descs[i]; in _extend_sdma_tx_descs()
3052 return -ENOMEM; in _extend_sdma_tx_descs()
3056 * ext_coal_sdma_tx_descs() - extend or coalesce sdma tx descriptors
3067 * <0 - error
3068 * 0 - coalescing, don't populate descriptor
3069 * 1 - continue with populating descriptor
3085 if (tx->coalesce_buf) { in ext_coal_sdma_tx_descs()
3088 return -EINVAL; in ext_coal_sdma_tx_descs()
3096 return -EINVAL; in ext_coal_sdma_tx_descs()
3099 memcpy(tx->coalesce_buf + tx->coalesce_idx, kvaddr, len); in ext_coal_sdma_tx_descs()
3100 tx->coalesce_idx += len; in ext_coal_sdma_tx_descs()
3105 if (tx->tlen - tx->coalesce_idx) in ext_coal_sdma_tx_descs()
3109 pad_len = tx->packet_len & (sizeof(u32) - 1); in ext_coal_sdma_tx_descs()
3111 pad_len = sizeof(u32) - pad_len; in ext_coal_sdma_tx_descs()
3112 memset(tx->coalesce_buf + tx->coalesce_idx, 0, pad_len); in ext_coal_sdma_tx_descs()
3114 tx->packet_len += pad_len; in ext_coal_sdma_tx_descs()
3115 tx->tlen += pad_len; in ext_coal_sdma_tx_descs()
3119 addr = dma_map_single(&dd->pcidev->dev, in ext_coal_sdma_tx_descs()
3120 tx->coalesce_buf, in ext_coal_sdma_tx_descs()
3121 tx->tlen, in ext_coal_sdma_tx_descs()
3124 if (unlikely(dma_mapping_error(&dd->pcidev->dev, addr))) { in ext_coal_sdma_tx_descs()
3126 return -ENOSPC; in ext_coal_sdma_tx_descs()
3130 tx->desc_limit = MAX_DESC; in ext_coal_sdma_tx_descs()
3132 addr, tx->tlen); in ext_coal_sdma_tx_descs()
3150 for (i = 0; i < dd->num_sdma; i++) { in sdma_update_lmc()
3153 sde = &dd->per_sdma[i]; in sdma_update_lmc()
3158 /* tx not dword sized - pad */
3163 tx->num_desc++; in _pad_sdma_tx_descs()
3164 if ((unlikely(tx->num_desc == tx->desc_limit))) { in _pad_sdma_tx_descs()
3175 dd->sdma_pad_phys, in _pad_sdma_tx_descs()
3176 sizeof(u32) - (tx->packet_len & (sizeof(u32) - 1))); in _pad_sdma_tx_descs()
3206 tx->num_desc++; in _sdma_txreq_ahgadd()
3210 tx->num_desc++; in _sdma_txreq_ahgadd()
3211 tx->descs[2].qw[0] = 0; in _sdma_txreq_ahgadd()
3212 tx->descs[2].qw[1] = 0; in _sdma_txreq_ahgadd()
3215 tx->num_desc++; in _sdma_txreq_ahgadd()
3216 tx->descs[1].qw[0] = 0; in _sdma_txreq_ahgadd()
3217 tx->descs[1].qw[1] = 0; in _sdma_txreq_ahgadd()
3221 tx->descs[0].qw[1] |= in _sdma_txreq_ahgadd()
3230 for (i = 0; i < (num_ahg - 1); i++) { in _sdma_txreq_ahgadd()
3233 tx->descs[desc].qw[!!(i & 2)] |= in _sdma_txreq_ahgadd()
3241 * sdma_ahg_alloc - allocate an AHG entry
3245 * 0-31 when successful, -EOPNOTSUPP if AHG is not enabled,
3246 * -ENOSPC if an entry is not available
3254 trace_hfi1_ahg_allocate(sde, -EINVAL); in sdma_ahg_alloc()
3255 return -EINVAL; in sdma_ahg_alloc()
3258 nr = ffz(READ_ONCE(sde->ahg_bits)); in sdma_ahg_alloc()
3260 trace_hfi1_ahg_allocate(sde, -ENOSPC); in sdma_ahg_alloc()
3261 return -ENOSPC; in sdma_ahg_alloc()
3263 oldbit = test_and_set_bit(nr, &sde->ahg_bits); in sdma_ahg_alloc()
3273 * sdma_ahg_free - free an AHG entry
3286 clear_bit(ahg_index, &sde->ahg_bits); in sdma_ahg_free()
3304 atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma); in sdma_freeze_notify()
3307 for (i = 0; i < dd->num_sdma; i++) in sdma_freeze_notify()
3308 sdma_process_event(&dd->per_sdma[i], event); in sdma_freeze_notify()
3326 ret = wait_event_interruptible(dd->sdma_unfreeze_wq, in sdma_freeze()
3327 atomic_read(&dd->sdma_unfreeze_count) <= in sdma_freeze()
3329 /* interrupted or count is negative, then unloading - just exit */ in sdma_freeze()
3330 if (ret || atomic_read(&dd->sdma_unfreeze_count) < 0) in sdma_freeze()
3334 atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma); in sdma_freeze()
3337 for (i = 0; i < dd->num_sdma; i++) in sdma_freeze()
3338 sdma_process_event(&dd->per_sdma[i], sdma_event_e81_hw_frozen); in sdma_freeze()
3345 (void)wait_event_interruptible(dd->sdma_unfreeze_wq, in sdma_freeze()
3346 atomic_read(&dd->sdma_unfreeze_count) <= 0); in sdma_freeze()
3347 /* no need to check results - done no matter what */ in sdma_freeze()
3363 for (i = 0; i < dd->num_sdma; i++) in sdma_unfreeze()
3364 sdma_process_event(&dd->per_sdma[i], in sdma_unfreeze()
3369 * _sdma_engine_progress_schedule() - schedule progress on engine
3376 trace_hfi1_sdma_engine_progress(sde, sde->progress_mask); in _sdma_engine_progress_schedule()
3378 write_csr(sde->dd, in _sdma_engine_progress_schedule()
3380 sde->progress_mask); in _sdma_engine_progress_schedule()