71 	struct cfhsi *cfhsi = from_timer(cfhsi, t, inactivity_timer);  in cfhsi_inactivity_tout()  local
73 	netdev_dbg(cfhsi->ndev, "%s.\n",  in cfhsi_inactivity_tout()
77 	if (!test_bit(CFHSI_SHUTDOWN, &cfhsi->bits))  in cfhsi_inactivity_tout()
78 		queue_work(cfhsi->wq, &cfhsi->wake_down_work);  in cfhsi_inactivity_tout()
81 static void cfhsi_update_aggregation_stats(struct cfhsi *cfhsi,  in cfhsi_update_aggregation_stats()  argument
89 	hpad = 1 + PAD_POW2((info->hdr_len + 1), cfhsi->cfg.head_align);  in cfhsi_update_aggregation_stats()
90 	tpad = PAD_POW2((skb->len + hpad), cfhsi->cfg.tail_align);  in cfhsi_update_aggregation_stats()
94 		cfhsi->aggregation_len += len;  in cfhsi_update_aggregation_stats()
96 		cfhsi->aggregation_len -= len;  in cfhsi_update_aggregation_stats()
99 static bool cfhsi_can_send_aggregate(struct cfhsi *cfhsi)  in cfhsi_can_send_aggregate()  argument
103 	if (cfhsi->cfg.aggregation_timeout == 0)  in cfhsi_can_send_aggregate()
107 		if (cfhsi->qhead[i].qlen)  in cfhsi_can_send_aggregate()
112 	if (cfhsi->qhead[CFHSI_PRIO_BEBK].qlen >= CFHSI_MAX_PKTS)  in cfhsi_can_send_aggregate()
118 static struct sk_buff *cfhsi_dequeue(struct cfhsi *cfhsi)  in cfhsi_dequeue()  argument
124 		skb = skb_dequeue(&cfhsi->qhead[i]);  in cfhsi_dequeue()
132 static int cfhsi_tx_queue_len(struct cfhsi *cfhsi)  in cfhsi_tx_queue_len()  argument
136 		len += skb_queue_len(&cfhsi->qhead[i]);  in cfhsi_tx_queue_len()
140 static void cfhsi_abort_tx(struct cfhsi *cfhsi)  in cfhsi_abort_tx()  argument
145 		spin_lock_bh(&cfhsi->lock);  in cfhsi_abort_tx()
146 		skb = cfhsi_dequeue(cfhsi);  in cfhsi_abort_tx()
150 		cfhsi->ndev->stats.tx_errors++;  in cfhsi_abort_tx()
151 		cfhsi->ndev->stats.tx_dropped++;  in cfhsi_abort_tx()
152 		cfhsi_update_aggregation_stats(cfhsi, skb, -1);  in cfhsi_abort_tx()
153 		spin_unlock_bh(&cfhsi->lock);  in cfhsi_abort_tx()
156 	cfhsi->tx_state = CFHSI_TX_STATE_IDLE;  in cfhsi_abort_tx()
157 	if (!test_bit(CFHSI_SHUTDOWN, &cfhsi->bits))  in cfhsi_abort_tx()
158 		mod_timer(&cfhsi->inactivity_timer,  in cfhsi_abort_tx()
159 			jiffies + cfhsi->cfg.inactivity_timeout);  in cfhsi_abort_tx()
160 	spin_unlock_bh(&cfhsi->lock);  in cfhsi_abort_tx()
163 static int cfhsi_flush_fifo(struct cfhsi *cfhsi)  in cfhsi_flush_fifo()  argument
169 	netdev_dbg(cfhsi->ndev, "%s.\n",  in cfhsi_flush_fifo()
173 		ret = cfhsi->ops->cfhsi_fifo_occupancy(cfhsi->ops,  in cfhsi_flush_fifo()
176 			netdev_warn(cfhsi->ndev,  in cfhsi_flush_fifo()
185 		set_bit(CFHSI_FLUSH_FIFO, &cfhsi->bits);  in cfhsi_flush_fifo()
186 		ret = cfhsi->ops->cfhsi_rx(buffer, fifo_occupancy,  in cfhsi_flush_fifo()
187 				cfhsi->ops);  in cfhsi_flush_fifo()
189 			clear_bit(CFHSI_FLUSH_FIFO, &cfhsi->bits);  in cfhsi_flush_fifo()
190 			netdev_warn(cfhsi->ndev,  in cfhsi_flush_fifo()
197 		ret = wait_event_interruptible_timeout(cfhsi->flush_fifo_wait,  in cfhsi_flush_fifo()
198 			 !test_bit(CFHSI_FLUSH_FIFO, &cfhsi->bits), ret);  in cfhsi_flush_fifo()
201 			netdev_warn(cfhsi->ndev,  in cfhsi_flush_fifo()
207 			netdev_warn(cfhsi->ndev,  in cfhsi_flush_fifo()
217 static int cfhsi_tx_frm(struct cfhsi_desc *desc, struct cfhsi *cfhsi)  in cfhsi_tx_frm()  argument
224 	skb = cfhsi_dequeue(cfhsi);  in cfhsi_tx_frm()
240 		hpad = 1 + PAD_POW2((info->hdr_len + 1), cfhsi->cfg.head_align);  in cfhsi_tx_frm()
241 		tpad = PAD_POW2((skb->len + hpad), cfhsi->cfg.tail_align);  in cfhsi_tx_frm()
251 			spin_lock_bh(&cfhsi->lock);  in cfhsi_tx_frm()
252 			cfhsi->ndev->stats.tx_packets++;  in cfhsi_tx_frm()
253 			cfhsi->ndev->stats.tx_bytes += skb->len;  in cfhsi_tx_frm()
254 			cfhsi_update_aggregation_stats(cfhsi, skb, -1);  in cfhsi_tx_frm()
255 			spin_unlock_bh(&cfhsi->lock);  in cfhsi_tx_frm()
273 			skb = cfhsi_dequeue(cfhsi);  in cfhsi_tx_frm()
281 		hpad = 1 + PAD_POW2((info->hdr_len + 1), cfhsi->cfg.head_align);  in cfhsi_tx_frm()
282 		tpad = PAD_POW2((skb->len + hpad), cfhsi->cfg.tail_align);  in cfhsi_tx_frm()
292 		spin_lock_bh(&cfhsi->lock);  in cfhsi_tx_frm()
293 		cfhsi->ndev->stats.tx_packets++;  in cfhsi_tx_frm()
294 		cfhsi->ndev->stats.tx_bytes += skb->len;  in cfhsi_tx_frm()
295 		cfhsi_update_aggregation_stats(cfhsi, skb, -1);  in cfhsi_tx_frm()
296 		spin_unlock_bh(&cfhsi->lock);  in cfhsi_tx_frm()
322 	if (cfhsi_can_send_aggregate(cfhsi))  in cfhsi_tx_frm()
330 static void cfhsi_start_tx(struct cfhsi *cfhsi)  in cfhsi_start_tx()  argument
332 	struct cfhsi_desc *desc = (struct cfhsi_desc *)cfhsi->tx_buf;  in cfhsi_start_tx()
335 	netdev_dbg(cfhsi->ndev, "%s.\n", __func__);  in cfhsi_start_tx()
337 	if (test_bit(CFHSI_SHUTDOWN, &cfhsi->bits))  in cfhsi_start_tx()
342 		len = cfhsi_tx_frm(desc, cfhsi);  in cfhsi_start_tx()
344 			spin_lock_bh(&cfhsi->lock);  in cfhsi_start_tx()
345 			if (unlikely(cfhsi_tx_queue_len(cfhsi))) {  in cfhsi_start_tx()
346 				spin_unlock_bh(&cfhsi->lock);  in cfhsi_start_tx()
350 			cfhsi->tx_state = CFHSI_TX_STATE_IDLE;  in cfhsi_start_tx()
352 			mod_timer(&cfhsi->inactivity_timer,  in cfhsi_start_tx()
353 				jiffies + cfhsi->cfg.inactivity_timeout);  in cfhsi_start_tx()
354 			spin_unlock_bh(&cfhsi->lock);  in cfhsi_start_tx()
359 		res = cfhsi->ops->cfhsi_tx(cfhsi->tx_buf, len, cfhsi->ops);  in cfhsi_start_tx()
361 			netdev_err(cfhsi->ndev, "%s: TX error %d.\n",  in cfhsi_start_tx()
366 static void cfhsi_tx_done(struct cfhsi *cfhsi)  in cfhsi_tx_done()  argument
368 	netdev_dbg(cfhsi->ndev, "%s.\n", __func__);  in cfhsi_tx_done()
370 	if (test_bit(CFHSI_SHUTDOWN, &cfhsi->bits))  in cfhsi_tx_done()
377 	spin_lock_bh(&cfhsi->lock);  in cfhsi_tx_done()
378 	if (cfhsi->flow_off_sent &&  in cfhsi_tx_done()
379 			cfhsi_tx_queue_len(cfhsi) <= cfhsi->cfg.q_low_mark &&  in cfhsi_tx_done()
380 			cfhsi->cfdev.flowctrl) {  in cfhsi_tx_done()
382 		cfhsi->flow_off_sent = 0;  in cfhsi_tx_done()
383 		cfhsi->cfdev.flowctrl(cfhsi->ndev, ON);  in cfhsi_tx_done()
386 	if (cfhsi_can_send_aggregate(cfhsi)) {  in cfhsi_tx_done()
387 		spin_unlock_bh(&cfhsi->lock);  in cfhsi_tx_done()
388 		cfhsi_start_tx(cfhsi);  in cfhsi_tx_done()
390 		mod_timer(&cfhsi->aggregation_timer,  in cfhsi_tx_done()
391 			jiffies + cfhsi->cfg.aggregation_timeout);  in cfhsi_tx_done()
392 		spin_unlock_bh(&cfhsi->lock);  in cfhsi_tx_done()
400 	struct cfhsi *cfhsi;  in cfhsi_tx_done_cb()  local
402 	cfhsi = container_of(cb_ops, struct cfhsi, cb_ops);  in cfhsi_tx_done_cb()
403 	netdev_dbg(cfhsi->ndev, "%s.\n",  in cfhsi_tx_done_cb()
406 	if (test_bit(CFHSI_SHUTDOWN, &cfhsi->bits))  in cfhsi_tx_done_cb()
408 	cfhsi_tx_done(cfhsi);  in cfhsi_tx_done_cb()
411 static int cfhsi_rx_desc(struct cfhsi_desc *desc, struct cfhsi *cfhsi)  in cfhsi_rx_desc()  argument
420 		netdev_err(cfhsi->ndev, "%s: Invalid descriptor.\n",  in cfhsi_rx_desc()
441 			netdev_err(cfhsi->ndev, "%s: Invalid length.\n",  in cfhsi_rx_desc()
449 			netdev_err(cfhsi->ndev, "%s: Out of memory !\n",  in cfhsi_rx_desc()
459 		skb->dev = cfhsi->ndev;  in cfhsi_rx_desc()
472 		cfhsi->ndev->stats.rx_packets++;  in cfhsi_rx_desc()
473 		cfhsi->ndev->stats.rx_bytes += len;  in cfhsi_rx_desc()
489 		netdev_err(cfhsi->ndev,  in cfhsi_rx_desc()
526 static int cfhsi_rx_pld(struct cfhsi_desc *desc, struct cfhsi *cfhsi)  in cfhsi_rx_pld()  argument
536 		netdev_err(cfhsi->ndev, "%s: Invalid descriptor.\n",  in cfhsi_rx_pld()
546 	while (nfrms < cfhsi->rx_state.nfrms) {  in cfhsi_rx_pld()
569 			netdev_err(cfhsi->ndev, "%s: Invalid length.\n",  in cfhsi_rx_pld()
577 			netdev_err(cfhsi->ndev, "%s: Out of memory !\n",  in cfhsi_rx_pld()
579 			cfhsi->rx_state.nfrms = nfrms;  in cfhsi_rx_pld()
588 		skb->dev = cfhsi->ndev;  in cfhsi_rx_pld()
600 		cfhsi->ndev->stats.rx_packets++;  in cfhsi_rx_pld()
601 		cfhsi->ndev->stats.rx_bytes += len;  in cfhsi_rx_pld()
612 static void cfhsi_rx_done(struct cfhsi *cfhsi)  in cfhsi_rx_done()  argument
620 	desc = (struct cfhsi_desc *)cfhsi->rx_buf;  in cfhsi_rx_done()
622 	netdev_dbg(cfhsi->ndev, "%s\n", __func__);  in cfhsi_rx_done()
624 	if (test_bit(CFHSI_SHUTDOWN, &cfhsi->bits))  in cfhsi_rx_done()
628 	spin_lock_bh(&cfhsi->lock);  in cfhsi_rx_done()
629 	mod_timer_pending(&cfhsi->inactivity_timer,  in cfhsi_rx_done()
630 			jiffies + cfhsi->cfg.inactivity_timeout);  in cfhsi_rx_done()
631 	spin_unlock_bh(&cfhsi->lock);  in cfhsi_rx_done()
633 	if (cfhsi->rx_state.state == CFHSI_RX_STATE_DESC) {  in cfhsi_rx_done()
639 		rx_buf = cfhsi->rx_buf;  in cfhsi_rx_done()
644 			rx_buf = cfhsi->rx_flip_buf;  in cfhsi_rx_done()
646 		rx_buf = cfhsi->rx_flip_buf;  in cfhsi_rx_done()
649 		if (cfhsi->rx_state.pld_len > 0 &&  in cfhsi_rx_done()
654 						cfhsi->rx_state.pld_len);  in cfhsi_rx_done()
656 			cfhsi->rx_state.piggy_desc = true;  in cfhsi_rx_done()
688 	if (test_bit(CFHSI_AWAKE, &cfhsi->bits)) {  in cfhsi_rx_done()
690 		netdev_dbg(cfhsi->ndev, "%s: Start RX.\n",  in cfhsi_rx_done()
693 		res = cfhsi->ops->cfhsi_rx(rx_ptr, rx_len,  in cfhsi_rx_done()
694 				cfhsi->ops);  in cfhsi_rx_done()
696 			netdev_err(cfhsi->ndev, "%s: RX error %d.\n",  in cfhsi_rx_done()
698 			cfhsi->ndev->stats.rx_errors++;  in cfhsi_rx_done()
699 			cfhsi->ndev->stats.rx_dropped++;  in cfhsi_rx_done()
703 	if (cfhsi->rx_state.state == CFHSI_RX_STATE_DESC) {  in cfhsi_rx_done()
705 		if (cfhsi_rx_desc(desc, cfhsi) < 0)  in cfhsi_rx_done()
709 		if (cfhsi_rx_pld(desc, cfhsi) < 0)  in cfhsi_rx_done()
713 			if (cfhsi_rx_desc(piggy_desc, cfhsi) < 0)  in cfhsi_rx_done()
721 	memset(&cfhsi->rx_state, 0, sizeof(cfhsi->rx_state));  in cfhsi_rx_done()
722 	cfhsi->rx_state.state = rx_state;  in cfhsi_rx_done()
723 	cfhsi->rx_ptr = rx_ptr;  in cfhsi_rx_done()
724 	cfhsi->rx_len = rx_len;  in cfhsi_rx_done()
725 	cfhsi->rx_state.pld_len = desc_pld_len;  in cfhsi_rx_done()
726 	cfhsi->rx_state.piggy_desc = desc->header & CFHSI_PIGGY_DESC;  in cfhsi_rx_done()
728 	if (rx_buf != cfhsi->rx_buf)  in cfhsi_rx_done()
729 		swap(cfhsi->rx_buf, cfhsi->rx_flip_buf);  in cfhsi_rx_done()
733 	netdev_err(cfhsi->ndev, "%s: Out of sync.\n", __func__);  in cfhsi_rx_done()
735 			cfhsi->rx_buf, CFHSI_DESC_SZ);  in cfhsi_rx_done()
736 	schedule_work(&cfhsi->out_of_sync_work);  in cfhsi_rx_done()
741 	struct cfhsi *cfhsi = from_timer(cfhsi, t, rx_slowpath_timer);  in cfhsi_rx_slowpath()  local
743 	netdev_dbg(cfhsi->ndev, "%s.\n",  in cfhsi_rx_slowpath()
746 	cfhsi_rx_done(cfhsi);  in cfhsi_rx_slowpath()
751 	struct cfhsi *cfhsi;  in cfhsi_rx_done_cb()  local
753 	cfhsi = container_of(cb_ops, struct cfhsi, cb_ops);  in cfhsi_rx_done_cb()
754 	netdev_dbg(cfhsi->ndev, "%s.\n",  in cfhsi_rx_done_cb()
757 	if (test_bit(CFHSI_SHUTDOWN, &cfhsi->bits))  in cfhsi_rx_done_cb()
760 	if (test_and_clear_bit(CFHSI_FLUSH_FIFO, &cfhsi->bits))  in cfhsi_rx_done_cb()
761 		wake_up_interruptible(&cfhsi->flush_fifo_wait);  in cfhsi_rx_done_cb()
763 		cfhsi_rx_done(cfhsi);  in cfhsi_rx_done_cb()
768 	struct cfhsi *cfhsi = NULL;  in cfhsi_wake_up()  local
773 	cfhsi = container_of(work, struct cfhsi, wake_up_work);  in cfhsi_wake_up()
775 	if (test_bit(CFHSI_SHUTDOWN, &cfhsi->bits))  in cfhsi_wake_up()
778 	if (unlikely(test_bit(CFHSI_AWAKE, &cfhsi->bits))) {  in cfhsi_wake_up()
781 		clear_bit(CFHSI_WAKE_UP, &cfhsi->bits);  in cfhsi_wake_up()
782 		clear_bit(CFHSI_WAKE_UP_ACK, &cfhsi->bits);  in cfhsi_wake_up()
787 	cfhsi->ops->cfhsi_wake_up(cfhsi->ops);  in cfhsi_wake_up()
789 	netdev_dbg(cfhsi->ndev, "%s: Start waiting.\n",  in cfhsi_wake_up()
794 	ret = wait_event_interruptible_timeout(cfhsi->wake_up_wait,  in cfhsi_wake_up()
796 							&cfhsi->bits), ret);  in cfhsi_wake_up()
799 		netdev_err(cfhsi->ndev, "%s: Signalled: %ld.\n",  in cfhsi_wake_up()
802 		clear_bit(CFHSI_WAKE_UP, &cfhsi->bits);  in cfhsi_wake_up()
803 		cfhsi->ops->cfhsi_wake_down(cfhsi->ops);  in cfhsi_wake_up()
810 		netdev_dbg(cfhsi->ndev, "%s: Timeout.\n",  in cfhsi_wake_up()
814 		WARN_ON(cfhsi->ops->cfhsi_fifo_occupancy(cfhsi->ops,  in cfhsi_wake_up()
817 		netdev_dbg(cfhsi->ndev, "%s: Bytes in FIFO: %u.\n",  in cfhsi_wake_up()
821 		WARN_ON(cfhsi->ops->cfhsi_get_peer_wake(cfhsi->ops,  in cfhsi_wake_up()
825 			netdev_err(cfhsi->ndev, "%s: CA Wake missed !.\n",  in cfhsi_wake_up()
829 			clear_bit(CFHSI_WAKE_UP_ACK, &cfhsi->bits);  in cfhsi_wake_up()
835 		clear_bit(CFHSI_WAKE_UP, &cfhsi->bits);  in cfhsi_wake_up()
836 		cfhsi->ops->cfhsi_wake_down(cfhsi->ops);  in cfhsi_wake_up()
840 	netdev_dbg(cfhsi->ndev, "%s: Woken.\n",  in cfhsi_wake_up()
844 	set_bit(CFHSI_AWAKE, &cfhsi->bits);  in cfhsi_wake_up()
845 	clear_bit(CFHSI_WAKE_UP, &cfhsi->bits);  in cfhsi_wake_up()
848 	netdev_dbg(cfhsi->ndev, "%s: Start RX.\n", __func__);  in cfhsi_wake_up()
849 	res = cfhsi->ops->cfhsi_rx(cfhsi->rx_ptr, cfhsi->rx_len, cfhsi->ops);  in cfhsi_wake_up()
852 		netdev_err(cfhsi->ndev, "%s: RX err %d.\n", __func__, res);  in cfhsi_wake_up()
855 	clear_bit(CFHSI_WAKE_UP_ACK, &cfhsi->bits);  in cfhsi_wake_up()
857 	spin_lock_bh(&cfhsi->lock);  in cfhsi_wake_up()
860 	if (!cfhsi_tx_queue_len(cfhsi)) {  in cfhsi_wake_up()
861 		netdev_dbg(cfhsi->ndev, "%s: Peer wake, start timer.\n",  in cfhsi_wake_up()
864 		mod_timer(&cfhsi->inactivity_timer,  in cfhsi_wake_up()
865 				jiffies + cfhsi->cfg.inactivity_timeout);  in cfhsi_wake_up()
866 		spin_unlock_bh(&cfhsi->lock);  in cfhsi_wake_up()
870 	netdev_dbg(cfhsi->ndev, "%s: Host wake.\n",  in cfhsi_wake_up()
873 	spin_unlock_bh(&cfhsi->lock);  in cfhsi_wake_up()
876 	len = cfhsi_tx_frm((struct cfhsi_desc *)cfhsi->tx_buf, cfhsi);  in cfhsi_wake_up()
880 		res = cfhsi->ops->cfhsi_tx(cfhsi->tx_buf, len, cfhsi->ops);  in cfhsi_wake_up()
882 			netdev_err(cfhsi->ndev, "%s: TX error %d.\n",  in cfhsi_wake_up()
884 			cfhsi_abort_tx(cfhsi);  in cfhsi_wake_up()
887 		netdev_err(cfhsi->ndev,  in cfhsi_wake_up()
896 	struct cfhsi *cfhsi = NULL;  in cfhsi_wake_down()  local
900 	cfhsi = container_of(work, struct cfhsi, wake_down_work);  in cfhsi_wake_down()
901 	netdev_dbg(cfhsi->ndev, "%s.\n", __func__);  in cfhsi_wake_down()
903 	if (test_bit(CFHSI_SHUTDOWN, &cfhsi->bits))  in cfhsi_wake_down()
907 	cfhsi->ops->cfhsi_wake_down(cfhsi->ops);  in cfhsi_wake_down()
911 	ret = wait_event_interruptible_timeout(cfhsi->wake_down_wait,  in cfhsi_wake_down()
913 							&cfhsi->bits), ret);  in cfhsi_wake_down()
916 		netdev_err(cfhsi->ndev, "%s: Signalled: %ld.\n",  in cfhsi_wake_down()
923 		netdev_err(cfhsi->ndev, "%s: Timeout.\n", __func__);  in cfhsi_wake_down()
926 		WARN_ON(cfhsi->ops->cfhsi_get_peer_wake(cfhsi->ops,  in cfhsi_wake_down()
929 			netdev_err(cfhsi->ndev, "%s: CA Wake missed !.\n",  in cfhsi_wake_down()
935 		WARN_ON(cfhsi->ops->cfhsi_fifo_occupancy(cfhsi->ops,  in cfhsi_wake_down()
947 		netdev_err(cfhsi->ndev, "%s: FIFO Timeout.\n", __func__);  in cfhsi_wake_down()
950 	clear_bit(CFHSI_AWAKE, &cfhsi->bits);  in cfhsi_wake_down()
953 	cfhsi->ops->cfhsi_rx_cancel(cfhsi->ops);  in cfhsi_wake_down()
958 	struct cfhsi *cfhsi = NULL;  in cfhsi_out_of_sync()  local
960 	cfhsi = container_of(work, struct cfhsi, out_of_sync_work);  in cfhsi_out_of_sync()
963 	dev_close(cfhsi->ndev);  in cfhsi_out_of_sync()
969 	struct cfhsi *cfhsi = NULL;  in cfhsi_wake_up_cb()  local
971 	cfhsi = container_of(cb_ops, struct cfhsi, cb_ops);  in cfhsi_wake_up_cb()
972 	netdev_dbg(cfhsi->ndev, "%s.\n",  in cfhsi_wake_up_cb()
975 	set_bit(CFHSI_WAKE_UP_ACK, &cfhsi->bits);  in cfhsi_wake_up_cb()
976 	wake_up_interruptible(&cfhsi->wake_up_wait);  in cfhsi_wake_up_cb()
978 	if (test_bit(CFHSI_SHUTDOWN, &cfhsi->bits))  in cfhsi_wake_up_cb()
982 	if (!test_and_set_bit(CFHSI_WAKE_UP, &cfhsi->bits))  in cfhsi_wake_up_cb()
983 		queue_work(cfhsi->wq, &cfhsi->wake_up_work);  in cfhsi_wake_up_cb()
988 	struct cfhsi *cfhsi = NULL;  in cfhsi_wake_down_cb()  local
990 	cfhsi = container_of(cb_ops, struct cfhsi, cb_ops);  in cfhsi_wake_down_cb()
991 	netdev_dbg(cfhsi->ndev, "%s.\n",  in cfhsi_wake_down_cb()
995 	set_bit(CFHSI_WAKE_DOWN_ACK, &cfhsi->bits);  in cfhsi_wake_down_cb()
996 	wake_up_interruptible(&cfhsi->wake_down_wait);  in cfhsi_wake_down_cb()
1001 	struct cfhsi *cfhsi = from_timer(cfhsi, t, aggregation_timer);  in cfhsi_aggregation_tout()  local
1003 	netdev_dbg(cfhsi->ndev, "%s.\n",  in cfhsi_aggregation_tout()
1006 	cfhsi_start_tx(cfhsi);  in cfhsi_aggregation_tout()
1011 	struct cfhsi *cfhsi = NULL;  in cfhsi_xmit()  local
1019 	cfhsi = netdev_priv(dev);  in cfhsi_xmit()
1039 	spin_lock_bh(&cfhsi->lock);  in cfhsi_xmit()
1042 	cfhsi_update_aggregation_stats(cfhsi, skb, 1);  in cfhsi_xmit()
1045 	skb_queue_tail(&cfhsi->qhead[prio], skb);  in cfhsi_xmit()
1048 	if (WARN_ON(test_bit(CFHSI_SHUTDOWN, &cfhsi->bits))) {  in cfhsi_xmit()
1049 		spin_unlock_bh(&cfhsi->lock);  in cfhsi_xmit()
1050 		cfhsi_abort_tx(cfhsi);  in cfhsi_xmit()
1055 	if (!cfhsi->flow_off_sent &&  in cfhsi_xmit()
1056 		cfhsi_tx_queue_len(cfhsi) > cfhsi->cfg.q_high_mark &&  in cfhsi_xmit()
1057 		cfhsi->cfdev.flowctrl) {  in cfhsi_xmit()
1058 		cfhsi->flow_off_sent = 1;  in cfhsi_xmit()
1059 		cfhsi->cfdev.flowctrl(cfhsi->ndev, OFF);  in cfhsi_xmit()
1062 	if (cfhsi->tx_state == CFHSI_TX_STATE_IDLE) {  in cfhsi_xmit()
1063 		cfhsi->tx_state = CFHSI_TX_STATE_XFER;  in cfhsi_xmit()
1070 			cfhsi_can_send_aggregate(cfhsi) &&  in cfhsi_xmit()
1071 			del_timer(&cfhsi->aggregation_timer) > 0;  in cfhsi_xmit()
1072 		spin_unlock_bh(&cfhsi->lock);  in cfhsi_xmit()
1074 			cfhsi_start_tx(cfhsi);  in cfhsi_xmit()
1079 	timer_active = del_timer_sync(&cfhsi->inactivity_timer);  in cfhsi_xmit()
1081 	spin_unlock_bh(&cfhsi->lock);  in cfhsi_xmit()
1084 		struct cfhsi_desc *desc = (struct cfhsi_desc *)cfhsi->tx_buf;  in cfhsi_xmit()
1089 		len = cfhsi_tx_frm(desc, cfhsi);  in cfhsi_xmit()
1093 		res = cfhsi->ops->cfhsi_tx(cfhsi->tx_buf, len, cfhsi->ops);  in cfhsi_xmit()
1095 			netdev_err(cfhsi->ndev, "%s: TX error %d.\n",  in cfhsi_xmit()
1097 			cfhsi_abort_tx(cfhsi);  in cfhsi_xmit()
1101 		if (!test_and_set_bit(CFHSI_WAKE_UP, &cfhsi->bits))  in cfhsi_xmit()
1102 			queue_work(cfhsi->wq, &cfhsi->wake_up_work);  in cfhsi_xmit()
1113 	struct cfhsi *cfhsi = netdev_priv(dev);  in cfhsi_setup()  local
1122 		skb_queue_head_init(&cfhsi->qhead[i]);  in cfhsi_setup()
1123 	cfhsi->cfdev.link_select = CAIF_LINK_HIGH_BANDW;  in cfhsi_setup()
1124 	cfhsi->cfdev.use_frag = false;  in cfhsi_setup()
1125 	cfhsi->cfdev.use_stx = false;  in cfhsi_setup()
1126 	cfhsi->cfdev.use_fcs = false;  in cfhsi_setup()
1127 	cfhsi->ndev = dev;  in cfhsi_setup()
1128 	cfhsi->cfg = hsi_default_config;  in cfhsi_setup()
1133 	struct cfhsi *cfhsi = netdev_priv(ndev);  in cfhsi_open()  local
1136 	clear_bit(CFHSI_SHUTDOWN, &cfhsi->bits);  in cfhsi_open()
1139 	cfhsi->tx_state = CFHSI_TX_STATE_IDLE;  in cfhsi_open()
1140 	cfhsi->rx_state.state = CFHSI_RX_STATE_DESC;  in cfhsi_open()
1143 	cfhsi->flow_off_sent = 0;  in cfhsi_open()
1149 	cfhsi->tx_buf = kzalloc(CFHSI_BUF_SZ_TX, GFP_KERNEL);  in cfhsi_open()
1150 	if (!cfhsi->tx_buf) {  in cfhsi_open()
1159 	cfhsi->rx_buf = kzalloc(CFHSI_BUF_SZ_RX, GFP_KERNEL);  in cfhsi_open()
1160 	if (!cfhsi->rx_buf) {  in cfhsi_open()
1165 	cfhsi->rx_flip_buf = kzalloc(CFHSI_BUF_SZ_RX, GFP_KERNEL);  in cfhsi_open()
1166 	if (!cfhsi->rx_flip_buf) {  in cfhsi_open()
1172 	cfhsi->cfg.aggregation_timeout = hsi_default_config.aggregation_timeout;  in cfhsi_open()
1175 	cfhsi->rx_ptr = cfhsi->rx_buf;  in cfhsi_open()
1176 	cfhsi->rx_len = CFHSI_DESC_SZ;  in cfhsi_open()
1179 	spin_lock_init(&cfhsi->lock);  in cfhsi_open()
1182 	cfhsi->cb_ops.tx_done_cb = cfhsi_tx_done_cb;  in cfhsi_open()
1183 	cfhsi->cb_ops.rx_done_cb = cfhsi_rx_done_cb;  in cfhsi_open()
1184 	cfhsi->cb_ops.wake_up_cb = cfhsi_wake_up_cb;  in cfhsi_open()
1185 	cfhsi->cb_ops.wake_down_cb = cfhsi_wake_down_cb;  in cfhsi_open()
1188 	INIT_WORK(&cfhsi->wake_up_work, cfhsi_wake_up);  in cfhsi_open()
1189 	INIT_WORK(&cfhsi->wake_down_work, cfhsi_wake_down);  in cfhsi_open()
1190 	INIT_WORK(&cfhsi->out_of_sync_work, cfhsi_out_of_sync);  in cfhsi_open()
1193 	clear_bit(CFHSI_WAKE_UP_ACK, &cfhsi->bits);  in cfhsi_open()
1194 	clear_bit(CFHSI_WAKE_DOWN_ACK, &cfhsi->bits);  in cfhsi_open()
1195 	clear_bit(CFHSI_WAKE_UP, &cfhsi->bits);  in cfhsi_open()
1196 	clear_bit(CFHSI_AWAKE, &cfhsi->bits);  in cfhsi_open()
1199 	cfhsi->wq = alloc_ordered_workqueue(cfhsi->ndev->name, WQ_MEM_RECLAIM);  in cfhsi_open()
1200 	if (!cfhsi->wq) {  in cfhsi_open()
1201 		netdev_err(cfhsi->ndev, "%s: Failed to create work queue.\n",  in cfhsi_open()
1208 	init_waitqueue_head(&cfhsi->wake_up_wait);  in cfhsi_open()
1209 	init_waitqueue_head(&cfhsi->wake_down_wait);  in cfhsi_open()
1210 	init_waitqueue_head(&cfhsi->flush_fifo_wait);  in cfhsi_open()
1213 	timer_setup(&cfhsi->inactivity_timer, cfhsi_inactivity_tout, 0);  in cfhsi_open()
1215 	timer_setup(&cfhsi->rx_slowpath_timer, cfhsi_rx_slowpath, 0);  in cfhsi_open()
1217 	timer_setup(&cfhsi->aggregation_timer, cfhsi_aggregation_tout, 0);  in cfhsi_open()
1220 	res = cfhsi->ops->cfhsi_up(cfhsi->ops);  in cfhsi_open()
1222 		netdev_err(cfhsi->ndev,  in cfhsi_open()
1229 	res = cfhsi_flush_fifo(cfhsi);  in cfhsi_open()
1231 		netdev_err(cfhsi->ndev, "%s: Can't flush FIFO: %d.\n",  in cfhsi_open()
1238 	cfhsi->ops->cfhsi_down(cfhsi->ops);  in cfhsi_open()
1240 	destroy_workqueue(cfhsi->wq);  in cfhsi_open()
1242 	kfree(cfhsi->rx_flip_buf);  in cfhsi_open()
1244 	kfree(cfhsi->rx_buf);  in cfhsi_open()
1246 	kfree(cfhsi->tx_buf);  in cfhsi_open()
1253 	struct cfhsi *cfhsi = netdev_priv(ndev);  in cfhsi_close()  local
1257 	set_bit(CFHSI_SHUTDOWN, &cfhsi->bits);  in cfhsi_close()
1260 	del_timer_sync(&cfhsi->inactivity_timer);  in cfhsi_close()
1261 	del_timer_sync(&cfhsi->rx_slowpath_timer);  in cfhsi_close()
1262 	del_timer_sync(&cfhsi->aggregation_timer);  in cfhsi_close()
1265 	cfhsi->ops->cfhsi_rx_cancel(cfhsi->ops);  in cfhsi_close()
1268 	destroy_workqueue(cfhsi->wq);  in cfhsi_close()
1271 	tx_buf = cfhsi->tx_buf;  in cfhsi_close()
1272 	rx_buf = cfhsi->rx_buf;  in cfhsi_close()
1273 	flip_buf = cfhsi->rx_flip_buf;  in cfhsi_close()
1275 	cfhsi_abort_tx(cfhsi);  in cfhsi_close()
1278 	cfhsi->ops->cfhsi_down(cfhsi->ops);  in cfhsi_close()
1289 	struct cfhsi *cfhsi = netdev_priv(dev);  in cfhsi_uninit()  local
1292 	list_del(&cfhsi->list);  in cfhsi_uninit()
1302 static void cfhsi_netlink_parms(struct nlattr *data[], struct cfhsi *cfhsi)  in cfhsi_netlink_parms()  argument
1319 		cfhsi->cfg.inactivity_timeout =	inactivity_timeout * HZ / 1000;  in cfhsi_netlink_parms()
1320 		if (cfhsi->cfg.inactivity_timeout == 0)  in cfhsi_netlink_parms()
1321 			cfhsi->cfg.inactivity_timeout = 1;  in cfhsi_netlink_parms()
1322 		else if (cfhsi->cfg.inactivity_timeout > NEXT_TIMER_MAX_DELTA)  in cfhsi_netlink_parms()
1323 			cfhsi->cfg.inactivity_timeout = NEXT_TIMER_MAX_DELTA;  in cfhsi_netlink_parms()
1328 		cfhsi->cfg.aggregation_timeout = nla_get_u32(data[i]);  in cfhsi_netlink_parms()
1332 		cfhsi->cfg.head_align = nla_get_u32(data[i]);  in cfhsi_netlink_parms()
1336 		cfhsi->cfg.tail_align = nla_get_u32(data[i]);  in cfhsi_netlink_parms()
1340 		cfhsi->cfg.q_high_mark = nla_get_u32(data[i]);  in cfhsi_netlink_parms()
1344 		cfhsi->cfg.q_low_mark = nla_get_u32(data[i]);  in cfhsi_netlink_parms()
1376 	struct cfhsi *cfhsi = netdev_priv(dev);  in caif_hsi_fill_info()  local
1379 			cfhsi->cfg.inactivity_timeout) ||  in caif_hsi_fill_info()
1381 			cfhsi->cfg.aggregation_timeout) ||  in caif_hsi_fill_info()
1383 			cfhsi->cfg.head_align) ||  in caif_hsi_fill_info()
1385 			cfhsi->cfg.tail_align) ||  in caif_hsi_fill_info()
1387 			cfhsi->cfg.q_high_mark) ||  in caif_hsi_fill_info()
1389 			cfhsi->cfg.q_low_mark))  in caif_hsi_fill_info()
1399 	struct cfhsi *cfhsi = NULL;  in caif_hsi_newlink()  local
1404 	cfhsi = netdev_priv(dev);  in caif_hsi_newlink()
1405 	cfhsi_netlink_parms(data, cfhsi);  in caif_hsi_newlink()
1414 	cfhsi->ops = (*get_ops)();  in caif_hsi_newlink()
1415 	if (!cfhsi->ops) {  in caif_hsi_newlink()
1421 	cfhsi->ops->cb_ops = &cfhsi->cb_ops;  in caif_hsi_newlink()
1427 	list_add_tail(&cfhsi->list, &cfhsi_list);  in caif_hsi_newlink()
1437 	.priv_size	= sizeof(struct cfhsi),
1451 	struct cfhsi *cfhsi;  in cfhsi_exit_module()  local
1457 		cfhsi = list_entry(list_node, struct cfhsi, list);  in cfhsi_exit_module()
1458 		unregister_netdev(cfhsi->ndev);  in cfhsi_exit_module()