3  * Generic (non-bus specific) TX handling
6  * Copyright (C) 2007-2008 Intel Corporation. All rights reserved.
35  * Intel Corporation <linux-wimax@intel.com>
37  *  - Initial implementation
39  * Intel Corporation <linux-wimax@intel.com>
40  * Inaky Perez-Gonzalez <inaky.perez-gonzalez@intel.com>
41  *  - Rewritten to use a single FIFO to lower the memory allocation
43  *    well as splitting out bus-specific code.
47  * software FIFO, as data/control frames can be coalesced (while the
50  * A FIFO is used because at the end it is resource-cheaper that trying
56  * the transmission of that. This is split between a bus-independent
57  * part that just prepares everything and a bus-specific part that
59  * bus-specific driver).
62  * The general format of a device-host transaction is MSG-HDR, PLD1,
67  * per node), it becomes cheaper to append all the data to a FIFO
68  * (copying to a FIFO potentially in cache is cheaper).
70  * Then the bus-specific code takes the parts of that FIFO that are
75  * We use a FIFO to queue the data in a linear buffer. We first append
76  * a MSG-HDR, space for I2400M_TX_PLD_MAX payload descriptors and then
79  * multiple of i2400m->bus_tx_block_size (as defined by the bus layer).
81  * - A TX message: a combination of a message header, payload
84  *     Open: it is marked as active (i2400m->tx_msg is valid) and we
90  *       length is aligned to i2400m->bus_tx_block_size (as set by the
93  * - Most of the time we keep a TX message open to which we append
96  * - If we are going to append and there is no more space (we are at
97  *   the end of the FIFO), we close the message, mark the rest of the
98  *   FIFO space unusable (skip_tail), create a new message at the
99  *   beginning of the FIFO (if there is space) and append the message
103  *   engine. So we don't write a message to the remaining FIFO space
106  * - We overload one of the fields in the message header to use it as
107  *   'size' of the TX message, so we can iterate over them. It also
109  *   When we send the buffer, we update that to its real on-the-wire
112  * - The MSG-HDR PLD1...PLD2 stuff has to be a size multiple of 16.
114  *   It follows that if MSG-HDR says we have N messages, the whole
130  *   We do this in i2400m_tx_close(). See 'i2400m_msg_hdr->offset'.
132  * - Each payload has to be size-padded to 16 bytes; before appending
135  * - The whole message has to be padded to i2400m->bus_tx_block_size;
141  * TXs. It will TX (in parallel) until the buffer is exhausted--hence
144  * course, when the bus-specific driver attempts to TX a message that
148  * partially full FIFO, with a closed message ready to be delivered
149  * (with a moved message header to make sure it is size-aligned to
158  *    |  msg_hdr to skip (size |= 0x80000)                |
159  *    |---------------------------------------------------|-------
165  *    |- - - - - - - - - - - - - - - - - - - - - - - - - -|   |
170  *    |- - - - - - - - - - - - - - - - - - - - - - - - - -|   |
175  *    |- - - - - - - - - - - - - - - - - - - - - - - - - -|- -|- - - -
181  *    |- - - - - - - - - - - - - - - - - - - - - - - - - -|- - -   |
185  *    |- - - - - - - - - - - - - - - - - - - - - - - - - -|        |
186  *    |  msg_hdr (size X)       [this message is closed]  |       \|/
209  *    |- - - - - - - - - - - - - - - - - - - - - - - - - -|
214  *    |- - - - - - - - - - - - - - - - - - - - - - - - - -|
219  *    |  msg_hdr (size X)           \|/ [message is open] |
220  * 0   ---------------------------------------------------
229  *    i2400m_tx_fifo_push()     Allocates append-space in the FIFO
230  *    i2400m_tx_new()           Opens a new message in the FIFO
232  *    i2400m_tx_close()         Closes an open message in the FIFO
233  *    i2400m_tx_skip_tail()     Marks unusable FIFO tail space
234  *    i2400m->bus_tx_kick()
236  * Now i2400m->bus_tx_kick() is the the bus-specific driver backend
239  * i2400m->bus_tx_kick()
244  *                               (FIFO empty).
253 #include "debug-levels.h"
257 	 * TX Buffer size
263 	 * tx buffer size to 64KiB.
269 	 * packets are MTU size (~1400-~1500) it follows that we could
270 	 * fit at most 10-11 payloads in one transaction. To meet the
288 	 * documents, the maximum size of each message can be up to 16KiB.
305  *   |<-  IN   ->|
309  *   |<-  OUT  ->|
312  * 0  -----------
315  * i2400m->tx_in is right at the end of the buffer (really full
319  * i2400m->tx_in being zero would fail, so we treat is an a special
328 	if (unlikely(i2400m->tx_in == 0))  in __i2400m_tx_tail_room()
330 	tx_in = i2400m->tx_in % I2400M_TX_BUF_SIZE;  in __i2400m_tx_tail_room()
331 	tail_room = I2400M_TX_BUF_SIZE - tx_in;  in __i2400m_tx_tail_room()
338  * Allocate @size bytes in the TX fifo, return a pointer to it
341  * @size: size of the buffer we need to allocate
343  *     contiguous space in the fifo. This is needed because later on
346  *     in the TX FIFO. This boolean is required to avoids a system hang
360  * These are the two basic cases we need to keep an eye for -- it is
368  *   |<-  IN   ->|        |<-  OUT  ->|
372  *   |<-  OUT  ->|        |<-  IN   ->|
375  * 0  -----------          -----------
386  * In certain situations, tx_in would have reached on the top of TX FIFO
390  *   |<-  IN   ->|
395  *   |<-  OUT  ->|
399  * 0  -----------
400  * During such a time, where tail room is zero in the TX FIFO and if there
401  * is a request to add a payload to TX FIFO, which calls:
403  *         ->calls i2400m_tx_close()
404  *         ->calls i2400m_tx_skip_tail()
406  *         ->calls i2400m_tx_new()
407  *                    |----> [try_head:]
408  *     infinite loop  |     ->calls i2400m_tx_fifo_push()
412  *                    |<----  goto try_head;
420  * The i2400m_tx_new() keeps re-retrying by calling i2400m_tx_fifo_push()
428  *     Assumes i2400m->tx_lock is taken, and we use that as a barrier
434 void *i2400m_tx_fifo_push(struct i2400m *i2400m, size_t size,  in i2400m_tx_fifo_push()  argument
441 	needed_size = size + padding;  in i2400m_tx_fifo_push()
442 	room = I2400M_TX_BUF_SIZE - (i2400m->tx_in - i2400m->tx_out);  in i2400m_tx_fifo_push()
444 		d_printf(2, dev, "fifo push %zu/%zu: no space\n",  in i2400m_tx_fifo_push()
445 			 size, padding);  in i2400m_tx_fifo_push()
453 		 * in the TX FIFO, then there are two possibilities:  in i2400m_tx_fifo_push()
455 		 * this message in the TX FIFO.  in i2400m_tx_fifo_push()
457 		 * in tail room of the TX FIFO to accommodate the message.  in i2400m_tx_fifo_push()
461 		 * In the case (2), return NULL, indicating that the TX FIFO  in i2400m_tx_fifo_push()
464 		if (room - tail_room >= needed_size) {  in i2400m_tx_fifo_push()
465 			d_printf(2, dev, "fifo push %zu/%zu: tail full\n",  in i2400m_tx_fifo_push()
466 				 size, padding);  in i2400m_tx_fifo_push()
469 			d_printf(2, dev, "fifo push %zu/%zu: no head space\n",  in i2400m_tx_fifo_push()
470 				 size, padding);  in i2400m_tx_fifo_push()
474 	ptr = i2400m->tx_buf + i2400m->tx_in % I2400M_TX_BUF_SIZE;  in i2400m_tx_fifo_push()
475 	d_printf(2, dev, "fifo push %zu/%zu: at @%zu\n", size, padding,  in i2400m_tx_fifo_push()
476 		 i2400m->tx_in % I2400M_TX_BUF_SIZE);  in i2400m_tx_fifo_push()
477 	i2400m->tx_in += size;  in i2400m_tx_fifo_push()
483  * Mark the tail of the FIFO buffer as 'to-skip'
486  * the FIFO are padded to be a multiple of 16 -- the size of *msg
491  * space only for a header. _tx_close() will mark it as to-skip (as it
499  *     Assumes i2400m->tx_lock is taken, and we use that as a barrier
501  *     This path is only taken for Case A FIFO situations [see
508 	size_t tx_in = i2400m->tx_in % I2400M_TX_BUF_SIZE;  in i2400m_tx_skip_tail()
510 	struct i2400m_msg_hdr *msg = i2400m->tx_buf + tx_in;  in i2400m_tx_skip_tail()
514 	msg->size = tail_room | I2400M_TX_SKIP;  in i2400m_tx_skip_tail()
517 	i2400m->tx_in += tail_room;  in i2400m_tx_skip_tail()
530  *     Assumes a TX message is active (i2400m->tx_msg).
532  *     Assumes i2400m->tx_lock is taken, and we use that as a barrier
537 	struct i2400m_msg_hdr *msg_hdr = i2400m->tx_msg;  in i2400m_tx_fits()
538 	return le16_to_cpu(msg_hdr->num_pls) < I2400M_TX_PLD_MAX;  in i2400m_tx_fits()
546  * Reserve memory from the base FIFO engine and then just initialize
550  * fit I2400M_TX_PLD_MAX payloads) -- when it is closed it will be
558  *     Assumes i2400m->tx_lock is taken, and we use that as a barrier
566 	BUG_ON(i2400m->tx_msg != NULL);  in i2400m_tx_new()
573 	 * size <= bus_tx_room_min.  in i2400m_tx_new()
577 				     i2400m->bus_tx_room_min, try_head);  in i2400m_tx_new()
587 	tx_msg->size = I2400M_TX_PLD_SIZE;  in i2400m_tx_new()
589 	i2400m->tx_msg = tx_msg;  in i2400m_tx_new()
591 		 tx_msg, (void *) tx_msg - i2400m->tx_buf);  in i2400m_tx_new()
605  * in the FIFO for that. If there are no payloads, just pass, as it
616 	struct i2400m_msg_hdr *tx_msg = i2400m->tx_msg;  in i2400m_tx_close()
622 	if (tx_msg->size & I2400M_TX_SKIP)	/* a skipper? nothing to do */  in i2400m_tx_close()
624 	num_pls = le16_to_cpu(tx_msg->num_pls);  in i2400m_tx_close()
629 		tx_msg->size |= I2400M_TX_SKIP;  in i2400m_tx_close()
634 	 * Find the current header size, align it to 16 and if we need  in i2400m_tx_close()
643 	hdr_size = struct_size(tx_msg, pld, le16_to_cpu(tx_msg->num_pls));  in i2400m_tx_close()
645 	tx_msg->offset = I2400M_TX_PLD_SIZE - hdr_size;  in i2400m_tx_close()
646 	tx_msg_moved = (void *) tx_msg + tx_msg->offset;  in i2400m_tx_close()
648 	tx_msg_moved->size -= tx_msg->offset;  in i2400m_tx_close()
651 	 * message so the size is a multiple of i2400m->bus_tx_block_size.  in i2400m_tx_close()
653 	aligned_size = ALIGN(tx_msg_moved->size, i2400m->bus_tx_block_size);  in i2400m_tx_close()
654 	padding = aligned_size - tx_msg_moved->size;  in i2400m_tx_close()
658 			/* This should not happen -- append should verify  in i2400m_tx_close()
663 				"device should not read for padding - "  in i2400m_tx_close()
664 				"size %lu aligned_size %zu tx_buf %p in "  in i2400m_tx_close()
666 				(unsigned long) tx_msg_moved->size,  in i2400m_tx_close()
667 				aligned_size, i2400m->tx_buf, i2400m->tx_in,  in i2400m_tx_close()
668 				i2400m->tx_out);  in i2400m_tx_close()
672 	tx_msg_moved->padding = cpu_to_le16(padding);  in i2400m_tx_close()
673 	tx_msg_moved->size += padding;  in i2400m_tx_close()
675 		tx_msg->size += padding;  in i2400m_tx_close()
677 	i2400m->tx_msg = NULL;  in i2400m_tx_close()
682  * i2400m_tx - send the data in a buffer to the device
686  * @buf_len: buffer size
691  *     0 if ok, < 0 errno code on error (-ENOSPC, if there is no more
694  * Appends the buffer to the TX FIFO and notifies the bus-specific
702  * Whenever we write to a message, we increase msg->size, so it
706  * size with the real barker before sending).
716 	int result = -ENOSPC;  in i2400m_tx()
732 	spin_lock_irqsave(&i2400m->tx_lock, flags);  in i2400m_tx()
734 	if (i2400m->tx_buf == NULL) {  in i2400m_tx()
735 		result = -ESHUTDOWN;  in i2400m_tx()
739 	if (unlikely(i2400m->tx_msg == NULL))  in i2400m_tx()
742 			  || (is_singleton && i2400m->tx_msg->num_pls != 0))) {  in i2400m_tx()
745 			 is_singleton, i2400m->tx_msg->num_pls);  in i2400m_tx()
749 	if (i2400m->tx_msg == NULL)  in i2400m_tx()
753 	 * TX message. The total message size must not exceed the maximum  in i2400m_tx()
754 	 * size of each message I2400M_TX_MSG_SIZE. If it exceeds,  in i2400m_tx()
757 	if (i2400m->tx_msg->size + padded_len > I2400M_TX_MSG_SIZE) {  in i2400m_tx()
762 	if (i2400m->tx_msg == NULL)  in i2400m_tx()
765 	 * the message -- if there is not enough, try the head */  in i2400m_tx()
767 				  i2400m->bus_tx_block_size, try_head);  in i2400m_tx()
775 		result = -ENOSPC;  in i2400m_tx()
778 		struct i2400m_msg_hdr *tx_msg = i2400m->tx_msg;  in i2400m_tx()
779 		unsigned num_pls = le16_to_cpu(tx_msg->num_pls);  in i2400m_tx()
781 		memset(ptr + buf_len, 0xad, padded_len - buf_len);  in i2400m_tx()
782 		i2400m_pld_set(&tx_msg->pld[num_pls], buf_len, pl_type);  in i2400m_tx()
784 			 le32_to_cpu(tx_msg->pld[num_pls].val),  in i2400m_tx()
786 		tx_msg->num_pls = le16_to_cpu(num_pls+1);  in i2400m_tx()
787 		tx_msg->size += padded_len;  in i2400m_tx()
789 			padded_len, tx_msg->size, num_pls+1);  in i2400m_tx()
792 			 (void *)tx_msg - i2400m->tx_buf, (size_t)tx_msg->size,  in i2400m_tx()
793 			 num_pls+1, ptr - i2400m->tx_buf, buf_len, padded_len);  in i2400m_tx()
799 	spin_unlock_irqrestore(&i2400m->tx_lock, flags);  in i2400m_tx()
802 	if (likely(result != -ESHUTDOWN))  in i2400m_tx()
803 		i2400m->bus_tx_kick(i2400m);  in i2400m_tx()
812  * i2400m_tx_msg_get - Get the first TX message in the FIFO to start sending it
815  * @bus_size: where to place the size of the TX message
817  * Called by the bus-specific driver to get the first TX message at
820  * It sets the state in @i2400m to indicate the bus-specific driver is
821  * transferring that message (i2400m->tx_msg_size).
827  *     The size of the TX message to be transmitted might be smaller than
828  *     that of the TX message in the FIFO (in case the header was
830  *     use. We keep the message's size in i2400m->tx_msg_size so that
832  *     advance the fifo.
845 	spin_lock_irqsave(&i2400m->tx_lock, flags);  in i2400m_tx_msg_get()
847 	if (i2400m->tx_buf == NULL)  in i2400m_tx_msg_get()
851 	if (i2400m->tx_in == i2400m->tx_out) {	/* Empty FIFO? */  in i2400m_tx_msg_get()
852 		i2400m->tx_in = 0;  in i2400m_tx_msg_get()
853 		i2400m->tx_out = 0;  in i2400m_tx_msg_get()
854 		d_printf(2, dev, "TX: FIFO empty: resetting\n");  in i2400m_tx_msg_get()
857 	tx_msg = i2400m->tx_buf + i2400m->tx_out % I2400M_TX_BUF_SIZE;  in i2400m_tx_msg_get()
858 	if (tx_msg->size & I2400M_TX_SKIP) {	/* skip? */  in i2400m_tx_msg_get()
860 			 i2400m->tx_out % I2400M_TX_BUF_SIZE,  in i2400m_tx_msg_get()
861 			 (size_t) tx_msg->size & ~I2400M_TX_SKIP);  in i2400m_tx_msg_get()
862 		i2400m->tx_out += tx_msg->size & ~I2400M_TX_SKIP;  in i2400m_tx_msg_get()
866 	if (tx_msg->num_pls == 0) {		/* No payloads? */  in i2400m_tx_msg_get()
867 		if (tx_msg == i2400m->tx_msg) {	/* open, we are done */  in i2400m_tx_msg_get()
869 				 "TX: FIFO empty: open msg w/o payloads @%zu\n",  in i2400m_tx_msg_get()
870 				 (void *) tx_msg - i2400m->tx_buf);  in i2400m_tx_msg_get()
876 				 (void *) tx_msg - i2400m->tx_buf,  in i2400m_tx_msg_get()
877 				 (size_t) tx_msg->size);  in i2400m_tx_msg_get()
878 			i2400m->tx_out += tx_msg->size & ~I2400M_TX_SKIP;  in i2400m_tx_msg_get()
882 	if (tx_msg == i2400m->tx_msg)		/* open msg? */  in i2400m_tx_msg_get()
886 	tx_msg_moved = (void *) tx_msg + tx_msg->offset;  in i2400m_tx_msg_get()
887 	i2400m->tx_msg_size = tx_msg->size;  in i2400m_tx_msg_get()
888 	*bus_size = tx_msg_moved->size;  in i2400m_tx_msg_get()
890 		 "size %zu bus_size %zu\n",  in i2400m_tx_msg_get()
891 		 current->pid, (void *) tx_msg - i2400m->tx_buf,  in i2400m_tx_msg_get()
892 		 (size_t) tx_msg->offset, (size_t) tx_msg->size,  in i2400m_tx_msg_get()
893 		 (size_t) tx_msg_moved->size);  in i2400m_tx_msg_get()
894 	tx_msg_moved->barker = le32_to_cpu(I2400M_H2D_PREVIEW_BARKER);  in i2400m_tx_msg_get()
895 	tx_msg_moved->sequence = le32_to_cpu(i2400m->tx_sequence++);  in i2400m_tx_msg_get()
897 	pls = le32_to_cpu(tx_msg_moved->num_pls);  in i2400m_tx_msg_get()
898 	i2400m->tx_pl_num += pls;		/* Update stats */  in i2400m_tx_msg_get()
899 	if (pls > i2400m->tx_pl_max)  in i2400m_tx_msg_get()
900 		i2400m->tx_pl_max = pls;  in i2400m_tx_msg_get()
901 	if (pls < i2400m->tx_pl_min)  in i2400m_tx_msg_get()
902 		i2400m->tx_pl_min = pls;  in i2400m_tx_msg_get()
903 	i2400m->tx_num++;  in i2400m_tx_msg_get()
904 	i2400m->tx_size_acc += *bus_size;  in i2400m_tx_msg_get()
905 	if (*bus_size < i2400m->tx_size_min)  in i2400m_tx_msg_get()
906 		i2400m->tx_size_min = *bus_size;  in i2400m_tx_msg_get()
907 	if (*bus_size > i2400m->tx_size_max)  in i2400m_tx_msg_get()
908 		i2400m->tx_size_max = *bus_size;  in i2400m_tx_msg_get()
910 	spin_unlock_irqrestore(&i2400m->tx_lock, flags);  in i2400m_tx_msg_get()
919  * i2400m_tx_msg_sent - indicate the transmission of a TX message
923  * Called by the bus-specific driver when a message has been sent;
924  * this pops it from the FIFO; and as there is space, start the queue
937 	spin_lock_irqsave(&i2400m->tx_lock, flags);  in i2400m_tx_msg_sent()
938 	if (i2400m->tx_buf == NULL)  in i2400m_tx_msg_sent()
940 	i2400m->tx_out += i2400m->tx_msg_size;  in i2400m_tx_msg_sent()
941 	d_printf(2, dev, "TX: sent %zu b\n", (size_t) i2400m->tx_msg_size);  in i2400m_tx_msg_sent()
942 	i2400m->tx_msg_size = 0;  in i2400m_tx_msg_sent()
943 	BUG_ON(i2400m->tx_out > i2400m->tx_in);  in i2400m_tx_msg_sent()
944 	/* level them FIFO markers off */  in i2400m_tx_msg_sent()
945 	n = i2400m->tx_out / I2400M_TX_BUF_SIZE;  in i2400m_tx_msg_sent()
946 	i2400m->tx_out %= I2400M_TX_BUF_SIZE;  in i2400m_tx_msg_sent()
947 	i2400m->tx_in -= n * I2400M_TX_BUF_SIZE;  in i2400m_tx_msg_sent()
949 	spin_unlock_irqrestore(&i2400m->tx_lock, flags);  in i2400m_tx_msg_sent()
956  * i2400m_tx_setup - Initialize the TX queue and infrastructure
969 	/* Do this here only once -- can't do on  in i2400m_tx_setup()
972 	INIT_WORK(&i2400m->wake_tx_ws, i2400m_wake_tx_work);  in i2400m_tx_setup()
976 		result = -ENOMEM;  in i2400m_tx_setup()
981 	 * Fail the build if we can't fit at least two maximum size messages  in i2400m_tx_setup()
982 	 * on the TX FIFO [one being delivered while one is constructed].  in i2400m_tx_setup()
985 	spin_lock_irqsave(&i2400m->tx_lock, flags);  in i2400m_tx_setup()
986 	i2400m->tx_sequence = 0;  in i2400m_tx_setup()
987 	i2400m->tx_in = 0;  in i2400m_tx_setup()
988 	i2400m->tx_out = 0;  in i2400m_tx_setup()
989 	i2400m->tx_msg_size = 0;  in i2400m_tx_setup()
990 	i2400m->tx_msg = NULL;  in i2400m_tx_setup()
991 	i2400m->tx_buf = tx_buf;  in i2400m_tx_setup()
992 	spin_unlock_irqrestore(&i2400m->tx_lock, flags);  in i2400m_tx_setup()
994 	BUG_ON(i2400m->bus_tx_block_size == 0);  in i2400m_tx_setup()
1002  * i2400m_tx_release - Tear down the TX queue and infrastructure
1007 	spin_lock_irqsave(&i2400m->tx_lock, flags);  in i2400m_tx_release()
1008 	kfree(i2400m->tx_buf);  in i2400m_tx_release()
1009 	i2400m->tx_buf = NULL;  in i2400m_tx_release()
1010 	spin_unlock_irqrestore(&i2400m->tx_lock, flags);  in i2400m_tx_release()