340 static int add_td_to_list(struct ci_hw_ep *hwep, struct ci_hw_req *hwreq,  in add_td_to_list()  argument
361 		u32 mul = hwreq->req.length / hwep->ep.maxpacket;  in add_td_to_list()
363 		if (hwreq->req.length == 0  in add_td_to_list()
364 				|| hwreq->req.length % hwep->ep.maxpacket)  in add_td_to_list()
369 	temp = (u32) (hwreq->req.dma + hwreq->req.actual);  in add_td_to_list()
379 	hwreq->req.actual += length;  in add_td_to_list()
381 	if (!list_empty(&hwreq->tds)) {  in add_td_to_list()
383 		lastnode = list_entry(hwreq->tds.prev,  in add_td_to_list()
389 	list_add_tail(&node->td, &hwreq->tds);  in add_td_to_list()
410 static int _hardware_enqueue(struct ci_hw_ep *hwep, struct ci_hw_req *hwreq)  in _hardware_enqueue()  argument
414 	unsigned rest = hwreq->req.length;  in _hardware_enqueue()
419 	if (hwreq->req.status == -EALREADY)  in _hardware_enqueue()
422 	hwreq->req.status = -EALREADY;  in _hardware_enqueue()
425 					    &hwreq->req, hwep->dir);  in _hardware_enqueue()
433 	if (hwreq->req.dma % PAGE_SIZE)  in _hardware_enqueue()
437 		ret = add_td_to_list(hwep, hwreq, 0);  in _hardware_enqueue()
443 		unsigned count = min(hwreq->req.length - hwreq->req.actual,  in _hardware_enqueue()
445 		ret = add_td_to_list(hwep, hwreq, count);  in _hardware_enqueue()
452 	if (hwreq->req.zero && hwreq->req.length && hwep->dir == TX  in _hardware_enqueue()
453 	    && (hwreq->req.length % hwep->ep.maxpacket == 0)) {  in _hardware_enqueue()
454 		ret = add_td_to_list(hwep, hwreq, 0);  in _hardware_enqueue()
459 	firstnode = list_first_entry(&hwreq->tds, struct td_node, td);  in _hardware_enqueue()
461 	lastnode = list_entry(hwreq->tds.prev,  in _hardware_enqueue()
465 	if (!hwreq->req.no_interrupt)  in _hardware_enqueue()
469 	hwreq->req.actual = 0;  in _hardware_enqueue()
501 		u32 mul = hwreq->req.length / hwep->ep.maxpacket;  in _hardware_enqueue()
503 		if (hwreq->req.length == 0  in _hardware_enqueue()
504 				|| hwreq->req.length % hwep->ep.maxpacket)  in _hardware_enqueue()
546 static int _hardware_dequeue(struct ci_hw_ep *hwep, struct ci_hw_req *hwreq)  in _hardware_dequeue()  argument
551 	unsigned actual = hwreq->req.length;  in _hardware_dequeue()
554 	if (hwreq->req.status != -EALREADY)  in _hardware_dequeue()
557 	hwreq->req.status = 0;  in _hardware_dequeue()
559 	list_for_each_entry_safe(node, tmpnode, &hwreq->tds, td) {  in _hardware_dequeue()
567 			hwreq->req.status = -EALREADY;  in _hardware_dequeue()
575 		hwreq->req.status = tmptoken & TD_STATUS;  in _hardware_dequeue()
576 		if ((TD_STATUS_HALTED & hwreq->req.status)) {  in _hardware_dequeue()
577 			hwreq->req.status = -EPIPE;  in _hardware_dequeue()
579 		} else if ((TD_STATUS_DT_ERR & hwreq->req.status)) {  in _hardware_dequeue()
580 			hwreq->req.status = -EPROTO;  in _hardware_dequeue()
582 		} else if ((TD_STATUS_TR_ERR & hwreq->req.status)) {  in _hardware_dequeue()
583 			hwreq->req.status = -EILSEQ;  in _hardware_dequeue()
589 				hwreq->req.status = -EPROTO;  in _hardware_dequeue()
606 					&hwreq->req, hwep->dir);  in _hardware_dequeue()
608 	hwreq->req.actual += actual;  in _hardware_dequeue()
610 	if (hwreq->req.status)  in _hardware_dequeue()
611 		return hwreq->req.status;  in _hardware_dequeue()
613 	return hwreq->req.actual;  in _hardware_dequeue()
636 		struct ci_hw_req *hwreq = list_entry(hwep->qh.queue.next,  in _ep_nuke()  local
639 		list_for_each_entry_safe(node, tmpnode, &hwreq->tds, td) {  in _ep_nuke()
646 		list_del_init(&hwreq->queue);  in _ep_nuke()
647 		hwreq->req.status = -ESHUTDOWN;  in _ep_nuke()
649 		if (hwreq->req.complete != NULL) {  in _ep_nuke()
651 			usb_gadget_giveback_request(&hwep->ep, &hwreq->req);  in _ep_nuke()
805 	struct ci_hw_req *hwreq = container_of(req, struct ci_hw_req, req);  in _ep_queue()  local
824 	    hwreq->req.length > hwep->ep.mult * hwep->ep.maxpacket) {  in _ep_queue()
830 	if (!list_empty(&hwreq->queue)) {  in _ep_queue()
836 	hwreq->req.status = -EINPROGRESS;  in _ep_queue()
837 	hwreq->req.actual = 0;  in _ep_queue()
839 	retval = _hardware_enqueue(hwep, hwreq);  in _ep_queue()
844 		list_add_tail(&hwreq->queue, &hwep->qh.queue);  in _ep_queue()
974 	struct ci_hw_req *hwreq, *hwreqtemp;  in isr_tr_complete_low()  local
978 	list_for_each_entry_safe(hwreq, hwreqtemp, &hwep->qh.queue,  in isr_tr_complete_low()
980 		retval = _hardware_dequeue(hwep, hwreq);  in isr_tr_complete_low()
983 		list_del_init(&hwreq->queue);  in isr_tr_complete_low()
984 		if (hwreq->req.complete != NULL) {  in isr_tr_complete_low()
987 					hwreq->req.length)  in isr_tr_complete_low()
989 			usb_gadget_giveback_request(&hweptemp->ep, &hwreq->req);  in isr_tr_complete_low()
1336 	struct ci_hw_req *hwreq = NULL;  in ep_alloc_request()  local
1341 	hwreq = kzalloc(sizeof(struct ci_hw_req), gfp_flags);  in ep_alloc_request()
1342 	if (hwreq != NULL) {  in ep_alloc_request()
1343 		INIT_LIST_HEAD(&hwreq->queue);  in ep_alloc_request()
1344 		INIT_LIST_HEAD(&hwreq->tds);  in ep_alloc_request()
1347 	return (hwreq == NULL) ? NULL : &hwreq->req;  in ep_alloc_request()
1358 	struct ci_hw_req *hwreq = container_of(req, struct ci_hw_req, req);  in ep_free_request()  local
1364 	} else if (!list_empty(&hwreq->queue)) {  in ep_free_request()
1371 	list_for_each_entry_safe(node, tmpnode, &hwreq->tds, td) {  in ep_free_request()
1378 	kfree(hwreq);  in ep_free_request()
1416 	struct ci_hw_req *hwreq = container_of(req, struct ci_hw_req, req);  in ep_dequeue()  local
1420 	if (ep == NULL || req == NULL || hwreq->req.status != -EALREADY ||  in ep_dequeue()
1421 		hwep->ep.desc == NULL || list_empty(&hwreq->queue) ||  in ep_dequeue()
1429 	list_for_each_entry_safe(node, tmpnode, &hwreq->tds, td) {  in ep_dequeue()
1436 	list_del_init(&hwreq->queue);  in ep_dequeue()
1442 	if (hwreq->req.complete != NULL) {  in ep_dequeue()
1444 		usb_gadget_giveback_request(&hwep->ep, &hwreq->req);  in ep_dequeue()