if (unlikely(ep->dma >= 0) && !list_empty(&ep->queue)) {DMSG("%s pio2dma\n", ep->ep.name);				req = list_entry(ep->queue.next,						struct pxa2xx_request, queue);				kick_dma(ep,req);				return 0;			}#endif			return 1;		}		// TODO experiment: how robust can fifo mode tweaking be?		// the double buffering could speed up I/O a bunch.	} while (*ep->reg_udccs & UDCCS_BI_TFS);	return 0;}/* caller asserts req->pending (ep0 irq status nyet cleared); starts * ep0 data stage.  these chips want very simple state transitions. */static inlinevoid ep0start(struct pxa2xx_udc *dev, u32 flags, const char *tag){	UDCCS0 = flags|UDCCS0_SA|UDCCS0_OPR;	USIR0 = USIR0_IR0;	dev->req_pending = 0;	DBG(DBG_VERY_NOISY, "%s %s, %02x/%02x\n",		__FUNCTION__, tag, UDCCS0, flags);}static intwrite_ep0_fifo (struct pxa2xx_ep *ep, struct pxa2xx_request *req){	unsigned	count;	int		is_short;	count = write_packet(&UDDR0, req, EP0_FIFO_SIZE);	ep->dev->stats.write.bytes += count;	/* last packet "must be" short (or a zlp) */	is_short = (count != EP0_FIFO_SIZE);	DBG(DBG_VERY_NOISY, "ep0in %d bytes %d left %p\n", count,		req->req.length - req->req.actual, req);	if (unlikely (is_short)) {		if (ep->dev->req_pending)			ep0start(ep->dev, UDCCS0_IPR, "short IN");		else			UDCCS0 = UDCCS0_IPR;		count = req->req.length;		done (ep, req, 0);		ep0_idle(ep->dev);#if 1		/* This seems to get rid of lost status irqs in some cases:		 * host responds quickly, or next request involves config		 * change automagic, or should have been hidden, or ...		 *		 * FIXME get rid of all udelays possible...		 */		if (count >= EP0_FIFO_SIZE) {			count = 100;			do {				if ((UDCCS0 & UDCCS0_OPR) != 0) {					/* clear OPR, generate ack */					UDCCS0 = UDCCS0_OPR;					break;				}				count--;				udelay(1);			} while (count);		}#endif	} else if (ep->dev->req_pending)		ep0start(ep->dev, 0, "IN");	return is_short;}/* * read_fifo -  unload packet(s) from the fifo we use for usb OUT * transfers and put them into the request.  caller should have made * sure there's at least one packet ready. * * returns true if the request completed because of short packet or the * request buffer having filled (and maybe overran till end-of-packet). */static intread_fifo (struct pxa2xx_ep *ep, struct pxa2xx_request *req){	for (;;) {		u32		udccs;		u8		*buf;		unsigned	bufferspace, count, is_short;		/* make sure there's a packet in the FIFO.		 * UDCCS_{BO,IO}_RPC are all the same bit value.		 * UDCCS_{BO,IO}_RNE are all the same bit value.		 */		udccs = *ep->reg_udccs;		if (unlikely ((udccs & UDCCS_BO_RPC) == 0))			break;		buf = req->req.buf + req->req.actual;		prefetchw(buf);		bufferspace = req->req.length - req->req.actual;		/* read all bytes from this packet */		if (likely (udccs & UDCCS_BO_RNE)) {			count = 1 + (0x0ff & *ep->reg_ubcr);			req->req.actual += min (count, bufferspace);		} else /* zlp */			count = 0;		is_short = (count < ep->ep.maxpacket);		DBG(DBG_VERY_NOISY, "read %s %02x, %d bytes%s req %p %d/%d\n",			ep->ep.name, udccs, count,			is_short ? "/S" : "",			req, req->req.actual, req->req.length);		while (likely (count-- != 0)) {			u8	byte = (u8) *ep->reg_uddr;			if (unlikely (bufferspace == 0)) {				/* this happens when the driver's buffer				 * is smaller than what the host sent.				 * discard the extra data.				 */				if (req->req.status != -EOVERFLOW)					DMSG("%s overflow %d\n",						ep->ep.name, count);				req->req.status = -EOVERFLOW;			} else {				*buf++ = byte;				bufferspace--;			}		}		*ep->reg_udccs =  UDCCS_BO_RPC;		/* RPC/RSP/RNE could now reflect the other packet buffer */		/* iso is one request per packet */		if (ep->bmAttributes == USB_ENDPOINT_XFER_ISOC) {			if (udccs & UDCCS_IO_ROF)				req->req.status = -EHOSTUNREACH;			/* more like "is_done" */			is_short = 1;		}		/* completion */		if (is_short || req->req.actual == req->req.length) {			done (ep, req, 0);			if (list_empty(&ep->queue))				pio_irq_disable (ep->bEndpointAddress);			return 1;		}		/* finished that packet.  the next one may be waiting... */	}	return 0;}/* * special ep0 version of the above.  no UBCR0 or double buffering; status * handshaking is magic.  most device protocols don't need control-OUT. * CDC vendor commands (and RNDIS), mass storage CB/CBI, and some other * protocols do use them. */static intread_ep0_fifo (struct pxa2xx_ep *ep, struct pxa2xx_request *req){	u8		*buf, byte;	unsigned	bufferspace;	buf = req->req.buf + req->req.actual;	bufferspace = req->req.length - req->req.actual;	while (UDCCS0 & UDCCS0_RNE) {		byte = (u8) UDDR0;		if (unlikely (bufferspace == 0)) {			/* this happens when the driver's buffer			 * is smaller than what the host sent.			 * discard the extra data.			 */			if (req->req.status != -EOVERFLOW)				DMSG("%s overflow\n", ep->ep.name);			req->req.status = -EOVERFLOW;		} else {			*buf++ = byte;			req->req.actual++;			bufferspace--;		}	}		UDCCS0 = UDCCS0_OPR | UDCCS0_IPR;		/* completion */	if (req->req.actual >= req->req.length)		return 1;		/* finished that packet.  the next one may be waiting... */	return 0;}#ifdef	USE_DMAstatic inline voidstart_dma_nodesc(struct pxa2xx_ep *ep, struct pxa2xx_request *req, int is_in){	u32	dcmd = req->req.length;	u32	buf = virt_to_bus (req->req.buf);	u32	fifo = io_v2p ((u32)ep->reg_uddr);	/* no-descriptor mode can be simple for bulk-in, iso-in, iso-out */	DCSR(ep->dma) = DCSR_NODESC;	dcmd |= DCMD_BURST32 | DCMD_ENDIRQEN | DCMD_WIDTH1;	if (is_in) {		DSADR(ep->dma) = buf;		DTADR(ep->dma) = fifo;		dcmd |= DCMD_FLOWTRG | DCMD_INCSRCADDR;	} else {		DSADR(ep->dma) = fifo;		DTADR(ep->dma) = buf;		dcmd |= DCMD_FLOWSRC | DCMD_INCTRGADDR;	}	DCMD(ep->dma) = dcmd;	DCSR(ep->dma) = DCSR_RUN | DCSR_STOPIRQEN | DCSR_NODESC;	/* and later the dma handler gets called */}static void kick_dma(struct pxa2xx_ep *ep, struct pxa2xx_request *req){	if (ep->bEndpointAddress & USB_DIR_IN) {		/* docs imply we can't preload with pio */		if ((((u32)req->req.buf) & 0x0f) != 0) {// VERBOSE			DMSG("%s bad DMA align %p\n",				ep->ep.name, req->req.buf);pio_in:// FIXME PIO fallback doesn't work right yet (recovery?)DMSG("%s dma2pio\n", ep->ep.name);			pio_irq_enable(ep->bEndpointAddress);			if ((*ep->reg_udccs & UDCCS_BI_TFS) != 0)				(void) write_fifo(ep, req);			return;		}		/* dmacount 0 means end-of-transfer */		if (unlikely((req->req.length - req->req.actual) == 0)) {// VERBOSE			DMSG("%s zlp dma write...\n", ep->ep.name);			goto pio_in;		}		start_dma_nodesc(ep, req, USB_DIR_IN);	} else {		// if ISO, use no-descriptor DMA		BUG();	}}static void cancel_dma(struct pxa2xx_ep *ep){	struct pxa2xx_request	*req;	u32			tmp;	if (DCSR(ep->dma) == 0 || list_empty(&ep->queue))		return;	DCSR(ep->dma) = 0;	while ((DCSR(ep->dma) & DCSR_STOPSTATE) == 0)		cpu_relax();	req = list_entry(ep->queue.next, struct pxa2xx_request, queue);	tmp = DCMD(ep->dma) & DCMD_LENGTH;	req->req.actual = req->req.length - (tmp & DCMD_LENGTH);	/* the last tx packet may be incomplete, so flush the fifo.	 * FIXME correct req.actual if we can	 */	if (ep->bEndpointAddress & USB_DIR_IN)		*ep->reg_udccs = UDCCS_BI_FTF;}static void dma_nodesc_handler(int dmach, void *_ep, struct pt_regs *r){	struct pxa2xx_ep	*ep = _ep;	struct pxa2xx_request	*req;	u32			tmp;	req = list_entry(ep->queue.next, struct pxa2xx_request, queue);	ep->dev->stats.irqs++;	HEX_DISPLAY(ep->dev->stats.irqs);	/* ack/clear */	tmp = DCSR(ep->dma);	DCSR(ep->dma) = tmp;	if ((tmp & DCSR_STOPSTATE) == 0			|| (DDADR(ep->dma) & DDADR_STOP) != 0) {		DBG(DBG_VERBOSE, "%s, dcsr %08x ddadr %08x\n",			ep->ep.name, DCSR(ep->dma), DDADR(ep->dma));		return;	}	DCSR(ep->dma) = 0;	/* clear DCSR_STOPSTATE */	/* wrap up the transfer, and collect status */	if (unlikely(tmp & DCSR_BUSERR))		req->req.status = -EIO;	tmp = DCMD(ep->dma);	req->req.actual = req->req.length - (tmp & DCMD_LENGTH);	tmp = 1;	/* normally this is the last packet */	if (ep->bEndpointAddress & USB_DIR_IN) {		/* maybe validate final short packet */		if ((ep->bmAttributes == USB_ENDPOINT_XFER_BULK					&& req->req.actual % BULK_FIFO_SIZE)				|| (ep->bmAttributes == USB_ENDPOINT_XFER_ISOC					&& req->req.actual % ISO_FIFO_SIZE))			*ep->reg_udccs = UDCCS_BI_TSP /*|UDCCS_BI_TPC*/;		/* or force a zlp, with pio ... */		else if (ep->bmAttributes == USB_ENDPOINT_XFER_BULK				&& req->req.zero) {			tmp = 0;		}		// if iso, maybe report underrun (TUR)	} else {		BUG();	}	if (likely(tmp != 0))		done(ep, req, 0);	/* maybe re-activate after completion */	if (ep->stopped || list_empty(&ep->queue))		return;	req = list_entry(ep->queue.next, struct pxa2xx_request, queue);	kick_dma(ep, req);}#endif/*-------------------------------------------------------------------------*/static intpxa2xx_ep_queue(struct usb_ep *_ep, struct usb_request *_req, int gfp_flags){	struct pxa2xx_request	*req;	struct pxa2xx_ep	*ep;	struct pxa2xx_udc	*dev;	unsigned long		flags;	req = container_of(_req, struct pxa2xx_request, req);	if (unlikely (!_req || !_req->complete || !_req->buf			|| !list_empty(&req->queue))) {		DMSG("%s, bad params\n", __FUNCTION__);		return -EINVAL;	}	ep = container_of(_ep, struct pxa2xx_ep, ep);	if (unlikely (!_ep || (!ep->desc && ep->ep.name != ep0name))) {		DMSG("%s, bad ep\n", __FUNCTION__);		return -EINVAL;	}	dev = ep->dev;	if (unlikely (!dev->driver			|| dev->gadget.speed == USB_SPEED_UNKNOWN)) {		DMSG("%s, bogus device state\n", __FUNCTION__);		return -ESHUTDOWN;	}	/* iso is always one packet per request, that's the only way	 * we can report per-packet status.  that also helps with dma.	 */	if (unlikely (ep->bmAttributes == USB_ENDPOINT_XFER_ISOC			&& req->req.length > le16_to_cpu						(ep->desc->wMaxPacketSize)))		return -EMSGSIZE;#ifdef	USE_DMA	if (ep->dma >= 0) {		unsigned long	start = (unsigned long) _req->buf;		clean_dcache_range(start, start + _req->length);		/* or for USB_DIR_OUT, invalidate_dcache_range (...) */	}#endif	DBG(DBG_NOISY, "%s queue req %p, len %d buf %p\n",	     _ep->name, _req, _req->length, _req->buf);	local_irq_save(flags);	_req->status = -EINPROGRESS;	_req->actual = 0;	/* kickstart this i/o queue? */	if (list_empty(&ep->queue) && !ep->stopped) {		if (ep->desc == 0 /* ep0 */) {			unsigned	length = _req->length;			switch (dev->ep0state) {			case EP0_IN_DATA_PHASE:				dev->stats.write.ops++;				if (write_ep0_fifo(ep, req))					req = 0;				break;			case EP0_OUT_DATA_PHASE:				dev->stats.read.ops++;				/* messy ... */				if (dev->req_config) {					DBG(DBG_VERBOSE, "ep0 config ack%s\n",						dev->has_cfr ?  "" : " raced");					if (dev->has_cfr)						UDCCFR = UDCCFR_AREN|UDCCFR_ACM;					done(ep, req, 0);					dev->ep0state = EP0_END_XFER;					local_irq_restore (flags);					return 0;				}				if (dev->req_pending)					ep0start(dev, UDCCS0_IPR, "OUT");				if (length == 0 || ((UDCCS0 & UDCCS0_RNE) != 0						&& read_ep0_fifo(ep, req))) {					ep0_idle(dev);					done(ep, req, 0);					req = 0;				}				break;			default:				DMSG("ep0 i/o, odd state %d\n", dev->ep0state);				local_irq_restore (flags);				return -EL2HLT;			}#ifdef	USE_DMA		/* either start dma or prime pio pump */		} else if (ep->dma >= 0) {			kick_dma(ep, req);#endif		/* can the FIFO can satisfy the request immediately? */		} else if ((ep->bEndpointAddress & USB_DIR_IN) != 0				&& (*ep->reg_udccs & UDCCS_BI_TFS) != 0				&& write_fifo(ep, req)) {			req = 0;		} else if ((*ep->reg_udccs & UDCCS_BO_RFS) != 0				&& read_fifo(ep, req)) {			req = 0;		}		if (likely (req && ep->desc) && ep->dma < 0)			pio_irq_enable(ep->bEndpointAddress);	}	/* pio or dma irq handler advances the queue. */	if (likely (req != 0))		list_add_tail(&req->queue, &ep->queue);	local_irq_restore(flags);	return 0;}/* * 	nuke - dequeue ALL requests */static void nuke(struct pxa2xx_ep *ep, int status){	struct pxa2xx_request *req;	/* called with irqs blocked */#ifdef	USE_DMA	if (ep->dma >= 0 && !ep->stopped)		cancel_dma(ep);#endif	while (!list_empty(&ep->queue)) {		req = list_entry(ep->queue.next,				struct pxa2xx_request,				queue);		done(ep, req, status);	}	if (ep->desc)		pio_irq_disable (ep->bEndpointAddress);}/* dequeue JUST ONE request */static int pxa2xx_ep_dequeue(struct usb_ep *_ep, struct usb_request *_req){	struct pxa2xx_ep	*ep;	struct pxa2xx_request	*req;	unsigned long		flags;	ep = container_of(_ep, struct pxa2xx_ep, ep);	if (!_ep || ep->ep.name == ep0name)		return -EINVAL;	local_irq_save(flags);