/* * Copyright (C) 2001-2004 by David Brownell *  * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License * for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software Foundation, * Inc., 675 Mass Ave, Cambridge, MA 02139, USA. *//* this file is part of ehci-hcd.c *//*-------------------------------------------------------------------------*//* * EHCI hardware queue manipulation ... the core.  QH/QTD manipulation. * * Control, bulk, and interrupt traffic all use "qh" lists.  They list "qtd" * entries describing USB transactions, max 16-20kB/entry (with 4kB-aligned * buffers needed for the larger number).  We use one QH per endpoint, queue * multiple urbs (all three types) per endpoint.  URBs may need several qtds. * * ISO traffic uses "ISO TD" (itd, and sitd) records, and (along with * interrupts) needs careful scheduling.  Performance improvements can be * an ongoing challenge.  That's in "ehci-sched.c". *  * USB 1.1 devices are handled (a) by "companion" OHCI or UHCI root hubs, * or otherwise through transaction translators (TTs) in USB 2.0 hubs using * (b) special fields in qh entries or (c) split iso entries.  TTs will * buffer low/full speed data so the host collects it at high speed. *//*-------------------------------------------------------------------------*//* fill a qtd, returning how much of the buffer we were able to queue up */static intqtd_fill (struct ehci_qtd *qtd, dma_addr_t buf, size_t len,		int token, int maxpacket){	int	i, count;	u64	addr = buf;	/* one buffer entry per 4K ... first might be short or unaligned */	qtd->hw_buf [0] = cpu_to_le32 ((u32)addr);	qtd->hw_buf_hi [0] = cpu_to_le32 ((u32)(addr >> 32));	count = 0x1000 - (buf & 0x0fff);	/* rest of that page */	if (likely (len < count))		/* ... iff needed */		count = len;	else {		buf +=  0x1000;		buf &= ~0x0fff;		/* per-qtd limit: from 16K to 20K (best alignment) */		for (i = 1; count < len && i < 5; i++) {			addr = buf;			qtd->hw_buf [i] = cpu_to_le32 ((u32)addr);			qtd->hw_buf_hi [i] = cpu_to_le32 ((u32)(addr >> 32));			buf += 0x1000;			if ((count + 0x1000) < len)				count += 0x1000;			else				count = len;		}		/* short packets may only terminate transfers */		if (count != len)			count -= (count % maxpacket);	}	qtd->hw_token = cpu_to_le32 ((count << 16) | token);	qtd->length = count;	return count;}/*-------------------------------------------------------------------------*/static inline voidqh_update (struct ehci_hcd *ehci, struct ehci_qh *qh, struct ehci_qtd *qtd){	/* writes to an active overlay are unsafe */	BUG_ON(qh->qh_state != QH_STATE_IDLE);	qh->hw_qtd_next = QTD_NEXT (qtd->qtd_dma);	qh->hw_alt_next = EHCI_LIST_END;	/* Except for control endpoints, we make hardware maintain data	 * toggle (like OHCI) ... here (re)initialize the toggle in the QH,	 * and set the pseudo-toggle in udev. Only usb_clear_halt() will	 * ever clear it.	 */	if (!(qh->hw_info1 & cpu_to_le32(1 << 14))) {		unsigned	is_out, epnum;		is_out = !(qtd->hw_token & cpu_to_le32(1 << 8));		epnum = (le32_to_cpup(&qh->hw_info1) >> 8) & 0x0f;		if (unlikely (!rtdm_usb_gettoggle (qh->dev, epnum, is_out))) {			qh->hw_token &= ~__constant_cpu_to_le32 (QTD_TOGGLE);			rtdm_usb_settoggle (qh->dev, epnum, is_out, 1);		}	}	/* HC must see latest qtd and qh data before we clear ACTIVE+HALT */	wmb ();	qh->hw_token &= __constant_cpu_to_le32 (QTD_TOGGLE | QTD_STS_PING);}/* if it weren't for a common silicon quirk (writing the dummy into the qh * overlay, so qh->hw_token wrongly becomes inactive/halted), only fault * recovery (including urb dequeue) would need software changes to a QH... */static voidqh_refresh (struct ehci_hcd *ehci, struct ehci_qh *qh){	struct ehci_qtd *qtd;	if (list_empty (&qh->qtd_list))		qtd = qh->dummy;	else {		qtd = list_entry (qh->qtd_list.next,				struct ehci_qtd, qtd_list);		/* first qtd may already be partially processed */		if (cpu_to_le32 (qtd->qtd_dma) == qh->hw_current)			qtd = NULL;	}	if (qtd)		qh_update (ehci, qh, qtd);}/*-------------------------------------------------------------------------*/static void qtd_copy_status (	struct ehci_hcd *ehci,	struct rtdm_urb *urb,	size_t length,	u32 token){	/* count IN/OUT bytes, not SETUP (even short packets) */	if (likely (QTD_PID (token) != 2))		urb->actual_length += length - QTD_LENGTH (token);	/* don't modify error codes */	if (unlikely (urb->status != -EINPROGRESS))		return;	/* force cleanup after short read; not always an error */	if (unlikely (IS_SHORT_READ (token)))		urb->status = -EREMOTEIO;	/* serious "can't proceed" faults reported by the hardware */	if (token & QTD_STS_HALT) {		if (token & QTD_STS_BABBLE) {			/* FIXME "must" disable babbling device's port too */			urb->status = -EOVERFLOW;		} else if (token & QTD_STS_MMF) {			/* fs/ls interrupt xfer missed the complete-split */			urb->status = -EPROTO;		} else if (token & QTD_STS_DBE) {			urb->status = (QTD_PID (token) == 1) /* IN ? */				? -ENOSR  /* hc couldn't read data */				: -ECOMM; /* hc couldn't write data */		} else if (token & QTD_STS_XACT) {			/* timeout, bad crc, wrong PID, etc; retried */			if (QTD_CERR (token))				urb->status = -EPIPE;			else {				ehci_dbg (ehci, "devpath %s ep%d%s 3strikes\n",					urb->dev->devpath,					rtdm_usb_pipeendpoint (urb->pipe),					rtdm_usb_pipein (urb->pipe) ? "in" : "out");				urb->status = -EPROTO;			}		/* CERR nonzero + no errors + halt --> stall */		} else if (QTD_CERR (token))			urb->status = -EPIPE;		else	/* unknown */			urb->status = -EPROTO;		ehci_vdbg (ehci,			"dev%d ep%d%s qtd token %08x --> status %d\n",			rtdm_usb_pipedevice (urb->pipe),			rtdm_usb_pipeendpoint (urb->pipe),			rtdm_usb_pipein (urb->pipe) ? "in" : "out",			token, urb->status);		/* if async CSPLIT failed, try cleaning out the TT buffer */		if (urb->status != -EPIPE				  && urb->dev->tt && !rtdm_usb_pipeint (urb->pipe)				&& ((token & QTD_STS_MMF) != 0					|| QTD_CERR(token) == 0)				&& (!ehci_is_TDI(ehci)                	                || urb->dev->tt->hub !=					   ehci_to_hcd(ehci)->self.root_hub)) {#ifdef DEBUG			struct rtdm_usb_device *tt = urb->dev->tt->hub;			rtdm_dev_dbg(&tt->dev,				"clear tt buffer port %d, a%d ep%d t%08x\n",				urb->dev->ttport, urb->dev->devnum,				rtdm_usb_pipeendpoint (urb->pipe), token);#endif /* DEBUG */			rtdm_usb_hub_tt_clear_buffer (urb->dev, urb->pipe);		}	}}static voidehci_urb_done (struct ehci_hcd *ehci, struct rtdm_urb *urb){	if (likely (urb->hcpriv != NULL)) {		struct ehci_qh	*qh = (struct ehci_qh *) urb->hcpriv;		/* S-mask in a QH means it's an interrupt urb */		if ((qh->hw_info2 & __constant_cpu_to_le32 (QH_SMASK)) != 0) {			/* ... update hc-wide periodic stats (for usbfs) */			ehci_to_hcd(ehci)->self.bandwidth_int_reqs--;		}		qh_put (qh);	}	rtdm_lock_get (&urb->rt_lock);	urb->hcpriv = NULL;	switch (urb->status) {	case -EINPROGRESS:		/* success */		urb->status = 0;	default:			/* fault */		COUNT (ehci->stats.complete);		break;	case -EREMOTEIO:		/* fault or normal */		if (!(urb->transfer_flags & RTDM_URB_SHORT_NOT_OK))			urb->status = 0;		COUNT (ehci->stats.complete);		break;	case -ECONNRESET:		/* canceled */	case -ENOENT:		COUNT (ehci->stats.unlink);		break;	}	rtdm_lock_put (&urb->rt_lock);#ifdef EHCI_URB_TRACE	ehci_dbg(ehci,		"%s %s urb %p ep%d%s status %d len %d/%d\n",		__FUNCTION__, urb->dev->devpath, urb,		rtdm_usb_pipeendpoint (urb->pipe),		rtdm_usb_pipein (urb->pipe) ? "in" : "out",		urb->status,		urb->actual_length, urb->transfer_buffer_length);#endif	/* complete() can reenter this HCD */	rtdm_lock_put(&ehci->rt_lock);	rtdm_usb_hcd_giveback_urb (ehci_to_hcd(ehci), urb);	rtdm_lock_get (&ehci->rt_lock);}static void start_unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh);static void unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh);static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh);static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh);/* * Process and free completed qtds for a qh, returning URBs to drivers. * Chases up to qh->hw_current.  Returns number of completions called, * indicating how much "real" work we did. */#define HALT_BIT __constant_cpu_to_le32(QTD_STS_HALT)static unsignedqh_completions (struct ehci_hcd *ehci, struct ehci_qh *qh){	struct ehci_qtd		*last = NULL, *end = qh->dummy;	struct list_head	*entry, *tmp;	int			stopped;	unsigned		count = 0;	int			do_status = 0;	u8			state;	if (unlikely (list_empty (&qh->qtd_list)))		return count;	/* completions (or tasks on other cpus) must never clobber HALT	 * till we've gone through and cleaned everything up, even when	 * they add urbs to this qh's queue or mark them for unlinking.	 *	 * NOTE:  unlinking expects to be done in queue order.	 */	state = qh->qh_state;	qh->qh_state = QH_STATE_COMPLETING;	stopped = (state == QH_STATE_IDLE);	/* remove de-activated QTDs from front of queue.	 * after faults (including short reads), cleanup this urb	 * then let the queue advance.	 * if queue is stopped, handles unlinks.	 */	list_for_each_safe (entry, tmp, &qh->qtd_list) {		struct ehci_qtd	*qtd;		struct rtdm_urb	*urb;		u32		token = 0;		qtd = list_entry (entry, struct ehci_qtd, qtd_list);		urb = qtd->urb;		/* clean up any state from previous QTD ...*/		if (last) {			if (likely (last->urb != urb)) {				ehci_urb_done (ehci, last->urb);				count++;			}			ehci_qtd_free (ehci, last);			last = NULL;		}		/* ignore urbs submitted during completions we reported */		if (qtd == end)			break;		/* hardware copies qtd out of qh overlay */		rmb ();		token = le32_to_cpu (qtd->hw_token);		/* always clean up qtds the hc de-activated */		if ((token & QTD_STS_ACTIVE) == 0) {			if ((token & QTD_STS_HALT) != 0) {				stopped = 1;			/* magic dummy for some short reads; qh won't advance.			 * that silicon quirk can kick in with this dummy too.			 */			} else if (IS_SHORT_READ (token)					&& !(qtd->hw_alt_next & EHCI_LIST_END)) {				stopped = 1;				goto halt;			}		/* stop scanning when we reach qtds the hc is using */		} else if (likely (!stopped				&& HC_IS_RUNNING (ehci_to_hcd(ehci)->state))) {			break;		} else {			stopped = 1;			if (unlikely (!HC_IS_RUNNING (ehci_to_hcd(ehci)->state)))				urb->status = -ESHUTDOWN;			/* ignore active urbs unless some previous qtd			 * for the urb faulted (including short read) or			 * its urb was canceled.  we may patch qh or qtds.			 */			if (likely (urb->status == -EINPROGRESS))				continue;						/* issue status after short control reads */			if (unlikely (do_status != 0)					&& QTD_PID (token) == 0 /* OUT */) {				do_status = 0;				continue;			}			/* token in overlay may be most current */			if (state == QH_STATE_IDLE					&& cpu_to_le32 (qtd->qtd_dma)						== qh->hw_current)				token = le32_to_cpu (qh->hw_token);			/* force halt for unlinked or blocked qh, so we'll			 * patch the qh later and so that completions can't			 * activate it while we "know" it's stopped.			 */			if ((HALT_BIT & qh->hw_token) == 0) {halt:				qh->hw_token |= HALT_BIT;				wmb ();			}		} 		/* remove it from the queue */		rtdm_lock_get (&urb->rt_lock);		qtd_copy_status (ehci, urb, qtd->length, token);		do_status = (urb->status == -EREMOTEIO)					&& rtdm_usb_pipecontrol (urb->pipe);		rtdm_lock_put (&urb->rt_lock);		if (stopped && qtd->qtd_list.prev != &qh->qtd_list) {			last = list_entry (qtd->qtd_list.prev,					struct ehci_qtd, qtd_list);			last->hw_next = qtd->hw_next;		}		list_del (&qtd->qtd_list);		last = qtd;	}	/* last urb's completion might still need calling */	if (likely (last != NULL)) {		ehci_urb_done (ehci, last->urb);		count++;		ehci_qtd_free (ehci, last);	}	/* restore original state; caller must unlink or relink */	qh->qh_state = state;	/* be sure the hardware's done with the qh before refreshing	 * it after fault cleanup, or recovering from silicon wrongly	 * overlaying the dummy qtd (which reduces DMA chatter).	 */	if (stopped != 0 || qh->hw_qtd_next == EHCI_LIST_END) {		switch (state) {		case QH_STATE_IDLE:			qh_refresh(ehci, qh);			break;		case QH_STATE_LINKED:			/* should be rare for periodic transfers,			 * except maybe high bandwidth ...			 */			if ((__constant_cpu_to_le32 (QH_SMASK)					& qh->hw_info2) != 0) {				intr_deschedule (ehci, qh);				(void) qh_schedule (ehci, qh);			} else				unlink_async (ehci, qh);			break;		/* otherwise, unlink already started */		}	}	return count;}/*-------------------------------------------------------------------------*/// high bandwidth multiplier, as encoded in highspeed endpoint descriptors#define hb_mult(wMaxPacketSize) (1 + (((wMaxPacketSize) >> 11) & 0x03))// ... and packet size, for any kind of endpoint descriptor#define max_packet(wMaxPacketSize) ((wMaxPacketSize) & 0x07ff)/* * reverse of qh_urb_transaction:  free a list of TDs. * used for cleanup after errors, before HC sees an URB's TDs. */static void qtd_list_free (	struct ehci_hcd		*ehci,	struct rtdm_urb		*urb,	struct list_head	*qtd_list) {	struct list_head	*entry, *temp;	list_for_each_safe (entry, temp, qtd_list) {		struct ehci_qtd	*qtd;		qtd = list_entry (entry, struct ehci_qtd, qtd_list);		list_del (&qtd->qtd_list);		ehci_qtd_free (ehci, qtd);	}}/* * create a list of filled qtds for this URB; won't link into qh. */static struct list_head *qh_urb_transaction (	struct ehci_hcd		*ehci,	struct rtdm_urb		*urb,	struct list_head	*head//,//	int			flags) {	struct ehci_qtd		*qtd, *qtd_prev;	dma_addr_t		buf;	int			len, maxpacket;	int			is_input;	u32			token;	/*	 * URBs map to sequences of QTDs:  one logical transaction	 */	qtd = ehci_qtd_alloc (ehci);	if (unlikely (!qtd))		return NULL;	list_add_tail (&qtd->qtd_list, head);	qtd->urb = urb;	token = QTD_STS_ACTIVE;	token |= (EHCI_TUNE_CERR << 10);	/* for split transactions, SplitXState initialized to zero */	len = urb->transfer_buffer_length;	is_input = rtdm_usb_pipein (urb->pipe);	if (rtdm_usb_pipecontrol (urb->pipe)) {		/* SETUP pid */		qtd_fill (qtd, urb->setup_dma, sizeof (struct usb_ctrlrequest),			token | (2 /* "setup" */ << 8), 8);		/* ... and always at least one more pid */		token ^= QTD_TOGGLE;		qtd_prev = qtd;		qtd = ehci_qtd_alloc (ehci);		if (unlikely (!qtd))			goto cleanup;		qtd->urb = urb;		qtd_prev->hw_next = QTD_NEXT (qtd->qtd_dma);		list_add_tail (&qtd->qtd_list, head);	} 	/*	 * data transfer stage:  buffer setup	 */	if (likely (len > 0))		buf = urb->transfer_dma;	else		buf = 0;	/* for zero length DATA stages, STATUS is always IN */	if (!buf || is_input)		token |= (1 /* "in" */ << 8);	/* else it's already initted to "out" pid (0 << 8) */	maxpacket = max_packet(rtdm_usb_maxpacket(urb->dev, urb->pipe, !is_input));	/*	 * buffer gets wrapped in one or more qtds;	 * last one may be "short" (including zero len)	 * and may serve as a control status ack	 */	for (;;) {		int this_qtd_len;		this_qtd_len = qtd_fill (qtd, buf, len, token, maxpacket);		len -= this_qtd_len;		buf += this_qtd_len;		if (is_input)			qtd->hw_alt_next = ehci->async->hw_alt_next;		/* qh makes control packets use qtd toggle; maybe switch it */