/* * Copyright (C) 2001-2004 by David Brownell * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License * for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software Foundation, * Inc., 675 Mass Ave, Cambridge, MA 02139, USA. *//* this file is part of ehci-hcd.c *//*-------------------------------------------------------------------------*//* * EHCI hardware queue manipulation ... the core.  QH/QTD manipulation. * * Control, bulk, and interrupt traffic all use "qh" lists.  They list "qtd" * entries describing USB transactions, max 16-20kB/entry (with 4kB-aligned * buffers needed for the larger number).  We use one QH per endpoint, queue * multiple urbs (all three types) per endpoint.  URBs may need several qtds. * * ISO traffic uses "ISO TD" (itd, and sitd) records, and (along with * interrupts) needs careful scheduling.  Performance improvements can be * an ongoing challenge.  That's in "ehci-sched.c". * * USB 1.1 devices are handled (a) by "companion" OHCI or UHCI root hubs, * or otherwise through transaction translators (TTs) in USB 2.0 hubs using * (b) special fields in qh entries or (c) split iso entries.  TTs will * buffer low/full speed data so the host collects it at high speed. *//*-------------------------------------------------------------------------*//* fill a qtd, returning how much of the buffer we were able to queue up */static intqtd_fill (struct ehci_qtd *qtd, dma_addr_t buf, size_t len,		int token, int maxpacket){	int	i, count;	u64	addr = buf;	/* one buffer entry per 4K ... first might be short or unaligned */	qtd->hw_buf [0] = cpu_to_le32 ((u32)addr);	qtd->hw_buf_hi [0] = cpu_to_le32 ((u32)(addr >> 32));	count = 0x1000 - (buf & 0x0fff);	/* rest of that page */	if (likely (len < count))		/* ... iff needed */		count = len;	else {		buf +=  0x1000;		buf &= ~0x0fff;		/* per-qtd limit: from 16K to 20K (best alignment) */		for (i = 1; count < len && i < 5; i++) {			addr = buf;			qtd->hw_buf [i] = cpu_to_le32 ((u32)addr);			qtd->hw_buf_hi [i] = cpu_to_le32 ((u32)(addr >> 32));			buf += 0x1000;			if ((count + 0x1000) < len)				count += 0x1000;			else				count = len;		}		/* short packets may only terminate transfers */		if (count != len)			count -= (count % maxpacket);	}	qtd->hw_token = cpu_to_le32 ((count << 16) | token);	qtd->length = count;	return count;}/*-------------------------------------------------------------------------*/static inline voidqh_update (struct ehci_hcd *ehci, struct ehci_qh *qh, struct ehci_qtd *qtd){	/* writes to an active overlay are unsafe */	BUG_ON(qh->qh_state != QH_STATE_IDLE);	qh->hw_qtd_next = QTD_NEXT (qtd->qtd_dma);	qh->hw_alt_next = EHCI_LIST_END;	/* Except for control endpoints, we make hardware maintain data	 * toggle (like OHCI) ... here (re)initialize the toggle in the QH,	 * and set the pseudo-toggle in udev. Only usb_clear_halt() will	 * ever clear it.	 */	if (!(qh->hw_info1 & cpu_to_le32(1 << 14))) {		unsigned	is_out, epnum;		is_out = !(qtd->hw_token & cpu_to_le32(1 << 8));		epnum = (le32_to_cpup(&qh->hw_info1) >> 8) & 0x0f;		if (unlikely (!usb_gettoggle (qh->dev, epnum, is_out))) {			qh->hw_token &= ~__constant_cpu_to_le32 (QTD_TOGGLE);			usb_settoggle (qh->dev, epnum, is_out, 1);		}	}	/* HC must see latest qtd and qh data before we clear ACTIVE+HALT */	wmb ();	qh->hw_token &= __constant_cpu_to_le32 (QTD_TOGGLE | QTD_STS_PING);}/* if it weren't for a common silicon quirk (writing the dummy into the qh * overlay, so qh->hw_token wrongly becomes inactive/halted), only fault * recovery (including urb dequeue) would need software changes to a QH... */static voidqh_refresh (struct ehci_hcd *ehci, struct ehci_qh *qh){	struct ehci_qtd *qtd;	if (list_empty (&qh->qtd_list))		qtd = qh->dummy;	else {		qtd = list_entry (qh->qtd_list.next,				struct ehci_qtd, qtd_list);		/* first qtd may already be partially processed */		if (cpu_to_le32 (qtd->qtd_dma) == qh->hw_current)			qtd = NULL;	}	if (qtd)		qh_update (ehci, qh, qtd);}/*-------------------------------------------------------------------------*/static void qtd_copy_status (	struct ehci_hcd *ehci,	struct urb *urb,	size_t length,	u32 token){	/* count IN/OUT bytes, not SETUP (even short packets) */	if (likely (QTD_PID (token) != 2))		urb->actual_length += length - QTD_LENGTH (token);	/* don't modify error codes */	if (unlikely (urb->status != -EINPROGRESS))		return;	/* force cleanup after short read; not always an error */	if (unlikely (IS_SHORT_READ (token)))		urb->status = -EREMOTEIO;	/* serious "can't proceed" faults reported by the hardware */	if (token & QTD_STS_HALT) {		if (token & QTD_STS_BABBLE) {			/* FIXME "must" disable babbling device's port too */			urb->status = -EOVERFLOW;		} else if (token & QTD_STS_MMF) {			/* fs/ls interrupt xfer missed the complete-split */			urb->status = -EPROTO;		} else if (token & QTD_STS_DBE) {			urb->status = (QTD_PID (token) == 1) /* IN ? */				? -ENOSR  /* hc couldn't read data */				: -ECOMM; /* hc couldn't write data */		} else if (token & QTD_STS_XACT) {			/* timeout, bad crc, wrong PID, etc; retried */			if (QTD_CERR (token))				urb->status = -EPIPE;			else {				ehci_dbg (ehci, "devpath %s ep%d%s 3strikes\n",					urb->dev->devpath,					usb_pipeendpoint (urb->pipe),					usb_pipein (urb->pipe) ? "in" : "out");				urb->status = -EPROTO;			}		/* CERR nonzero + no errors + halt --> stall */		} else if (QTD_CERR (token))			urb->status = -EPIPE;		else	/* unknown */			urb->status = -EPROTO;		ehci_vdbg (ehci,			"dev%d ep%d%s qtd token %08x --> status %d\n",			usb_pipedevice (urb->pipe),			usb_pipeendpoint (urb->pipe),			usb_pipein (urb->pipe) ? "in" : "out",			token, urb->status);		/* if async CSPLIT failed, try cleaning out the TT buffer */		if (urb->status != -EPIPE				&& urb->dev->tt && !usb_pipeint (urb->pipe)				&& ((token & QTD_STS_MMF) != 0					|| QTD_CERR(token) == 0)				&& (!ehci_is_TDI(ehci)			                || urb->dev->tt->hub !=					   ehci_to_hcd(ehci)->self.root_hub)) {#ifdef DEBUG			struct usb_device *tt = urb->dev->tt->hub;			dev_dbg (&tt->dev,				"clear tt buffer port %d, a%d ep%d t%08x\n",				urb->dev->ttport, urb->dev->devnum,				usb_pipeendpoint (urb->pipe), token);#endif /* DEBUG */			usb_hub_tt_clear_buffer (urb->dev, urb->pipe);		}	}}static voidehci_urb_done (struct ehci_hcd *ehci, struct urb *urb)__releases(ehci->lock)__acquires(ehci->lock){	if (likely (urb->hcpriv != NULL)) {		struct ehci_qh	*qh = (struct ehci_qh *) urb->hcpriv;		/* S-mask in a QH means it's an interrupt urb */		if ((qh->hw_info2 & __constant_cpu_to_le32 (QH_SMASK)) != 0) {			/* ... update hc-wide periodic stats (for usbfs) */			ehci_to_hcd(ehci)->self.bandwidth_int_reqs--;		}		qh_put (qh);	}	spin_lock (&urb->lock);	urb->hcpriv = NULL;	switch (urb->status) {	case -EINPROGRESS:		/* success */		urb->status = 0;	default:			/* fault */		COUNT (ehci->stats.complete);		break;	case -EREMOTEIO:		/* fault or normal */		if (!(urb->transfer_flags & URB_SHORT_NOT_OK))			urb->status = 0;		COUNT (ehci->stats.complete);		break;	case -ECONNRESET:		/* canceled */	case -ENOENT:		COUNT (ehci->stats.unlink);		break;	}	spin_unlock (&urb->lock);#ifdef EHCI_URB_TRACE	ehci_dbg (ehci,		"%s %s urb %p ep%d%s status %d len %d/%d\n",		__FUNCTION__, urb->dev->devpath, urb,		usb_pipeendpoint (urb->pipe),		usb_pipein (urb->pipe) ? "in" : "out",		urb->status,		urb->actual_length, urb->transfer_buffer_length);#endif	/* complete() can reenter this HCD */	spin_unlock (&ehci->lock);	usb_hcd_giveback_urb (ehci_to_hcd(ehci), urb);	spin_lock (&ehci->lock);}static void start_unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh);static void unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh);static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh);static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh);/* * Process and free completed qtds for a qh, returning URBs to drivers. * Chases up to qh->hw_current.  Returns number of completions called, * indicating how much "real" work we did. */#define HALT_BIT __constant_cpu_to_le32(QTD_STS_HALT)static unsignedqh_completions (struct ehci_hcd *ehci, struct ehci_qh *qh){	struct ehci_qtd		*last = NULL, *end = qh->dummy;	struct list_head	*entry, *tmp;	int			stopped;	unsigned		count = 0;	int			do_status = 0;	u8			state;	if (unlikely (list_empty (&qh->qtd_list)))		return count;	/* completions (or tasks on other cpus) must never clobber HALT	 * till we've gone through and cleaned everything up, even when	 * they add urbs to this qh's queue or mark them for unlinking.	 *	 * NOTE:  unlinking expects to be done in queue order.	 */	state = qh->qh_state;	qh->qh_state = QH_STATE_COMPLETING;	stopped = (state == QH_STATE_IDLE);	/* remove de-activated QTDs from front of queue.	 * after faults (including short reads), cleanup this urb	 * then let the queue advance.	 * if queue is stopped, handles unlinks.	 */	list_for_each_safe (entry, tmp, &qh->qtd_list) {		struct ehci_qtd	*qtd;		struct urb	*urb;		u32		token = 0;		qtd = list_entry (entry, struct ehci_qtd, qtd_list);		urb = qtd->urb;		/* clean up any state from previous QTD ...*/		if (last) {			if (likely (last->urb != urb)) {				ehci_urb_done (ehci, last->urb);				count++;			}			ehci_qtd_free (ehci, last);			last = NULL;		}		/* ignore urbs submitted during completions we reported */		if (qtd == end)			break;		/* hardware copies qtd out of qh overlay */		rmb ();		token = le32_to_cpu (qtd->hw_token);		/* always clean up qtds the hc de-activated */		if ((token & QTD_STS_ACTIVE) == 0) {			if ((token & QTD_STS_HALT) != 0) {				stopped = 1;			/* magic dummy for some short reads; qh won't advance.			 * that silicon quirk can kick in with this dummy too.			 */			} else if (IS_SHORT_READ (token)					&& !(qtd->hw_alt_next & EHCI_LIST_END)) {				stopped = 1;				goto halt;			}		/* stop scanning when we reach qtds the hc is using */		} else if (likely (!stopped				&& HC_IS_RUNNING (ehci_to_hcd(ehci)->state))) {			break;		} else {			stopped = 1;			if (unlikely (!HC_IS_RUNNING (ehci_to_hcd(ehci)->state)))				urb->status = -ESHUTDOWN;			/* ignore active urbs unless some previous qtd			 * for the urb faulted (including short read) or			 * its urb was canceled.  we may patch qh or qtds.			 */			if (likely (urb->status == -EINPROGRESS))				continue;			/* issue status after short control reads */			if (unlikely (do_status != 0)					&& QTD_PID (token) == 0 /* OUT */) {				do_status = 0;				continue;			}