trans |= length << 16;		packet->transaction = cpu_to_le32 (trans);		/* might need to cross a buffer page within a td */		packet->bufp = buf;		packet->buf1 = (buf + length) & ~0x0fff;		if (packet->buf1 != (buf & ~(u64)0x0fff))			packet->cross = 1;		/* OUT uses multiple start-splits */ 		if (stream->bEndpointAddress & USB_DIR_IN)			continue;		length = (length + 187) / 188;		if (length > 1) /* BEGIN vs ALL */			length |= 1 << 3;		packet->buf1 |= length;	}}static intsitd_urb_transaction (	struct ehci_iso_stream	*stream,	struct ehci_hcd		*ehci,	struct rtdm_urb		*urb//,){	struct ehci_sitd	*sitd;	dma_addr_t		sitd_dma = 0;	int			i, j;	struct ehci_iso_sched	*iso_sched;	unsigned long		context;	iso_sched = iso_sched_alloc (urb->number_of_packets);	if (iso_sched == NULL)		return -ENOMEM;	sitd_sched_init (iso_sched, stream, urb);	/* allocate/init sITDs */	rtdm_lock_get_irqsave(&ehci->rt_lock, context);	for (i = 0; i < urb->number_of_packets; i++) {		/* NOTE:  for now, we don't try to handle wraparound cases		 * for IN (using sitd->hw_backpointer, like a FSTN), which		 * means we never need two sitds for full speed packets.		 */		/* free_list.next might be cache-hot ... but maybe		 * the HC caches it too. avoid that issue for now.		 */		/* prefer previously-allocated sitds */		if (!list_empty(&stream->free_list)) {			sitd = list_entry (stream->free_list.prev,					 struct ehci_sitd, sitd_list);			list_del (&sitd->sitd_list);			sitd_dma = sitd->sitd_dma;		} else			sitd = NULL;		if (!sitd) {			rtdm_lock_put_irqrestore(&ehci->rt_lock, context);			rtdm_sem_down(&ehci->sitd_buffer_semaphore);			for(j = 0; j < EHCI_MAX_SITD; j++){				if(!ehci->sitd_buffer_pool[j].in_use)break;			}				if(j != EHCI_MAX_SITD){				ehci->sitd_buffer_pool[j].in_use = ~0;				sitd = ehci->sitd_buffer_pool[j].buffer;				sitd_dma = ehci->sitd_buffer_pool[j].dma_address;			}			else{				sitd = NULL;				ehci_err (ehci, "unable to allocate sitd\n");			}			rtdm_sem_up(&ehci->sitd_buffer_semaphore);			rtdm_lock_get_irqsave(&ehci->rt_lock, context);		}		if (!sitd) {			iso_sched_free (stream, iso_sched);			rtdm_lock_put_irqrestore(&ehci->rt_lock, context);			return -ENOMEM;		}		memset (sitd, 0, sizeof *sitd);		sitd->sitd_dma = sitd_dma;		list_add (&sitd->sitd_list, &iso_sched->td_list);	}	/* temporarily store schedule info in hcpriv */	urb->hcpriv = iso_sched;	urb->error_count = 0;	 rtdm_lock_put_irqrestore(&ehci->rt_lock, context);	return 0;}/*-------------------------------------------------------------------------*/static inline voidsitd_patch (	struct ehci_iso_stream	*stream,	struct ehci_sitd	*sitd,	struct ehci_iso_sched	*iso_sched,	unsigned		index){	struct ehci_iso_packet	*uf = &iso_sched->packet [index];	u64			bufp = uf->bufp;	sitd->hw_next = EHCI_LIST_END;	sitd->hw_fullspeed_ep = stream->address;	sitd->hw_uframe = stream->splits;	sitd->hw_results = uf->transaction;	sitd->hw_backpointer = EHCI_LIST_END;	bufp = uf->bufp;	sitd->hw_buf [0] = cpu_to_le32 (bufp);	sitd->hw_buf_hi [0] = cpu_to_le32 (bufp >> 32);	sitd->hw_buf [1] = cpu_to_le32 (uf->buf1);	if (uf->cross)		bufp += 4096;	sitd->hw_buf_hi [1] = cpu_to_le32 (bufp >> 32);	sitd->index = index;}static inline voidsitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd){	/* note: sitd ordering could matter (CSPLIT then SSPLIT) */	sitd->sitd_next = ehci->pshadow [frame];	sitd->hw_next = ehci->periodic [frame];	ehci->pshadow [frame].sitd = sitd;	sitd->frame = frame;	wmb ();	ehci->periodic [frame] = cpu_to_le32 (sitd->sitd_dma) | Q_TYPE_SITD;}/* fit urb's sitds into the selected schedule slot; activate as needed */static intsitd_link_urb (	struct ehci_hcd		*ehci,	struct rtdm_urb		*urb,	unsigned		mod,	struct ehci_iso_stream	*stream){	int			packet;	unsigned		next_uframe;	struct ehci_iso_sched	*sched = urb->hcpriv;	struct ehci_sitd	*sitd;	next_uframe = stream->next_uframe;	if (list_empty(&stream->td_list)) {		/* usbfs ignores TT bandwidth */		ehci_to_hcd(ehci)->self.bandwidth_allocated				+= stream->bandwidth;		ehci_vdbg (ehci,			"sched devp %s ep%d%s-iso [%d] %dms/%04x\n",			urb->dev->devpath, stream->bEndpointAddress & 0x0f,			(stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",			(next_uframe >> 3) % ehci->periodic_size,			stream->interval, le32_to_cpu (stream->splits));		stream->start = jiffies;	}	ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;	/* fill sITDs frame by frame */	for (packet = 0, sitd = NULL;			packet < urb->number_of_packets;			packet++) {		/* ASSERT:  we have all necessary sitds */		BUG_ON (list_empty (&sched->td_list));		/* ASSERT:  no itds for this endpoint in this frame */		sitd = list_entry (sched->td_list.next,				struct ehci_sitd, sitd_list);		list_move_tail (&sitd->sitd_list, &stream->td_list);		sitd->stream = iso_stream_get (stream);		sitd->urb = rtdm_usb_get_urb (urb);		sitd_patch (stream, sitd, sched, packet);		sitd_link (ehci, (next_uframe >> 3) % ehci->periodic_size,				sitd);		next_uframe += stream->interval << 3;		stream->depth += stream->interval << 3;	}	stream->next_uframe = next_uframe % mod;	/* don't need that schedule data any more */	iso_sched_free (stream, sched);	urb->hcpriv = NULL;	timer_action (ehci, TIMER_IO_WATCHDOG);	if (!ehci->periodic_sched++)		return enable_periodic (ehci);	return 0;}/*-------------------------------------------------------------------------*/#define	SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \	       			| SITD_STS_XACT | SITD_STS_MMF)static unsignedsitd_complete (	struct ehci_hcd		*ehci,	struct ehci_sitd	*sitd) {	struct rtdm_urb				*urb = sitd->urb;	struct rtdm_usb_iso_packet_descriptor	*desc;	u32					t;	int					urb_index = -1;	struct ehci_iso_stream			*stream = sitd->stream;	struct rtdm_usb_device			*dev;	urb_index = sitd->index;	desc = &urb->iso_frame_desc [urb_index];	t = le32_to_cpup (&sitd->hw_results);	/* report transfer status */	if (t & SITD_ERRS) {		urb->error_count++;		if (t & SITD_STS_DBE)			desc->status = rtdm_usb_pipein (urb->pipe)				? -ENOSR  /* hc couldn't read */				: -ECOMM; /* hc couldn't write */		else if (t & SITD_STS_BABBLE)			desc->status = -EOVERFLOW;		else /* XACT, MMF, etc */			desc->status = -EPROTO;	} else {		desc->status = 0;		desc->actual_length = desc->length - SITD_LENGTH (t);	}	rtdm_usb_put_urb (urb);	sitd->urb = NULL;	sitd->stream = NULL;	list_move (&sitd->sitd_list, &stream->free_list);	stream->depth -= stream->interval << 3;	iso_stream_put (ehci, stream);	/* handle completion now? */	if ((urb_index + 1) != urb->number_of_packets)		return 0;	/* ASSERT: it's really the last sitd for this urb	list_for_each_entry (sitd, &stream->td_list, sitd_list)		BUG_ON (sitd->urb == urb);	 */	/* give urb back to the driver */	dev = rtdm_usb_get_dev (urb->dev);	ehci_urb_done (ehci, urb);	urb = NULL;	/* defer stopping schedule; completion can submit */	ehci->periodic_sched--;	if (!ehci->periodic_sched)		(void) disable_periodic (ehci);	ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;	if (list_empty (&stream->td_list)) {		ehci_to_hcd(ehci)->self.bandwidth_allocated				-= stream->bandwidth;		ehci_vdbg (ehci,			"deschedule devp %s ep%d%s-iso\n",			dev->devpath, stream->bEndpointAddress & 0x0f,			(stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");	}	iso_stream_put (ehci, stream);	rtdm_usb_put_dev (dev);	return 1;}static int sitd_submit (struct ehci_hcd *ehci, struct rtdm_urb *urb){	int			status = -EINVAL;	unsigned long		context;	struct ehci_iso_stream	*stream;	/* Get iso_stream head */	stream = iso_stream_find (ehci, urb);	if (stream == NULL) {		ehci_dbg (ehci, "can't get iso stream\n");		return -ENOMEM;	}	if (urb->interval != stream->interval) {		ehci_dbg (ehci, "can't change iso interval %d --> %d\n",			stream->interval, urb->interval);		goto done;	}#ifdef EHCI_URB_TRACE	ehci_dbg (ehci,		"submit %p dev%s ep%d%s-iso len %d\n",		urb, urb->dev->devpath,		rtdm_usb_pipeendpoint (urb->pipe),		rtdm_usb_pipein (urb->pipe) ? "in" : "out",		urb->transfer_buffer_length);#endif	/* allocate SITDs */	status = sitd_urb_transaction (stream, ehci, urb);	if (status < 0) {		ehci_dbg(ehci, "can't init sitds\n");		goto done;	}	/* schedule ... need to lock */	rtdm_lock_get_irqsave(&ehci->rt_lock, context);	status = iso_stream_schedule (ehci, urb, stream); 	if (status == 0)		sitd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);	 rtdm_lock_put_irqrestore(&ehci->rt_lock, context);done:	if (status < 0)		iso_stream_put (ehci, stream);	return status;}#elsestatic inline intsitd_submit (struct ehci_hcd *ehci, struct rtdm_urb *urb){	ehci_dbg (ehci, "split iso support is disabled\n");	return -ENOSYS;}static inline unsignedsitd_complete (	struct ehci_hcd		*ehci,	struct ehci_sitd	*sitd) {	ehci_err (ehci, "sitd_complete %p?\n", sitd);	return 0;}#endif /* USB_EHCI_SPLIT_ISO *//*-------------------------------------------------------------------------*/static voidscan_periodic (struct ehci_hcd *ehci){	unsigned	frame, clock, now_uframe, mod;	unsigned	modified;	mod = ehci->periodic_size << 3;	/*	 * When running, scan from last scan point up to "now"	 * else clean up by scanning everything that's left.	 * Touches as few pages as possible:  cache-friendly.	 */	now_uframe = ehci->next_uframe;	if (HC_IS_RUNNING (ehci_to_hcd(ehci)->state))		clock = readl (&ehci->regs->frame_index);	else		clock = now_uframe + mod - 1;	clock %= mod;	for (;;) {		union ehci_shadow	q, *q_p;		__le32			type, *hw_p;		unsigned		uframes;		/* don't scan past the live uframe */		frame = now_uframe >> 3;		if (frame == (clock >> 3))			uframes = now_uframe & 0x07;		else {			/* safe to scan the whole frame at once */			now_uframe |= 0x07;			uframes = 8;		}restart:		/* scan each element in frame's queue for completions */		q_p = &ehci->pshadow [frame];		hw_p = &ehci->periodic [frame];		q.ptr = q_p->ptr;		type = Q_NEXT_TYPE (*hw_p);		modified = 0;		while (q.ptr != NULL) {			unsigned		uf;			union ehci_shadow	temp;			int			live;			live = HC_IS_RUNNING (ehci_to_hcd(ehci)->state);			switch (type) {			case Q_TYPE_QH:				/* handle any completions */				temp.qh = qh_get (q.qh);				type = Q_NEXT_TYPE (q.qh->hw_next);				q = q.qh->qh_next;				modified = qh_completions (ehci, temp.qh);				if (unlikely (list_empty (&temp.qh->qtd_list)))					intr_deschedule (ehci, temp.qh);				qh_put (temp.qh);				break;			case Q_TYPE_FSTN:				/* for "save place" FSTNs, look at QH entries				 * in the previous frame for completions.				 */				if (q.fstn->hw_prev != EHCI_LIST_END) {					dbg ("ignoring completions from FSTNs");				}				type = Q_NEXT_TYPE (q.fstn->hw_next);				q = q.fstn->fstn_next;				break;			case Q_TYPE_ITD:				/* skip itds for later in the frame */				rmb ();				for (uf = live ? uframes : 8; uf < 8; uf++) {					if (0 == (q.itd->hw_transaction [uf]							& ITD_ACTIVE))						continue;					q_p = &q.itd->itd_next;					hw_p = &q.itd->hw_next;					type = Q_NEXT_TYPE (q.itd->hw_next);					q = *q_p;					break;				}				if (uf != 8)					break;				/* this one's ready ... HC won't cache the				 * pointer for much longer, if at all.				 */				*q_p = q.itd->itd_next;				*hw_p = q.itd->hw_next;				type = Q_NEXT_TYPE (q.itd->hw_next);				wmb();				modified = itd_complete (ehci, q.itd);				q = *q_p;				break;			case Q_TYPE_SITD:				if ((q.sitd->hw_results & SITD_ACTIVE)						&& live) {					q_p = &q.sitd->sitd_next;					hw_p = &q.sitd->hw_next;					type = Q_NEXT_TYPE (q.sitd->hw_next);					q = *q_p;					break;				}				*q_p = q.sitd->sitd_next;				*hw_p = q.sitd->hw_next;				type = Q_NEXT_TYPE (q.sitd->hw_next);				wmb();				modified = sitd_complete (ehci, q.sitd);				q = *q_p;				break;			default:				dbg ("corrupt type %d frame %d shadow %p",					type, frame, q.ptr);				// BUG ();				q.ptr = NULL;			}			/* assume completion callbacks modify the queue */			if (unlikely (modified))				goto restart;		}		/* stop when we catch up to the HC */		// FIXME:  this assumes we won't get lapped when		// latencies climb; that should be rare, but...		// detect it, and just go all the way around.		// FLR might help detect this case, so long as latencies		// don't exceed periodic_size msec (default 1.024 sec).		// FIXME:  likewise assumes HC doesn't halt mid-scan		if (now_uframe == clock) {			unsigned	now;			if (!HC_IS_RUNNING (ehci_to_hcd(ehci)->state))				break;			ehci->next_uframe = now_uframe;			now = readl (&ehci->regs->frame_index) % mod;			if (now_uframe == now)				break;			/* rescan the rest of this frame, then ... */			clock = now;		} else {			now_uframe++;			now_uframe %= mod;		}	} }