unsigned long cr_end = mfctl(16);		unsigned long tmp = cr_end - cr_start;		/* check for roll over */		cr_start = (cr_end < cr_start) ?  -(tmp) : (tmp);	}	ioc->avg_search[ioc->avg_idx++] = cr_start;	ioc->avg_idx &= CCIO_SEARCH_SAMPLE - 1;#endif#ifdef CCIO_MAP_STATS	ioc->used_pages += pages_needed;#endif	/* 	** return the bit address.	*/	return res_idx << 3;}#define CCIO_FREE_MAPPINGS(ioc, res_idx, mask, size) \        u##size *res_ptr = (u##size *)&((ioc)->res_map[res_idx]); \        BUG_ON((*res_ptr & mask) != mask); \        *res_ptr &= ~(mask);/** * ccio_free_range - Free pages from the ioc's resource map. * @ioc: The I/O Controller. * @iova: The I/O Virtual Address. * @pages_mapped: The requested number of pages to be freed from the * I/O Pdir. * * This function frees the resouces allocated for the iova. */static voidccio_free_range(struct ioc *ioc, dma_addr_t iova, unsigned long pages_mapped){	unsigned long iovp = CCIO_IOVP(iova);	unsigned int res_idx = PDIR_INDEX(iovp) >> 3;	BUG_ON(pages_mapped == 0);	BUG_ON((pages_mapped * IOVP_SIZE) > DMA_CHUNK_SIZE);	BUG_ON(pages_mapped > BITS_PER_LONG);	DBG_RES("%s():  res_idx: %d pages_mapped %d\n", 		__FUNCTION__, res_idx, pages_mapped);#ifdef CCIO_MAP_STATS	ioc->used_pages -= pages_mapped;#endif	if(pages_mapped <= 8) {#if 0		/* see matching comments in alloc_range */		unsigned long mask = ~(~0UL >> pages_mapped);		CCIO_FREE_MAPPINGS(ioc, res_idx, mask, 8);#else		CCIO_FREE_MAPPINGS(ioc, res_idx, 0xff, 8);#endif	} else if(pages_mapped <= 16) {		CCIO_FREE_MAPPINGS(ioc, res_idx, 0xffff, 16);	} else if(pages_mapped <= 32) {		CCIO_FREE_MAPPINGS(ioc, res_idx, ~(unsigned int)0, 32);#ifdef __LP64__	} else if(pages_mapped <= 64) {		CCIO_FREE_MAPPINGS(ioc, res_idx, ~0UL, 64);#endif	} else {		panic("%s:%s() Too many pages to unmap.\n", __FILE__,		      __FUNCTION__);	}}/********************************************************************          CCIO dma_ops support routines*******************************************************************/typedef unsigned long space_t;#define KERNEL_SPACE 0/*** DMA "Page Type" and Hints ** o if SAFE_DMA isn't set, mapping is for FAST_DMA. SAFE_DMA should be**   set for subcacheline DMA transfers since we don't want to damage the**   other part of a cacheline.** o SAFE_DMA must be set for "memory" allocated via pci_alloc_consistent().**   This bit tells U2 to do R/M/W for partial cachelines. "Streaming"**   data can avoid this if the mapping covers full cache lines.** o STOP_MOST is needed for atomicity across cachelines.**   Apparently only "some EISA devices" need this.**   Using CONFIG_ISA is hack. Only the IOA with EISA under it needs**   to use this hint iff the EISA devices needs this feature.**   According to the U2 ERS, STOP_MOST enabled pages hurt performance.** o PREFETCH should *not* be set for cases like Multiple PCI devices**   behind GSCtoPCI (dino) bus converter. Only one cacheline per GSC**   device can be fetched and multiply DMA streams will thrash the**   prefetch buffer and burn memory bandwidth. See 6.7.3 "Prefetch Rules**   and Invalidation of Prefetch Entries".**** FIXME: the default hints need to be per GSC device - not global.** ** HP-UX dorks: linux device driver programming model is totally different**    than HP-UX's. HP-UX always sets HINT_PREFETCH since it's drivers**    do special things to work on non-coherent platforms...linux has to**    be much more careful with this.*/#define IOPDIR_VALID    0x01UL#define HINT_SAFE_DMA   0x02UL	/* used for pci_alloc_consistent() pages */#ifdef CONFIG_EISA#define HINT_STOP_MOST  0x04UL	/* LSL support */#else#define HINT_STOP_MOST  0x00UL	/* only needed for "some EISA devices" */#endif#define HINT_UDPATE_ENB 0x08UL  /* not used/supported by U2 */#define HINT_PREFETCH   0x10UL	/* for outbound pages which are not SAFE *//*** Use direction (ie PCI_DMA_TODEVICE) to pick hint.** ccio_alloc_consistent() depends on this to get SAFE_DMA** when it passes in BIDIRECTIONAL flag.*/static u32 hint_lookup[] = {	[PCI_DMA_BIDIRECTIONAL]	= HINT_STOP_MOST | HINT_SAFE_DMA | IOPDIR_VALID,	[PCI_DMA_TODEVICE]	= HINT_STOP_MOST | HINT_PREFETCH | IOPDIR_VALID,	[PCI_DMA_FROMDEVICE]	= HINT_STOP_MOST | IOPDIR_VALID,};/** * ccio_io_pdir_entry - Initialize an I/O Pdir. * @pdir_ptr: A pointer into I/O Pdir. * @sid: The Space Identifier. * @vba: The virtual address. * @hints: The DMA Hint. * * Given a virtual address (vba, arg2) and space id, (sid, arg1), * load the I/O PDIR entry pointed to by pdir_ptr (arg0). Each IO Pdir * entry consists of 8 bytes as shown below (MSB == bit 0): * * * WORD 0: * +------+----------------+-----------------------------------------------+ * | Phys | Virtual Index  |               Phys                            | * | 0:3  |     0:11       |               4:19                            | * |4 bits|   12 bits      |              16 bits                          | * +------+----------------+-----------------------------------------------+ * WORD 1: * +-----------------------+-----------------------------------------------+ * |      Phys    |  Rsvd  | Prefetch |Update |Rsvd  |Lock  |Safe  |Valid  | * |     20:39    |        | Enable   |Enable |      |Enable|DMA   |       | * |    20 bits   | 5 bits | 1 bit    |1 bit  |2 bits|1 bit |1 bit |1 bit  | * +-----------------------+-----------------------------------------------+ * * The virtual index field is filled with the results of the LCI * (Load Coherence Index) instruction.  The 8 bits used for the virtual * index are bits 12:19 of the value returned by LCI. */ void CCIO_INLINEccio_io_pdir_entry(u64 *pdir_ptr, space_t sid, unsigned long vba,		   unsigned long hints){	register unsigned long pa;	register unsigned long ci; /* coherent index */	/* We currently only support kernel addresses */	BUG_ON(sid != KERNEL_SPACE);	mtsp(sid,1);	/*	** WORD 1 - low order word	** "hints" parm includes the VALID bit!	** "dep" clobbers the physical address offset bits as well.	*/	pa = virt_to_phys(vba);	asm volatile("depw  %1,31,12,%0" : "+r" (pa) : "r" (hints));	((u32 *)pdir_ptr)[1] = (u32) pa;	/*	** WORD 0 - high order word	*/#ifdef __LP64__	/*	** get bits 12:15 of physical address	** shift bits 16:31 of physical address	** and deposit them	*/	asm volatile ("extrd,u %1,15,4,%0" : "=r" (ci) : "r" (pa));	asm volatile ("extrd,u %1,31,16,%0" : "+r" (pa) : "r" (pa));	asm volatile ("depd  %1,35,4,%0" : "+r" (pa) : "r" (ci));#else	pa = 0;#endif	/*	** get CPU coherency index bits	** Grab virtual index [0:11]	** Deposit virt_idx bits into I/O PDIR word	*/	asm volatile ("lci %%r0(%%sr1, %1), %0" : "=r" (ci) : "r" (vba));	asm volatile ("extru %1,19,12,%0" : "+r" (ci) : "r" (ci));	asm volatile ("depw  %1,15,12,%0" : "+r" (pa) : "r" (ci));	((u32 *)pdir_ptr)[0] = (u32) pa;	/* FIXME: PCX_W platforms don't need FDC/SYNC. (eg C360)	**        PCX-U/U+ do. (eg C200/C240)	**        PCX-T'? Don't know. (eg C110 or similar K-class)	**	** See PDC_MODEL/option 0/SW_CAP word for "Non-coherent IO-PDIR bit".	** Hopefully we can patch (NOP) these out at boot time somehow.	**	** "Since PCX-U employs an offset hash that is incompatible with	** the real mode coherence index generation of U2, the PDIR entry	** must be flushed to memory to retain coherence."	*/	asm volatile("fdc %%r0(%0)" : : "r" (pdir_ptr));	asm volatile("sync");}/** * ccio_clear_io_tlb - Remove stale entries from the I/O TLB. * @ioc: The I/O Controller. * @iovp: The I/O Virtual Page. * @byte_cnt: The requested number of bytes to be freed from the I/O Pdir. * * Purge invalid I/O PDIR entries from the I/O TLB. * * FIXME: Can we change the byte_cnt to pages_mapped? */static CCIO_INLINE voidccio_clear_io_tlb(struct ioc *ioc, dma_addr_t iovp, size_t byte_cnt){	u32 chain_size = 1 << ioc->chainid_shift;	iovp &= IOVP_MASK;	/* clear offset bits, just want pagenum */	byte_cnt += chain_size;	while(byte_cnt > chain_size) {		WRITE_U32(CMD_TLB_PURGE | iovp, &ioc->ioc_regs->io_command);		iovp += chain_size;		byte_cnt -= chain_size;	}}/** * ccio_mark_invalid - Mark the I/O Pdir entries invalid. * @ioc: The I/O Controller. * @iova: The I/O Virtual Address. * @byte_cnt: The requested number of bytes to be freed from the I/O Pdir. * * Mark the I/O Pdir entries invalid and blow away the corresponding I/O * TLB entries. * * FIXME: at some threshhold it might be "cheaper" to just blow *        away the entire I/O TLB instead of individual entries. * * FIXME: Uturn has 256 TLB entries. We don't need to purge every *        PDIR entry - just once for each possible TLB entry. *        (We do need to maker I/O PDIR entries invalid regardless). * * FIXME: Can we change byte_cnt to pages_mapped? */ static CCIO_INLINE voidccio_mark_invalid(struct ioc *ioc, dma_addr_t iova, size_t byte_cnt){	u32 iovp = (u32)CCIO_IOVP(iova);	size_t saved_byte_cnt;	/* round up to nearest page size */	saved_byte_cnt = byte_cnt = ALIGN(byte_cnt, IOVP_SIZE);	while(byte_cnt > 0) {		/* invalidate one page at a time */		unsigned int idx = PDIR_INDEX(iovp);		char *pdir_ptr = (char *) &(ioc->pdir_base[idx]);		BUG_ON(idx >= (ioc->pdir_size / sizeof(u64)));		pdir_ptr[7] = 0;	/* clear only VALID bit */ 		/*		** FIXME: PCX_W platforms don't need FDC/SYNC. (eg C360)		**   PCX-U/U+ do. (eg C200/C240)		** See PDC_MODEL/option 0/SW_CAP for "Non-coherent IO-PDIR bit".		**		** Hopefully someone figures out how to patch (NOP) the		** FDC/SYNC out at boot time.		*/		asm volatile("fdc %%r0(%0)" : : "r" (pdir_ptr[7]));		iovp     += IOVP_SIZE;		byte_cnt -= IOVP_SIZE;	}	asm volatile("sync");	ccio_clear_io_tlb(ioc, CCIO_IOVP(iova), saved_byte_cnt);}/********************************************************************          CCIO dma_ops*******************************************************************//** * ccio_dma_supported - Verify the IOMMU supports the DMA address range. * @dev: The PCI device. * @mask: A bit mask describing the DMA address range of the device. * * This function implements the pci_dma_supported function. */static int ccio_dma_supported(struct device *dev, u64 mask){	if(dev == NULL) {		printk(KERN_ERR MODULE_NAME ": EISA/ISA/et al not supported\n");		BUG();		return 0;	}	/* only support 32-bit devices (ie PCI/GSC) */	return (int)(mask == 0xffffffffUL);}/** * ccio_map_single - Map an address range into the IOMMU. * @dev: The PCI device. * @addr: The start address of the DMA region. * @size: The length of the DMA region. * @direction: The direction of the DMA transaction (to/from device). * * This function implements the pci_map_single function. */static dma_addr_t ccio_map_single(struct device *dev, void *addr, size_t size,		enum dma_data_direction direction){	int idx;	struct ioc *ioc;	unsigned long flags;	dma_addr_t iovp;	dma_addr_t offset;	u64 *pdir_start;	unsigned long hint = hint_lookup[(int)direction];	BUG_ON(!dev);	ioc = GET_IOC(dev);	BUG_ON(size <= 0);	/* save offset bits */	offset = ((unsigned long) addr) & ~IOVP_MASK;	/* round up to nearest IOVP_SIZE */	size = ALIGN(size + offset, IOVP_SIZE);	spin_lock_irqsave(&ioc->res_lock, flags);#ifdef CCIO_MAP_STATS	ioc->msingle_calls++;	ioc->msingle_pages += size >> IOVP_SHIFT;#endif	idx = ccio_alloc_range(ioc, size);	iovp = (dma_addr_t)MKIOVP(idx);	pdir_start = &(ioc->pdir_base[idx]);	DBG_RUN("%s() 0x%p -> 0x%lx size: %0x%x\n",		__FUNCTION__, addr, (long)iovp | offset, size);	/* If not cacheline aligned, force SAFE_DMA on the whole mess */	if((size % L1_CACHE_BYTES) || ((unsigned long)addr % L1_CACHE_BYTES))		hint |= HINT_SAFE_DMA;	while(size > 0) {		ccio_io_pdir_entry(pdir_start, KERNEL_SPACE, (unsigned long)addr, hint);		DBG_RUN(" pdir %p %08x%08x\n",			pdir_start,			(u32) (((u32 *) pdir_start)[0]),			(u32) (((u32 *) pdir_start)[1]));		++pdir_start;		addr += IOVP_SIZE;		size -= IOVP_SIZE;	}	spin_unlock_irqrestore(&ioc->res_lock, flags);	/* form complete address */	return CCIO_IOVA(iovp, offset);}/** * ccio_unmap_single - Unmap an address range from the IOMMU. * @dev: The PCI device. * @addr: The start address of the DMA region. * @size: The length of the DMA region. * @direction: The direction of the DMA transaction (to/from device). * * This function implements the pci_unmap_single function. */static void