idx = dma->srci;	for (i = 0; i < src->nsegs; i++) {		if (i == src->nsegs - 1)			last = HIFN_D_LAST;		dma->srcr[idx].p = htole32(src->segs[i].ds_addr);		dma->srcr[idx].l = htole32(src->segs[i].ds_len |		    HIFN_D_MASKDONEIRQ | last);		dma->srcr[idx].l |= htole32(HIFN_D_VALID);		HIFN_SRCR_SYNC(sc, idx,		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);		if (++idx == HIFN_D_SRC_RSIZE) {			dma->srcr[idx].l = htole32(HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);			dma->srcr[idx].l |= htole32(HIFN_D_VALID);			HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,			    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);			idx = 0;		}	}	dma->srci = idx;	dma->srcu += src->nsegs;	return (idx);} static int hifn_crypto(	struct hifn_softc *sc,	struct hifn_command *cmd,	struct cryptop *crp,	int hint){	struct	hifn_dma *dma = sc->sc_dma;	u_int32_t cmdlen;	int cmdi, resi, err = 0;	int l_flags;	DPRINTF("%s()\n", __FUNCTION__);	/*	 * need 1 cmd, and 1 res	 *	 * NB: check this first since it's easy.	 */	HIFN_LOCK(sc);	if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE ||	    (dma->resu + 1) > HIFN_D_RES_RSIZE) {#ifdef HIFN_DEBUG		if (hifn_debug) {			device_printf(sc->sc_dev,				"cmd/result exhaustion, cmdu %u resu %u\n",				dma->cmdu, dma->resu);		}#endif		hifnstats.hst_nomem_cr++;		HIFN_UNLOCK(sc);		return (ERESTART);	}	if (crp->crp_flags & CRYPTO_F_IMBUF) {#if NOTYET		if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map,		    cmd->src_m, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {			hifnstats.hst_nomem_load++;			err = ENOMEM;			goto err_srcmap1;		}#else		device_printf(sc->sc_dev, "CRYPTO_F_IMBUF no implemented\n");#endif	} else if (crp->crp_flags & CRYPTO_F_IOV) {		if (pci_map_uio(sc, &cmd->src, cmd->src_io)) {			hifnstats.hst_nomem_load++;			err = ENOMEM;			goto err_srcmap1;		}	} else {		err = EINVAL;		goto err_srcmap1;	}	if (hifn_dmamap_aligned(&cmd->src)) {		cmd->sloplen = cmd->src_mapsize & 3;		cmd->dst = cmd->src;	} else {		if (crp->crp_flags & CRYPTO_F_IOV) {			err = EINVAL;			goto err_srcmap;		} else if (crp->crp_flags & CRYPTO_F_IMBUF) {#ifdef NOTYET			int totlen, len;			struct mbuf *m, *m0, *mlast;			KASSERT(cmd->dst_m == cmd->src_m,				("hifn_crypto: dst_m initialized improperly"));			hifnstats.hst_unaligned++;			/*			 * Source is not aligned on a longword boundary.			 * Copy the data to insure alignment.  If we fail			 * to allocate mbufs or clusters while doing this			 * we return ERESTART so the operation is requeued			 * at the crypto later, but only if there are			 * ops already posted to the hardware; otherwise we			 * have no guarantee that we'll be re-entered.			 */			totlen = cmd->src_mapsize;			if (cmd->src_m->m_flags & M_PKTHDR) {				len = MHLEN;				MGETHDR(m0, M_DONTWAIT, MT_DATA);				if (m0 && !m_dup_pkthdr(m0, cmd->src_m, M_DONTWAIT)) {					m_free(m0);					m0 = NULL;				}			} else {				len = MLEN;				MGET(m0, M_DONTWAIT, MT_DATA);			}			if (m0 == NULL) {				hifnstats.hst_nomem_mbuf++;				err = dma->cmdu ? ERESTART : ENOMEM;				goto err_srcmap;			}			if (totlen >= MINCLSIZE) {				MCLGET(m0, M_DONTWAIT);				if ((m0->m_flags & M_EXT) == 0) {					hifnstats.hst_nomem_mcl++;					err = dma->cmdu ? ERESTART : ENOMEM;					m_freem(m0);					goto err_srcmap;				}				len = MCLBYTES;			}			totlen -= len;			m0->m_pkthdr.len = m0->m_len = len;			mlast = m0;			while (totlen > 0) {				MGET(m, M_DONTWAIT, MT_DATA);				if (m == NULL) {					hifnstats.hst_nomem_mbuf++;					err = dma->cmdu ? ERESTART : ENOMEM;					m_freem(m0);					goto err_srcmap;				}				len = MLEN;				if (totlen >= MINCLSIZE) {					MCLGET(m, M_DONTWAIT);					if ((m->m_flags & M_EXT) == 0) {						hifnstats.hst_nomem_mcl++;						err = dma->cmdu ? ERESTART : ENOMEM;						mlast->m_next = m;						m_freem(m0);						goto err_srcmap;					}					len = MCLBYTES;				}				m->m_len = len;				m0->m_pkthdr.len += len;				totlen -= len;				mlast->m_next = m;				mlast = m;			}			cmd->dst_m = m0;#else			device_printf(sc->sc_dev, "CRYPTO_F_IMBUF no implemented\n");#endif		}	}	if (cmd->dst_map == NULL) {		if (crp->crp_flags & CRYPTO_F_IMBUF) {#ifdef NOTYET			if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,			    cmd->dst_m, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {				hifnstats.hst_nomem_map++;				err = ENOMEM;				goto err_dstmap1;			}#else			device_printf(sc->sc_dev, "CRYPTO_F_IMBUF no implemented\n");#endif		} else if (crp->crp_flags & CRYPTO_F_IOV) {			if (pci_map_uio(sc, &cmd->dst, cmd->dst_io)) {				hifnstats.hst_nomem_load++;				err = ENOMEM;				goto err_dstmap1;			}		}	}#ifdef HIFN_DEBUG	if (hifn_debug) {		device_printf(sc->sc_dev,		    "Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n",		    READ_REG_1(sc, HIFN_1_DMA_CSR),		    READ_REG_1(sc, HIFN_1_DMA_IER),		    dma->cmdu, dma->srcu, dma->dstu, dma->resu,		    cmd->src_nsegs, cmd->dst_nsegs);	}#endif#if 0	if (cmd->src_map == cmd->dst_map) {		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,		    BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);	} else {		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,		    BUS_DMASYNC_PREWRITE);		bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,		    BUS_DMASYNC_PREREAD);	}#endif	/*	 * need N src, and N dst	 */	if ((dma->srcu + cmd->src_nsegs) > HIFN_D_SRC_RSIZE ||	    (dma->dstu + cmd->dst_nsegs + 1) > HIFN_D_DST_RSIZE) {#ifdef HIFN_DEBUG		if (hifn_debug) {			device_printf(sc->sc_dev,				"src/dst exhaustion, srcu %u+%u dstu %u+%u\n",				dma->srcu, cmd->src_nsegs,				dma->dstu, cmd->dst_nsegs);		}#endif		hifnstats.hst_nomem_sd++;		err = ERESTART;		goto err_dstmap;	}	if (dma->cmdi == HIFN_D_CMD_RSIZE) {		dma->cmdi = 0;		dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_JUMP|HIFN_D_MASKDONEIRQ);		dma->cmdr[HIFN_D_CMD_RSIZE].l |= htole32(HIFN_D_VALID);		HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);	}	cmdi = dma->cmdi++;	cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]);	HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE);	/* .p for command/result already set */	dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_LAST |	    HIFN_D_MASKDONEIRQ);	dma->cmdr[cmdi].l |= htole32(HIFN_D_VALID);	HIFN_CMDR_SYNC(sc, cmdi,	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);	dma->cmdu++;	if (sc->sc_c_busy == 0) {		WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);		sc->sc_c_busy = 1;	}	/*	 * We don't worry about missing an interrupt (which a "command wait"	 * interrupt salvages us from), unless there is more than one command	 * in the queue.	 */	if (dma->cmdu > 1) {		sc->sc_dmaier |= HIFN_DMAIER_C_WAIT;		WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);	}	hifnstats.hst_ipackets++;	hifnstats.hst_ibytes += cmd->src_mapsize;	hifn_dmamap_load_src(sc, cmd);	if (sc->sc_s_busy == 0) {		WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);		sc->sc_s_busy = 1;	}	/*	 * Unlike other descriptors, we don't mask done interrupt from	 * result descriptor.	 */#ifdef HIFN_DEBUG	if (hifn_debug)		device_printf(sc->sc_dev"load res\n");#endif	if (dma->resi == HIFN_D_RES_RSIZE) {		dma->resi = 0;		dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_JUMP|HIFN_D_MASKDONEIRQ);		dma->resr[HIFN_D_RES_RSIZE].l |= htole32(HIFN_D_VALID);		HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);	}	resi = dma->resi++;	KASSERT(dma->hifn_commands[resi] == NULL,		("hifn_crypto: command slot %u busy", resi));	dma->hifn_commands[resi] = cmd;	HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD);	if ((hint & CRYPTO_HINT_MORE) && sc->sc_curbatch < hifn_maxbatch) {		dma->resr[resi].l = htole32(HIFN_MAX_RESULT |		    HIFN_D_LAST | HIFN_D_MASKDONEIRQ);		dma->resr[resi].l |= htole32(HIFN_D_VALID);		sc->sc_curbatch++;		if (sc->sc_curbatch > hifnstats.hst_maxbatch)			hifnstats.hst_maxbatch = sc->sc_curbatch;		hifnstats.hst_totbatch++;	} else {		dma->resr[resi].l = htole32(HIFN_MAX_RESULT | HIFN_D_LAST);		dma->resr[resi].l |= htole32(HIFN_D_VALID);		sc->sc_curbatch = 0;	}	HIFN_RESR_SYNC(sc, resi,	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);	dma->resu++;	if (sc->sc_r_busy == 0) {		WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);		sc->sc_r_busy = 1;	}	if (cmd->sloplen)		cmd->slopidx = resi;	hifn_dmamap_load_dst(sc, cmd);	if (sc->sc_d_busy == 0) {		WRITE_REG_1(sc, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);		sc->sc_d_busy = 1;	}#ifdef HIFN_DEBUG	if (hifn_debug) {		device_printf(sc->sc_dev, "command: stat %8x ier %8x\n",		    READ_REG_1(sc, HIFN_1_DMA_CSR),		    READ_REG_1(sc, HIFN_1_DMA_IER));	}#endif	sc->sc_active = 5;	HIFN_UNLOCK(sc);	KASSERT(err == 0, ("hifn_crypto: success with error %u", err));	return (err);		/* success */err_dstmap:	if (cmd->src_map != cmd->dst_map)		pci_unmap_iov(sc, &cmd->dst);err_dstmap1:err_srcmap:	if (crp->crp_flags & CRYPTO_F_IMBUF) {#ifdef NOTYET		if (cmd->src_m != cmd->dst_m)			m_freem(cmd->dst_m);#else		device_printf(sc->sc_dev, "CRYPTO_F_IMBUF no implemented\n");#endif	}	pci_unmap_iov(sc, &cmd->src);err_srcmap1:	HIFN_UNLOCK(sc);	return (err);}static voidhifn_tick(unsigned long arg){	struct hifn_softc *sc = (struct hifn_softc *) arg;	int l_flags;	HIFN_LOCK(sc);	if (sc->sc_active == 0) {		struct hifn_dma *dma = sc->sc_dma;		u_int32_t r = 0;		if (dma->cmdu == 0 && sc->sc_c_busy) {			sc->sc_c_busy = 0;			r |= HIFN_DMACSR_C_CTRL_DIS;		}		if (dma->srcu == 0 && sc->sc_s_busy) {			sc->sc_s_busy = 0;			r |= HIFN_DMACSR_S_CTRL_DIS;		}		if (dma->dstu == 0 && sc->sc_d_busy) {			sc->sc_d_busy = 0;			r |= HIFN_DMACSR_D_CTRL_DIS;		}		if (dma->resu == 0 && sc->sc_r_busy) {			sc->sc_r_busy = 0;			r |= HIFN_DMACSR_R_CTRL_DIS;		}		if (r)			WRITE_REG_1(sc, HIFN_1_DMA_CSR, r);	} else		sc->sc_active--;	HIFN_UNLOCK(sc);	mod_timer(&sc->sc_tickto, jiffies + HZ);}static irqreturn_thifn_intr(int irq, void *arg, struct pt_regs *regs){	struct hifn_softc *sc = arg;	struct hifn_dma *dma;	u_int32_t dmacsr, restart;	int i, u;	int l_flags;	dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR);	/* Nothing in the DMA unit interrupted */	if ((dmacsr & sc->sc_dmaier) == 0)		return IRQ_NONE;	HIFN_LOCK(sc);	dma = sc->sc_dma;#ifdef HIFN_DEBUG	if (hifn_debug) {		device_printf(sc->sc_dev,		    "irq: stat %08x ien %08x damier %08x i %d/%d/%d/%d k %d/%d/%d/%d u %d/%d/%d/%d\n",		    dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER), sc->sc_dmaier,		    dma->cmdi, dma->srci, dma->dsti, dma->resi,		    dma->cmdk, dma->srck, dma->dstk, dma->resk,		    dma->cmdu, dma->srcu, dma->dstu, dma->resu);	}#endif	WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier);	if ((sc->sc_flags & HIFN_HAS_PUBLIC) &&	    (dmacsr & HIFN_DMACSR_PUBDONE))		WRITE_REG_1(sc, HIFN_1_PUB_STATUS,		    READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);	restart = dmacsr & (HIFN_DMACSR_D_OVER | HIFN_DMACSR_R_OVER);	if (restart)		device_printf(sc->sc_dev, "overrun %x\n", dmacsr);	if (sc->sc_flags & HIFN_IS_7811) {		if (dmacsr & HIFN_DMACSR_ILLR)			device_printf(sc->sc_dev, "illegal read\n");		if (dmacsr & HIFN_DMACSR_ILLW)			device_printf(sc->sc_dev, "illegal write\n");	}	restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |	    HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);	if (restart) {		device_printf(sc->sc_dev, "abort, resetting.\n");		hifnstats.hst_abort++;		hifn_abort(sc);		HIFN_UNLOCK(sc);		return IRQ_HANDLED;	}	if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) {		/*		 * If no slots to process and we receive a "waiting on		 * command" interrupt, we disable the "waiting on command"		 * (by clearing it).		 */		sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;		WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);	}	/* clear the rings */	i = dma->resk; u = dma->resu;	while (u != 0) {		HIFN_RESR_SYNC(sc, i,		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);		if (dma->resr[i].l & htole32(HIFN_D_VALID)) {			HIFN_RESR_SYNC(sc, i,			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);			break;		}		if (i != HIFN_D_RES_RSIZE) {			struct hifn_command *cmd;			u_int8_t *macbuf = NULL;			HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD);			cmd = dma->hifn_commands[i];			KASSERT(cmd != NULL,				("hifn_intr: null command slot %u", i));			dma->hifn_commands[i] = NULL;			if (cmd->base_masks & HIFN_BASE_CMD_MAC) {				macbuf = dma->result_bufs[i];				macbuf += 12;			}