/* * An OCF module that uses Intels IXP CryptACC API to do the crypto. * This driver requires the IXP400 Access Library that is available * from Intel in order to operate. * * Copyright (C) 2004 David McCullough <davidm@snapgear.com> * All rights reserved. * * LICENSE TERMS * * The free distribution and use of this software in both source and binary * form is allowed (with or without changes) provided that: * *   1. distributions of this source code include the above copyright *      notice, this list of conditions and the following disclaimer; * *   2. distributions in binary form include the above copyright *      notice, this list of conditions and the following disclaimer *      in the documentation and/or other associated materials; * *   3. the copyright holder's name is not used to endorse products *      built using this software without specific written permission. * * ALTERNATIVELY, provided that this notice is retained in full, this product * may be distributed under the terms of the GNU General Public License (GPL), * in which case the provisions of the GPL apply INSTEAD OF those given above. * * DISCLAIMER * * This software is provided 'as is' with no explicit or implied warranties * in respect of its properties, including, but not limited to, correctness * and/or fitness for purpose. * --------------------------------------------------------------------------- * * NOTES: *     The IXP driver is creating/destroying a context for every packet. *     At first glance this would seem like a bad idea,  but it isn't. * *     First some details,  the CryptACC engine allows 10000 contexts. *     It we allocate a context for the life of a session we are limited to *     10000 sessions,  where as the current implementation is only limited *     to 10000 outstanding requests at anyone point in time,  and new ones *     will become available quite quickly. * *     Second,  you cannot change a context from encoding to decoding after *     registration (well it wasn't working for me so I assumed you *     couldn't).  Thus we really need 2 contexts (encode/decode),  reducing *     out sessions to 5000, a number which is not all that large IMO. * *     Third,  after benchmarking the "single" context for the life of a *     session the drop in throughput was insignificant.  The largest drop I *     saw was 1.7%,  but there were also increases as well,  so it seems to *     me that there is no significant difference. * */#include <linux/config.h>#include <linux/module.h>#include <linux/init.h>#include <linux/list.h>#include <linux/slab.h>#include <linux/sched.h>#include <linux/wait.h>#include <linux/crypto.h>#include <asm/scatterlist.h>#include <cryptodev.h>#include <uio.h>#include <IxTypes.h>#include <IxOsBuffMgt.h>#include <IxNpeDl.h>#include <IxCryptoAcc.h>#include <IxQMgr.h>#include <IxOsServices.h>#include <IxOsCacheMMU.h>struct ixp_data {	int					 ixp_cipher_alg;	int					 ixp_auth_alg;	struct cryptop		*ixp_crp;	UINT32				 ixp_ctx_id;	IxCryptoAccCtx		 ixp_ctx;	IX_MBUF				 ixp_mbuf;	int					 ixp_mbuf_len;	unsigned char		*ixp_iv;	IX_MBUF				 ixp_pri_mbuf;	IX_MBUF				 ixp_sec_mbuf;};static int32_t			 ixp_id = -1;static struct ixp_data **ixp_sessions = NULL;static u_int32_t		 ixp_sesnum = 0;static int ixp_process(void *, struct cryptop *, int);static int ixp_newsession(void *, u_int32_t *, struct cryptoini *);static int ixp_freesession(void *, u_int64_t);static int debug = 0;MODULE_PARM(debug, "i");MODULE_PARM_DESC(debug, "Enable debug");/* * Generate a new software session. */static intixp_newsession(void *arg, u_int32_t *sid, struct cryptoini *cri){	struct ixp_data *ixp;	u_int32_t i;	dprintk("%s()\n", __FUNCTION__);	if (sid == NULL || cri == NULL) {		dprintk("%s,%d - EINVAL\n", __FILE__, __LINE__);		return EINVAL;	}	if (ixp_sessions) {		for (i = 1; i < ixp_sesnum; i++)			if (ixp_sessions[i] == NULL)				break;	} else		i = 1;		/* NB: to silence compiler warning */	if (ixp_sessions == NULL || i == ixp_sesnum) {		struct ixp_data **ixpd;		if (ixp_sessions == NULL) {			i = 1; /* We leave ixp_sessions[0] empty */			ixp_sesnum = CRYPTO_SW_SESSIONS;		} else			ixp_sesnum *= 2;		ixpd = kmalloc(ixp_sesnum * sizeof(struct ixp_data *), GFP_ATOMIC);		if (ixpd == NULL) {			/* Reset session number */			if (ixp_sesnum == CRYPTO_SW_SESSIONS)				ixp_sesnum = 0;			else				ixp_sesnum /= 2;			dprintk("%s,%d: ENOBUFS\n", __FILE__, __LINE__);			return ENOBUFS;		}		memset(ixpd, 0, ixp_sesnum * sizeof(struct ixp_data *));		/* Copy existing sessions */		if (ixp_sessions) {			memcpy(ixpd, ixp_sessions,			    (ixp_sesnum / 2) * sizeof(struct ixp_data *));			kfree(ixp_sessions);		}		ixp_sessions = ixpd;	}	ixp_sessions[i] = (struct ixp_data *) kmalloc(sizeof(struct ixp_data),			GFP_ATOMIC);	if (ixp_sessions[i] == NULL) {		ixp_freesession(NULL, i);		dprintk("%s,%d: EINVAL\n", __FILE__, __LINE__);		return ENOBUFS;	}	*sid = i;	ixp = ixp_sessions[i];	memset(ixp, 0, sizeof(*ixp));	ixp->ixp_cipher_alg = -1;	ixp->ixp_auth_alg = -1;	ixp->ixp_ctx_id = -1;	while (cri) {		switch (cri->cri_alg) {		case CRYPTO_DES_CBC:			ixp->ixp_cipher_alg = cri->cri_alg;			ixp->ixp_ctx.cipherCtx.cipherAlgo = IX_CRYPTO_ACC_CIPHER_DES;			ixp->ixp_ctx.cipherCtx.cipherMode = IX_CRYPTO_ACC_MODE_CBC;			ixp->ixp_ctx.cipherCtx.cipherKeyLen = cri->cri_klen / 8;			ixp->ixp_ctx.cipherCtx.cipherBlockLen = 8;			ixp->ixp_ctx.cipherCtx.cipherInitialVectorLen = 8;			memcpy(ixp->ixp_ctx.cipherCtx.key.cipherKey,					cri->cri_key, cri->cri_klen / 8);			break;		case CRYPTO_3DES_CBC:			ixp->ixp_cipher_alg = cri->cri_alg;			ixp->ixp_ctx.cipherCtx.cipherAlgo = IX_CRYPTO_ACC_CIPHER_3DES;			ixp->ixp_ctx.cipherCtx.cipherMode = IX_CRYPTO_ACC_MODE_CBC;			ixp->ixp_ctx.cipherCtx.cipherKeyLen = cri->cri_klen / 8;			ixp->ixp_ctx.cipherCtx.cipherBlockLen = 8;			ixp->ixp_ctx.cipherCtx.cipherInitialVectorLen = 8;			memcpy(ixp->ixp_ctx.cipherCtx.key.cipherKey,					cri->cri_key, cri->cri_klen / 8);			break;		case CRYPTO_RIJNDAEL128_CBC:			ixp->ixp_cipher_alg = cri->cri_alg;			ixp->ixp_ctx.cipherCtx.cipherAlgo = IX_CRYPTO_ACC_CIPHER_AES;			ixp->ixp_ctx.cipherCtx.cipherMode = IX_CRYPTO_ACC_MODE_CBC;			ixp->ixp_ctx.cipherCtx.cipherKeyLen = cri->cri_klen / 8;			ixp->ixp_ctx.cipherCtx.cipherBlockLen = 16;			ixp->ixp_ctx.cipherCtx.cipherInitialVectorLen = 16;			memcpy(ixp->ixp_ctx.cipherCtx.key.cipherKey,					cri->cri_key, cri->cri_klen / 8);			break;		case CRYPTO_MD5_HMAC:			ixp->ixp_auth_alg = cri->cri_alg;			ixp->ixp_ctx.authCtx.authAlgo = IX_CRYPTO_ACC_AUTH_MD5;			ixp->ixp_ctx.authCtx.authDigestLen = 0;			ixp->ixp_ctx.authCtx.aadLen = 0;			ixp->ixp_ctx.authCtx.authKeyLen = cri->cri_klen / 8;			memcpy(ixp->ixp_ctx.authCtx.key.authKey,					cri->cri_key, cri->cri_klen / 8);			break;		case CRYPTO_SHA1_HMAC:			ixp->ixp_auth_alg = cri->cri_alg;			ixp->ixp_ctx.authCtx.authAlgo = IX_CRYPTO_ACC_AUTH_SHA1;			ixp->ixp_ctx.authCtx.authDigestLen = 0;			ixp->ixp_ctx.authCtx.aadLen = 0;			ixp->ixp_ctx.authCtx.authKeyLen = cri->cri_klen / 8;			memcpy(ixp->ixp_ctx.authCtx.key.authKey,					cri->cri_key, cri->cri_klen / 8);			break;		default:			printk("ixp: unknown algo 0x%x\n", cri->cri_alg);			ixp_freesession(NULL, i);			return EINVAL;		}		cri = cri->cri_next;	}	return 0;}/* * Free a session. */static intixp_freesession(void *arg, u_int64_t tid){	u_int32_t sid = CRYPTO_SESID2LID(tid);	dprintk("%s()\n", __FUNCTION__);	if (sid > ixp_sesnum || ixp_sessions == NULL ||			ixp_sessions[sid] == NULL) {		dprintk("%s,%d: EINVAL\n", __FILE__, __LINE__);		return EINVAL;	}	/* Silently accept and return */	if (sid == 0)		return 0;	if (ixp_sessions[sid]) {		if (ixp_sessions[sid]->ixp_ctx_id != -1) {			ixCryptoAccCtxUnregister(ixp_sessions[sid]->ixp_ctx_id);			ixp_sessions[sid]->ixp_ctx_id = -1;		}		kfree(ixp_sessions[sid]);	}	ixp_sessions[sid] = NULL;	return 0;}static voidixp_register_cb(UINT32 ctx_id, IX_MBUF *bufp, IxCryptoAccStatus status){	int i;	struct ixp_data *ixp;	dprintk("%s()\n", __FUNCTION__);	for (i = 0; i < ixp_sesnum; i++) {		ixp = ixp_sessions[i];		if (ixp && ixp->ixp_ctx_id == ctx_id)			break;	}	if (i >= ixp_sesnum) {		printk("ixp: invalid context id 0x%x\n", ctx_id);		return;	}	if (IX_CRYPTO_ACC_STATUS_WAIT == status) {		printk("ixp: register not finished yet!\n");		return;	}	if (IX_CRYPTO_ACC_STATUS_SUCCESS != status) {		printk("ixp: register failed 0x%x\n", status);		ixp->ixp_crp->crp_etype = EINVAL;		crypto_done(ixp->ixp_crp);		return;	}	status = ixCryptoAccAuthCryptPerform(			ixp->ixp_ctx_id,			&ixp->ixp_mbuf,			NULL,			0,			ixp->ixp_mbuf_len,			0,			ixp->ixp_mbuf_len,			ixp->ixp_mbuf_len,			ixp->ixp_iv);	if (IX_CRYPTO_ACC_STATUS_SUCCESS != status) {		printk("ixp: ixCryptoAccAuthCryptPerform failed 0x%x\n", status);		ixp->ixp_crp->crp_etype = EINVAL;		crypto_done(ixp->ixp_crp);	}}static voidixp_perform_cb(	UINT32 ctx_id,	IX_MBUF *sbufp,	IX_MBUF *dbufp,	IxCryptoAccStatus status){	int i;	struct ixp_data *ixp;	dprintk("%s()\n", __FUNCTION__);	if (sbufp == NULL) {		printk("ixp: error ixp_perform_cb(0x%x, %p, %p, 0x%x)\n",				ctx_id, sbufp, dbufp, status);		return;	}	for (i = 0; i < ixp_sesnum; i++) {		ixp = ixp_sessions[i];		if (ixp && ixp->ixp_ctx_id == ctx_id)			break;	}	if (i >= ixp_sesnum) {		printk("ixp: ixp_perform_cb invalid context id 0x%x\n", ctx_id);		return;	}	if (IX_CRYPTO_ACC_STATUS_SUCCESS != status) {		printk("ixp: perform failed 0x%x\n", status);		ixp->ixp_crp->crp_etype = EINVAL;	}	/*	 * DAVIDM this doesn't work, most likely because we are in the callback	 * and it hasn't been removed from the queue yet. Do it later when we	 * get another request or close the session.	 * ixCryptoAccCtxUnregister(ctx_id);	 */	crypto_done(ixp->ixp_crp);}/* * Process a software request. */static intixp_process(void *arg, struct cryptop *crp, int hint){	struct cryptodesc *crd1, *crd2;	struct ixp_data *ixp;	unsigned int lid;	struct uio *uiop;	IX_MBUF *pri = NULL, *sec = NULL;	int status;	dprintk("%s()\n", __FUNCTION__);	/* Sanity check */	if (crp == NULL) {		dprintk("%s,%d: EINVAL\n", __FILE__, __LINE__);		return EINVAL;	}	crp->crp_etype = 0;	if (crp->crp_desc == NULL || crp->crp_buf == NULL) {		dprintk("%s,%d: EINVAL\n", __FILE__, __LINE__);		crp->crp_etype = EINVAL;		goto done;	}	lid = crp->crp_sid & 0xffffffff;	if (lid >= ixp_sesnum || lid == 0 || ixp_sessions == NULL ||			ixp_sessions[lid] == NULL) {		crp->crp_etype = ENOENT;		dprintk("%s,%d: ENOENT\n", __FILE__, __LINE__);		goto done;	}	ixp = ixp_sessions[lid];	ixp->ixp_iv = NULL;	crd1 = crp->crp_desc;	crd2 = crd1->crd_next;	/* swap crd1/2 so 1 is the cipher */	if (crd2 && crd1->crd_alg == ixp->ixp_auth_alg) {		crd2 = crd1;		crd1 = crd2->crd_next;	}	if (crd1->crd_alg == ixp->ixp_cipher_alg) {		if (crd2 && crd2->crd_alg != ixp->ixp_auth_alg) {			crp->crp_etype = ENOENT;			dprintk("%s,%d: ENOENT\n", __FILE__, __LINE__);			goto done;		}		if (crd1->crd_flags & CRD_F_ENCRYPT)			ixp->ixp_ctx.operation = crd2 ? IX_CRYPTO_ACC_OP_ENCRYPT_AUTH :												IX_CRYPTO_ACC_OP_ENCRYPT;		else			ixp->ixp_ctx.operation = crd2 ? IX_CRYPTO_ACC_OP_AUTH_DECRYPT :												IX_CRYPTO_ACC_OP_DECRYPT;		if (crd1->crd_flags & CRD_F_IV_EXPLICIT)			/* copy in IV ??? */;		if (crd2) {			pri = &ixp->ixp_pri_mbuf;			sec = &ixp->ixp_sec_mbuf;		}		ixp->ixp_iv = crd1->crd_iv;	} else if (crd1->crd_alg == ixp->ixp_auth_alg) {		ixp->ixp_ctx.operation = IX_CRYPTO_ACC_OP_AUTH_CALC;		/* or is it ixp->ixp_ctx.operation = IX_CRYPTO_ACC_OP_AUTH_CHECK; */		pri = &ixp->ixp_pri_mbuf;		sec = &ixp->ixp_sec_mbuf;	} else {		crp->crp_etype = ENOENT;		dprintk("%s,%d: ENOENT\n", __FILE__, __LINE__);		goto done;	}	if (crp->crp_flags & CRYPTO_F_IMBUF) {		printk("ixp: CRYPTO_F_IMBUF not implemented");		crp->crp_etype = ENOENT;		goto done;	} else if (crp->crp_flags & CRYPTO_F_IOV) {		uiop = (struct uio *) crp->crp_buf;		if (uiop->uio_iovcnt != 1) {			/*			 * DAVIDM fix this limitation one day by using			 * a buffer pool and chaining,  it is not currently			 * needed for user space acceleration			 */			printk("ixp: Cannot handle more than 1 iovec yet !\n");			crp->crp_etype = ENOENT;			goto done;		}		ixp->ixp_mbuf.m_len = uiop->uio_iov[0].iov_len;		ixp->ixp_mbuf.m_data = uiop->uio_iov[0].iov_base;		ixp->ixp_mbuf_len = uiop->uio_iov[0].iov_len;	} else /* contig buffer */ {		ixp->ixp_mbuf.m_len  = crp->crp_ilen;		ixp->ixp_mbuf.m_data = crp->crp_buf;		ixp->ixp_mbuf_len    = crp->crp_olen;	}	ixp->ixp_crp = crp;	/* if we were previously registered,  unregister */	if (ixp->ixp_ctx_id != -1)		ixCryptoAccCtxUnregister(ixp->ixp_ctx_id);	ixp->ixp_ctx_id = -1;	status = ixCryptoAccCtxRegister(					&ixp->ixp_ctx,					pri, sec,					ixp_register_cb,					ixp_perform_cb,					&ixp->ixp_ctx_id);	if (IX_CRYPTO_ACC_STATUS_EXCEED_MAX_TUNNELS == status) {		printk("ixp: ixCryptoAccCtxRegister failed (out of tunnels)\n");		ixp->ixp_ctx_id = -1;		return ERESTART;	}	if (IX_CRYPTO_ACC_STATUS_SUCCESS != status) {		printk("ixp: ixCryptoAccCtxRegister failed %d\n", status);		crp->crp_etype = ENOENT;		ixp->ixp_ctx_id = -1;		goto done;	}	return 0;done:	crypto_done(crp);	return 0;}static intixp_init(void){	dprintk("%s(%p)\n", __FUNCTION__, ixp_init);	ixp_id = crypto_get_driverid(0);	if (ixp_id < 0)		panic("IXP/OCF crypto device cannot initialize!");	crypto_register(ixp_id, CRYPTO_DES_CBC,	    0, 0, ixp_newsession, ixp_freesession, ixp_process, NULL);#define	REGISTER(alg) \	crypto_register(ixp_id,alg,0,0,NULL,NULL,NULL,NULL)	REGISTER(CRYPTO_3DES_CBC);	REGISTER(CRYPTO_RIJNDAEL128_CBC);	REGISTER(CRYPTO_MD5_HMAC);	REGISTER(CRYPTO_SHA1_HMAC);#undef REGISTER	return 0;}static voidixp_exit(void){	dprintk("%s()\n", __FUNCTION__);	crypto_unregister_all(ixp_id);	ixp_id = -1;}module_init(ixp_init);module_exit(ixp_exit);MODULE_LICENSE("Dual BSD/GPL");MODULE_AUTHOR("davidm@snapgear.com");MODULE_DESCRIPTION("ixp (OCF module for IXP4xx crypto)");