libsec: remove flawed aes() digest and hmac_aes() implementations (thanks aiju)

4 files changed, 5 insertions, 186 deletions

diff --git a/sys/include/libsec.h b/sys/include/libsec.h
index 9a6a6a54c..d275d6d47 100644
--- a/sys/include/libsec.h
+++ b/sys/include/libsec.h
@@ -141,7 +141,6 @@ enum
 	SHA2_512dlen=	64,	/* SHA-512 digest length */
 	MD4dlen=	16,	/* MD4 digest length */
 	MD5dlen=	16,	/* MD5 digest length */
-	AESdlen=	16,	/* TODO: see rfc */
 
 	Hmacblksz	= 64,	/* in bytes; from rfc2104 */
 };
@@ -167,7 +166,6 @@ typedef struct DigestState SHA2_384state;
 typedef struct DigestState SHA2_512state;
 typedef struct DigestState MD5state;
 typedef struct DigestState MD4state;
-typedef struct DigestState AEShstate;
 
 DigestState*	md4(uchar*, ulong, uchar*, DigestState*);
 DigestState*	md5(uchar*, ulong, uchar*, DigestState*);
@@ -176,7 +174,6 @@ DigestState*	sha2_224(uchar*, ulong, uchar*, DigestState*);
 DigestState*	sha2_256(uchar*, ulong, uchar*, DigestState*);
 DigestState*	sha2_384(uchar*, ulong, uchar*, DigestState*);
 DigestState*	sha2_512(uchar*, ulong, uchar*, DigestState*);
-DigestState*	aes(uchar*, ulong, uchar*, DigestState*);
 DigestState*	hmac_x(uchar *p, ulong len, uchar *key, ulong klen,
 			uchar *digest, DigestState *s,
 			DigestState*(*x)(uchar*, ulong, uchar*, DigestState*),
@@ -187,7 +184,6 @@ DigestState*	hmac_sha2_224(uchar*, ulong, uchar*, ulong, uchar*, DigestState*);
 DigestState*	hmac_sha2_256(uchar*, ulong, uchar*, ulong, uchar*, DigestState*);
 DigestState*	hmac_sha2_384(uchar*, ulong, uchar*, ulong, uchar*, DigestState*);
 DigestState*	hmac_sha2_512(uchar*, ulong, uchar*, ulong, uchar*, DigestState*);
-DigestState*	hmac_aes(uchar*, ulong, uchar*, ulong, uchar*, DigestState*);
 char*		md5pickle(MD5state*);
 MD5state*	md5unpickle(char*);
 char*		sha1pickle(SHA1state*);
diff --git a/sys/man/2/sechash b/sys/man/2/sechash
index fa8f198e7..4fd9f4f08 100644
--- a/sys/man/2/sechash
+++ b/sys/man/2/sechash
@@ -3,9 +3,9 @@
 md4, md5,
 sha1, sha2_224, sha2_256, sha2_384, sha2_512,
 ripemd160,
-aes, hmac_x, hmac_md5,
+hmac_x, hmac_md5,
 hmac_sha1, hmac_sha2_224, hmac_sha2_256, hmac_sha2_384, hmac_sha2_512,
-hmac_aes, md5pickle, md5unpickle,
+md5pickle, md5unpickle,
 sha1pickle, sha1unpickle \- cryptographically secure hashes
 .SH SYNOPSIS
 .nr Wd \w'\fLDS* \fP'u
@@ -58,8 +58,6 @@ DS*	sha2_512(uchar *data, ulong dlen, uchar *digest, DS *state)
 .Ti
 DS*	ripemd160(uchar *data, ulong dlen, uchar *digest, DS *state)
 .Ti
-DS*	aes(uchar *data, ulong dlen, uchar *digest, DS *state)
-.Ti
 DS*	hmac_x(uchar *p, ulong len, uchar *key, ulong klen, uchar *digest, DS *s, DS*(*x)(uchar*, ulong, uchar*, DS*), int xlen)
 .Ti
 DS*	hmac_md5(uchar *data, ulong dlen, uchar *key, ulong klen, uchar *digest, DS *state)
@@ -73,8 +71,6 @@ DS*	hmac_sha2_256(uchar *data, ulong dlen, uchar *key, ulong klen, uchar *digest
 DS*	hmac_sha2_384(uchar *data, ulong dlen, uchar *key, ulong klen, uchar *digest, DS *state)
 .Ti
 DS*	hmac_sha2_512(uchar *data, ulong dlen, uchar *key, ulong klen, uchar *digest, DS *state)
-.Ti
-DS*	hmac_aes(uchar *data, ulong dlen, uchar *key, ulong klen, uchar *digest, DS *state)
 .SH DESCRIPTION
 .DT
 We support several secure hash functions.  The output of a
@@ -96,15 +92,13 @@ The routines
 .IR sha2_384 ,
 .IR sha2_512 ,
 .IR ripemd160 ,
-.IR aes ,
 .IR hmac_md5 ,
 .IR hmac_sha1 ,
 .IR hmac_sha2_224 ,
 .IR hmac_sha2_256 ,
 .IR hmac_sha2_384 ,
-.IR hmac_sha2_512 ,
 and
-.I hmac_aes
+.IR hmac_sha2_512
 differ only in the length of the resulting digest
 and in the security of the hash.
 .I Sha2_*
@@ -143,9 +137,8 @@ define the lengths of the digests.
 .IR hmac_sha2_224 ,
 .IR hmac_sha2_256 ,
 .IR hmac_sha2_384 ,
-.IR hmac_sha2_512 ,
 and
-.I hmac_aes
+.IR hmac_sha2_512
 are used slightly differently.  These hash algorithms are keyed and require
 a key to be specified on every call.
 The digest lengths for these hashes are the obvious ones from
@@ -194,7 +187,6 @@ md5("drowssap ym", 11, digest, s);
 .SH SOURCE
 .B /sys/src/libsec
 .SH SEE ALSO
-.IR aes (2),
 .IR blowfish (2),
 .IR des (2),
 .IR elgamal (2),
diff --git a/sys/src/9/ip/esp.c b/sys/src/9/ip/esp.c
index fb3ab2338..19a6554f3 100644
--- a/sys/src/9/ip/esp.c
+++ b/sys/src/9/ip/esp.c
@@ -162,7 +162,6 @@ static	void desespinit(Espcb *ecb, char *name, uchar *k, unsigned n);
 
 static	void nullahinit(Espcb*, char*, uchar *key, unsigned keylen);
 static	void shaahinit(Espcb*, char*, uchar *key, unsigned keylen);
-static	void aesahinit(Espcb*, char*, uchar *key, unsigned keylen);
 static	void md5ahinit(Espcb*, char*, uchar *key, unsigned keylen);
 
 static Algorithm espalg[] =
@@ -172,8 +171,6 @@ static Algorithm espalg[] =
 	"aes_128_cbc",	128,	aescbcespinit,	/* new rfc3602 */
 	"aes_ctr",	128,	aesctrespinit,	/* new rfc3686 */
 	"des_56_cbc",	64,	desespinit,	/* rfc2405, deprecated */
-	/* rc4 was never required, was used in original bandt */
-//	"rc4_128",	128,	rc4espinit,
 	nil,		0,	nil,
 };
 
@@ -181,7 +178,6 @@ static Algorithm ahalg[] =
 {
 	"null",		0,	nullahinit,
 	"hmac_sha1_96",	128,	shaahinit,	/* rfc2404 */
-	"aes_xcbc_mac_96", 128,	aesahinit,	/* new rfc3566 */
 	"hmac_md5_96",	128,	md5ahinit,	/* rfc2403 */
 	nil,		0,	nil,
 };
@@ -803,37 +799,6 @@ shaahinit(Espcb *ecb, char *name, uchar *key, unsigned klen)
 /*
  * aes
  */
-
-/* ah_aes_xcbc_mac_96, rfc3566 */
-static int
-aesahauth(Espcb *ecb, uchar *t, int tlen, uchar *auth)
-{
-	int r;
-	uchar hash[AESdlen];
-
-	memset(hash, 0, AESdlen);
-	ecb->ds = hmac_aes(t, tlen, (uchar*)ecb->ahstate, BITS2BYTES(96), hash,
-		ecb->ds);
-	r = memcmp(auth, hash, ecb->ahlen) == 0;
-	memmove(auth, hash, ecb->ahlen);
-	return r;
-}
-
-static void
-aesahinit(Espcb *ecb, char *name, uchar *key, unsigned klen)
-{
-	if(klen != 128)
-		panic("aesahinit: keylen not 128");
-	klen /= BI2BY;
-
-	ecb->ahalg = name;
-	ecb->ahblklen = 1;
-	ecb->ahlen = BITS2BYTES(96);
-	ecb->auth = aesahauth;
-	ecb->ahstate = smalloc(klen);
-	memmove(ecb->ahstate, key, klen);
-}
-
 static int
 aescbccipher(Espcb *ecb, uchar *p, int n)	/* 128-bit blocks */
 {
diff --git a/sys/src/libsec/port/aes.c b/sys/src/libsec/port/aes.c
index 9dcf26547..4ce653784 100644
--- a/sys/src/libsec/port/aes.c
+++ b/sys/src/libsec/port/aes.c
@@ -59,7 +59,7 @@ static uchar basekey[3][16] = {
 	},
 };
 
-int aes_setupEnc(ulong rk[/*4*(Nr + 1)*/], const uchar cipherKey[],
+static int aes_setupEnc(ulong rk[/*4*(Nr + 1)*/], const uchar cipherKey[],
 		int keyBits);
 static int aes_setupDec(ulong rk[/*4*(Nr + 1)*/], const uchar cipherKey[],
 		int keyBits);
@@ -220,140 +220,6 @@ aesCBCdecrypt(uchar *p, int len, AESstate *s)
 	}
 }
 
-/* taken from sha1; TODO: verify suitability (esp. byte order) for aes */
-/*
- *	encodes input (ulong) into output (uchar). Assumes len is
- *	a multiple of 4.
- */
-static void
-encode(uchar *output, ulong *input, ulong len)
-{
-	ulong x;
-	uchar *e;
-
-	for(e = output + len; output < e;) {
-		x = *input++;
-		*output++ = x >> 24;
-		*output++ = x >> 16;
-		*output++ = x >> 8;
-		*output++ = x;
-	}
-}
-
-/* TODO: verify use of aes_encrypt here */
-AEShstate*
-aes(uchar *p, ulong len, uchar *digest, AEShstate *s)
-{
-	uchar buf[128];
-	ulong x[16];
-	int i;
-	uchar *e;
-
-	if(s == nil){
-		s = malloc(sizeof(*s));
-		if(s == nil)
-			return nil;
-		memset(s, 0, sizeof(*s));
-		s->malloced = 1;
-	}
-
-	if(s->seeded == 0){
-		/* seed the state, these constants would look nicer big-endian */
-		s->state[0] = 0x67452301;
-		s->state[1] = 0xefcdab89;
-		s->state[2] = 0x98badcfe;
-		s->state[3] = 0x10325476;
-		/* in sha1 (20-byte digest), but not md5 (16 bytes)*/
-		s->state[4] = 0xc3d2e1f0;
-		s->seeded = 1;
-	}
-
-	/* fill out the partial 64 byte block from previous calls */
-	if(s->blen){
-		i = 64 - s->blen;
-		if(len < i)
-			i = len;
-		memmove(s->buf + s->blen, p, i);
-		len -= i;
-		s->blen += i;
-		p += i;
-		if(s->blen == 64){
-			/* encrypt s->buf into s->state */
-			// _sha1block(s->buf, s->blen, s->state);
-			aes_encrypt((ulong *)s->buf, 1, s->buf, (uchar *)s->state);
-			s->len += s->blen;
-			s->blen = 0;
-		}
-	}
-
-	/* do 64 byte blocks */
-	i = len & ~0x3f;
-	if(i){
-		/* encrypt p into s->state */
-		// _sha1block(p, i, s->state);
-		aes_encrypt((ulong *)s->buf, 1, p, (uchar *)s->state);
-		s->len += i;
-		len -= i;
-		p += i;
-	}
-
-	/* save the left overs if not last call */
-	if(digest == 0){
-		if(len){
-			memmove(s->buf, p, len);
-			s->blen += len;
-		}
-		return s;
-	}
-
-	/*
-	 *  this is the last time through, pad what's left with 0x80,
-	 *  0's, and the input count to create a multiple of 64 bytes
-	 */
-	if(s->blen){
-		p = s->buf;
-		len = s->blen;
-	} else {
-		memmove(buf, p, len);
-		p = buf;
-	}
-	s->len += len;
-	e = p + len;
-	if(len < 56)
-		i = 56 - len;
-	else
-		i = 120 - len;
-	memset(e, 0, i);
-	*e = 0x80;
-	len += i;
-
-	/* append the count */
-	x[0] = s->len>>29;		/* byte-order dependent */
-	x[1] = s->len<<3;
-	encode(p+len, x, 8);
-
-	/* digest the last part */
-	/* encrypt p into s->state */
-	// _sha1block(p, len+8, s->state);
-	aes_encrypt((ulong *)s->buf, 1, p, (uchar *)s->state);
-	s->len += len+8;		/* sha1: +8 */
-
-	/* return result and free state */
-	encode((uchar *)digest, (ulong *)s->state, AESdlen);
-	if(s->malloced == 1)
-		free(s);
-	return nil;
-}
-
-DigestState*
-hmac_aes(uchar *p, ulong len, uchar *key, ulong klen, uchar *digest,
-	DigestState *s)
-{
-	return hmac_x(p, len, key, klen, digest, s, aes, AESdlen);
-}
-
-
-
 /*
  * this function has been changed for plan 9.
  * Expand the cipher key into the encryption and decryption key schedules.