smtp, smtpd: fix use of enc64(), fix memory leaks in doauth()

the approximation of n*2 to calculate the number of output bytes
for enc64() fails for inputs of size < 3. this is fixed by using
encodefmt() which gets the calculation right and also simplifies
the code avoiding the allocation and freeing of intermediate string
buffers.

2 files changed, 28 insertions, 40 deletions

diff --git a/sys/src/cmd/upas/smtp/smtp.c b/sys/src/cmd/upas/smtp/smtp.c
index 37fb5fbd7..96f2d65ab 100644
--- a/sys/src/cmd/upas/smtp/smtp.c
+++ b/sys/src/cmd/upas/smtp/smtp.c
@@ -114,6 +114,7 @@ main(int argc, char **argv)
 
 	alarmscale = 60*1000;	/* minutes */
 	quotefmtinstall();
+	fmtinstall('[', encodefmt);
 	errs = malloc(argc*sizeof(char*));
 	reply = s_new();
 	host = 0;
@@ -398,10 +399,10 @@ dotls(char *me)
 static char *
 doauth(char *methods)
 {
-	char *buf, *base64;
+	char *buf, *err;
+	UserPasswd *p;
 	int n;
 	DS ds;
-	UserPasswd *p;
 
 	dial_string_parse(ddomain, &ds);
 
@@ -414,51 +415,44 @@ doauth(char *methods)
 	if (p == nil)
 		return Giveup;
 
+	err = Retry;
 	if (strstr(methods, "LOGIN")){
 		dBprint("AUTH LOGIN\r\n");
 		if (getreply() != 3)
-			return Retry;
+			goto out;
 
-		n = strlen(p->user);
-		base64 = malloc(2*n);
-		if (base64 == nil)
-			return Retry;	/* Out of memory */
-		enc64(base64, 2*n, (uchar *)p->user, n);
-		dBprint("%s\r\n", base64);
+		dBprint("%.*[\r\n", strlen(p->user), p->user);
 		if (getreply() != 3)
-			return Retry;
+			goto out;
 
-		n = strlen(p->passwd);
-		base64 = malloc(2*n);
-		if (base64 == nil)
-			return Retry;	/* Out of memory */
-		enc64(base64, 2*n, (uchar *)p->passwd, n);
-		dBprint("%s\r\n", base64);
+		dBprint("%.*[\r\n", strlen(p->passwd), p->passwd);
 		if (getreply() != 2)
-			return Retry;
+			goto out;
 
-		free(base64);
+		err = nil;
 	}
 	else
 	if (strstr(methods, "PLAIN")){
-		n = strlen(p->user) + strlen(p->passwd) + 3;
-		buf = malloc(n);
-		base64 = malloc(2 * n);
-		if (buf == nil || base64 == nil) {
+		n = strlen(p->user) + strlen(p->passwd) + 2;
+		buf = malloc(n+1);
+		if (buf == nil) {
 			free(buf);
-			return Retry;	/* Out of memory */
+			goto out;	/* Out of memory */
 		}
 		snprint(buf, n, "%c%s%c%s", 0, p->user, 0, p->passwd);
-		enc64(base64, 2 * n, (uchar *)buf, n - 1);
+		dBprint("AUTH PLAIN %.*[\r\n", n, buf);
+		memset(buf, 0, n);
 		free(buf);
-		dBprint("AUTH PLAIN %s\r\n", base64);
-		free(base64);
 		if (getreply() != 2)
-			return Retry;
-	}
-	else
-		return "No supported AUTH method";
-	return(0);
+			goto out;
+		err = nil;
+	} else
+		err = "No supported AUTH method";
+out:
+	memset(p->user, 0, strlen(p->user));
+	memset(p->passwd, 0, strlen(p->passwd));
+	free(p);
+	return err;
 }
 
 char *
diff --git a/sys/src/cmd/upas/smtp/smtpd.c b/sys/src/cmd/upas/smtp/smtpd.c
index c73e822be..84fabbd21 100644
--- a/sys/src/cmd/upas/smtp/smtpd.c
+++ b/sys/src/cmd/upas/smtp/smtpd.c
@@ -120,6 +120,7 @@ main(int argc, char **argv)
 	netdir = nil;
 	quotefmtinstall();
 	fmtinstall('I', eipfmt);
+	fmtinstall('[', encodefmt);
 	starttime = time(0);
 	ARGBEGIN{
 	case 'a':
@@ -1587,7 +1588,7 @@ starttls(void)
 void
 auth(String *mech, String *resp)
 {
-	char *user, *pass, *scratch = nil;
+	char *user, *pass;
 	AuthInfo *ai = nil;
 	Chalstate *chs = nil;
 	String *s_resp1_64 = nil, *s_resp2_64 = nil, *s_resp1 = nil;
@@ -1674,7 +1675,6 @@ windup:
 		goto bomb_out;
 	}
 	else if (cistrcmp(s_to_c(mech), "cram-md5") == 0) {
-		int chal64n;
 		char *resp, *t;
 
 		chs = auth_challenge("proto=cram role=server");
@@ -1683,11 +1683,7 @@ windup:
 			reply("501 5.7.5 Couldn't get CRAM-MD5 challenge\r\n");
 			goto bomb_out;
 		}
-		scratch = malloc(chs->nchal * 2 + 1);
-		chal64n = enc64(scratch, chs->nchal * 2, (uchar *)chs->chal,
-			chs->nchal);
-		scratch[chal64n] = 0;
-		reply("334 %s\r\n", scratch);
+		reply("334 %.*[\r\n", chs->nchal, chs->chal);
 		s_resp1_64 = s_new();
 		if (getcrnl(s_resp1_64, &bin) <= 0)
 			goto bad_sequence;
@@ -1720,8 +1716,6 @@ bomb_out:
 		auth_freeAI(ai);
 	if (chs)
 		auth_freechal(chs);
-	if (scratch)
-		free(scratch);
 	if (s_resp1)
 		s_free(s_resp1);
 	if (s_resp2)