draw: add badrect() function to reject zero, negative size or orverly huge rectangles

not checking the rectangle dimensions causes integer overflows
and memory corruption. adding a new badrect() function that checks
for these cases.

1 files changed, 9 insertions, 4 deletions

diff --git a/sys/src/libmemdraw/alloc.c b/sys/src/libmemdraw/alloc.c
index 14abdf53d..d2852c3c4 100644
--- a/sys/src/libmemdraw/alloc.c
+++ b/sys/src/libmemdraw/alloc.c
@@ -27,14 +27,14 @@ allocmemimaged(Rectangle r, ulong chan, Memdata *md)
 	ulong l;
 	Memimage *i;
 
-	if(Dx(r) <= 0 || Dy(r) <= 0){
-		werrstr("bad rectangle %R", r);
-		return nil;
-	}
 	if((d = chantodepth(chan)) == 0) {
 		werrstr("bad channel descriptor %.8lux", chan);
 		return nil;
 	}
+	if(badrect(r)){
+		werrstr("bad rectangle %R", r);
+		return nil;
+	}
 
 	l = wordsperline(r, d);
 
@@ -76,8 +76,13 @@ allocmemimage(Rectangle r, ulong chan)
 		werrstr("bad channel descriptor %.8lux", chan);
 		return nil;
 	}
+	if(badrect(r)){
+		werrstr("bad rectangle %R", r);
+		return nil;
+	}
 
 	l = wordsperline(r, d);
+
 	nw = l*Dy(r);
 	md = malloc(sizeof(Memdata));
 	if(md == nil)