draw: add badrect() function to reject zero, negative size or orverly huge rectangles

not checking the rectangle dimensions causes integer overflows
and memory corruption. adding a new badrect() function that checks
for these cases.

5 files changed, 13 insertions, 8 deletions

diff --git a/sys/src/libmemdraw/alloc.c b/sys/src/libmemdraw/alloc.c
index 14abdf53d..d2852c3c4 100644
--- a/sys/src/libmemdraw/alloc.c
+++ b/sys/src/libmemdraw/alloc.c
@@ -27,14 +27,14 @@ allocmemimaged(Rectangle r, ulong chan, Memdata *md)
 	ulong l;
 	Memimage *i;
 
-	if(Dx(r) <= 0 || Dy(r) <= 0){
-		werrstr("bad rectangle %R", r);
-		return nil;
-	}
 	if((d = chantodepth(chan)) == 0) {
 		werrstr("bad channel descriptor %.8lux", chan);
 		return nil;
 	}
+	if(badrect(r)){
+		werrstr("bad rectangle %R", r);
+		return nil;
+	}
 
 	l = wordsperline(r, d);
 
@@ -76,8 +76,13 @@ allocmemimage(Rectangle r, ulong chan)
 		werrstr("bad channel descriptor %.8lux", chan);
 		return nil;
 	}
+	if(badrect(r)){
+		werrstr("bad rectangle %R", r);
+		return nil;
+	}
 
 	l = wordsperline(r, d);
+
 	nw = l*Dy(r);
 	md = malloc(sizeof(Memdata));
 	if(md == nil)
diff --git a/sys/src/libmemdraw/cload.c b/sys/src/libmemdraw/cload.c
index 5e068cba0..b70c23603 100644
--- a/sys/src/libmemdraw/cload.c
+++ b/sys/src/libmemdraw/cload.c
@@ -9,7 +9,7 @@ cloadmemimage(Memimage *i, Rectangle r, uchar *data, int ndata)
 	int y, bpl, c, cnt, offs;
 	uchar mem[NMEM], *memp, *omemp, *emem, *linep, *elinep, *u, *eu;
 
-	if(!rectinrect(r, i->r))
+	if(badrect(r) || !rectinrect(r, i->r))
 		return -1;
 	bpl = bytesperline(r, i->depth);
 	u = data;
diff --git a/sys/src/libmemdraw/draw.c b/sys/src/libmemdraw/draw.c
index 9416e0059..5a15e4dae 100644
--- a/sys/src/libmemdraw/draw.c
+++ b/sys/src/libmemdraw/draw.c
@@ -228,7 +228,7 @@ drawclip(Memimage *dst, Rectangle *r, Memimage *src, Point *p0, Memimage *mask,
 	int splitcoords;
 	Rectangle omr;
 
-	if(r->min.x>=r->max.x || r->min.y>=r->max.y)
+	if(badrect(*r))
 		return 0;
 	splitcoords = (p0->x!=p1->x) || (p0->y!=p1->y);
 	/* clip to destination */
diff --git a/sys/src/libmemdraw/load.c b/sys/src/libmemdraw/load.c
index 984efa958..9c17eb01b 100644
--- a/sys/src/libmemdraw/load.c
+++ b/sys/src/libmemdraw/load.c
@@ -10,7 +10,7 @@ loadmemimage(Memimage *i, Rectangle r, uchar *data, int ndata)
 	Memdrawparam par;
 	uchar *q;
 
-	if(!rectinrect(r, i->r))
+	if(badrect(r) || !rectinrect(r, i->r))
 		return -1;
 
 	memset(&par, 0, sizeof par);
diff --git a/sys/src/libmemdraw/unload.c b/sys/src/libmemdraw/unload.c
index b3f25c93a..9e638a93e 100644
--- a/sys/src/libmemdraw/unload.c
+++ b/sys/src/libmemdraw/unload.c
@@ -9,7 +9,7 @@ unloadmemimage(Memimage *i, Rectangle r, uchar *data, int ndata)
 	int y, l;
 	uchar *q;
 
-	if(!rectinrect(r, i->r))
+	if(badrect(r) || !rectinrect(r, i->r))
 		return -1;
 	l = bytesperline(r, i->depth);
 	if(ndata < l*Dy(r))