draw: add badrect() function to reject zero, negative size or orverly huge rectangles

not checking the rectangle dimensions causes integer overflows
and memory corruption. adding a new badrect() function that checks
for these cases.

8 files changed, 40 insertions, 8 deletions

diff --git a/sys/src/libdraw/alloc.c b/sys/src/libdraw/alloc.c
index 7884321ac..256378f36 100644
--- a/sys/src/libdraw/alloc.c
+++ b/sys/src/libdraw/alloc.c
@@ -26,6 +26,10 @@ _allocimage(Image *ai, Display *d, Rectangle r, ulong chan, int repl, ulong val,
 	err = 0;
 	i = 0;
 
+	if(badrect(r)){
+		werrstr("bad rectangle");
+		return nil;
+	}
 	if(chan == 0){
 		werrstr("bad channel descriptor");
 		return nil;
diff --git a/sys/src/libdraw/badrect.c b/sys/src/libdraw/badrect.c
new file mode 100644
index 000000000..5ec53edb4
--- /dev/null
+++ b/sys/src/libdraw/badrect.c
@@ -0,0 +1,22 @@
+#include <u.h>
+#include <libc.h>
+#include <draw.h>
+
+/*
+ * check for zero, negative size or insanely huge rectangle.
+ */
+int
+badrect(Rectangle r)
+{
+	int x, y;
+	uint z;
+
+	x = Dx(r);
+	y = Dy(r);
+	if(x > 0 && y > 0){
+		z = x*y;
+		if(z/x == y && z < 0x10000000)
+			return 0;
+	}
+	return 1;
+}
diff --git a/sys/src/libdraw/mkfile b/sys/src/libdraw/mkfile
index 0a70c3a48..ef76b522b 100644
--- a/sys/src/libdraw/mkfile
+++ b/sys/src/libdraw/mkfile
@@ -6,6 +6,7 @@ OFILES=\
 	alloc.$O\
 	allocimagemix.$O\
 	arith.$O\
+	badrect.$O\
 	bezier.$O\
 	border.$O\
 	buildfont.$O\
diff --git a/sys/src/libmemdraw/alloc.c b/sys/src/libmemdraw/alloc.c
index 14abdf53d..d2852c3c4 100644
--- a/sys/src/libmemdraw/alloc.c
+++ b/sys/src/libmemdraw/alloc.c
@@ -27,14 +27,14 @@ allocmemimaged(Rectangle r, ulong chan, Memdata *md)
 	ulong l;
 	Memimage *i;
 
-	if(Dx(r) <= 0 || Dy(r) <= 0){
-		werrstr("bad rectangle %R", r);
-		return nil;
-	}
 	if((d = chantodepth(chan)) == 0) {
 		werrstr("bad channel descriptor %.8lux", chan);
 		return nil;
 	}
+	if(badrect(r)){
+		werrstr("bad rectangle %R", r);
+		return nil;
+	}
 
 	l = wordsperline(r, d);
 
@@ -76,8 +76,13 @@ allocmemimage(Rectangle r, ulong chan)
 		werrstr("bad channel descriptor %.8lux", chan);
 		return nil;
 	}
+	if(badrect(r)){
+		werrstr("bad rectangle %R", r);
+		return nil;
+	}
 
 	l = wordsperline(r, d);
+
 	nw = l*Dy(r);
 	md = malloc(sizeof(Memdata));
 	if(md == nil)
diff --git a/sys/src/libmemdraw/cload.c b/sys/src/libmemdraw/cload.c
index 5e068cba0..b70c23603 100644
--- a/sys/src/libmemdraw/cload.c
+++ b/sys/src/libmemdraw/cload.c
@@ -9,7 +9,7 @@ cloadmemimage(Memimage *i, Rectangle r, uchar *data, int ndata)
 	int y, bpl, c, cnt, offs;
 	uchar mem[NMEM], *memp, *omemp, *emem, *linep, *elinep, *u, *eu;
 
-	if(!rectinrect(r, i->r))
+	if(badrect(r) || !rectinrect(r, i->r))
 		return -1;
 	bpl = bytesperline(r, i->depth);
 	u = data;
diff --git a/sys/src/libmemdraw/draw.c b/sys/src/libmemdraw/draw.c
index 9416e0059..5a15e4dae 100644
--- a/sys/src/libmemdraw/draw.c
+++ b/sys/src/libmemdraw/draw.c
@@ -228,7 +228,7 @@ drawclip(Memimage *dst, Rectangle *r, Memimage *src, Point *p0, Memimage *mask,
 	int splitcoords;
 	Rectangle omr;
 
-	if(r->min.x>=r->max.x || r->min.y>=r->max.y)
+	if(badrect(*r))
 		return 0;
 	splitcoords = (p0->x!=p1->x) || (p0->y!=p1->y);
 	/* clip to destination */
diff --git a/sys/src/libmemdraw/load.c b/sys/src/libmemdraw/load.c
index 984efa958..9c17eb01b 100644
--- a/sys/src/libmemdraw/load.c
+++ b/sys/src/libmemdraw/load.c
@@ -10,7 +10,7 @@ loadmemimage(Memimage *i, Rectangle r, uchar *data, int ndata)
 	Memdrawparam par;
 	uchar *q;
 
-	if(!rectinrect(r, i->r))
+	if(badrect(r) || !rectinrect(r, i->r))
 		return -1;
 
 	memset(&par, 0, sizeof par);
diff --git a/sys/src/libmemdraw/unload.c b/sys/src/libmemdraw/unload.c
index b3f25c93a..9e638a93e 100644
--- a/sys/src/libmemdraw/unload.c
+++ b/sys/src/libmemdraw/unload.c
@@ -9,7 +9,7 @@ unloadmemimage(Memimage *i, Rectangle r, uchar *data, int ndata)
 	int y, l;
 	uchar *q;
 
-	if(!rectinrect(r, i->r))
+	if(badrect(r) || !rectinrect(r, i->r))
 		return -1;
 	l = bytesperline(r, i->depth);
 	if(ndata < l*Dy(r))