draw: fix drawing of replicated source image on memlayer with a clip rectangle

when a replicated source image with a clipr with clipr.min > Pt(0, 0),
drawclip() would properly translate the src->clipr on the dstr
but then clamp the source rectangle back on src->r.

while traversing down multiple layers, this would cause the translation to
be applied multiple times to the dst rectangle giving the wrong image result.

this change adds a new drawclipnorepl() function that avoids the clamping
of source and mask rectangles to src->r and mask->r. this is then used in
libmemlayer.

the final memimagedraw() call will call drawclip() which will do the final
claming.

a testcase is provided:

#include <u.h>
#include <libc.h>
#include <draw.h>

Image *blue;
Image *red;

void
main(int, char *argv[])
{
	Image *i;

	if(initdraw(nil, nil, argv[0]) < 0)
		sysfatal("initdraw: %r");
	i = allocimage(display, screen->r, screen->chan, 1, DWhite);

	red = allocimage(display, Rect(0,0,1,1), screen->chan, 1, DRed);
	blue = allocimage(display, Rect(0,0,1,1), screen->chan, 1, DPaleblue);
	replclipr(red, 1, Rect(10, 10, 110, 110));
	replclipr(blue, 1, Rect(11, 11, 111, 111));

	/* draw on non-layer, works correctly */
	draw(i, i->r, red, nil, ZP);
	draw(i, i->r, blue, nil, ZP);
	draw(screen, screen->r, i, nil, i->r.min);
	flushimage(display, 1);

	/* draw on (screen) layer is too far to the right */
	draw(screen, screen->r, red, nil, ZP);
	draw(screen, screen->r, blue, nil, ZP);
	flushimage(display, 1);

	for(;;){
		sleep(1000);
	}
}

4 files changed, 46 insertions, 13 deletions

diff --git a/sys/include/memdraw.h b/sys/include/memdraw.h
index 997cbbea5..ccbc2abae 100644
--- a/sys/include/memdraw.h
+++ b/sys/include/memdraw.h
@@ -127,6 +127,7 @@ extern int		cloadmemimage(Memimage*, Rectangle, uchar*, int);
 extern int		unloadmemimage(Memimage*, Rectangle, uchar*, int);
 extern ulong*	wordaddr(Memimage*, Point);
 extern uchar*	byteaddr(Memimage*, Point);
+extern int		drawclipnorepl(Memimage*, Rectangle*, Memimage*, Point*, Memimage*, Point*, Rectangle*, Rectangle*);
 extern int		drawclip(Memimage*, Rectangle*, Memimage*, Point*, Memimage*, Point*, Rectangle*, Rectangle*);
 extern void	memfillcolor(Memimage*, ulong);
 extern int		memsetchan(Memimage*, ulong);
diff --git a/sys/man/2/memdraw b/sys/man/2/memdraw
index df68b699d..733ae0e78 100644
--- a/sys/man/2/memdraw
+++ b/sys/man/2/memdraw
@@ -25,6 +25,7 @@ memfillpoly,
 memimageline,
 memimagedraw,
 drawclip,
+drawclipnorepl,
 memlinebbox,
 memlineendsize,
 allocmemsubfont,
@@ -139,6 +140,9 @@ void	memimagedraw(Memimage *dst, Rectangle r, Memimage *src,
 int	drawclip(Memimage *dst, Rectangle *dr, Memimage *src,
 	   Point *sp, Memimage *mask, Point *mp,
 	   Rectangle *sr, Rectangle *mr)
+int	drawclipnorepl(Memimage *dst, Rectangle *dr, Memimage *src,
+	   Point *sp, Memimage *mask, Point *mp,
+	   Rectangle *sr, Rectangle *mr)
 Rectangle	memlinebbox(Point p0, Point p1, int end0, int end1,
 	   int radius)
 int	memlineendsize(int end)
@@ -396,8 +400,18 @@ but translated so the upper left corners are the returned
 .B sp
 and
 .BR mp .
+.I Drawclipnorepl
+does the same as
+.B drawclip
+but avoids clamping
+.B sp
+and
+.B mr
+within the image rectangle of source and mask when replicated.
 .I Drawclip
-returns zero when the clipped rectangle is empty.
+and
+.I drawclipnorepl
+return zero when the clipped rectangle is empty.
 .I Memlinebbox
 returns a conservative bounding box containing a line between
 two points
diff --git a/sys/src/libmemdraw/draw.c b/sys/src/libmemdraw/draw.c
index 96cb01e44..3bd5ff0e7 100644
--- a/sys/src/libmemdraw/draw.c
+++ b/sys/src/libmemdraw/draw.c
@@ -213,17 +213,16 @@ DBG print("alphadraw handled\n");
 }
 #undef DBG
 
+
 /*
  * Clip the destination rectangle further based on the properties of the 
  * source and mask rectangles.  Once the destination rectangle is properly
  * clipped, adjust the source and mask rectangles to be the same size.
- * Then if source or mask is replicated, move its clipped rectangle
- * so that its minimum point falls within the repl rectangle.
  *
  * Return zero if the final rectangle is null.
  */
 int
-drawclip(Memimage *dst, Rectangle *r, Memimage *src, Point *p0, Memimage *mask, Point *p1, Rectangle *sr, Rectangle *mr)
+drawclipnorepl(Memimage *dst, Rectangle *r, Memimage *src, Point *p0, Memimage *mask, Point *p1, Rectangle *sr, Rectangle *mr)
 {
 	Point rmin, delta;
 	int splitcoords;
@@ -270,15 +269,13 @@ drawclip(Memimage *dst, Rectangle *r, Memimage *src, Point *p0, Memimage *mask,
 		sr->min.y += mr->min.y-omr.min.y;
 		sr->max.x += mr->max.x-omr.max.x;
 		sr->max.y += mr->max.y-omr.max.y;
-		*p1 = mr->min;
 	}else{
 		if(!(mask->flags&Frepl) && !rectclip(sr, mask->r))
 			return 0;
 		if(!rectclip(sr, mask->clipr))
 			return 0;
-		*p1 = sr->min;
+		*mr = *sr;
 	}
-
 	/* move source clipping back to destination */
 	delta.x = r->min.x - p0->x;
 	delta.y = r->min.y - p0->y;
@@ -286,6 +283,30 @@ drawclip(Memimage *dst, Rectangle *r, Memimage *src, Point *p0, Memimage *mask,
 	r->min.y = sr->min.y + delta.y;
 	r->max.x = sr->max.x + delta.x;
 	r->max.y = sr->max.y + delta.y;
+	*p0 = sr->min;
+	*p1 = mr->min;
+
+	assert(Dx(*sr) == Dx(*mr) && Dx(*mr) == Dx(*r));
+	assert(Dy(*sr) == Dy(*mr) && Dy(*mr) == Dy(*r));
+	assert(ptinrect(r->min, dst->r));
+
+	return 1;
+}
+
+/*
+ * like drawclipnorepl() above, but if source or mask is replicated,
+ * move its clipped rectangle so that its minimum point falls within
+ * the repl rectangle.
+ *
+ * Return zero if the final rectangle is null.
+ */
+int
+drawclip(Memimage *dst, Rectangle *r, Memimage *src, Point *p0, Memimage *mask, Point *p1, Rectangle *sr, Rectangle *mr)
+{
+	Point delta;
+
+	if(!drawclipnorepl(dst, r, src, p0, mask, p1, sr, mr))
+		return 0;
 
 	/* move source rectangle so sr->min is in src->r */
 	if(src->flags&Frepl) {
@@ -295,8 +316,8 @@ drawclip(Memimage *dst, Rectangle *r, Memimage *src, Point *p0, Memimage *mask,
 		sr->min.y += delta.y;
 		sr->max.x += delta.x;
 		sr->max.y += delta.y;
+		*p0 = sr->min;
 	}
-	*p0 = sr->min;
 
 	/* move mask point so it is in mask->r */
 	*p1 = drawrepl(mask->r, *p1);
@@ -304,11 +325,8 @@ drawclip(Memimage *dst, Rectangle *r, Memimage *src, Point *p0, Memimage *mask,
 	mr->max.x = p1->x+Dx(*sr);
 	mr->max.y = p1->y+Dy(*sr);
 
-	assert(Dx(*sr) == Dx(*mr) && Dx(*mr) == Dx(*r));
-	assert(Dy(*sr) == Dy(*mr) && Dy(*mr) == Dy(*r));
 	assert(ptinrect(*p0, src->r));
 	assert(ptinrect(*p1, mask->r));
-	assert(ptinrect(r->min, dst->r));
 
 	return 1;
 }
diff --git a/sys/src/libmemlayer/draw.c b/sys/src/libmemlayer/draw.c
index c352a0b2c..a9c546bd9 100644
--- a/sys/src/libmemlayer/draw.c
+++ b/sys/src/libmemlayer/draw.c
@@ -42,7 +42,7 @@ ldrawop(Memimage *dst, Rectangle screenr, Rectangle clipr, void *etc, int insave
 	if(!rectinrect(r, clipr)){
 		oclipr = dst->clipr;
 		dst->clipr = clipr;
-		ok = drawclip(dst, &r, d->src, &p0, d->mask, &p1, &srcr, &mr);
+		ok = drawclipnorepl(dst, &r, d->src, &p0, d->mask, &p1, &srcr, &mr);
 		dst->clipr = oclipr;
 		if(!ok)
 			return;
@@ -74,7 +74,7 @@ if(drawdebug)	iprint("mask->layer != nil\n");
 		return;
 	}
 
-	if(drawclip(dst, &r, src, &p0, mask, &p1, &srcr, &mr) == 0){
+	if(drawclipnorepl(dst, &r, src, &p0, mask, &p1, &srcr, &mr) == 0){
 if(drawdebug)	iprint("drawclip dstcr %R srccr %R maskcr %R\n", dst->clipr, src->clipr, mask->clipr);
 		return;
 	}