ethervirtio: fix queue notifications and interrupt flags, avoid useless notifications

bug: Rnointerrupt was used on Vqueue.used.flags instead of
Vqueue.avail.flags.

introduce vqnotify() function that notifies the device
about available ring advancement.

avoid queue notifications there that can be slow by
checking Unonotify flag in Vqueue.used.flags.

keep track of the number of notifications in the queue.

1 files changed, 26 insertions, 10 deletions

diff --git a/sys/src/9/pc/ethervirtio.c b/sys/src/9/pc/ethervirtio.c
index ca3dc055f..170425a0e 100644
--- a/sys/src/9/pc/ethervirtio.c
+++ b/sys/src/9/pc/ethervirtio.c
@@ -137,6 +137,7 @@ struct Vqueue
 	u16int	lastused;
 
 	uint	nintr;
+	uint	nnote;
 };
 
 struct Ctlr {
@@ -169,6 +170,19 @@ vhasroom(void *v)
 }
 
 static void
+vqnotify(Ctlr *ctlr, int x)
+{
+	Vqueue *q;
+
+	coherence();
+	q = &ctlr->queue[x];
+	if(q->used->flags & Unonotify)
+		return;
+	q->nnote++;
+	outs(ctlr->port+Qnotify, x);
+}
+
+static void
 txproc(void *v)
 {
 	Vheader *header;
@@ -201,7 +215,7 @@ txproc(void *v)
 		q->desc[j].flags = 0;
 	}
 
-	q->used->flags &= ~Rnointerrupt;
+	q->avail->flags &= ~Rnointerrupt;
 
 	while(waserror())
 		;
@@ -236,7 +250,7 @@ txproc(void *v)
 		q->desc[j].len = BLEN(b);
 		coherence();
 		q->avail->idx++;
-		outs(ctlr->port+Qnotify, Vtxq);
+		vqnotify(ctlr, Vtxq);
 	}
 
 	pexit("ether out queue closed", 1);
@@ -275,7 +289,7 @@ rxproc(void *v)
 		q->desc[j].flags = Dwrite;
 	}
 
-	q->used->flags &= ~Rnointerrupt;
+	q->avail->flags &= ~Rnointerrupt;
 
 	while(waserror())
 		;
@@ -294,8 +308,8 @@ rxproc(void *v)
 			q->desc[j].len = BALLOC(b);
 			coherence();
 			q->avail->idx++;
-			outs(ctlr->port+Qnotify, Vrxq);
 		} while(q->avail->idx != q->used->idx);
+		vqnotify(ctlr, Vrxq);
 
 		/* wait for any packets to complete */
 		if(!vhasroom(q))
@@ -359,13 +373,13 @@ vctlcmd(Ether *edev, uchar class, uchar cmd, uchar *data, int ndata)
 	q->availent[i] = 0;
 	coherence();
 
-	q->used->flags &= ~Rnointerrupt;
+	q->avail->flags &= ~Rnointerrupt;
 	q->avail->idx++;
-	outs(ctlr->port+Qnotify, Vctlq);
+	vqnotify(ctlr, Vctlq);
 	while(!vhasroom(q))
 		sleep(q, vhasroom, q);
 	q->lastused = q->used->idx;
-	q->used->flags |= Rnointerrupt;
+	q->avail->flags |= Rnointerrupt;
 
 	qunlock(&ctlr->ctllock);
 	poperror();
@@ -440,8 +454,9 @@ ifstat(Ether *edev, void *a, long n, ulong offset)
 
 	for(i = 0; i < ctlr->nqueue; i++){
 		q = &ctlr->queue[i];
-		l += snprint(p+l, READSTR-l, "vq%d %#p size %d avail->idx %d used->idx %d lastused %hud nintr %ud\n",
-			i, q, q->qsize, q->avail->idx, q->used->idx, q->lastused, q->nintr);
+		l += snprint(p+l, READSTR-l,
+			"vq%d %#p size %d avail->idx %d used->idx %d lastused %hud nintr %ud nnote %ud\n",
+			i, q, q->qsize, q->avail->idx, q->used->idx, q->lastused, q->nintr, q->nnote);
 	}
 
 	n = readstr(offset, a, n, p);
@@ -520,7 +535,8 @@ initqueue(Vqueue *q, int size)
 	q->qmask = q->qsize - 1;
 
 	q->lastused = q->avail->idx = q->used->idx = 0;
-	q->used->flags |= Rnointerrupt;
+
+	q->avail->flags |= Rnointerrupt;
 
 	return 0;
 }