devip: cache arp entry in Routehint

Instead of having to do an arp hash table lookup for each
outgoing ip packet, forward the Routehint pointer to the
medium's bwrite() function and let it cache the arp entry
pointer.

This avoids route and arp hash table lookups for tcp, il
and connection oriented udp.

It also allows us to avoid multiple route and arp table
lookups for the retransmits once an arp/neighbour solicitation
response arrives.

11 files changed, 76 insertions, 46 deletions

diff --git a/sys/src/9/ip/arp.c b/sys/src/9/ip/arp.c
index f74d1b6f8..08c934dbe 100644
--- a/sys/src/9/ip/arp.c
+++ b/sys/src/9/ip/arp.c
@@ -19,6 +19,8 @@ enum
 
 	AOK		= 1,
 	AWAIT		= 2,
+
+	MAXAGE_TIMER	= 15*60*1000,
 };
 
 char *arpstate[] =
@@ -182,7 +184,7 @@ arplookup(Arp *arp, Ipifc *ifc, uchar *ip)
  *  waiting for ip->mac to be resolved.
  */
 Arpent*
-arpget(Arp *arp, Block *bp, int version, Ipifc *ifc, uchar *ip, uchar *mac)
+arpget(Arp *arp, Block *bp, int version, Ipifc *ifc, uchar *ip, uchar *mac, Routehint *rh)
 {
 	uchar v6ip[IPaddrlen];
 	Arpent *a;
@@ -191,15 +193,32 @@ arpget(Arp *arp, Block *bp, int version, Ipifc *ifc, uchar *ip, uchar *mac)
 		v4tov6(v6ip, ip);
 		ip = v6ip;
 	}
+
+	if(rh != nil
+	&& (a = rh->a) != nil
+	&& a->state == AOK
+	&& a->ifc == ifc
+	&& a->ifcid == ifc->ifcid
+	&& ipcmp(ip, a->ip) == 0){
+		memmove(mac, a->mac, ifc->m->maclen);
+		a->utime = NOW;
+		if(a->utime - a->ctime < MAXAGE_TIMER)
+			return nil;
+	}
+
 	rlock(arp);
 	if((a = arplookup(arp, ifc, ip)) != nil && a->state == AOK){
 		memmove(mac, a->mac, ifc->m->maclen);
 		a->utime = NOW;
-		if(a->utime - a->ctime < 15*60*1000){
+		if(a->utime - a->ctime < MAXAGE_TIMER){
+			if(rh != nil)
+				rh->a = a;
 			runlock(arp);
 			return nil;
 		}
 	}
+	if(rh != nil)
+		rh->a = nil;
 	runlock(arp);
 	wlock(arp);
 	if((a = arplookup(arp, ifc, ip)) == nil)
@@ -269,11 +288,11 @@ arprelease(Arp *arp, Arpent*)
  * called with arp locked
  */
 Block*
-arpresolve(Arp *arp, Arpent *a, Medium *type, uchar *mac)
+arpresolve(Arp *arp, Arpent *a, uchar *mac, Routehint *rh)
 {
 	Block *bp;
 
-	memmove(a->mac, mac, type->maclen);
+	memmove(a->mac, mac, a->ifc->m->maclen);
 	if(a->state == AWAIT) {
 		rxmtunchain(arp, a);
 		a->rxtsrem = 0;
@@ -282,6 +301,8 @@ arpresolve(Arp *arp, Arpent *a, Medium *type, uchar *mac)
 	a->hold = a->last = nil;
 	a->ctime = a->utime = NOW;
 	a->state = AOK;
+	if(rh != nil)
+		rh->a = a;
 	wunlock(arp);
 
 	return bp;
@@ -290,6 +311,7 @@ arpresolve(Arp *arp, Arpent *a, Medium *type, uchar *mac)
 int
 arpenter(Fs *fs, int version, uchar *ip, uchar *mac, int n, uchar *ia, Ipifc *ifc, int refresh)
 {
+	Routehint rh;
 	uchar v6ip[IPaddrlen];
 	Block *bp, *next;
 	Arpent *a;
@@ -299,14 +321,16 @@ arpenter(Fs *fs, int version, uchar *ip, uchar *mac, int n, uchar *ia, Ipifc *if
 	if(ifc->m == nil || ifc->m->maclen != n || ifc->m->maclen == 0)
 		return -1;
 
+	rh.r = nil;
+	rh.a = nil;
 	switch(version){
 	case V4:
-		r = v4lookup(fs, ip, ia, nil);
+		r = v4lookup(fs, ip, ia, &rh);
 		v4tov6(v6ip, ip);
 		ip = v6ip;
 		break;
 	case V6:
-		r = v6lookup(fs, ip, ia, nil);
+		r = v6lookup(fs, ip, ia, &rh);
 		break;
 	default:
 		panic("arpenter: version %d", version);
@@ -326,9 +350,9 @@ arpenter(Fs *fs, int version, uchar *ip, uchar *mac, int n, uchar *ia, Ipifc *if
 		}
 		a = newarpent(arp, ip, ifc);
 	}
+	bp = arpresolve(arp, a, mac, &rh);	/* unlocks arp */
 	if(version == V4)
 		ip += IPv4off;
-	bp = arpresolve(arp, a, ifc->m, mac);	/* unlocks arp */
 	for(; bp != nil; bp = next){
 		next = bp->list;
 		bp->list = nil;
@@ -336,7 +360,7 @@ arpenter(Fs *fs, int version, uchar *ip, uchar *mac, int n, uchar *ia, Ipifc *if
 			freeblistchain(next);
 			break;
 		}
-		ipifcoput(ifc, bp, version, ip);
+		ipifcoput(ifc, bp, version, ip, &rh);
 		poperror();
 	}
 	return 1;
@@ -496,8 +520,9 @@ arpread(Arp *arp, char *s, ulong offset, int len)
 }
 
 void
-ndpsendsol(Fs *f, Ipifc *ifc, Arpent *a)
+ndpsendsol(Fs *f, Arpent *a)
 {
+	Ipifc *ifc = a->ifc;
 	uchar targ[IPaddrlen], src[IPaddrlen];
 
 	if(a->rxtsrem == 0)
@@ -538,7 +563,7 @@ rxmt(Arp *arp)
 	while((a = arp->rxmt[0]) != nil && NOW - a->ctime > 3*RETRANS_TIMER/4){
 		if(a->rxtsrem > 0 && (ifc = a->ifc) != nil && canrlock(ifc)){
 			if(a->ifcid == ifc->ifcid){
-				ndpsendsol(arp->f, ifc, a);	/* unlocks arp */
+				ndpsendsol(arp->f, a);	/* unlocks arp */
 				runlock(ifc);
 
 				wlock(arp);
diff --git a/sys/src/9/ip/devip.c b/sys/src/9/ip/devip.c
index c7fc0c72f..403faa083 100644
--- a/sys/src/9/ip/devip.c
+++ b/sys/src/9/ip/devip.c
@@ -1332,6 +1332,7 @@ retry:
 	c->state = Idle;
 	ipmove(c->laddr, IPnoaddr);
 	ipmove(c->raddr, IPnoaddr);
+	c->a = nil;
 	c->r = nil;
 	c->rgen = 0;
 	c->lport = 0;
diff --git a/sys/src/9/ip/ethermedium.c b/sys/src/9/ip/ethermedium.c
index 964b71a72..fb6d05279 100644
--- a/sys/src/9/ip/ethermedium.c
+++ b/sys/src/9/ip/ethermedium.c
@@ -21,12 +21,12 @@ static void	etherread4(void *a);
 static void	etherread6(void *a);
 static void	etherbind(Ipifc *ifc, int argc, char **argv);
 static void	etherunbind(Ipifc *ifc);
-static void	etherbwrite(Ipifc *ifc, Block *bp, int version, uchar *ip);
+static void	etherbwrite(Ipifc *ifc, Block *bp, int version, uchar *ip, Routehint *rh);
 static void	etheraddmulti(Ipifc *ifc, uchar *a, uchar *ia);
 static void	etherremmulti(Ipifc *ifc, uchar *a, uchar *ia);
 static void	etherareg(Fs *f, Ipifc *ifc, Iplifc *lifc, uchar *ip);
-static Block*	multicastarp(Fs *f, Arpent *a, Medium*, uchar *mac);
-static void	sendarpreq(Ipifc *ifc, Arpent *a);
+static Block*	multicastarp(Fs *f, Arpent *a, uchar *mac, Routehint *rh);
+static void	sendarpreq(Fs *f, Arpent *a);
 static int	multicastea(uchar *ea, uchar *ip);
 static void	recvarpproc(void*);
 static void	etherpref2addr(uchar *pref, uchar *ea);
@@ -253,7 +253,7 @@ etherunbind(Ipifc *ifc)
  *  called by ipoput with a single block to write with ifc rlock'd
  */
 static void
-etherbwrite(Ipifc *ifc, Block *bp, int version, uchar *ip)
+etherbwrite(Ipifc *ifc, Block *bp, int version, uchar *ip, Routehint *rh)
 {
 	Etherhdr *eh;
 	Arpent *a;
@@ -261,22 +261,22 @@ etherbwrite(Ipifc *ifc, Block *bp, int version, uchar *ip)
 	Etherrock *er = ifc->arg;
 
 	/* get mac address of destination */
-	a = arpget(er->f->arp, bp, version, ifc, ip, mac);
+	a = arpget(er->f->arp, bp, version, ifc, ip, mac, rh);
 	if(a != nil){
 		/* check for broadcast or multicast */
-		bp = multicastarp(er->f, a, ifc->m, mac);
+		bp = multicastarp(er->f, a, mac, rh);
 		if(bp == nil){
 			/* don't do anything if it's been less than a second since the last */
 			if(a->utime - a->ctime < RETRANS_TIMER){
-				arprelease(er->f->arp, a);
+				arprelease(er->f->arp, a);	/* unlocks arp */
 				return;
 			}
 			switch(version){
 			case V4:
-				sendarpreq(ifc, a);		/* unlocks arp */
+				sendarpreq(er->f, a);	/* unlocks arp */
 				break;
 			case V6:
-				ndpsendsol(er->f, ifc, a);	/* unlocks arp */
+				ndpsendsol(er->f, a);	/* unlocks arp */
 				break;
 			default:
 				panic("etherbwrite: version %d", version);
@@ -440,16 +440,17 @@ etherremmulti(Ipifc *ifc, uchar *a, uchar *)
  *  (only v4, v6 uses the neighbor discovery, rfc1970)
  */
 static void
-sendarpreq(Ipifc *ifc, Arpent *a)
+sendarpreq(Fs *f, Arpent *a)
 {
 	int n;
 	Block *bp;
 	Etherarp *e;
+	Ipifc *ifc = a->ifc;
 	Etherrock *er = ifc->arg;
 	uchar targ[IPv4addrlen], src[IPv4addrlen];
 
 	memmove(targ, a->ip+IPv4off, IPv4addrlen);
-	arpcontinue(er->f->arp, a);
+	arpcontinue(f->arp, a);
 
 	if(!ipv4local(ifc, src, 0, targ))
 		return;
@@ -668,19 +669,19 @@ multicastea(uchar *ea, uchar *ip)
  *  IP address.
  */
 static Block*
-multicastarp(Fs *f, Arpent *a, Medium *medium, uchar *mac)
+multicastarp(Fs *f, Arpent *a, uchar *mac, Routehint *rh)
 {
 	/* is it broadcast? */
 	if(ipforme(f, a->ip) == Rbcast){
-		memset(mac, 0xff, medium->maclen);
-		return arpresolve(f->arp, a, medium, mac);
+		memset(mac, 0xff, a->ifc->m->maclen);
+		return arpresolve(f->arp, a, mac, rh);
 	}
 
 	/* if multicast, fill in mac */
 	switch(multicastea(mac, a->ip)){
 	case V4:
 	case V6:
-		return arpresolve(f->arp, a, medium, mac);
+		return arpresolve(f->arp, a, mac, rh);
 	}
 
 	/* let arp take care of it */
diff --git a/sys/src/9/ip/ip.c b/sys/src/9/ip/ip.c
index 7f197b22b..284ef0b77 100644
--- a/sys/src/9/ip/ip.c
+++ b/sys/src/9/ip/ip.c
@@ -153,7 +153,7 @@ ipoput4(Fs *f, Block *bp, int gating, int ttl, int tos, Routehint *rh)
 		eh->cksum[1] = 0;
 		hnputs(eh->cksum, ipcsum(&eh->vihl));
 
-		ipifcoput(ifc, bp, V4, gate);
+		ipifcoput(ifc, bp, V4, gate, rh);
 		runlock(ifc);
 		poperror();
 		return 0;
@@ -237,7 +237,7 @@ ipoput4(Fs *f, Block *bp, int gating, int ttl, int tos, Routehint *rh)
 		feh->cksum[1] = 0;
 		hnputs(feh->cksum, ipcsum(&feh->vihl));
 
-		ipifcoput(ifc, nb, V4, gate);
+		ipifcoput(ifc, nb, V4, gate, rh);
 		ip->stats[FragCreates]++;
 	}
 	ip->stats[FragOKs]++;
@@ -308,8 +308,8 @@ ipiput4(Fs *f, Ipifc *ifc, Block *bp)
 	/* route */
 	v4tov6(v6dst, h->dst);
 	if(!ipforme(f, v6dst)) {
-		Route *r;
 		Routehint rh;
+		Route *r;
 		Ipifc *nifc;
 
 		if(!ip->iprouting)
@@ -317,6 +317,7 @@ ipiput4(Fs *f, Ipifc *ifc, Block *bp)
 
 		/* don't forward to source's network */
 		rh.r = nil;
+		rh.a = nil;
 		r = v4lookup(f, h->dst, h->src, &rh);
 		if(r == nil || (nifc = r->ifc) == nil
 		|| (nifc == ifc && !ifc->reflect)){
diff --git a/sys/src/9/ip/ip.h b/sys/src/9/ip/ip.h
index c3be02f6b..e9f6fcf18 100644
--- a/sys/src/9/ip/ip.h
+++ b/sys/src/9/ip/ip.h
@@ -168,6 +168,7 @@ struct Routehint
 {
 	Route	*r;			/* last route used */
 	ulong	rgen;			/* routetable generation for *r */
+	Arpent	*a;			/* last arp entry used */
 };
 
 /*
@@ -232,7 +233,7 @@ struct Medium
 	int	maclen;		/* mac address length  */
 	void	(*bind)(Ipifc*, int, char**);
 	void	(*unbind)(Ipifc*);
-	void	(*bwrite)(Ipifc *ifc, Block *b, int version, uchar *ip);
+	void	(*bwrite)(Ipifc *ifc, Block *b, int version, uchar *ip, Routehint *rh);
 
 	/* for arming interfaces to receive multicast */
 	void	(*addmulti)(Ipifc *ifc, uchar *a, uchar *ia);
@@ -565,8 +566,8 @@ struct Route
 
 extern void	addroute(Fs *f, uchar *a, uchar *mask, uchar *s, uchar *smask, uchar *gate, int type, Ipifc *ifc, char *tag);
 extern void	remroute(Fs *f, uchar *a, uchar *mask, uchar *s, uchar *smask, uchar *gate, int type, Ipifc *ifc, char *tag);
-extern Route*	v4lookup(Fs *f, uchar *a, uchar *s, Routehint *h);
-extern Route*	v6lookup(Fs *f, uchar *a, uchar *s, Routehint *h);
+extern Route*	v4lookup(Fs *f, uchar *a, uchar *s, Routehint *rh);
+extern Route*	v6lookup(Fs *f, uchar *a, uchar *s, Routehint *rh);
 extern Route*	v4source(Fs *f, uchar *a, uchar *s);
 extern Route*	v6source(Fs *f, uchar *a, uchar *s);
 extern long	routeread(Fs *f, char*, ulong, int);
@@ -611,12 +612,12 @@ struct Arpent
 extern void	arpinit(Fs*);
 extern int	arpread(Arp*, char*, ulong, int);
 extern int	arpwrite(Fs*, char*, int);
-extern Arpent*	arpget(Arp*, Block *bp, int version, Ipifc *ifc, uchar *ip, uchar *h);
+extern Arpent*	arpget(Arp*, Block *bp, int version, Ipifc *ifc, uchar *ip, uchar *mac, Routehint *rh);
 extern void	arprelease(Arp*, Arpent *a);
 extern void	arpcontinue(Arp*, Arpent *a);
-extern Block*	arpresolve(Arp*, Arpent *a, Medium *type, uchar *mac);
+extern Block*	arpresolve(Arp*, Arpent *a, uchar *mac, Routehint *rh);
 extern int	arpenter(Fs*, int version, uchar *ip, uchar *mac, int n, uchar *ia, Ipifc *ifc, int refresh);
-extern void	ndpsendsol(Fs*, Ipifc*, Arpent*);
+extern void	ndpsendsol(Fs*, Arpent*);
 
 /*
  * ipaux.c
@@ -661,7 +662,7 @@ extern Medium	pktmedium;
  */
 extern Medium*	ipfindmedium(char *name);
 extern void	addipmedium(Medium *med);
-extern void	ipifcoput(Ipifc *ifc, Block *bp, int version, uchar *ip);
+extern void	ipifcoput(Ipifc *ifc, Block *bp, int version, uchar *ip, Routehint *rh);
 extern int	ipforme(Fs*, uchar *addr);
 extern int	ipismulticast(uchar *ip);
 extern Ipifc*	findipifc(Fs*, uchar *local, uchar *remote, int type);
diff --git a/sys/src/9/ip/ipifc.c b/sys/src/9/ip/ipifc.c
index 458ae6763..b125ddf0b 100644
--- a/sys/src/9/ip/ipifc.c
+++ b/sys/src/9/ip/ipifc.c
@@ -347,7 +347,7 @@ ipifcsetspeed(Ipifc *ifc, int speed)
 }
 
 void
-ipifcoput(Ipifc *ifc, Block *bp, int version, uchar *ip)
+ipifcoput(Ipifc *ifc, Block *bp, int version, uchar *ip, Routehint *rh)
 {
 	if(ifc->speed){
 		ulong now = MACHP(0)->ticks;
@@ -363,7 +363,7 @@ ipifcoput(Ipifc *ifc, Block *bp, int version, uchar *ip)
 	}
 	bp = concatblock(bp);
 	ifc->load += BLEN(bp);
-	ifc->m->bwrite(ifc, bp, version, ip);
+	ifc->m->bwrite(ifc, bp, version, ip, rh);
 }
 
 
diff --git a/sys/src/9/ip/ipv6.c b/sys/src/9/ip/ipv6.c
index 779499a47..0ad8d8e86 100644
--- a/sys/src/9/ip/ipv6.c
+++ b/sys/src/9/ip/ipv6.c
@@ -99,7 +99,7 @@ ipoput6(Fs *f, Block *bp, int gating, int ttl, int tos, Routehint *rh)
 	medialen = ifc->maxtu - ifc->m->hsize;
 	if(len <= medialen) {
 		hnputs(eh->ploadlen, len - IP6HDR);
-		ipifcoput(ifc, bp, V6, gate);
+		ipifcoput(ifc, bp, V6, gate, rh);
 		runlock(ifc);
 		poperror();
 		return 0;
@@ -195,7 +195,7 @@ ipoput6(Fs *f, Block *bp, int gating, int ttl, int tos, Routehint *rh)
 			if(xp->rp == xp->wp)
 				xp = xp->next;
 		}
-		ipifcoput(ifc, nb, V6, gate);
+		ipifcoput(ifc, nb, V6, gate, rh);
 		ip->stats[FragCreates]++;
 	}
 	ip->stats[FragOKs]++;
@@ -252,8 +252,8 @@ ipiput6(Fs *f, Ipifc *ifc, Block *bp)
 
 	/* route */
 	if(!ipforme(f, h->dst)) {
-		Route *r;
 		Routehint rh;
+		Route *r;
 		Ipifc *nifc;
 
 		if(!ip->iprouting)
@@ -268,6 +268,7 @@ ipiput6(Fs *f, Ipifc *ifc, Block *bp)
 			
 		/* don't forward to source's network */
 		rh.r = nil;
+		rh.a = nil;
 		r  = v6lookup(f, h->dst, h->src, &rh);
 		if(r == nil || (nifc = r->ifc) == nil || (r->type & Rv4) != 0
 		|| (nifc == ifc && !ifc->reflect)){
diff --git a/sys/src/9/ip/loopbackmedium.c b/sys/src/9/ip/loopbackmedium.c
index 2bb4271be..036de1137 100644
--- a/sys/src/9/ip/loopbackmedium.c
+++ b/sys/src/9/ip/loopbackmedium.c
@@ -71,7 +71,7 @@ loopbackunbind(Ipifc *ifc)
 }
 
 static void
-loopbackbwrite(Ipifc *ifc, Block *bp, int, uchar*)
+loopbackbwrite(Ipifc *ifc, Block *bp, int, uchar*, Routehint*)
 {
 	LB *lb;
 
diff --git a/sys/src/9/ip/netdevmedium.c b/sys/src/9/ip/netdevmedium.c
index 6d56420fd..9e265d19b 100644
--- a/sys/src/9/ip/netdevmedium.c
+++ b/sys/src/9/ip/netdevmedium.c
@@ -9,7 +9,7 @@
 
 static void	netdevbind(Ipifc *ifc, int argc, char **argv);
 static void	netdevunbind(Ipifc *ifc);
-static void	netdevbwrite(Ipifc *ifc, Block *bp, int version, uchar *ip);
+static void	netdevbwrite(Ipifc *ifc, Block *bp, int version, uchar *ip, Routehint*);
 static void	netdevread(void *a);
 
 typedef struct	Netdevrock Netdevrock;
@@ -99,7 +99,7 @@ netdevunbind(Ipifc *ifc)
  *  called by ipoput with a single block to write
  */
 static void
-netdevbwrite(Ipifc *ifc, Block *bp, int, uchar*)
+netdevbwrite(Ipifc *ifc, Block *bp, int, uchar*, Routehint*)
 {
 	Netdevrock *er = ifc->arg;
 
diff --git a/sys/src/9/ip/nullmedium.c b/sys/src/9/ip/nullmedium.c
index 21cb44354..094e103f6 100644
--- a/sys/src/9/ip/nullmedium.c
+++ b/sys/src/9/ip/nullmedium.c
@@ -19,7 +19,7 @@ nullunbind(Ipifc*)
 }
 
 static void
-nullbwrite(Ipifc*, Block *bp, int, uchar*)
+nullbwrite(Ipifc*, Block *bp, int, uchar*, Routehint*)
 {
 	freeb(bp);
 	error("nullbwrite");
diff --git a/sys/src/9/ip/pktmedium.c b/sys/src/9/ip/pktmedium.c
index b26868af0..15ac6405a 100644
--- a/sys/src/9/ip/pktmedium.c
+++ b/sys/src/9/ip/pktmedium.c
@@ -10,7 +10,7 @@
 
 static void	pktbind(Ipifc*, int, char**);
 static void	pktunbind(Ipifc*);
-static void	pktbwrite(Ipifc*, Block*, int, uchar*);
+static void	pktbwrite(Ipifc*, Block*, int, uchar*, Routehint*);
 static void	pktin(Fs*, Ipifc*, Block*);
 
 Medium pktmedium =
@@ -49,7 +49,7 @@ pktunbind(Ipifc*)
  *  called by ipoput with a single packet to write
  */
 static void
-pktbwrite(Ipifc *ifc, Block *bp, int, uchar*)
+pktbwrite(Ipifc *ifc, Block *bp, int, uchar*, Routehint*)
 {
 	/* enqueue onto the conversation's rq */
 	if(ifc->conv->snoopers.ref > 0)