bcm64: switch to 64k page size

1 files changed, 61 insertions, 54 deletions

diff --git a/sys/src/9/bcm64/mmu.c b/sys/src/9/bcm64/mmu.c
index 29313d3ac..b71d6097b 100644
--- a/sys/src/9/bcm64/mmu.c
+++ b/sys/src/9/bcm64/mmu.c
@@ -8,44 +8,64 @@
 void
 mmu0init(uintptr *l1)
 {
-	uintptr va, pa, pe;
+	uintptr va, pa, pe, attr;
 
-	/* 0 identity map */
+	/* KZERO */
+	attr = PTEWRITE | PTEAF | PTEKERNEL | PTESH(SHARE_INNER);
 	pe = PHYSDRAM + soc.dramsize;
 	if(pe > (uintptr)-KZERO)
 		pe = (uintptr)-KZERO;
-
-	for(pa = PHYSDRAM; pa < pe; pa += PGLSZ(1))
-		l1[PTL1X(pa, 1)] = pa | PTEVALID | PTEBLOCK | PTEWRITE | PTEAF
-			 | PTEKERNEL | PTESH(SHARE_INNER);
+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
+		if(pe - pa < PGLSZ(1)){
+			l1[PTL1X(va, 1)] = (uintptr)l1 | PTEVALID | PTETABLE;
+			l1[PTL1X(pa, 1)] = (uintptr)l1 | PTEVALID | PTETABLE;
+			for(; pa < pe; pa += PGLSZ(0), va += PGLSZ(0))
+				l1[PTLX(va, 0)] = pa | PTEVALID | PTEPAGE | attr;
+			break;
+		}
+		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
+		l1[PTL1X(pa, 1)] = pa | PTEVALID | PTEBLOCK | attr;
+	}
+	pe = (uintptr)-KZERO;	/* populate top levels for mmukmap() */
 	if(PTLEVELS > 2)
-	for(pa = PHYSDRAM; pa < pe; pa += PGLSZ(2))
+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2)){
+		l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
 		l1[PTL1X(pa, 2)] = (uintptr)&l1[L1TABLEX(pa, 1)] | PTEVALID | PTETABLE;
+	}
 	if(PTLEVELS > 3)
-	for(pa = PHYSDRAM; pa < pe; pa += PGLSZ(3))
+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3)){
+		l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
 		l1[PTL1X(pa, 3)] = (uintptr)&l1[L1TABLEX(pa, 2)] | PTEVALID | PTETABLE;
+	}
 
-	/* KZERO */
-	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
-		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | PTEWRITE | PTEAF
-			| PTEKERNEL | PTESH(SHARE_INNER);
+	/* VIRTIO */
+	attr = PTEWRITE | PTEAF | PTEKERNEL | PTESH(SHARE_OUTER) | PTEDEVICE;
+	pe = soc.physio + IOSIZE;
+	for(pa = soc.physio, va = VIRTIO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
+		if(pe - pa < PGLSZ(1)){
+			l1[PTL1X(va, 1)] = (uintptr)l1 | PTEVALID | PTETABLE;
+			for(; pa < pe; pa += PGLSZ(0), va += PGLSZ(0))
+				l1[PTLX(va, 0)] = pa | PTEVALID | PTEPAGE | attr;
+			break;
+		}
+		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
+	}
 	if(PTLEVELS > 2)
-	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2))
+	for(pa = soc.physio, va = VIRTIO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2))
 		l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
 	if(PTLEVELS > 3)
-	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
+	for(pa = soc.physio, va = VIRTIO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
 		l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
 
-	/* VIRTIO */
-	pe = -VIRTIO + soc.physio;
-	for(pa = soc.physio, va = VIRTIO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
-		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | PTEWRITE | PTEAF
-			| PTEKERNEL | PTESH(SHARE_OUTER) | PTEDEVICE;
+	/* ARMLOCAL */
+	pe = soc.armlocal + MB;
+	for(pa = soc.armlocal, va = ARMLOCAL; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
+		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
 	if(PTLEVELS > 2)
-	for(pa = soc.physio, va = VIRTIO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2))
+	for(pa = soc.armlocal, va = ARMLOCAL; pa < pe; pa += PGLSZ(2), va += PGLSZ(2))
 		l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
 	if(PTLEVELS > 3)
-	for(pa = soc.physio, va = VIRTIO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
+	for(pa = soc.armlocal, va = ARMLOCAL; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
 		l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
 }
 
@@ -57,21 +77,17 @@ mmu0clear(uintptr *l1)
 	pe = PHYSDRAM + soc.dramsize;
 	if(pe > (uintptr)-KZERO)
 		pe = (uintptr)-KZERO;
-
-	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
 		if(PTL1X(pa, 1) != PTL1X(va, 1))
 			l1[PTL1X(pa, 1)] = 0;
-	}
 	if(PTLEVELS > 2)
-	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2)){
+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2))
 		if(PTL1X(pa, 2) != PTL1X(va, 2))
 			l1[PTL1X(pa, 2)] = 0;
-	}
 	if(PTLEVELS > 3)
-	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3)){
+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
 		if(PTL1X(pa, 3) != PTL1X(va, 3))
 			l1[PTL1X(pa, 3)] = 0;
-	}
 }
 
 void
@@ -85,32 +101,24 @@ mmuidmap(uintptr *l1)
 	pe = PHYSDRAM + soc.dramsize;
 	if(pe > (uintptr)-KZERO)
 		pe = (uintptr)-KZERO;
-
-	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
-		if(PTL1X(pa, 1) != PTL1X(va, 1))
-			l1[PTL1X(pa, 1)] = pa | PTEVALID | PTEBLOCK | PTEWRITE | PTEAF
-				 | PTEKERNEL | PTESH(SHARE_INNER);
-	}
+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
+		l1[PTL1X(pa, 1)] = l1[PTL1X(va, 1)];
 	if(PTLEVELS > 2)
-	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2)){
-		if(PTL1X(pa, 2) != PTL1X(va, 2))
-			l1[PTL1X(pa, 2)] = PADDR(&l1[L1TABLEX(pa, 1)]) | PTEVALID | PTETABLE;
-	}
+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2))
+		l1[PTL1X(pa, 2)] = l1[PTL1X(va, 2)];
 	if(PTLEVELS > 3)
-	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3)){
-		if(PTL1X(pa, 3) != PTL1X(va, 3))
-			l1[PTL1X(pa, 3)] = PADDR(&l1[L1TABLEX(pa, 2)]) | PTEVALID | PTETABLE;
-	}
+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
+		l1[PTL1X(pa, 3)] = l1[PTL1X(va, 3)];
 	setttbr(PADDR(&l1[L1TABLEX(0, PTLEVELS-1)]));
 }
 
 void
 mmu1init(void)
 {
-	m->mmul1 = mallocalign(L1SIZE+L1TOPSIZE, BY2PG, L1SIZE, 0);
-	if(m->mmul1 == nil)
-		panic("mmu1init: no memory for mmul1");
-	memset(m->mmul1, 0, L1SIZE+L1TOPSIZE);
+	m->mmutop = mallocalign(L1TOPSIZE, BY2PG, 0, 0);
+	if(m->mmutop == nil)
+		panic("mmu1init: no memory for mmutop");
+	memset(m->mmutop, 0, L1TOPSIZE);
 	mmuswitch(nil);
 }
 
@@ -187,7 +195,7 @@ mmuwalk(uintptr va, int level)
 	int i, x;
 
 	x = PTLX(va, PTLEVELS-1);
-	table = &m->mmul1[L1TABLEX(va, PTLEVELS-1)];
+	table = m->mmutop;
 	for(i = PTLEVELS-2; i >= level; i--){
 		pte = table[x];
 		if(pte & PTEVALID) {
@@ -262,12 +270,12 @@ putasid(Proc *p)
 {
 	/*
 	 * Prevent the following scenario:
-	 *	pX sleeps on cpuA, leaving its page tables in mmul1
+	 *	pX sleeps on cpuA, leaving its page tables in mmutop
 	 *	pX wakes up on cpuB, and exits, freeing its page tables
 	 *  pY on cpuB allocates a freed page table page and overwrites with data
 	 *  cpuA takes an interrupt, and is now running with bad page tables
 	 * In theory this shouldn't hurt because only user address space tables
-	 * are affected, and mmuswitch will clear mmul1 before a user process is
+	 * are affected, and mmuswitch will clear mmutop before a user process is
 	 * dispatched.  But empirically it correlates with weird problems, eg
 	 * resetting of the core clock at 0x4000001C which confuses local timers.
 	 */
@@ -287,7 +295,6 @@ putmmu(uintptr va, uintptr pa, Page *pg)
 	s = splhi();
 	while((pte = mmuwalk(va, 0)) == nil){
 		spllo();
-		assert(up->mmufree == nil);
 		up->mmufree = newpage(0, nil, 0);
 		splhi();
 	}
@@ -330,10 +337,10 @@ mmuswitch(Proc *p)
 	Page *t;
 
 	for(va = UZERO; va < USTKTOP; va += PGLSZ(PTLEVELS-1))
-		m->mmul1[PTL1X(va, PTLEVELS-1)] = 0;
+		m->mmutop[PTLX(va, PTLEVELS-1)] = 0;
 
 	if(p == nil){
-		setttbr(PADDR(&m->mmul1[L1TABLEX(0, PTLEVELS-1)]));
+		setttbr(PADDR(m->mmutop));
 		return;
 	}
 
@@ -344,13 +351,13 @@ mmuswitch(Proc *p)
 
 	for(t = p->mmuhead[PTLEVELS-1]; t != nil; t = t->next){
 		va = t->va;
-		m->mmul1[PTL1X(va, PTLEVELS-1)] = t->pa | PTEVALID | PTETABLE;
+		m->mmutop[PTLX(va, PTLEVELS-1)] = t->pa | PTEVALID | PTETABLE;
 	}
 
 	if(allocasid(p))
 		flushasid((uvlong)p->asid<<48);
 
-	setttbr((uvlong)p->asid<<48 | PADDR(&m->mmul1[L1TABLEX(0, PTLEVELS-1)]));
+	setttbr((uvlong)p->asid<<48 | PADDR(m->mmutop));
 }
 
 void