imx8: share generic arm64 mmu.c

Split layout specific code into mem.c from mmu.c,
so generic mmu code can be shared.

8 files changed, 211 insertions, 574 deletions

diff --git a/sys/src/9/arm64/fns.h b/sys/src/9/arm64/fns.h
index c1ccdb310..cc7a2bebf 100644
--- a/sys/src/9/arm64/fns.h
+++ b/sys/src/9/arm64/fns.h
@@ -69,18 +69,17 @@ extern void kmapinval(void);
 #define	VA(k)	((uintptr)(k))
 extern KMap *kmap(Page*);
 extern void kunmap(KMap*);
+extern void kmapram(uintptr, uintptr);
 extern uintptr mmukmap(uintptr, uintptr, usize);
 extern void* vmap(uvlong, vlong);
 extern void vunmap(void*, vlong);
-
-extern void mmu0init(uintptr*);
-extern void mmuidmap(uintptr*);
 extern void mmu1init(void);
-extern void meminit(void);
-
 extern void putasid(Proc*);
 
-extern void* ucalloc(usize);
+/* mem */
+extern void mmuidmap(uintptr*);
+extern void mmu0init(uintptr*);
+extern void meminit(void);
 
 /* clock */
 extern void clockinit(void);
diff --git a/sys/src/9/arm64/mem.c b/sys/src/9/arm64/mem.c
new file mode 100644
index 000000000..df2a0ca0f
--- /dev/null
+++ b/sys/src/9/arm64/mem.c
@@ -0,0 +1,80 @@
+#include "u.h"
+#include "../port/lib.h"
+#include "mem.h"
+#include "dat.h"
+#include "fns.h"
+#include "../arm64/sysreg.h"
+
+#define INITMAP	(ROUND((uintptr)end + BY2PG, PGLSZ(1))-KZERO)
+
+/*
+ * Create initial identity map in top-level page table
+ * (L1BOT) for TTBR0. This page table is only used until
+ * mmu1init() loads m->mmutop.
+ */
+void
+mmuidmap(uintptr *l1bot)
+{
+	uintptr pa, pe, attr;
+
+	/* VDRAM */
+	attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTESH(SHARE_INNER);
+	pe = -KZERO;
+	for(pa = VDRAM - KZERO; pa < pe; pa += PGLSZ(PTLEVELS-1))
+		l1bot[PTLX(pa, PTLEVELS-1)] = pa | PTEVALID | PTEBLOCK | attr;
+}
+
+/*
+ * Create initial shared kernel page table (L1) for TTBR1.
+ * This page table coveres the INITMAP and VIRTIO,
+ * and later we fill the ram mappings in meminit().
+ */
+void
+mmu0init(uintptr *l1)
+{
+	uintptr va, pa, pe, attr;
+
+	/* DRAM - INITMAP */
+	attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTESH(SHARE_INNER);
+	pe = INITMAP;
+	for(pa = VDRAM - KZERO, va = VDRAM; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
+		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
+
+	/* VIRTIO */
+	attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTEPXN | PTESH(SHARE_OUTER) | PTEDEVICE;
+	pe = PHYSIOEND;
+	for(pa = PHYSIO, va = VIRTIO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
+		if(((pa|va) & PGLSZ(1)-1) != 0){
+			l1[PTL1X(va, 1)] = (uintptr)l1 | PTEVALID | PTETABLE;
+			for(; pa < pe && ((va|pa) & PGLSZ(1)-1) != 0; pa += PGLSZ(0), va += PGLSZ(0)){
+				assert(l1[PTLX(va, 0)] == 0);
+				l1[PTLX(va, 0)] = pa | PTEVALID | PTEPAGE | attr;
+			}
+			break;
+		}
+		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
+	}
+
+	if(PTLEVELS > 2)
+	for(va = KSEG0; va != 0; va += PGLSZ(2))
+		l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
+
+	if(PTLEVELS > 3)
+	for(va = KSEG0; va != 0; va += PGLSZ(3))
+		l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
+}
+
+void
+meminit(void)
+{
+	char *p;
+
+	conf.mem[0].base = PGROUND((uintptr)end - KZERO);
+	conf.mem[0].limit = GiB + 128 * MiB;
+	if(p = getconf("*maxmem"))
+		conf.mem[0].limit = strtoull(p, 0, 0);
+
+	kmapram(conf.mem[0].base, conf.mem[0].limit);
+
+	conf.mem[0].npage = (conf.mem[0].limit - conf.mem[0].base)/BY2PG;
+}
diff --git a/sys/src/9/arm64/mkfile b/sys/src/9/arm64/mkfile
index 269252c1a..cbe7a1473 100644
--- a/sys/src/9/arm64/mkfile
+++ b/sys/src/9/arm64/mkfile
@@ -47,6 +47,7 @@ OBJ=\
 	fpu.$O\
 	main.$O\
 	mmu.$O\
+	mem.$O\
 	sysreg.$O\
 	random.$O\
 	trap.$O\
diff --git a/sys/src/9/arm64/mmu.c b/sys/src/9/arm64/mmu.c
index 2badd65bd..6e9192aa8 100644
--- a/sys/src/9/arm64/mmu.c
+++ b/sys/src/9/arm64/mmu.c
@@ -3,67 +3,9 @@
 #include "mem.h"
 #include "dat.h"
 #include "fns.h"
-#include "sysreg.h"
 
 #define INITMAP	(ROUND((uintptr)end + BY2PG, PGLSZ(1))-KZERO)
 
-/*
- * Create initial identity map in top-level page table
- * (L1BOT) for TTBR0. This page table is only used until
- * mmu1init() loads m->mmutop.
- */
-void
-mmuidmap(uintptr *l1bot)
-{
-	uintptr pa, pe, attr;
-
-	/* VDRAM */
-	attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTESH(SHARE_INNER);
-	pe = -KZERO;
-	for(pa = VDRAM - KZERO; pa < pe; pa += PGLSZ(PTLEVELS-1))
-		l1bot[PTLX(pa, PTLEVELS-1)] = pa | PTEVALID | PTEBLOCK | attr;
-}
-
-/*
- * Create initial shared kernel page table (L1) for TTBR1.
- * This page table coveres the INITMAP and VIRTIO,
- * and later we fill the ram mappings in meminit().
- */
-void
-mmu0init(uintptr *l1)
-{
-	uintptr va, pa, pe, attr;
-
-	/* DRAM - INITMAP */
-	attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTESH(SHARE_INNER);
-	pe = INITMAP;
-	for(pa = VDRAM - KZERO, va = VDRAM; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
-		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
-
-	/* VIRTIO */
-	attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTEPXN | PTESH(SHARE_OUTER) | PTEDEVICE;
-	pe = PHYSIOEND;
-	for(pa = PHYSIO, va = VIRTIO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
-		if(((pa|va) & PGLSZ(1)-1) != 0){
-			l1[PTL1X(va, 1)] = (uintptr)l1 | PTEVALID | PTETABLE;
-			for(; pa < pe && ((va|pa) & PGLSZ(1)-1) != 0; pa += PGLSZ(0), va += PGLSZ(0)){
-				assert(l1[PTLX(va, 0)] == 0);
-				l1[PTLX(va, 0)] = pa | PTEVALID | PTEPAGE | attr;
-			}
-			break;
-		}
-		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
-	}
-
-	if(PTLEVELS > 2)
-	for(va = KSEG0; va != 0; va += PGLSZ(2))
-		l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
-
-	if(PTLEVELS > 3)
-	for(va = KSEG0; va != 0; va += PGLSZ(3))
-		l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
-}
-
 void
 mmu1init(void)
 {
@@ -137,7 +79,6 @@ rampage(void)
 
 	pa = conf.mem[0].base;
 	assert((pa % BY2PG) == 0);
-	assert(pa < INITMAP);
 	conf.mem[0].base += BY2PG;
 	return KADDR(pa);
 }
@@ -179,7 +120,7 @@ l1map(uintptr va, uintptr pa, uintptr pe, uintptr attr)
 	}
 }
 
-static void
+void
 kmapram(uintptr base, uintptr limit)
 {
 	if(base < (uintptr)-KZERO && limit > (uintptr)-KZERO){
@@ -196,21 +137,6 @@ kmapram(uintptr base, uintptr limit)
 		PTEWRITE | PTEPXN | PTEUXN | PTESH(SHARE_INNER));
 }
 
-void
-meminit(void)
-{
-	char *p;
-
-	conf.mem[0].base = PGROUND((uintptr)end - KZERO);
-	conf.mem[0].limit = GiB + 128 * MiB;
-	if(p = getconf("*maxmem"))
-		conf.mem[0].limit = strtoull(p, 0, 0);
-
-	kmapram(conf.mem[0].base, conf.mem[0].limit);
-
-	conf.mem[0].npage = (conf.mem[0].limit - conf.mem[0].base)/BY2PG;
-}
-
 uintptr
 mmukmap(uintptr va, uintptr pa, usize size)
 {
diff --git a/sys/src/9/imx8/fns.h b/sys/src/9/imx8/fns.h
index 480edbfc3..81aaded6d 100644
--- a/sys/src/9/imx8/fns.h
+++ b/sys/src/9/imx8/fns.h
@@ -69,17 +69,18 @@ extern void kmapinval(void);
 #define	VA(k)	((uintptr)(k))
 extern KMap *kmap(Page*);
 extern void kunmap(KMap*);
+extern void kmapram(uintptr, uintptr);
 extern uintptr mmukmap(uintptr, uintptr, usize);
 extern void* vmap(uvlong, vlong);
 extern void vunmap(void*, vlong);
+extern void mmu1init(void);
+extern void putasid(Proc*);
 
-extern void mmu0init(uintptr*);
+/* mem */
 extern void mmuidmap(uintptr*);
-extern void mmu1init(void);
+extern void mmu0init(uintptr*);
 extern void meminit(void);
 
-extern void putasid(Proc*);
-
 extern void* ucalloc(usize);
 
 /* clock */
diff --git a/sys/src/9/imx8/mem.c b/sys/src/9/imx8/mem.c
new file mode 100644
index 000000000..7efcdd679
--- /dev/null
+++ b/sys/src/9/imx8/mem.c
@@ -0,0 +1,117 @@
+#include "u.h"
+#include "../port/lib.h"
+#include "mem.h"
+#include "dat.h"
+#include "fns.h"
+
+#define INITMAP	(ROUND((uintptr)end + BY2PG, PGLSZ(1))-KZERO)
+
+/*
+ * Create initial identity map in top-level page table
+ * (L1BOT) for TTBR0. This page table is only used until
+ * mmu1init() loads m->mmutop.
+ */
+void
+mmuidmap(uintptr *l1bot)
+{
+	uintptr pa, pe, attr;
+
+	/* VDRAM */
+	attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTESH(SHARE_INNER);
+	pe = -KZERO;
+	for(pa = VDRAM - KZERO; pa < pe; pa += PGLSZ(PTLEVELS-1))
+		l1bot[PTLX(pa, PTLEVELS-1)] = pa | PTEVALID | PTEBLOCK | attr;
+}
+
+/*
+ * Create initial shared kernel page table (L1) for TTBR1.
+ * This page table coveres the INITMAP and VIRTIO,
+ * and later we fill the ram mappings in meminit().
+ */
+void
+mmu0init(uintptr *l1)
+{
+	uintptr va, pa, pe, attr;
+
+	/* DRAM - INITMAP */
+	attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTESH(SHARE_INNER);
+	pe = INITMAP;
+	for(pa = VDRAM - KZERO, va = VDRAM; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
+		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
+
+	/* VIRTIO */
+	attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTEPXN | PTESH(SHARE_OUTER) | PTEDEVICE;
+	pe = VDRAM - KZERO;
+	for(pa = VIRTIO - KZERO, va = VIRTIO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
+		if(((pa|va) & PGLSZ(1)-1) != 0){
+			l1[PTL1X(va, 1)] = (uintptr)l1 | PTEVALID | PTETABLE;
+			for(; pa < pe && ((va|pa) & PGLSZ(1)-1) != 0; pa += PGLSZ(0), va += PGLSZ(0)){
+				assert(l1[PTLX(va, 0)] == 0);
+				l1[PTLX(va, 0)] = pa | PTEVALID | PTEPAGE | attr;
+			}
+			break;
+		}
+		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
+	}
+
+	if(PTLEVELS > 2)
+	for(va = KSEG0; va != 0; va += PGLSZ(2))
+		l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
+
+	if(PTLEVELS > 3)
+	for(va = KSEG0; va != 0; va += PGLSZ(3))
+		l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
+}
+
+void
+meminit(void)
+{
+	/* DDR Memory (All modules) */
+	conf.mem[0].base = PGROUND((uintptr)end - KZERO);
+
+	/* exclude uncached dram for ucalloc() */
+	conf.mem[0].limit = UCRAMBASE;
+	conf.mem[1].base = UCRAMBASE+UCRAMSIZE;
+
+	conf.mem[1].limit = 0x100000000ULL;
+
+	/* DDR Memory (Quad-A53 only) */
+	conf.mem[2].base =  0x100000000ULL;
+	conf.mem[2].limit = 0x140000000ULL;
+
+	kmapram(conf.mem[0].base, conf.mem[0].limit);
+	kmapram(conf.mem[1].base, conf.mem[1].limit);
+	kmapram(conf.mem[2].base, conf.mem[2].limit);
+
+	conf.mem[0].npage = (conf.mem[0].limit - conf.mem[0].base)/BY2PG;
+	conf.mem[1].npage = (conf.mem[1].limit - conf.mem[1].base)/BY2PG;
+	conf.mem[2].npage = (conf.mem[2].limit - conf.mem[2].base)/BY2PG;
+}
+
+static void*
+ucramalloc(usize size, uintptr align, uint attr)
+{
+	static uintptr top = UCRAMBASE + UCRAMSIZE;
+	static Lock lk;
+	uintptr va, pg;
+
+	lock(&lk);
+	top -= size;
+	size += top & align-1;
+	top &= -align;
+	if(top < UCRAMBASE)
+		panic("ucramalloc: need %zd bytes", size);
+	va = KZERO + top;
+	pg = va & -BY2PG;
+	if(pg != ((va+size) & -BY2PG))
+		mmukmap(pg | attr, pg - KZERO, PGROUND(size));
+	unlock(&lk);
+
+	return (void*)va;
+}
+
+void*
+ucalloc(usize size)
+{
+	return ucramalloc(size, 8, PTEUNCACHED);
+}
diff --git a/sys/src/9/imx8/mkfile b/sys/src/9/imx8/mkfile
index 04bc39d08..6898ed982 100644
--- a/sys/src/9/imx8/mkfile
+++ b/sys/src/9/imx8/mkfile
@@ -47,6 +47,7 @@ OBJ=\
 	fpu.$O\
 	main.$O\
 	mmu.$O\
+	mem.$O\
 	sysreg.$O\
 	random.$O\
 	trap.$O\
@@ -102,7 +103,7 @@ i2cimx.$O: ../port/i2c.h
 pciimx.$O: ../port/pci.h
 usbxhciimx.$O: ../port/usbxhci.h
 
-l.$O main.$O mmu.$O clock.$O gic.$O cache.v8.$O fpu.$O trap.$O rebootcode.$O: ../arm64/sysreg.h
+l.$O main.$O clock.$O gic.$O cache.v8.$O fpu.$O trap.$O rebootcode.$O: ../arm64/sysreg.h
 
 initcode.out:		init9.$O initcode.$O /$objtype/lib/libc.a
 	$LD -l -R1 -s -o $target $prereq
diff --git a/sys/src/9/imx8/mmu.c b/sys/src/9/imx8/mmu.c
deleted file mode 100644
index 130f0ea1a..000000000
--- a/sys/src/9/imx8/mmu.c
+++ /dev/null
@@ -1,488 +0,0 @@
-#include "u.h"
-#include "../port/lib.h"
-#include "mem.h"
-#include "dat.h"
-#include "fns.h"
-#include "../arm64/sysreg.h"
-
-#define INITMAP	(ROUND((uintptr)end + BY2PG, PGLSZ(1))-KZERO)
-
-/*
- * Create initial identity map in top-level page table
- * (L1BOT) for TTBR0. This page table is only used until
- * mmu1init() loads m->mmutop.
- */
-void
-mmuidmap(uintptr *l1bot)
-{
-	uintptr pa, pe, attr;
-
-	/* VDRAM */
-	attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTESH(SHARE_INNER);
-	pe = -KZERO;
-	for(pa = VDRAM - KZERO; pa < pe; pa += PGLSZ(PTLEVELS-1))
-		l1bot[PTLX(pa, PTLEVELS-1)] = pa | PTEVALID | PTEBLOCK | attr;
-}
-
-/*
- * Create initial shared kernel page table (L1) for TTBR1.
- * This page table coveres the INITMAP and VIRTIO,
- * and later we fill the ram mappings in meminit().
- */
-void
-mmu0init(uintptr *l1)
-{
-	uintptr va, pa, pe, attr;
-
-	/* DRAM - INITMAP */
-	attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTESH(SHARE_INNER);
-	pe = INITMAP;
-	for(pa = VDRAM - KZERO, va = VDRAM; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
-		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
-
-	/* VIRTIO */
-	attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTEPXN | PTESH(SHARE_OUTER) | PTEDEVICE;
-	pe = VDRAM - KZERO;
-	for(pa = VIRTIO - KZERO, va = VIRTIO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
-		if(((pa|va) & PGLSZ(1)-1) != 0){
-			l1[PTL1X(va, 1)] = (uintptr)l1 | PTEVALID | PTETABLE;
-			for(; pa < pe && ((va|pa) & PGLSZ(1)-1) != 0; pa += PGLSZ(0), va += PGLSZ(0)){
-				assert(l1[PTLX(va, 0)] == 0);
-				l1[PTLX(va, 0)] = pa | PTEVALID | PTEPAGE | attr;
-			}
-			break;
-		}
-		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
-	}
-
-	if(PTLEVELS > 2)
-	for(va = KSEG0; va != 0; va += PGLSZ(2))
-		l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
-
-	if(PTLEVELS > 3)
-	for(va = KSEG0; va != 0; va += PGLSZ(3))
-		l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
-}
-
-void
-mmu1init(void)
-{
-	m->mmutop = mallocalign(L1TOPSIZE, BY2PG, 0, 0);
-	if(m->mmutop == nil)
-		panic("mmu1init: no memory for mmutop");
-	memset(m->mmutop, 0, L1TOPSIZE);
-	mmuswitch(nil);
-}
-
-/* KZERO maps the first 1GB of ram */
-uintptr
-paddr(void *va)
-{
-	if((uintptr)va >= KZERO)
-		return (uintptr)va-KZERO;
-	panic("paddr: va=%#p pc=%#p", va, getcallerpc(&va));
-	return 0;
-}
-
-uintptr
-cankaddr(uintptr pa)
-{
-	if(pa < (uintptr)-KZERO)
-		return -KZERO - pa;
-	return 0;
-}
-
-void*
-kaddr(uintptr pa)
-{
-	if(pa < (uintptr)-KZERO)
-		return (void*)(pa + KZERO);
-	panic("kaddr: pa=%#p pc=%#p", pa, getcallerpc(&pa));
-	return nil;
-}
-
-static void*
-kmapaddr(uintptr pa)
-{
-	if(pa < (uintptr)-KZERO)
-		return (void*)(pa + KZERO);
-	if(pa < (VDRAM - KZERO) || pa >= (VDRAM - KZERO) + (KMAPEND - KMAP))
-		panic("kmapaddr: pa=%#p pc=%#p", pa, getcallerpc(&pa));
-	return (void*)(pa + KMAP - (VDRAM - KZERO));
-}
-
-KMap*
-kmap(Page *p)
-{
-	return kmapaddr(p->pa);
-}
-
-void
-kunmap(KMap*)
-{
-}
-
-void
-kmapinval(void)
-{
-}
-
-static void*
-rampage(void)
-{
-	uintptr pa;
-
-	if(conf.npage)
-		return mallocalign(BY2PG, BY2PG, 0, 0);
-
-	pa = conf.mem[0].base;
-	assert((pa % BY2PG) == 0);
-	assert(pa < INITMAP);
-	conf.mem[0].base += BY2PG;
-	return KADDR(pa);
-}
-
-static void
-l1map(uintptr va, uintptr pa, uintptr pe, uintptr attr)
-{
-	uintptr *l1, *l0;
-
-	assert(pa < pe);
-
-	va &= -BY2PG;
-	pa &= -BY2PG;
-	pe = PGROUND(pe);
-
-	attr |= PTEKERNEL | PTEAF;
-
-	l1 = (uintptr*)L1;
-
-	while(pa < pe){
-		if(l1[PTL1X(va, 1)] == 0 && (pe-pa) >= PGLSZ(1) && ((va|pa) & PGLSZ(1)-1) == 0){
-			l1[PTL1X(va, 1)] = PTEVALID | PTEBLOCK | pa | attr;
-			va += PGLSZ(1);
-			pa += PGLSZ(1);
-			continue;
-		}
-		if(l1[PTL1X(va, 1)] & PTEVALID) {
-			assert((l1[PTL1X(va, 1)] & PTETABLE) == PTETABLE);
-			l0 = KADDR(l1[PTL1X(va, 1)] & -PGLSZ(0));
-		} else {
-			l0 = rampage();
-			memset(l0, 0, BY2PG);
-			l1[PTL1X(va, 1)] = PTEVALID | PTETABLE | PADDR(l0);
-		}
-		assert(l0[PTLX(va, 0)] == 0);
-		l0[PTLX(va, 0)] = PTEVALID | PTEPAGE | pa | attr;
-		va += BY2PG;
-		pa += BY2PG;
-	}
-}
-
-static void
-kmapram(uintptr base, uintptr limit)
-{
-	if(base < (uintptr)-KZERO && limit > (uintptr)-KZERO){
-		kmapram(base, (uintptr)-KZERO);
-		kmapram((uintptr)-KZERO, limit);
-		return;
-	}
-	if(base < INITMAP)
-		base = INITMAP;
-	if(base >= limit || limit <= INITMAP)
-		return;
-
-	l1map((uintptr)kmapaddr(base), base, limit,
-		PTEWRITE | PTEPXN | PTEUXN | PTESH(SHARE_INNER));
-}
-
-void
-meminit(void)
-{
-	/* DDR Memory (All modules) */
-	conf.mem[0].base = PGROUND((uintptr)end - KZERO);
-
-	/* exclude uncached dram for ucalloc() */
-	conf.mem[0].limit = UCRAMBASE;
-	conf.mem[1].base = UCRAMBASE+UCRAMSIZE;
-
-	conf.mem[1].limit = 0x100000000ULL;
-
-	/* DDR Memory (Quad-A53 only) */
-	conf.mem[2].base =  0x100000000ULL;
-	conf.mem[2].limit = 0x140000000ULL;
-
-	kmapram(conf.mem[0].base, conf.mem[0].limit);
-	kmapram(conf.mem[1].base, conf.mem[1].limit);
-	kmapram(conf.mem[2].base, conf.mem[2].limit);
-
-	conf.mem[0].npage = (conf.mem[0].limit - conf.mem[0].base)/BY2PG;
-	conf.mem[1].npage = (conf.mem[1].limit - conf.mem[1].base)/BY2PG;
-	conf.mem[2].npage = (conf.mem[2].limit - conf.mem[2].base)/BY2PG;
-}
-
-uintptr
-mmukmap(uintptr va, uintptr pa, usize size)
-{
-	uintptr attr, off;
-
-	if(va == 0)
-		return 0;
-
-	off = pa & BY2PG-1;
-
-	attr = va & PTEMA(7);
-	attr |= PTEWRITE | PTEUXN | PTEPXN | PTESH(SHARE_OUTER);
-
-	va &= -BY2PG;
-	pa &= -BY2PG;
-
-	l1map(va, pa, pa + off + size, attr);
-	flushtlb();
-
-	return va + off;
-}
-
-void*
-vmap(uvlong pa, vlong size)
-{
-	static uintptr base = VMAP;
-	uvlong pe = pa + size;
-	uintptr va;
-
-	va = base;
-	base += PGROUND(pe) - (pa & -BY2PG);
-	
-	return (void*)mmukmap(va | PTEDEVICE, pa, size);
-}
-
-void
-vunmap(void *, vlong)
-{
-}
-
-static uintptr*
-mmuwalk(uintptr va, int level)
-{
-	uintptr *table, pte;
-	Page *pg;
-	int i, x;
-
-	x = PTLX(va, PTLEVELS-1);
-	table = m->mmutop;
-	for(i = PTLEVELS-2; i >= level; i--){
-		pte = table[x];
-		if(pte & PTEVALID) {
-			if(pte & (0xFFFFULL<<48))
-				iprint("strange pte %#p va %#p\n", pte, va);
-			pte &= ~(0xFFFFULL<<48 | BY2PG-1);
-		} else {
-			pg = up->mmufree;
-			if(pg == nil)
-				return nil;
-			up->mmufree = pg->next;
-			pg->va = va & -PGLSZ(i+1);
-			if((pg->next = up->mmuhead[i+1]) == nil)
-				up->mmutail[i+1] = pg;
-			up->mmuhead[i+1] = pg;
-			pte = pg->pa;
-			memset(kmapaddr(pte), 0, BY2PG);
-			coherence();
-			table[x] = pte | PTEVALID | PTETABLE;
-		}
-		table = kmapaddr(pte);
-		x = PTLX(va, (uintptr)i);
-	}
-	return &table[x];
-}
-
-static Proc *asidlist[256];
-
-static int
-allocasid(Proc *p)
-{
-	static Lock lk;
-	Proc *x;
-	int a;
-
-	lock(&lk);
-	a = p->asid;
-	if(a < 0)
-		a = -a;
-	if(a == 0)
-		a = p->pid;
-	for(;; a++){
-		a %= nelem(asidlist);
-		if(a == 0)
-			continue;	// reserved
-		x = asidlist[a];
-		if(x == p || x == nil || (x->asid < 0 && x->mach == nil))
-			break;
-	}
-	p->asid = a;
-	asidlist[a] = p;
-	unlock(&lk);
-
-	return x != p;
-}
-
-static void
-freeasid(Proc *p)
-{
-	int a;
-
-	a = p->asid;
-	if(a < 0)
-		a = -a;
-	if(a > 0 && asidlist[a] == p)
-		asidlist[a] = nil;
-	p->asid = 0;
-}
-
-void
-putasid(Proc *p)
-{
-	/*
-	 * Prevent the following scenario:
-	 *	pX sleeps on cpuA, leaving its page tables in mmutop
-	 *	pX wakes up on cpuB, and exits, freeing its page tables
-	 *  pY on cpuB allocates a freed page table page and overwrites with data
-	 *  cpuA takes an interrupt, and is now running with bad page tables
-	 * In theory this shouldn't hurt because only user address space tables
-	 * are affected, and mmuswitch will clear mmutop before a user process is
-	 * dispatched.  But empirically it correlates with weird problems, eg
-	 * resetting of the core clock at 0x4000001C which confuses local timers.
-	 */
-	if(conf.nmach > 1)
-		mmuswitch(nil);
-
-	if(p->asid > 0)
-		p->asid = -p->asid;
-}
-
-void
-putmmu(uintptr va, uintptr pa, Page *pg)
-{
-	uintptr *pte, old;
-	int s;
-
-	s = splhi();
-	while((pte = mmuwalk(va, 0)) == nil){
-		spllo();
-		up->mmufree = newpage(0, nil, 0);
-		splhi();
-	}
-	old = *pte;
-	*pte = 0;
-	if((old & PTEVALID) != 0)
-		flushasidvall((uvlong)up->asid<<48 | va>>12);
-	else
-		flushasidva((uvlong)up->asid<<48 | va>>12);
-	*pte = pa | PTEPAGE | PTEUSER | PTEPXN | PTENG | PTEAF |
-		(((pa & PTEMA(7)) == PTECACHED)? PTESH(SHARE_INNER): PTESH(SHARE_OUTER));
-	if(needtxtflush(pg)){
-		cachedwbinvse(kmap(pg), BY2PG);
-		cacheiinvse((void*)va, BY2PG);
-		donetxtflush(pg);
-	}
-	splx(s);
-}
-
-static void
-mmufree(Proc *p)
-{
-	int i;
-
-	freeasid(p);
-
-	for(i=1; i<PTLEVELS; i++){
-		if(p->mmuhead[i] == nil)
-			break;
-		p->mmutail[i]->next = p->mmufree;
-		p->mmufree = p->mmuhead[i];
-		p->mmuhead[i] = p->mmutail[i] = nil;
-	}
-}
-
-void
-mmuswitch(Proc *p)
-{
-	uintptr va;
-	Page *t;
-
-	for(va = UZERO; va < USTKTOP; va += PGLSZ(PTLEVELS-1))
-		m->mmutop[PTLX(va, PTLEVELS-1)] = 0;
-
-	if(p == nil){
-		setttbr(PADDR(m->mmutop));
-		return;
-	}
-
-	if(p->newtlb){
-		mmufree(p);
-		p->newtlb = 0;
-	}
-
-	if(allocasid(p))
-		flushasid((uvlong)p->asid<<48);
-
-	setttbr((uvlong)p->asid<<48 | PADDR(m->mmutop));
-
-	for(t = p->mmuhead[PTLEVELS-1]; t != nil; t = t->next){
-		va = t->va;
-		m->mmutop[PTLX(va, PTLEVELS-1)] = t->pa | PTEVALID | PTETABLE;
-	}
-}
-
-void
-mmurelease(Proc *p)
-{
-	mmuswitch(nil);
-	mmufree(p);
-	freepages(p->mmufree, nil, 0);
-	p->mmufree = nil;
-}
-
-void
-flushmmu(void)
-{
-	int x;
-
-	x = splhi();
-	up->newtlb = 1;
-	mmuswitch(up);
-	splx(x);
-}
-
-void
-checkmmu(uintptr, uintptr)
-{
-}
-
-static void*
-ucramalloc(usize size, uintptr align, uint attr)
-{
-	static uintptr top = UCRAMBASE + UCRAMSIZE;
-	static Lock lk;
-	uintptr va, pg;
-
-	lock(&lk);
-	top -= size;
-	size += top & align-1;
-	top &= -align;
-	if(top < UCRAMBASE)
-		panic("ucramalloc: need %zd bytes", size);
-	va = KZERO + top;
-	pg = va & -BY2PG;
-	if(pg != ((va+size) & -BY2PG))
-		mmukmap(pg | attr, pg - KZERO, PGROUND(size));
-	unlock(&lk);
-
-	return (void*)va;
-}
-
-void*
-ucalloc(usize size)
-{
-	return ucramalloc(size, 8, PTEUNCACHED);
-}