pc, pc64: make mtrr() callable from interrupt context and before mpinit

to make it possible to mark the bootscreen framebuffer
as write combining in early initialization, mtrr() is
changed not not to error() but to return an error string.

as bootscreen() is used before multiprocessor initialization,
we have to synchronize the mtrr's for every processor as
it comes online. for this, a new mtrrsync() function is
provided that is called from cpuidentify() if mtrr support
is indicated.

the boot processor runs mtrrsync() which snarfs the
registers. later, mtrrsync() is run again from the
application processors which apply the values from the
boot processor.

checkmtrr() from mp.c was removed as its task is also
done by mtrrsync() now.

10 files changed, 132 insertions, 163 deletions

diff --git a/sys/src/9/pc/dat.h b/sys/src/9/pc/dat.h
index a8f46aaca..8bcc5053a 100644
--- a/sys/src/9/pc/dat.h
+++ b/sys/src/9/pc/dat.h
@@ -256,11 +256,6 @@ struct Mach
 	int	pdballoc;
 	int	pdbfree;
 
-	vlong	mtrrcap;
-	vlong	mtrrdef;
-	vlong	mtrrfix[11];
-	vlong	mtrrvar[32];		/* 256 max. */
-
 	int	stack[1];
 };
 
diff --git a/sys/src/9/pc/devarch.c b/sys/src/9/pc/devarch.c
index fce3c5bc5..9265e2696 100644
--- a/sys/src/9/pc/devarch.c
+++ b/sys/src/9/pc/devarch.c
@@ -883,6 +883,9 @@ cpuidentify(void)
 			rdmsr(0x01, &mct);
 	}
 
+	if(m->cpuiddx & Mtrr)
+		mtrrsync();
+
 	if(m->cpuiddx & Fxsr){			/* have sse fp? */
 		fpsave = fpssesave;
 		fprestore = fpsserestore;
@@ -1024,7 +1027,9 @@ archctlwrite(Chan*, void *a, long n, vlong)
 		size = strtoull(cb->f[2], &ep, 0);
 		if(*ep)
 			error("cache: parse error: size not a number?");
-		mtrr(base, size, cb->f[3]);
+		ep = mtrr(base, size, cb->f[3]);
+		if(ep != nil)
+			error(ep);
 		break;
 	}
 	free(cb);
diff --git a/sys/src/9/pc/fns.h b/sys/src/9/pc/fns.h
index 99fa4b671..8ecaa0f7d 100644
--- a/sys/src/9/pc/fns.h
+++ b/sys/src/9/pc/fns.h
@@ -111,9 +111,10 @@ void	mfence(void);
 #define mmuflushtlb(pdb) putcr3(pdb)
 void	mmuinit(void);
 ulong*	mmuwalk(ulong*, ulong, int, int);
-int	mtrr(uvlong, uvlong, char *);
+char*	mtrr(uvlong, uvlong, char *);
 void	mtrrclock(void);
 int	mtrrprint(char *, long);
+void	mtrrsync(void);
 uchar	nvramread(int);
 void	nvramwrite(int, uchar);
 void	outb(int, int);
diff --git a/sys/src/9/pc/mp.c b/sys/src/9/pc/mp.c
index ef027ece6..765fa30b6 100644
--- a/sys/src/9/pc/mp.c
+++ b/sys/src/9/pc/mp.c
@@ -90,58 +90,6 @@ mpintrinit(Bus* bus, PCMPintr* intr, int vno, int /*irq*/)
 	return v;
 }
 
-void
-checkmtrr(void)
-{
-	int i, vcnt;
-	Mach *mach0;
-
-	/*
-	 * If there are MTRR registers, snarf them for validation.
-	 */
-	if(!(m->cpuiddx & Mtrr))
-		return;
-
-	rdmsr(0x0FE, &m->mtrrcap);
-	rdmsr(0x2FF, &m->mtrrdef);
-	if(m->mtrrcap & 0x0100){
-		rdmsr(0x250, &m->mtrrfix[0]);
-		rdmsr(0x258, &m->mtrrfix[1]);
-		rdmsr(0x259, &m->mtrrfix[2]);
-		for(i = 0; i < 8; i++)
-			rdmsr(0x268+i, &m->mtrrfix[(i+3)]);
-	}
-	vcnt = m->mtrrcap & 0x00FF;
-	if(vcnt > nelem(m->mtrrvar))
-		vcnt = nelem(m->mtrrvar);
-	for(i = 0; i < vcnt; i++)
-		rdmsr(0x200+i, &m->mtrrvar[i]);
-
-	/*
-	 * If not the bootstrap processor, compare.
-	 */
-	if(m->machno == 0)
-		return;
-
-	mach0 = MACHP(0);
-	if(mach0->mtrrcap != m->mtrrcap)
-		print("mtrrcap%d: %lluX %lluX\n",
-			m->machno, mach0->mtrrcap, m->mtrrcap);
-	if(mach0->mtrrdef != m->mtrrdef)
-		print("mtrrdef%d: %lluX %lluX\n",
-			m->machno, mach0->mtrrdef, m->mtrrdef);
-	for(i = 0; i < 11; i++){
-		if(mach0->mtrrfix[i] != m->mtrrfix[i])
-			print("mtrrfix%d: i%d: %lluX %lluX\n",
-				m->machno, i, mach0->mtrrfix[i], m->mtrrfix[i]);
-	}
-	for(i = 0; i < vcnt; i++){
-		if(mach0->mtrrvar[i] != m->mtrrvar[i])
-			print("mtrrvar%d: i%d: %lluX %lluX\n",
-				m->machno, i, mach0->mtrrvar[i], m->mtrrvar[i]);
-	}
-}
-
 uvlong
 tscticks(uvlong *hz)
 {
@@ -234,8 +182,6 @@ mpinit(void)
 	intrenable(IrqSPURIOUS, lapicspurious, 0, BUSUNKNOWN, "lapicspurious");
 	lapiconline();
 
-	checkmtrr();
-
 	/*
 	 * Initialise the application processors.
 	 */
diff --git a/sys/src/9/pc/mtrr.c b/sys/src/9/pc/mtrr.c
index 1a2715c53..c4ce1b176 100644
--- a/sys/src/9/pc/mtrr.c
+++ b/sys/src/9/pc/mtrr.c
@@ -69,10 +69,6 @@ struct Mtrreg {
 	vlong	base;
 	vlong	mask;
 };
-struct Mtrrop {
-	Mtrreg	*reg;
-	int	slot;
-};
 
 static char *types[] = {
 [Uncacheable]	"uc",
@@ -84,8 +80,9 @@ static char *types[] = {
 [Writeback]	"wb",
 		nil
 };
-static Mtrrop *postedop;
-static Rendez oprend;
+
+static int dosync;
+static Mtrreg mtrreg[Nmtrr];
 
 static char *
 type2str(int type)
@@ -163,8 +160,6 @@ mtrrenc(Mtrreg *mtrr, uvlong ptr, uvlong size, int type, int ok)
 static void
 mtrrget(Mtrreg *mtrr, uint i)
 {
-	if (i >= Nmtrr)
-		error("mtrr index out of range");
 	rdmsr(MTRRPhysBase0 + 2*i, &mtrr->base);
 	rdmsr(MTRRPhysMask0 + 2*i, &mtrr->mask);
 	sanity(mtrr);
@@ -173,30 +168,43 @@ mtrrget(Mtrreg *mtrr, uint i)
 static void
 mtrrput(Mtrreg *mtrr, uint i)
 {
-	if (i >= Nmtrr)
-		error("mtrr index out of range");
 	sanity(mtrr);
 	wrmsr(MTRRPhysBase0 + 2*i, mtrr->base);
 	wrmsr(MTRRPhysMask0 + 2*i, mtrr->mask);
 }
 
+static int
+mtrrvcnt(void)
+{
+	vlong cap;
+	int vcnt;
+
+	rdmsr(MTRRCap, &cap);
+	vcnt = cap & Capvcnt;
+	if(vcnt > Nmtrr)
+		vcnt = Nmtrr;
+	return vcnt;
+}
+
+static int
+mtrrgetall(void)
+{
+	int i, vcnt;
+
+	vcnt = mtrrvcnt();
+	for(i = 0; i < vcnt; i++)
+		mtrrget(&mtrreg[i], i);
+	return vcnt;
+}
+
 static void
-mtrrop(Mtrrop **op)
+mtrrputall(void)
 {
-	int s;
+	int s, i, vcnt;
 	ulong cr0, cr4;
 	vlong def;
-	static Ref bar1, bar2;
 
-	s = splhi();		/* avoid race with mtrrclock */
-
-	/*
-	 * wait for all CPUs to sync here, so that the MTRR setup gets
-	 * done at roughly the same time on all processors.
-	 */
-	incref(&bar1);
-	while(bar1.ref < conf.nmach)
-		microdelay(10);
+	s = splhi();
 
 	cr4 = getcr4();
 	putcr4(cr4 & ~CR4PageGlobalEnable);
@@ -207,13 +215,39 @@ mtrrop(Mtrrop **op)
 	rdmsr(MTRRDefaultType, &def);
 	wrmsr(MTRRDefaultType, def & ~(vlong)Defena);
 
-	mtrrput((*op)->reg, (*op)->slot);
+	vcnt = mtrrvcnt();
+	for(i=0; i<vcnt; i++)
+		mtrrput(&mtrreg[i], i);
 
 	wbinvd();
 	wrmsr(MTRRDefaultType, def);
 	putcr0(cr0);
 	putcr4(cr4);
 
+	splx(s);
+}
+
+void
+mtrrclock(void)				/* called from clock interrupt */
+{
+	static Ref bar1, bar2;
+	int s;
+
+	if(dosync == 0)
+		return;
+
+	s = splhi();
+
+	/*
+	 * wait for all CPUs to sync here, so that the MTRR setup gets
+	 * done at roughly the same time on all processors.
+	 */
+	incref(&bar1);
+	while(bar1.ref < conf.nmach)
+		microdelay(10);
+
+	mtrrputall();
+
 	/*
 	 * wait for all CPUs to sync up again, so that we don't continue
 	 * executing while the MTRRs are still being set up.
@@ -225,60 +259,41 @@ mtrrop(Mtrrop **op)
 	while(bar1.ref > 0)
 		microdelay(10);
 	decref(&bar2);
-	*op = nil;
-	wakeup(&oprend);
-	splx(s);
-}
-
-void
-mtrrclock(void)				/* called from clock interrupt */
-{
-	if(postedop != nil)
-		mtrrop(&postedop);
-}
 
-/* if there's an operation still pending, keep sleeping */
-static int
-opavail(void *)
-{
-	return postedop == nil;
+	dosync = 0;
+	splx(s);
 }
 
-int
-mtrr(uvlong base, uvlong size, char *tstr)
+static char*
+mtrr0(uvlong base, uvlong size, char *tstr)
 {
 	int i, vcnt, slot, type, mtype, mok;
 	vlong def, cap;
 	uvlong mp, msize;
-	Mtrreg entry, mtrr;
-	Mtrrop op;
-	static int tickreg;
-	static QLock mtrrlk;
 
 	if(!(m->cpuiddx & Mtrr))
-		error("mtrrs not supported");
+		return "mtrrs not supported";
 	if(base & (BY2PG-1) || size & (BY2PG-1) || size == 0)
-		error("mtrr base or size not 4k aligned or zero size");
+		return "mtrr base or size not 4k aligned or zero size";
 	if(base + size >= Paerange)
-		error("mtrr range exceeds 36 bits");
+		return "mtrr range exceeds 36 bits";
 	if(!ispow2(size))
-		error("mtrr size not power of 2");
+		return "mtrr size not power of 2";
 	if(base & (size - 1))
-		error("mtrr base not naturally aligned");
+		return "mtrr base not naturally aligned";
 
 	if((type = str2type(tstr)) == -1)
-		error("mtrr bad type");
+		return "mtrr bad type";
 
 	rdmsr(MTRRCap, &cap);
 	rdmsr(MTRRDefaultType, &def);
 
 	switch(type){
 	default:
-		error("mtrr unknown type");
-		break;
+		return "mtrr unknown type";
 	case Writecomb:
 		if(!(cap & Capwc))
-			error("mtrr type wc (write combining) unsupported");
+			return "mtrr type wc (write combining) unsupported";
 		/* fallthrough */
 	case Uncacheable:
 	case Writethru:
@@ -287,18 +302,11 @@ mtrr(uvlong base, uvlong size, char *tstr)
 		break;
 	}
 
-	qlock(&mtrrlk);
-	if(waserror()){
-		qunlock(&mtrrlk);
-		nexterror();
-	}
+	vcnt = mtrrgetall();
+
 	slot = -1;
-	vcnt = cap & Capvcnt;
-	if(vcnt > Nmtrr)
-		vcnt = Nmtrr;
 	for(i = 0; i < vcnt; i++){
-		mtrrget(&mtrr, i);
-		mok = mtrrdec(&mtrr, &mp, &msize, &mtype);
+		mok = mtrrdec(&mtrreg[i], &mp, &msize, &mtype);
 		if(slot == -1 && (!mok || mtype == (def & Deftype)))
 			slot = i;	/* good, but look further for exact match */
 		if(mok && mp == base && msize == size){
@@ -307,40 +315,45 @@ mtrr(uvlong base, uvlong size, char *tstr)
 		}
 	}
 	if(slot == -1)
-		error("no free mtrr slots");
+		return "no free mtrr slots";
+
+	mtrrenc(&mtrreg[slot], base, size, type, 1);
 
-	while(!opavail(0))
-		sleep(&oprend, opavail, 0);
+	coherence();
+
+	dosync = 1;
+	mtrrclock();
+
+	return nil;
+}
 
-	mtrrenc(&entry, base, size, type, 1);
-	op.reg = &entry;
-	op.slot = slot;
-	postedop = &op;
-	mtrrop(&postedop);
+char*
+mtrr(uvlong base, uvlong size, char *tstr)
+{
+	static QLock mtrrlk;
+	char *err;
+
+	qlock(&mtrrlk);
+	err = mtrr0(base, size, tstr);
 	qunlock(&mtrrlk);
-	poperror();
-	return 0;
+
+	return err;
 }
 
 int
 mtrrprint(char *buf, long bufsize)
 {
-	int i, vcnt, type;
-	long n;
+	int i, n, vcnt, type;
 	uvlong base, size;
-	vlong cap, def;
 	Mtrreg mtrr;
+	vlong def;
 
-	n = 0;
 	if(!(m->cpuiddx & Mtrr))
 		return 0;
-	rdmsr(MTRRCap, &cap);
 	rdmsr(MTRRDefaultType, &def);
-	n += snprint(buf+n, bufsize-n, "cache default %s\n",
+	n = snprint(buf, bufsize, "cache default %s\n",
 		type2str(def & Deftype));
-	vcnt = cap & Capvcnt;
-	if(vcnt > Nmtrr)
-		vcnt = Nmtrr;
+	vcnt = mtrrvcnt();
 	for(i = 0; i < vcnt; i++){
 		mtrrget(&mtrr, i);
 		if (mtrrdec(&mtrr, &base, &size, &type))
@@ -350,3 +363,28 @@ mtrrprint(char *buf, long bufsize)
 	}
 	return n;
 }
+
+void
+mtrrsync(void)
+{
+	static vlong cap0, def0;
+	vlong cap, def;
+
+	rdmsr(MTRRCap, &cap);
+	rdmsr(MTRRDefaultType, &def);
+
+	if(m->machno == 0){
+		cap0 = cap;
+		def0 = def;
+		mtrrgetall();
+		return;
+	}
+
+	if(cap0 != cap)
+		print("mtrrcap%d: %lluX %lluX\n",
+			m->machno, cap0, cap);
+	if(def0 != def)
+		print("mtrrdef%d: %lluX %lluX\n",
+			m->machno, def0, def);
+	mtrrputall();
+}
diff --git a/sys/src/9/pc/screen.c b/sys/src/9/pc/screen.c
index f076b4431..494c6e908 100644
--- a/sys/src/9/pc/screen.c
+++ b/sys/src/9/pc/screen.c
@@ -512,13 +512,7 @@ vgalinearaddr0(VGAscr *scr, ulong paddr, int size)
 	scr->paddr = paddr;
 	scr->apsize = nsize;
 
-	if(up != nil){
-		/* let mtrr harmlessly fail on old CPUs, e.g., P54C */
-		if(!waserror()){
-			mtrr(npaddr, nsize, "wc");
-			poperror();
-		}
-	}
+	mtrr(npaddr, nsize, "wc");
 
 	return nil;
 }
diff --git a/sys/src/9/pc/squidboy.c b/sys/src/9/pc/squidboy.c
index 03fccb691..593016595 100644
--- a/sys/src/9/pc/squidboy.c
+++ b/sys/src/9/pc/squidboy.c
@@ -8,8 +8,6 @@
 
 #include "mp.h"
 
-extern void checkmtrr(void);
-
 static void
 squidboy(Apic* apic)
 {
@@ -20,7 +18,6 @@ squidboy(Apic* apic)
 
 	cpuidentify();
 	cpuidprint();
-	checkmtrr();
 
 	apic->online = 1;
 	coherence();
diff --git a/sys/src/9/pc64/dat.h b/sys/src/9/pc64/dat.h
index 0bf421d79..5b4103610 100644
--- a/sys/src/9/pc64/dat.h
+++ b/sys/src/9/pc64/dat.h
@@ -223,11 +223,6 @@ struct Mach
 	int	pdballoc;
 	int	pdbfree;
 
-	vlong	mtrrcap;
-	vlong	mtrrdef;
-	vlong	mtrrfix[11];
-	vlong	mtrrvar[32];		/* 256 max. */
-
 	uintptr	stack[1];
 };
 
diff --git a/sys/src/9/pc64/fns.h b/sys/src/9/pc64/fns.h
index 04e1c5d69..fe67edded 100644
--- a/sys/src/9/pc64/fns.h
+++ b/sys/src/9/pc64/fns.h
@@ -102,9 +102,10 @@ void	mfence(void);
 #define mmuflushtlb() putcr3(getcr3())
 void	mmuinit(void);
 uintptr	*mmuwalk(uintptr*, uintptr, int, int);
-int	mtrr(uvlong, uvlong, char *);
+char*	mtrr(uvlong, uvlong, char *);
 void	mtrrclock(void);
 int	mtrrprint(char *, long);
+void	mtrrsync(void);
 void	noteret(void);
 uchar	nvramread(int);
 void	nvramwrite(int, uchar);
diff --git a/sys/src/9/pc64/squidboy.c b/sys/src/9/pc64/squidboy.c
index dab78ea2d..653802955 100644
--- a/sys/src/9/pc64/squidboy.c
+++ b/sys/src/9/pc64/squidboy.c
@@ -8,8 +8,6 @@
 
 #include "mp.h"
 
-extern void checkmtrr(void);
-
 static void
 squidboy(Apic* apic)
 {
@@ -17,7 +15,6 @@ squidboy(Apic* apic)
 	mmuinit();
 	cpuidentify();
 	cpuidprint();
-	checkmtrr();
 	apic->online = 1;
 	coherence();