usbxhci: split usbxhci in portable and pci / soc specific drivers

For the reform kernel, we used to have a slightly modified
copy of port/usbxhci.c as the controller was implemented in
soc specific registers and requires some rocket science
initialization.

Instead, we want to have a common generic xhci driver (usbxhci)
and separate drivers that deal with the specific implementation
such as usbxhcipci and usbxhciimx.

10 files changed, 277 insertions, 1935 deletions

diff --git a/sys/src/9/bcm64/pi4 b/sys/src/9/bcm64/pi4
index 07dbfd5b9..ace3a182f 100644
--- a/sys/src/9/bcm64/pi4
+++ b/sys/src/9/bcm64/pi4
@@ -29,7 +29,7 @@ link
 	gisb
 	pcibcm
 	archbcm4	pci
-	usbxhci		pci archbcm4
+	usbxhcipci	pci usbxhci archbcm4 
 	ethergenet	ethermii
 	ethermedium
 	loopbackmedium
diff --git a/sys/src/9/imx8/mkfile b/sys/src/9/imx8/mkfile
index 97d6a471a..ac21a312d 100644
--- a/sys/src/9/imx8/mkfile
+++ b/sys/src/9/imx8/mkfile
@@ -90,6 +90,7 @@ install:V: /$objtype/$p$CONF
 main.$O: rebootcode.i
 
 pciimx.$O: ../port/pci.h
+usbxhciimx.$O: ../port/usbxhci.h
 
 initcode.out:		init9.$O initcode.$O /$objtype/lib/libc.a
 	$LD -l -R1 -s -o $target $prereq
diff --git a/sys/src/9/imx8/reform b/sys/src/9/imx8/reform
index 4a3bd2257..c5899f890 100644
--- a/sys/src/9/imx8/reform
+++ b/sys/src/9/imx8/reform
@@ -26,7 +26,7 @@ dev
 	audio
 
 link
-	usbxhciimx
+	usbxhciimx	usbxhci
 	etherimx	ethermii
 	ethermedium
 	loopbackmedium
diff --git a/sys/src/9/imx8/usbxhciimx.c b/sys/src/9/imx8/usbxhciimx.c
index 82bbf52bc..c42ccaeac 100644
--- a/sys/src/9/imx8/usbxhciimx.c
+++ b/sys/src/9/imx8/usbxhciimx.c
@@ -6,1767 +6,7 @@
 #include	"io.h"
 #include	"../port/error.h"
 #include	"../port/usb.h"
-
-enum {
-	/* Capability Registers */
-	CAPLENGTH	= 0x00/4,	// 1
-	HCIVERSION	= 0x02/4,	// 2
-	HCSPARAMS1	= 0x04/4,
-	HCSPARAMS2	= 0x08/4,
-	HCSPARAMS3	= 0x0C/4,
-
-	HCCPARAMS	= 0x10/4,
-		AC64	= 1<<0,
-		BNC	= 1<<1,
-		CSZ	= 1<<2,
-		PPC	= 1<<3,
-		PIND	= 1<<4,
-		LHRC	= 1<<5,
-		LTC	= 1<<6,
-		NSS	= 1<<7,
-
-	DBOFF		= 0x14/4,
-	RTSOFF		= 0x18/4,
-
-	HCCPARAMS2	= 0x1C/4,
-
-	/* Operational Registers */
-	USBCMD		= 0x00/4,	/* USB Command Register */
-		RUNSTOP	= 1<<0,		/* Run/Stop - RW */
-		HCRST	= 1<<1,		/* Host Controller Reset - RW */
-		INTE	= 1<<2,		/* Interrupter Enable - RW */
-		HSEE	= 1<<3,		/* Host System Error Enable - RW */
-		LHCRST	= 1<<7,		/* Light Host Controller Reset - RO/RW */
-		CSS	= 1<<8,		/* Controller Save State - RW */
-		CRS	= 1<<9,		/* Controller Restore State - RW */
-		EWE	= 1<<10,	/* Enable Wrap Event - RW */
-		EU3S	= 1<<11,	/* Enable U3 MFINDEX Stop - RW */
-
-	USBSTS		= 0x04/4,	/* USB Status Register */
-		HCH	= 1<<0,		/* HCHalted - RO */
-		HSE	= 1<<2,		/* Host System Error - RW1C */
-		EINT	= 1<<3,		/* Event Interrupt - RW1C */
-		PCD	= 1<<4,		/* Port Change Detect - RW1C */
-		SSS	= 1<<8,		/* Save State Status - RO */
-		RSS	= 1<<9,		/* Restore State Status - RO */
-		SRE	= 1<<10,	/* Save/Restore Error - RW1C */
-		CNR	= 1<<11,	/* Controller Not Ready - RO */
-		HCE	= 1<<12,	/* Host Controller Error - RO */
-
-	PAGESIZE	= 0x08/4,	/* Page Size - RO */
-
-	DNCTRL		= 0x14/4,	/* Device Notification Control Register - RW */
-
-	CRCR		= 0x18/4,	/* Command Ring Control Register - RW */
-		RCS	= 1<<0,		/* Ring Cycle State - RW */
-		CS	= 1<<1,		/* Command Stop - RW1S */
-		CA	= 1<<2,		/* Command Abort - RW1S */
-		CRR	= 1<<3,		/* Command Ring Running - RO */
-
-	DCBAAP		= 0x30/4,	// 8
-
-	CONFIG		= 0x38/4,	/* Configure Register (MaxSlotEn[7:0]) */
-
-	/* Port Register Set */
-	PORTSC		= 0x00/4,	/* Port status and Control Register */
-		CCS	= 1<<0,		/* Current Connect Status - ROS */
-		PED	= 1<<1,		/* Port Enable/Disabled - RW1CS */
-		OCA	= 1<<3,		/* Over-current Active - RO */
-		PR	= 1<<4,		/* Port Reset - RW1S */
-		PLS	= 15<<5,	/* Port Link State - RWS */
-		PP	= 1<<9,		/* Port Power - RWS */
-		PS	= 15<<10,	/* Port Speed - ROS */
-		PIC	= 3<<14,	/* Port Indicator Control - RWS */
-		LWS	= 1<<16,	/* Port Link Write Strobe - RW */
-		CSC	= 1<<17,	/* Connect Status Change - RW1CS */
-		PEC	= 1<<18,	/* Port Enabled/Disabled Change - RW1CS */
-		WRC	= 1<<19,	/* Warm Port Reset Change - RW1CS */
-		OCC	= 1<<20,	/* Over-current Change - RW1CS */
-		PRC	= 1<<21,	/* Port Reset Change - RW1CS */
-		PLC	= 1<<22,	/* Port Link State Change - RW1CS */
-		CEC	= 1<<23,	/* Port Config Error Change - RW1CS */
-		CAS	= 1<<24,	/* Cold Attach Status - RO */
-		WCE	= 1<<25,	/* Wake on Connect Enable - RWS */
-		WDE	= 1<<26,	/* Wake on Disconnect Enable - RWS */
-		WOE	= 1<<27,	/* Wake on Over-current Enable - RWS */
-		DR	= 1<<30,	/* Device Removable - RO */
-		WPR	= 1<<31,	/* Warm Port Reset - RW1S */
-
-	PORTPMSC	= 0x04/4,
-	PORTLI		= 0x08/4,
-
-	/* Host Controller Runtime Register */
-	MFINDEX		= 0x0000/4,	/* Microframe Index */
-	IR0		= 0x0020/4,	/* Interrupt Register Set 0 */
-
-	/* Interrupter Registers */
-	IMAN		= 0x00/4,	/* Interrupter Management */
-	IMOD		= 0x04/4,	/* Interrupter Moderation */
-	ERSTSZ		= 0x08/4,	/* Event Ring Segment Table Size */
-	ERSTBA		= 0x10/4,	/* Event Ring Segment Table Base Address */
-	ERDP		= 0x18/4,	/* Event Ring Dequeue Pointer */
-
-	/* TRB flags */
-	TR_ENT		= 1<<1,
-	TR_ISP		= 1<<2,
-	TR_NS		= 1<<3,
-	TR_CH		= 1<<4,
-	TR_IOC		= 1<<5,
-	TR_IDT		= 1<<6,
-	TR_BEI		= 1<<9,
-	
-	/* TRB types */
-	TR_RESERVED	= 0<<10,
-	TR_NORMAL	= 1<<10,
-	TR_SETUPSTAGE	= 2<<10,
-	TR_DATASTAGE	= 3<<10,
-	TR_STATUSSTAGE	= 4<<10,
-	TR_ISOCH	= 5<<10,
-	TR_LINK		= 6<<10,
-	TR_EVENTDATA	= 7<<10,
-	TR_NOOP		= 8<<10,
-
-	CR_ENABLESLOT	= 9<<10,
-	CR_DISABLESLOT	= 10<<10,
-	CR_ADDRESSDEV	= 11<<10,
-	CR_CONFIGEP	= 12<<10,
-	CR_EVALCTX	= 13<<10,
-	CR_RESETEP	= 14<<10,
-	CR_STOPEP	= 15<<10,
-	CR_SETTRDQP	= 16<<10,
-	CR_RESETDEV	= 17<<10,
-	CR_FORCECMD	= 18<<10,
-	CR_NEGBW	= 19<<10,
-	CR_SETLAT	= 20<<10,
-	CR_GETPORTBW	= 21<<10,
-	CR_FORCEHDR	= 22<<10,
-	CR_NOOP		= 23<<10,
-
-	ER_TRANSFER	= 32<<10,
-	ER_CMDCOMPL	= 33<<10,
-	ER_PORTSC	= 34<<10,
-	ER_BWREQ	= 35<<10,
-	ER_DOORBELL	= 36<<10,
-	ER_HCE		= 37<<10,
-	ER_DEVNOTE	= 38<<10,
-	ER_MFINDEXWRAP	= 39<<10,
-};
-
-typedef struct Ctlr Ctlr;
-typedef struct Wait Wait;
-typedef struct Ring Ring;
-typedef struct Slot Slot;
-typedef struct Epio Epio;
-typedef struct Port Port;
-
-struct Wait
-{
-	Wait	*next;
-	Ring	*ring;
-	u32int	*td;
-	u32int	er[4];
-	Rendez	*z;
-};
-
-struct Ring
-{
-	int	id;
-
-	Slot	*slot;
-
-	u32int	*base;
-
-	u32int	mask;
-	u32int	shift;
-
-	u32int	rp;
-	u32int	wp;
-
-	u32int	*ctx;
-	u32int	*doorbell;
-
-	int	stopped;
-
-	int	*residue;
-	Wait	*pending;
-	Lock;
-};
-
-struct Slot
-{
-	int	id;
-
-	int	confval; // bConfigurationValue of SET_CONFIGURATION
-	int	iface;	// bInterfaceNumber of SET_INTERFACE
-	int	altc;	// bAlternateSetting of SET_INTERFACE
-
-	Ctlr	*ctlr;
-	Udev	*dev;
-
-	u32int	*ibase;
-	u32int	*obase;
-
-	/* endpoint rings */
-	int	nep;
-	Ring	epr[32];
-};
-
-struct Port
-{
-	char	spec[4];
-	int	proto;
-
-	u32int	*reg;
-};
-
-struct Ctlr
-{
-	u32int	*mmio;
-
-	u32int	*opr;	/* operational registers */
-	u32int	*rts;	/* runtime registers */
-	u32int	*dba;	/* doorbell array */
-
-	u64int	*dcba;	/* device context base array */
-
-	u64int	*sba;	/* scratchpad buffer array */
-	void	*sbp;	/* scratchpad buffer pages */
-
-	u32int	*erst[1];	/* event ring segment table */
-	Ring	er[1];		/* event ring segment */
-	Ring	cr[1];		/* command ring segment */
-	QLock	cmdlock;
-
-	u32int	µframe;
-
-	QLock	slotlock;
-	Slot	**slot;		/* slots by slot id */
-	Port	*port;
-	
-	u32int	hccparams;
-
-	int	csz;
-	int	pagesize;
-	int	nscratch;
-	int	nintrs;
-	int	nslots;
-
-	Rendez	recover;	
-	void	*active;
-};
-
-struct Epio
-{
-	QLock;
-
-	Ring	*ring;
-	Block	*b;
-
-	/* iso */
-	u32int	frame;
-	u32int	period;
-	u32int	incr;
-	u32int	tdsz;
-
-	/* isoread */
-	u32int	rp0;
-	u32int	frame0;
-
-	int	nleft;
-};
-
-static char Ebadlen[] = "bad usb request length";
-static char Enotconfig[] = "usb endpoint not configured";
-static char Erecover[] = "xhci controller needs reset";
-
-static char*
-ctlrcmd(Ctlr *ctlr, u32int c, u32int s, u64int p, u32int *er);
-
-static void
-setrptr(u32int *reg, u64int pa)
-{
-	coherence();
-	reg[0] = pa;
-	reg[1] = pa>>32;
-}
-
-static u32int
-µframe(Ctlr *ctlr)
-{
-	u32int µ;
-	do {
-		µ = (ctlr->rts[MFINDEX] & (1<<14)-1) |
-			(ctlr->µframe & ~((1<<14)-1));
-	} while((int)(µ - ctlr->µframe) < 0);
-	return µ;
-}
-
-static void
-freering(Ring *r)
-{
-	if(r == nil)
-		return;
-	if(r->base != nil){
-		dmaflush(0, r->base, 4*4<<r->shift);
-		free(r->base);
-	}
-	if(r->residue != nil)
-		free(r->residue);
-	memset(r, 0, sizeof(*r));
-}
-
-static Ring*
-initring(Ring *r, int shift)
-{
-	r->id = 0;
-	r->ctx = nil;
-	r->slot = nil;
-	r->doorbell = nil;
-	r->pending = nil;
-	r->residue = nil;
-	r->stopped = 0;
-	r->shift = shift;
-	r->mask = (1<<shift)-1;
-	r->rp = r->wp = 0;
-	r->base = mallocalign(4*4<<shift, 64, 0, 64*1024);
-	if(r->base == nil){
-		freering(r);
-		error(Enomem);
-	}
-	dmaflush(1, r->base, 4*4<<shift);
-	return r;
-}
-
-static void
-flushring(Ring *r)
-{
-	Rendez *z;
-	Wait *w;
-
-	while((w = r->pending) != nil){
-		r->pending = w->next;
-		w->next = nil;
-		if((z = w->z) != nil){
-			w->z = nil;
-			wakeup(z);
-		}
-	}
-}
-
-static u64int
-resetring(Ring *r)
-{
-	u64int pa;
-
-	ilock(r);
-	flushring(r);
-	r->rp = r->wp;
-	pa = PADDR(&r->base[4*(r->wp & r->mask)]) | ((~r->wp>>r->shift) & 1);
-	iunlock(r);
-
-	return pa;
-}
-
-static u32int*
-xecp(Ctlr *ctlr, uchar id, u32int *p)
-{
-	u32int x;
-
-	if(p == nil){
-		p = ctlr->mmio;
-		x = ctlr->hccparams>>16;
-	} else {
-		x = (*p>>8) & 255;
-	}
-	while(x != 0){
-		p += x;
-		x = *p;
-		if((x & 255) == id)
-			return p;
-		x >>= 8;
-		x &= 255;
-	}
-	return nil;
-}
-
-static void
-handoff(Ctlr *ctlr)
-{
-	u32int *r;
-	int i;
-
-	if((r = xecp(ctlr, 1, nil)) == nil)
-		return;
-	if(getconf("*noxhcihandoff") == nil){
-		r[0] |= 1<<24;		/* request ownership */
-		for(i = 0; (r[0] & (1<<16)) != 0 && i<100; i++)
-			tsleep(&up->sleep, return0, nil, 10);
-	}
-	/* disable SMI interrupts */
-	r[1] &= 7<<1 | 255<<5 | 7<<17 | 7<<29;
-
-	/* clear BIOS ownership in case of timeout */
-	r[0] &= ~(1<<16);
-}
-
-static void
-shutdown(Hci *hp)
-{
-	Ctlr *ctlr = hp->aux;
-	int i;
-
-	ctlr->opr[USBCMD] = 0;
-	for(i=0; (ctlr->opr[USBSTS] & HCH) == 0 && i < 10; i++)
-		delay(10);
-	intrdisable(hp->irq, hp->interrupt, hp, hp->tbdf, hp->type);
-}
-
-static void
-release(Ctlr *ctlr)
-{
-	int i;
-
-	freering(ctlr->cr);
-	for(i=0; i<nelem(ctlr->er); i++){
-		freering(&ctlr->er[i]);
-		free(ctlr->erst[i]);
-		ctlr->erst[i] = nil;
-	}
-	free(ctlr->port), ctlr->port = nil;
-	free(ctlr->slot), ctlr->slot = nil;
-	free(ctlr->dcba), ctlr->dcba = nil;
-	free(ctlr->sba), ctlr->sba = nil;
-	if(ctlr->sbp != nil){
-		dmaflush(0, ctlr->sbp, ctlr->nscratch*ctlr->pagesize);
-		free(ctlr->sbp);
-		ctlr->sbp = nil;
-	}
-}
-
-static void recover(void *arg);
-
-static void
-init(Hci *hp)
-{
-	Ctlr *ctlr;
-	Port *pp;
-	u32int *x;
-	uchar *p;
-	int i, j;
-
-	ctlr = hp->aux;
-	if(ctlr->mmio[CAPLENGTH] == -1){
-		error("controller vanished");
-	}
-
-	ctlr->opr = &ctlr->mmio[(ctlr->mmio[CAPLENGTH]&0xFF)/4];
-	ctlr->dba = &ctlr->mmio[ctlr->mmio[DBOFF]/4];
-	ctlr->rts = &ctlr->mmio[ctlr->mmio[RTSOFF]/4];
-
-	ctlr->hccparams = ctlr->mmio[HCCPARAMS];
-	handoff(ctlr);
-
-	for(i=0; (ctlr->opr[USBSTS] & CNR) != 0 && i<100; i++)
-		tsleep(&up->sleep, return0, nil, 10);
-
-	ctlr->opr[USBCMD] = HCRST;
-	delay(1);
-	for(i=0; (ctlr->opr[USBSTS] & (CNR|HCH)) != HCH && i<100; i++)
-		tsleep(&up->sleep, return0, nil, 10);
-
-	intrenable(hp->irq, hp->interrupt, hp, hp->tbdf, hp->type);
-
-	if(waserror()){
-		shutdown(hp);
-		release(ctlr);
-		nexterror();
-	}
-
-	ctlr->csz = (ctlr->hccparams & CSZ) != 0;
-	ctlr->pagesize = (ctlr->opr[PAGESIZE] & 0xFFFF) << 12;
-
-	ctlr->nscratch = (ctlr->mmio[HCSPARAMS2] >> 27) & 0x1F | (ctlr->mmio[HCSPARAMS2] >> 16) & 0x3E0;
-	ctlr->nintrs = (ctlr->mmio[HCSPARAMS1] >> 8) & 0x7FF;
-	ctlr->nslots = (ctlr->mmio[HCSPARAMS1] >> 0) & 0xFF;
-
-	hp->highspeed = 1;
-	hp->superspeed = 0;
-	hp->nports = (ctlr->mmio[HCSPARAMS1] >> 24) & 0xFF;
-	ctlr->port = malloc(hp->nports * sizeof(Port));
-	if(ctlr->port == nil)
-		error(Enomem);
-	for(i=0; i<hp->nports; i++)
-		ctlr->port[i].reg = &ctlr->opr[0x400/4 + i*4];
-
-	x = nil;
-	while((x = xecp(ctlr, 2, x)) != nil){
-		i = x[2]&255;
-		j = (x[2]>>8)&255;
-		while(j--){
-			if(i < 1 || i > hp->nports)
-				break;
-			pp = &ctlr->port[i-1];
-			pp->proto = x[0]>>16;
-			memmove(pp->spec, &x[1], 4);
-			if(memcmp(pp->spec, "USB ", 4) == 0 && pp->proto >= 0x0300)
-				hp->superspeed |= 1<<(i-1);
-			i++;
-		}
-	}
-
-	ctlr->slot = malloc((1+ctlr->nslots)*sizeof(ctlr->slot[0]));
-	ctlr->dcba = mallocalign((1+ctlr->nslots)*sizeof(ctlr->dcba[0]), 64, 0, ctlr->pagesize);
-	if(ctlr->slot == nil || ctlr->dcba == nil)
-		error(Enomem);
-	if(ctlr->nscratch != 0){
-		ctlr->sba = mallocalign(ctlr->nscratch*8, 64, 0, ctlr->pagesize);
-		ctlr->sbp = mallocalign(ctlr->nscratch*ctlr->pagesize, ctlr->pagesize, 0, 0);
-		if(ctlr->sba == nil || ctlr->sbp == nil)
-			error(Enomem);
-		for(i=0, p = ctlr->sbp; i<ctlr->nscratch; i++, p += ctlr->pagesize){
-			memset(p, 0, ctlr->pagesize);
-			ctlr->sba[i] = PADDR(p);
-		}
-		dmaflush(1, ctlr->sbp, ctlr->nscratch*ctlr->pagesize);
-		dmaflush(1, ctlr->sba, ctlr->nscratch*8);
-		ctlr->dcba[0] = PADDR(ctlr->sba);
-	} else {
-		ctlr->dcba[0] = 0;
-	}
-	for(i=1; i<=ctlr->nslots; i++)
-		ctlr->dcba[i] = 0;
-
-	ctlr->opr[CONFIG] = (ctlr->opr[CONFIG] & 0xFFFFFC00) | ctlr->nslots;	/* MaxSlotsEn */
-
-	dmaflush(1, ctlr->dcba, (1+ctlr->nslots)*sizeof(ctlr->dcba[0]));
-	setrptr(&ctlr->opr[DCBAAP], PADDR(ctlr->dcba));
-
-	initring(ctlr->cr, 8);		/* 256 entries */
-	ctlr->cr->id = 0;
-	ctlr->cr->doorbell = &ctlr->dba[0];
-	setrptr(&ctlr->opr[CRCR], resetring(ctlr->cr));
-
-	for(i=0; i<ctlr->nintrs; i++){
-		u32int *irs = &ctlr->rts[IR0 + i*8];
-
-		if(i >= nelem(ctlr->er)){
-			irs[ERSTSZ] = 0;	/* disable ring */
-			irs[IMAN] = 1;
-			irs[IMOD] = 0;
-			setrptr(&irs[ERSTBA], 0);
-			setrptr(&irs[ERDP], 0);
-			continue;
-		}
-
-		/* allocate and link into event ring segment table */
-		initring(&ctlr->er[i], 8);	/* 256 entries */
-		ctlr->erst[i] = mallocalign(4*4, 64, 0, 0);
-		if(ctlr->erst[i] == nil)
-			error(Enomem);
-		*((u64int*)ctlr->erst[i]) = PADDR(ctlr->er[i].base);
-		ctlr->erst[i][2] = ctlr->er[i].mask+1;
-		ctlr->erst[i][3] = 0;
-		dmaflush(1, ctlr->erst[i], 4*4);
-
-		irs[ERSTSZ] = 1;	/* just one segment */
-		irs[IMAN] = 3;
-		irs[IMOD] = 0;
-		setrptr(&irs[ERSTBA], PADDR(ctlr->erst[i]));
-		setrptr(&irs[ERDP], PADDR(ctlr->er[i].base) | (1<<3));
-	}
-	poperror();
-
-	ctlr->µframe = 0;
-	ctlr->opr[USBSTS] = ctlr->opr[USBSTS] & (HSE|EINT|PCD|SRE);
-	coherence();
-
-	ctlr->opr[USBCMD] = RUNSTOP|INTE|HSEE|EWE;
-	for(i=0; (ctlr->opr[USBSTS] & (CNR|HCH)) != 0 && i<100; i++)
-		tsleep(&up->sleep, return0, nil, 10);
-
-	kproc("xhcirecover", recover, hp);
-}
-
-static int
-needrecover(void *arg)
-{
-	Ctlr *ctlr = arg;
-	return 	ctlr->er->stopped || 
-		(ctlr->opr[USBSTS] & (HCH|HCE|HSE)) != 0;
-}
-
-static void
-recover(void *arg)
-{
-	Hci *hp = arg;
-	Ctlr *ctlr = hp->aux;
-
-	while(waserror())
-		;
-	while(!needrecover(ctlr))
-		tsleep(&ctlr->recover, needrecover, ctlr, 1000);
-	shutdown(hp);
-
-	/*
-	 * flush all transactions and wait until all devices have
-	 * been detached by usbd.
-	 */
-	for(;;){
-		int i, j, active;
-
-		ilock(ctlr->cr);
-		ctlr->cr->stopped = 1;
-		flushring(ctlr->cr);
-		iunlock(ctlr->cr);
-
-		active = 0;
-		qlock(&ctlr->slotlock);
-		for(i=1; i<=ctlr->nslots; i++){
-			Slot *slot = ctlr->slot[i];
-			if(slot == nil)
-				continue;
-			active++;
-			for(j=0; j < slot->nep; j++){
-				Ring *ring = &slot->epr[j];
-				if(ring->base == nil)
-					continue;
-				ilock(ring);
-				ring->stopped = 1;
-				flushring(ring);
-				iunlock(ring);
-			}
-		}
-		qunlock(&ctlr->slotlock);
-		if(active == 0)
-			break;
-
-		tsleep(&up->sleep, return0, nil, 100);
-	}
-
-	qlock(&ctlr->slotlock);
-	qlock(&ctlr->cmdlock);
-
-	release(ctlr);
-	if(waserror()) {
-		print("xhci recovery failed: %s\n", up->errstr);
-	} else {
-		init(hp);
-		poperror();
-	}
-
-	qunlock(&ctlr->cmdlock);
-	qunlock(&ctlr->slotlock);
-
-	pexit("", 1);
-}
-
-static void
-dump(Hci *)
-{
-}
-
-static void
-queuetd(Ring *r, u32int c, u32int s, u64int p, Wait *w)
-{
-	u32int *td, x;
-
-	x = r->wp++;
-	if((x & r->mask) == r->mask){
-		td = r->base + 4*(x & r->mask);
-		*(u64int*)td = PADDR(r->base);
-		td[2] = 0;
-		td[3] = ((~x>>r->shift)&1) | (1<<1) | TR_LINK;
-		dmaflush(1, td, 4*4);
-		x = r->wp++;
-	}
-	td = r->base + 4*(x & r->mask);
-	if(w != nil){
-		w->er[0] = w->er[1] = w->er[2] = w->er[3] = 0;
-		w->ring = r;
-		w->td = td;
-		w->z = &up->sleep;
-
-		ilock(r);
-		w->next = r->pending;
-		r->pending = w;
-		iunlock(r);
-	}
-	if(r->residue != nil)
-		r->residue[x & r->mask] = s;
-	coherence();
-	*(u64int*)td = p;
-	td[2] = s;
-	td[3] = ((~x>>r->shift)&1) | c;
-	dmaflush(1, td, 4*4);
-}
-
-static char *ccerrtab[] = {
-[2]	"Data Buffer Error",
-[3]	"Babble Detected Error",
-[4]	"USB Transaction Error",
-[5]	"TRB Error",
-[6]	"Stall Error",
-[7]	"Resume Error",
-[8]	"Bandwidth Error",
-[9]	"No Slots Available",
-[10]	"Invalid Stream Type",
-[11]	"Slot Not Enabled",
-[12]	"Endpoint Not Enabled",
-[13]	"Short Packet",
-[14]	"Ring Underrun",
-[15]	"Ring Overrun",
-[16]	"VF Event Ring Full",
-[17]	"Parameter Error",
-[18]	"Bandwidth Overrun Error",
-[19]	"Context State Error",
-[20]	"No Ping Response",
-[21]	"Event Ring Full",
-[22]	"Incompatible Device",
-[23]	"Missed Service Error",
-[24]	"Command Ring Stopped",
-[25]	"Command Aborted",
-[26]	"Stopped",
-[27]	"Stoppe - Length Invalid",
-[29]	"Max Exit Latency Too Large",
-[31]	"Isoch Buffer Overrun",
-[32]	"Event Lost Error",
-[33]	"Undefined Error",
-[34]	"Invalid Stream ID",
-[35]	"Secondary Bandwidth Error",
-[36]	"Split Transaction Error",
-};
-
-static char*
-ccerrstr(u32int cc)
-{
-	char *s;
-
-	if(cc == 1 || cc == 13)
-		return nil;
-	if(cc < nelem(ccerrtab) && ccerrtab[cc] != nil)
-		s = ccerrtab[cc];
-	else
-		s = "???";
-	return s;
-}
-
-static int
-waitdone(void *a)
-{
-	return ((Wait*)a)->z == nil;
-}
-
-static char*
-waittd(Ctlr *ctlr, Wait *w, int tmout)
-{
-	Ring *r = w->ring;
-
-	coherence();
-	*r->doorbell = r->id;
-
-	while(waserror()){
-		if(r->stopped) {
-			ctlr->er->stopped = 1;
-			wakeup(&ctlr->recover);
-
-			/* wait for rescue */
-			tmout = 0;
-			continue;
-		}
-
-		if(r == ctlr->cr)
-			ctlr->opr[CRCR] |= CA;
-		else
-			ctlrcmd(ctlr, CR_STOPEP | (r->id<<16) | (r->slot->id<<24), 0, 0, nil);
-		r->stopped = 1;
-
-		/* time to abort the transaction */
-		tmout = 5000;
-	}
-	if(tmout > 0){
-		tsleep(&up->sleep, waitdone, w, tmout);
-		if(!waitdone(w))
-			error("timed out");
-	} else {
-		while(!waitdone(w))
-			sleep(&up->sleep, waitdone, w);
-	}
-	poperror();
-	return ccerrstr(w->er[2]>>24);
-}
-
-static char*
-ctlrcmd(Ctlr *ctlr, u32int c, u32int s, u64int p, u32int *er)
-{
-	Wait w[1];
-	char *err;
-
-	qlock(&ctlr->cmdlock);
-	if(needrecover(ctlr)){
-		qunlock(&ctlr->cmdlock);
-		return Erecover;
-	}
-	ctlr->cr->stopped = 0;
-	queuetd(ctlr->cr, c, s, p, w);
-	err = waittd(ctlr, w, 5000);
-	qunlock(&ctlr->cmdlock);
-
-	if(er != nil)
-		memmove(er, w->er, 4*4);
-
-	return err;
-}
-
-static void
-completering(Ring *r, u32int *er)
-{
-	Wait *w, **wp;
-	u32int *td, x;
-	u64int pa;
-
-	pa = (*(u64int*)er) & ~15ULL;
-	ilock(r);
-
-	for(x = r->rp; (int)(r->wp - x) > 0; x++){
-		td = &r->base[4*(x & r->mask)];
-		if((u64int)PADDR(td) == pa){
-			if(r->residue != nil)
-				r->residue[x & r->mask] = er[2] & 0xFFFFFF;
-			r->rp = x+1;
-			break;
-		}
-	}
-
-	wp = &r->pending;
-	while(w = *wp){
-		if((u64int)PADDR(w->td) == pa){
-			Rendez *z = w->z;
-
-			memmove(w->er, er, 4*4);
-			*wp = w->next;
-			w->next = nil;
-
-			if(z != nil){
-				w->z = nil;
-				wakeup(z);
-			}
-			break;
-		} else {
-			wp = &w->next;
-		}
-	}
-
-	iunlock(r);
-}
-
-static void
-interrupt(Ureg*, void *arg)
-{
-	Hci *hp = arg;
-	Ctlr *ctlr = hp->aux;
-	Ring *ring = ctlr->er;
-	Slot *slot;
-	u32int *irs, *td, x;
-
-	if(ring->base == nil)
-		return;
-
-	irs = &ctlr->rts[IR0];
-	x = irs[IMAN];
-	if(x & 1) irs[IMAN] = x & 3;
-
-	for(x = ring->rp;; x=++ring->rp){
-		td = ring->base + 4*(x & ring->mask);
-		dmaflush(0, td, 4*4);
-
-		if((((x>>ring->shift)^td[3])&1) == 0)
-			break;
-
-		switch(td[3] & 0xFC00){
-		case ER_CMDCOMPL:
-			completering(ctlr->cr, td);
-			break;
-		case ER_TRANSFER:
-			x = td[3]>>24;
-			if(x == 0 || x > ctlr->nslots)
-				break;
-			slot = ctlr->slot[x];
-			if(slot == nil)
-				break;
-			completering(&slot->epr[(td[3]>>16)-1&31], td);
-			break;
-		case ER_MFINDEXWRAP:
-			ctlr->µframe = (ctlr->rts[MFINDEX] & (1<<14)-1) | 
-				(ctlr->µframe+(1<<14) & ~((1<<14)-1));
-			break;
-		case ER_HCE:
-			iprint("xhci: host controller error: %ux %ux %ux %ux\n",
-				td[0], td[1], td[2], td[3]);
-			ctlr->er->stopped = 1;
-			wakeup(&ctlr->recover);
-			return;
-		case ER_PORTSC:
-			break;
-		case ER_BWREQ:
-		case ER_DOORBELL:
-		case ER_DEVNOTE:
-		default:
-			iprint("xhci: event %ud: %ux %ux %ux %ux\n",
-				x, td[0], td[1], td[2], td[3]);
-		}
-	}
-
-	setrptr(&irs[ERDP], PADDR(td) | (1<<3));
-}
-
-static void
-freeslot(void *arg)
-{
-	Slot *slot;
-
-	if(arg == nil)
-		return;
-	slot = arg;
-	if(slot->id != 0){
-		Ctlr *ctlr = slot->ctlr;
-		qlock(&ctlr->slotlock);
-		if(ctlr->slot != nil && ctlr->slot[slot->id] == slot){
-			ctlrcmd(ctlr, CR_DISABLESLOT | (slot->id<<24), 0, 0, nil);
-			dmaflush(0, slot->obase, 32*32 << ctlr->csz);
-			ctlr->dcba[slot->id] = 0;
-			dmaflush(1, &ctlr->dcba[slot->id], sizeof(ctlr->dcba[0]));
-			ctlr->slot[slot->id] = nil;
-		}
-		qunlock(&ctlr->slotlock);
-	}
-	freering(&slot->epr[0]);
-	free(slot->ibase);
-	free(slot->obase);
-	free(slot);
-}
-
-static Slot*
-allocslot(Ctlr *ctlr, Udev *dev)
-{
-	u32int r[4];
-	Slot *slot;
-	char *err;
-
-	slot = malloc(sizeof(Slot));
-	if(slot == nil)
-		error(Enomem);
-
-	slot->ctlr = ctlr;
-	slot->dev = dev;
-	slot->nep = 0;
-	slot->id = 0;
-
-	slot->confval = 0;
-	slot->iface = 0;
-	slot->altc = 0;
-
-	qlock(&ctlr->slotlock);
-	if(waserror()){
-		qunlock(&ctlr->slotlock);
-		freeslot(slot);
-		nexterror();
-	}
-	if(ctlr->slot == nil)
-		error(Erecover);
-	slot->ibase = mallocalign(32*33 << ctlr->csz, 64, 0, ctlr->pagesize);
-	slot->obase = mallocalign(32*32 << ctlr->csz, 64, 0, ctlr->pagesize);
-	if(slot->ibase == nil || slot->obase == nil)
-		error(Enomem);
-
-	if((err = ctlrcmd(ctlr, CR_ENABLESLOT, 0, 0, r)) != nil)
-		error(err);
-	slot->id = r[3]>>24;
-	if(slot->id <= 0 || slot->id > ctlr->nslots || ctlr->slot[slot->id] != nil){
-		slot->id = 0;
-		error("bad slot id from controller");
-	}
-	poperror();
-
-	dmaflush(1, slot->obase, 32*32 << ctlr->csz);
-	ctlr->dcba[slot->id] = PADDR(slot->obase);
-	dmaflush(1, &ctlr->dcba[slot->id], sizeof(ctlr->dcba[0]));
-
-	ctlr->slot[slot->id] = slot;
-
-	qunlock(&ctlr->slotlock);
-
-	return slot;
-}
-
-static void
-setdebug(Hci *, int)
-{
-}
-
-static void
-epclose(Ep *ep)
-{
-	Ctlr *ctlr;
-	Slot *slot;
-	Ring *ring;
-	Epio *io;
-
-	if(ep->dev->isroot)
-		return;
-
-	io = ep->aux;
-	if(io == nil)
-		return;
-	ep->aux = nil;
-
-	ctlr = ep->hp->aux;
-	slot = ep->dev->aux;
-
-	if(ep->nb > 0 && (io[OREAD].ring != nil || io[OWRITE].ring != nil)){
-		u32int *w;
-
-		/* input control context */
-		w = slot->ibase;
-		memset(w, 0, 32<<ctlr->csz);
-		w[1] = 1;
-		if((ring = io[OREAD].ring) != nil){
-			w[0] |= 1 << ring->id;
-			if(ring->id == slot->nep)
-				slot->nep--;
-			ctlrcmd(ctlr, CR_STOPEP | (ring->id<<16) | (slot->id<<24), 0, 0, nil);
-		}
-		if((ring = io[OWRITE].ring) != nil){
-			w[0] |= 1 << ring->id;
-			if(ring->id == slot->nep)
-				slot->nep--;
-			ctlrcmd(ctlr, CR_STOPEP | (ring->id<<16) | (slot->id<<24), 0, 0, nil);
-		}
-
-		/* (input) slot context */
-		w += 8<<ctlr->csz;
-		w[0] = (w[0] & ~(0x1F<<27)) | slot->nep<<27;
-
-		/* (input) ep context */
-		w += (ep->nb&Epmax)*2*8<<ctlr->csz;
-		memset(w, 0, 2*32<<ctlr->csz);
-
-		dmaflush(1, slot->ibase, 32*33 << ctlr->csz);
-		ctlrcmd(ctlr, CR_CONFIGEP | (slot->id<<24), 0, PADDR(slot->ibase), nil);
-		dmaflush(0, slot->obase, 32*32 << ctlr->csz);
-
-		freering(io[OREAD].ring);
-		freering(io[OWRITE].ring);
-	}
-	freeb(io[OREAD].b);
-	freeb(io[OWRITE].b);
-	free(io);
-}
-
-static void
-initepctx(u32int *w, Ring *r, Ep *ep)
-{
-	int ival;
-
-	if(ep->dev->speed == Lowspeed || ep->dev->speed == Fullspeed){
-		for(ival=3; ival < 11 && (1<<ival) < ep->pollival; ival++)
-			;
-	} else {
-		for(ival=0; ival < 15 && (1<<ival) < ep->pollival; ival++)
-			;
-	}
-	w[0] = ival<<16;
-	w[1] = ((ep->ttype-Tctl) | (r->id&1)<<2)<<3 | (ep->ntds-1)<<8 | ep->maxpkt<<16;
-	if(ep->ttype != Tiso)
-		w[1] |= 3<<1;
-	*((u64int*)&w[2]) = PADDR(r->base) | 1;
-	w[4] = 2*ep->maxpkt;
-	if(ep->ttype == Tintr || ep->ttype == Tiso)
-		w[4] |= (ep->maxpkt*ep->ntds)<<16;
-}
-
-static void
-initisoio(Epio *io, Ep *ep)
-{
-	if(io->ring == nil)
-		return;
-	io->rp0 = io->ring->wp;
-	io->frame0 = io->frame = 0;
-	io->period = ep->pollival << 3*(ep->dev->speed == Fullspeed || ep->dev->speed == Lowspeed);
-	if(io->ring->id & 1){
-		io->ring->residue = smalloc((io->ring->mask+1)*sizeof(io->ring->residue[0]));
-		io->incr = 0;
-		io->tdsz = ep->maxpkt*ep->ntds;
-	} else {
-		io->incr = ((vlong)ep->hz*ep->pollival<<8)/1000;
-		io->tdsz = (io->incr+255>>8)*ep->samplesz;
-	}
-	io->b = allocb((io->ring->mask+1)*io->tdsz);
-}
-
-static void
-initep(Ep *ep)
-{
-	Epio *io;
-	Ctlr *ctlr;
-	Slot *slot;
-	Ring *ring;
-	u32int *w;
-	char *err;
-
-	io = ep->aux;
-	ctlr = ep->hp->aux;
-	slot = ep->dev->aux;
-
-	io[OREAD].ring = io[OWRITE].ring = nil;
-	if(ep->nb == 0){
-		io[OWRITE].ring = &slot->epr[0];
-		return;
-	}
-
-	/* (input) control context */
-	w = slot->ibase;
-	memset(w, 0, 32<<ctlr->csz);
-	w[1] = 1;
-	w[31] = slot->altc<<16 | slot->iface<<8 | slot->confval;
-
-	if(waserror()){
-		freering(io[OWRITE].ring), io[OWRITE].ring = nil;
-		freering(io[OREAD].ring), io[OREAD].ring = nil;
-		nexterror();
-	}
-	if(ep->mode != OREAD){
-		ring = initring(io[OWRITE].ring = &slot->epr[(ep->nb&Epmax)*2-1], 8);
-		ring->id = (ep->nb&Epmax)*2;
-		if(ring->id > slot->nep)
-			slot->nep = ring->id;
-		ring->slot = slot;
-		ring->doorbell = &ctlr->dba[slot->id];
-		ring->ctx = &slot->obase[ring->id*8<<ctlr->csz];
-		w[1] |= 1 << ring->id;
-	}
-	if(ep->mode != OWRITE){
-		ring = initring(io[OREAD].ring = &slot->epr[(ep->nb&Epmax)*2], 8);
-		ring->id = (ep->nb&Epmax)*2+1;
-		if(ring->id > slot->nep)
-			slot->nep = ring->id;
-		ring->slot = slot;
-		ring->doorbell = &ctlr->dba[slot->id];
-		ring->ctx = &slot->obase[ring->id*8<<ctlr->csz];
-		w[1] |= 1 << ring->id;
-	}
-
-	/* (input) slot context */
-	w += 8<<ctlr->csz;
-	w[0] = (w[0] & ~(0x1F<<27)) | slot->nep<<27;
-	if(!ep->dev->ishub)
-		w[0] &= ~(1<<25);	// MTT
-
-	/* (input) ep context */
-	w += (ep->nb&Epmax)*2*8<<ctlr->csz;
-	if(io[OWRITE].ring != nil){
-		memset(w, 0, 5*4);
-		initepctx(w, io[OWRITE].ring, ep);
-	}
-
-	w += 8<<ctlr->csz;
-	if(io[OREAD].ring != nil){
-		memset(w, 0, 5*4);
-		initepctx(w, io[OREAD].ring, ep);
-	}
-
-	dmaflush(1, slot->ibase, 32*33 << ctlr->csz);
-	err = ctlrcmd(ctlr, CR_CONFIGEP | (slot->id<<24), 0, PADDR(slot->ibase), nil);
-	dmaflush(0, slot->obase, 32*32 << ctlr->csz);
-	if(err != nil)
-		error(err);
-
-	if(ep->ttype == Tiso){
-		initisoio(io+OWRITE, ep);
-		initisoio(io+OREAD, ep);
-	}
-	poperror();
-}
-
-static int
-speedid(int speed)
-{
-	switch(speed){
-	case Fullspeed:		return 1;
-	case Lowspeed:		return 2;
-	case Highspeed:		return 3;
-	case Superspeed:	return 4;
-	}
-	return 0;
-}
-
-static void
-epopen(Ep *ep)
-{
-	Ctlr *ctlr = ep->hp->aux;
-	Slot *slot, *hub;
-	Ring *ring;
-	Epio *io;
-	Udev *dev;
-	char *err;
-	u32int *w;
-	int i;
-
-	if(ep->dev->isroot)
-		return;
-	if(needrecover(ctlr))
-		error(Erecover);
-	io = malloc(sizeof(Epio)*2);
-	if(io == nil)
-		error(Enomem);
-	ep->aux = io;
-	if(waserror()){
-		epclose(ep);
-		nexterror();
-	}
-	dev = ep->dev;
-	slot = dev->aux;
-	if(slot != nil && slot->dev == dev){
-		initep(ep);
-		poperror();
-		return;
-	}
-
-	/* first open has to be control endpoint */
-	if(ep->nb != 0)
-		error(Egreg);
-
-	slot = allocslot(ctlr, dev);
-	if(waserror()){
-		freeslot(slot);
-		nexterror();
-	}
-
-	/* allocate control ep 0 ring */
-	ring = initring(io[OWRITE].ring = &slot->epr[0], 4);
-	ring->id = 1;
-	slot->nep = 1;
-	ring->slot = slot;
-	ring->doorbell = &ctlr->dba[slot->id];
-	ring->ctx = &slot->obase[8<<ctlr->csz];
-
-	/* (input) control context */
-	w = slot->ibase;
-	memset(w, 0, 3*32<<ctlr->csz);
-	w[1] = 3;	/* A0, A1 */
-		
-	/* (input) slot context */
-	w += 8<<ctlr->csz;
-	w[2] = w[3] = 0;
-	w[0] = dev->routestr | speedid(dev->speed)<<20 |
-		(dev->speed == Highspeed && dev->ishub != 0)<<25 |	// MTT
-		(dev->ishub != 0)<<26 | slot->nep<<27;
-	w[1] = dev->rootport<<16;
-
-	/* find the parent hub that this device is conected to */
-	qlock(&ctlr->slotlock);
-	for(i=1; i<=ctlr->nslots; i++){
-		hub = ctlr->slot[i];
-		if(hub == nil || hub->dev == nil || hub->dev->aux != hub)
-			continue;
-		if(hub == slot || hub->dev == dev)
-			continue;
-		if(!hub->dev->ishub)
-			continue;
-		if(hub->dev->addr != dev->hub)
-			continue;
-		if(hub->dev->rootport != dev->rootport)
-			continue;
-
-		if(dev->speed < Highspeed && hub->dev->speed == Highspeed){
-			w[0] |= 1<<25;	// MTT
-			w[2] = hub->id | dev->port<<8;
-		}
-		break;
-	}
-	qunlock(&ctlr->slotlock);
-
-	/* (input) ep context 0 */
-	w += 8<<ctlr->csz;
-	initepctx(w, io[OWRITE].ring, ep);
-
-	dmaflush(1, slot->ibase, 32*33 << ctlr->csz);
-	err = ctlrcmd(ctlr, CR_ADDRESSDEV | (slot->id<<24), 0, PADDR(slot->ibase), nil);
-	dmaflush(0, slot->obase, 32*32 << ctlr->csz);
-	if(err != nil)
-		error(err);
-
-	/* (output) slot context */
-	w = slot->obase;
-
-	dev->addr = w[3] & 0xFF;
-
-	dev->aux = slot;
-	dev->free = freeslot;
-
-	poperror();
-	poperror();
-}
-
-static long
-isoread(Ep *ep, uchar *p, long n)
-{
-	uchar *s, *d;
-	Ctlr *ctlr;
-	Epio *io;
-	u32int i, µ;
-	long m;
-
-	s = p;
-
-	io = (Epio*)ep->aux + OREAD;
-	qlock(io);
-	if(waserror()){
-		qunlock(io);
-		nexterror();
-	}
-	µ = io->period;
-	ctlr = ep->hp->aux;
-	if(needrecover(ctlr))
-		error(Erecover);
-
-	for(i = io->frame0; (int)(io->ring->rp - io->rp0) > 0 && n > 0; i++) {
-		if((io->rp0 & io->ring->mask) == io->ring->mask)
-			io->rp0++;
-		m = io->tdsz - io->ring->residue[io->rp0 & io->ring->mask];
-		if(m > 0){
-			d = io->b->rp + (i&io->ring->mask)*io->tdsz;
-			d += io->nleft, m -= io->nleft;
-			if(n < m){
-				dmaflush(0, d, n);
-				memmove(p, d, n);
-				io->nleft += n;
-				p += n;
-				break;
-			}
-			dmaflush(0, d, m);
-			memmove(p, d, m);
-			p += m, n -= m;
-
-			if(ep->uframes == 1)
-				n = 0;
-		}
-		io->nleft = 0;
-		io->rp0++;
-	}
-	io->frame0 = i;
-
-	for(i = io->frame;; i++){
-		m = (int)(io->ring->wp - io->rp0);
-		if(m <= 0) {
-			i = (80 + µframe(ctlr))/µ;
-			io->frame0 = i;
-			io->rp0 = io->ring->wp;
-			io->nleft = 0;
-		} else if(m+1 >= io->ring->mask)
-			break;
-		m = io->tdsz;
-		d = io->b->rp + (i&io->ring->mask)*io->tdsz;
-		dmaflush(1, d, m);
-		queuetd(io->ring, TR_ISOCH | (i*µ/8 & 0x7ff)<<20 | TR_IOC, m, PADDR(d), nil);
-	}
-	io->frame = i;
-
-	*io->ring->doorbell = io->ring->id;
-	qunlock(io);
-	poperror();
-
-	return p - s;
-}
-
-static long
-isowrite(Ep *ep, uchar *p, long n)
-{
-	uchar *s, *d;
-	Ctlr *ctlr;
-	Epio *io;
-	u32int i, µ;
-	long m;
-
-	s = p;
-	io = (Epio*)ep->aux + OWRITE;
-	qlock(io);
-	if(waserror()){
-		qunlock(io);
-		nexterror();
-	}
-	µ = io->period;
-	ctlr = ep->hp->aux;
-
-	for(i = io->frame;; i++){
-		for(;;){
-			if(needrecover(ctlr))
-				error(Erecover);
-			m = (int)(io->ring->wp - io->ring->rp);
-			if(m <= 0)
-				i = (80 + µframe(ctlr))/µ;
-			if(m+1 < io->ring->mask)
-				break;
-
-			*io->ring->doorbell = io->ring->id;
-			tsleep(&up->sleep, return0, nil, 5);
-		}
-		m = ((io->incr + (i*io->incr&255))>>8)*ep->samplesz;
-		d = io->b->rp + (i&io->ring->mask)*io->tdsz;
-		m -= io->nleft, d += io->nleft;
-		if(n < m){
-			memmove(d, p, n);
-			p += n;
-			io->nleft += n;
-			break;
-		}
-		memmove(d, p, m);
-		p += m, n -= m;
-		m += io->nleft, d -= io->nleft;
-		io->nleft = 0;
-		dmaflush(1, d, m);
-		queuetd(io->ring, TR_ISOCH | (i*µ/8 & 0x7ff)<<20 | TR_IOC, m, PADDR(d), nil);
-	}
-	io->frame = i;
-
-	while(io->ring->rp != io->ring->wp){
-		int d = (int)(i*µ - µframe(ctlr))/8;
-		d -= ep->sampledelay*1000 / ep->hz;
-		if(d < 5)
-			break;
-
-		*io->ring->doorbell = io->ring->id;
-		tsleep(&up->sleep, return0, nil, d);
-		if(needrecover(ctlr))
-			error(Erecover);
-	}
-
-	qunlock(io);
-	poperror();
-
-	return p - s;
-}
-
-static char*
-unstall(Ep *ep, Ring *r)
-{
-	char *err;
-
-	switch(r->ctx[0]&7){
-	case 2:	/* halted */
-	case 4:	/* error */
-		ep->clrhalt = 1;
-	}
-	if(ep->clrhalt){
-		ep->clrhalt = 0;
-		err = ctlrcmd(r->slot->ctlr, CR_RESETEP | (r->id<<16) | (r->slot->id<<24), 0, 0, nil);
-		dmaflush(0, r->ctx, 8*4 << r->slot->ctlr->csz);
-		if(err != nil)
-			return err;
-		r->stopped = 1;
-	}
-	if(r->stopped){
-		err = ctlrcmd(r->slot->ctlr, CR_SETTRDQP | (r->id<<16) | (r->slot->id<<24), 0, resetring(r), nil);
-		dmaflush(0, r->ctx, 8*4 << r->slot->ctlr->csz);
-		if(err != nil)
-			return err;
-		r->stopped = 0;
-	}
-	if(r->wp - r->rp >= r->mask)
-		return "Ring Full";
-	return nil;
-}
-
-static long
-epread(Ep *ep, void *va, long n)
-{
-	Epio *io;
-	Ctlr *ctlr;
-	uchar *p;
-	char *err;
-	Wait w[1];
-
-	if(ep->dev->isroot)
-		error(Egreg);
-
-	p = va;
-	if(ep->ttype == Tctl){
-		io = (Epio*)ep->aux + OREAD;
-		qlock(io);
-		if(io->b == nil || BLEN(io->b) == 0){
-			qunlock(io);
-			return 0;
-		}
-		if(n > BLEN(io->b))
-			n = BLEN(io->b);
-		memmove(p, io->b->rp, n);
-		io->b->rp += n;
-		qunlock(io);
-		return n;
-	} else if(ep->ttype == Tiso)
-		return isoread(ep, p, n);
-
-	if((uintptr)p <= KZERO){
-		Block *b;
-
-		b = allocb(n);
-		if(waserror()){
-			freeb(b);
-			nexterror();
-		}
-		n = epread(ep, b->rp, n);
-		memmove(p, b->rp, n);
-		freeb(b);
-		poperror();
-		return n;
-	}
-
-	ctlr = (Ctlr*)ep->hp->aux;
-	io = (Epio*)ep->aux + OREAD;
-	qlock(io);
-	if(waserror()){
-		dmaflush(0, io->ring->ctx, 8*4 << ctlr->csz);
-		qunlock(io);
-		nexterror();
-	}
-
-	if((err = unstall(ep, io->ring)) != nil)
-		error(err);
-
-	dmaflush(1, p, n);
-	queuetd(io->ring, TR_NORMAL | TR_IOC, n, PADDR(p), w);
-	err = waittd(ctlr, w, ep->tmout);
-	dmaflush(0, p, n);
-	if(err != nil)
-		error(err);
-
-	qunlock(io);
-	poperror();
-
-	n -= (w->er[2] & 0xFFFFFF);
-	if(n < 0)
-		n = 0;
-
-	return n;
-}
-
-static long
-epwrite(Ep *ep, void *va, long n)
-{
-	Wait w[3];
-	Ctlr *ctlr;
-	Epio *io;
-	uchar *p;
-	char *err;
-
-	if(ep->dev->isroot)
-		error(Egreg);
-
-	p = va;
-	if(ep->ttype == Tctl){
-		int dir, len;
-		Ring *ring;
-		Slot *slot;
-
-		if(n < 8)
-			error(Eshort);
-
-		if(p[0] == 0x00 && p[1] == 0x05)
-			return n;
-
-		ctlr = (Ctlr*)ep->hp->aux;
-		io = (Epio*)ep->aux + OREAD;
-		ring = io[OWRITE-OREAD].ring;
-		slot = ring->slot;
-		qlock(io);
-		if(waserror()){
-			ilock(ring);
-			ring->pending = nil;
-			iunlock(ring);
-			dmaflush(0, ring->ctx, 8*4 << ctlr->csz);
-			qunlock(io);
-			nexterror();
-		}
-		if(io->b != nil){
-			freeb(io->b);
-			io->b = nil;
-		}
-		len = GET2(&p[6]);
-		dir = (p[0] & Rd2h) != 0;
-		if(len > 0){
-			io->b = allocb(len);		
-			if(dir == 0){	/* out */
-				assert(len >= n-8);
-				memmove(io->b->wp, p+8, n-8);
-			} else {
-				memset(io->b->wp, 0, len);
-				io->b->wp += len;
-			}
-		}
-		if((err = unstall(ep, ring)) != nil)
-			error(err);
-
-		if((ring->ctx[1]>>16) != ep->maxpkt){
-			u32int *w = slot->ibase;
-			w[0] = 0;
-			w[1] = 1<<ring->id;
-			w += (ring->id+1)*8<<ctlr->csz;
-			initepctx(w, ring, ep);
-			dmaflush(1, slot->ibase, 32*33 << ctlr->csz);
-			err = ctlrcmd(ctlr, CR_EVALCTX | (slot->id<<24), 0, PADDR(slot->ibase), nil);
-			dmaflush(0, slot->obase, 32*32 << ctlr->csz);
-			if(err != nil)
-				error(err);
-		}
-
-		queuetd(ring, TR_SETUPSTAGE | (len > 0 ? 2+dir : 0)<<16 | TR_IDT | TR_IOC, 8,
-			p[0] | p[1]<<8 | GET2(&p[2])<<16 |
-			(u64int)(GET2(&p[4]) | len<<16)<<32, &w[0]);
-		if(len > 0){
-			dmaflush(1, io->b->rp, len);
-			queuetd(ring, TR_DATASTAGE | dir<<16 | TR_IOC, len,
-				PADDR(io->b->rp), &w[1]);
-		}
-		queuetd(ring, TR_STATUSSTAGE | (len == 0 || !dir)<<16 | TR_IOC, 0, 0, &w[2]);
-
-		if((err = waittd(ctlr, &w[0], ep->tmout)) != nil)
-			error(err);
-		if(len > 0){
-			if((err = waittd(ctlr, &w[1], ep->tmout)) != nil)
-				error(err);
-			if(dir != 0){
-				dmaflush(0, io->b->rp, len);
-				io->b->wp -= (w[1].er[2] & 0xFFFFFF);
-				if(io->b->wp < io->b->rp)
-					io->b->wp = io->b->rp;
-			}
-		}
-		if((err = waittd(ctlr, &w[2], ep->tmout)) != nil)
-			error(err);
-
-		if(p[0] == 0x00 && p[1] == 0x09){
-			slot->confval = GET2(&p[2]);
-		} else if(p[0] == 0x01 && p[1] == 0x0d){
-			slot->altc = GET2(&p[2]);
-			slot->iface = GET2(&p[4]);
-		}
-
-		qunlock(io);
-		poperror();
-
-		return n;
-	} else if(ep->ttype == Tiso)
-		return isowrite(ep, p, n);
-
-	if((uintptr)p <= KZERO){
-		Block *b;
-
-		b = allocb(n);
-		if(waserror()){
-			freeb(b);
-			nexterror();
-		}
-		memmove(b->wp, p, n);
-		n = epwrite(ep, b->wp, n);
-		freeb(b);
-		poperror();
-		return n;
-	}
-
-	ctlr = (Ctlr*)ep->hp->aux;
-	io = (Epio*)ep->aux + OWRITE;
-	qlock(io);
-	if(waserror()){
-		dmaflush(0, io->ring->ctx, 8*4 << ctlr->csz);
-		qunlock(io);
-		nexterror();
-	}
-
-	if((err = unstall(ep, io->ring)) != nil)
-		error(err);
-
-	dmaflush(1, p, n);
-	queuetd(io->ring, TR_NORMAL | TR_IOC, n, PADDR(p), w);
-	if((err = waittd(ctlr, w, ep->tmout)) != nil)
-		error(err);
-
-	qunlock(io);
-	poperror();
-
-	return n;
-}
-
-static char*
-seprintep(char *s, char*, Ep*)
-{
-	return s;
-}
-
-static int
-portstatus(Hci *hp, int port)
-{
-	Ctlr *ctlr = hp->aux;
-	u32int psc, ps;
-
-	if(ctlr->port == nil || needrecover(ctlr))
-		return 0;
-
-	ps = 0;
-	psc = ctlr->port[port-1].reg[PORTSC];
-	if(psc & CCS)	ps |= HPpresent;
-	if(psc & PED)	ps |= HPenable;
-	if(psc & OCA)	ps |= HPovercurrent;
-	if(psc & PR)	ps |= HPreset;
-
-	if((hp->superspeed & (1<<(port-1))) != 0){
-		ps |= psc & (PLS|PP);
-		if(psc & CSC)	ps |= 1<<0+16;
-		if(psc & OCC)	ps |= 1<<3+16;
-		if(psc & PRC)	ps |= 1<<4+16;
-		if(psc & WRC)	ps |= 1<<5+16;
-		if(psc & PLC)	ps |= 1<<6+16;
-		if(psc & CEC)	ps |= 1<<7+16;
-	} else {
-		if((ps & HPreset) == 0){
-			switch((psc>>10)&15){
-			case 1:
-				/* full speed */
-				break;
-			case 2:
-				ps |= HPslow;
-				break;
-			case 3:
-				ps |= HPhigh;
-				break;
-			}
-		}
-		if(psc & PP)	ps |= HPpower;
-		if(psc & CSC)	ps |= HPstatuschg;
-		if(psc & PRC)	ps |= HPchange;
-	}
-
-	return ps;
-}
-	
-static int
-portenable(Hci*, int, int)
-{
-	return 0;
-}
-
-static int
-portreset(Hci *hp, int port, int on)
-{
-	Ctlr *ctlr = hp->aux;
-
-	if(ctlr->port == nil || needrecover(ctlr))
-		return 0;
-
-	if(on){
-		ctlr->port[port-1].reg[PORTSC] |= PR;
-		tsleep(&up->sleep, return0, nil, 200);
-	}
-	return 0;
-}
+#include	"../port/usbxhci.h"
 
 static void
 clkenable(int i, int on)
@@ -1826,25 +66,25 @@ static int
 reset(Hci *hp)
 {
 	static char *powerdom[] = { "usb_otg1", "usb_otg2" };
-	static Ctlr ctlrs[2];
-	Ctlr *ctlr;
+	static Xhci *ctlrs[2];
+	Xhci *ctlr;
 	int i;
 
 	for(i=0; i<nelem(ctlrs); i++){
-		ctlr = &ctlrs[i];
-		if(ctlr->active == nil){
-			ctlr->active = hp;
-			ctlr->mmio = (u32int*)(VIRTIO + 0x8100000 + i*0x100000);
-			hp->aux = ctlr;
-			hp->port = (uintptr)ctlr->mmio - KZERO;
-			hp->tbdf = BUSUNKNOWN;
-			hp->irq = IRQusb1 + i;
+		if(ctlrs[i] == nil){
+			uintptr base = VIRTIO + 0x8100000 + i*0x100000;
+			ctlr = xhcialloc((u32int*)base, base - KZERO, 0x100000);
+			ctlrs[i] = ctlr;
 			goto Found;
 		}
 	}
 	return -1;
 
 Found:
+	xhcilinkage(hp, ctlr);
+	hp->tbdf = BUSUNKNOWN;
+	hp->irq = IRQusb1 + i;
+
 	if(i == 0){
 		iomuxpad("pad_gpio1_io13", "usb1_otg_oc", "~LVTTL ~HYS ~PUE ~ODE FAST 45_OHM");
 		iomuxpad("pad_gpio1_io14", "gpio1_io14", "~LVTTL HYS PUE ~ODE FAST 45_OHM");
@@ -1865,21 +105,6 @@ Found:
 	phyinit(&ctlr->mmio[0xF0040/4]);
 	coreinit(ctlr->mmio);
 
-	hp->init = init;
-	hp->dump = dump;
-	hp->interrupt = interrupt;
-	hp->epopen = epopen;
-	hp->epclose = epclose;
-	hp->epread = epread;
-	hp->epwrite = epwrite;
-	hp->seprintep = seprintep;
-	hp->portenable = portenable;
-	hp->portreset = portreset;
-	hp->portstatus = portstatus;
-	hp->shutdown = shutdown;
-	hp->debug = setdebug;
-	hp->type = "xhci";
-
 	return 0;
 }
 
diff --git a/sys/src/9/pc/pc b/sys/src/9/pc/pc
index 2dad42576..ab909ee9f 100644
--- a/sys/src/9/pc/pc
+++ b/sys/src/9/pc/pc
@@ -86,8 +86,8 @@ link
 	pcmciamodem
 	usbuhci		pci
 	usbohci		pci
-	usbehci		usbehcipc
-	usbxhci		pci
+	usbehci		pci usbehcipc
+	usbxhci		pci usbxhcipci
 
 	audiosb16	dma
 	audioac97	pci audioac97mix
diff --git a/sys/src/9/pc64/pc64 b/sys/src/9/pc64/pc64
index e2fbb16f0..30b7d7ed7 100644
--- a/sys/src/9/pc64/pc64
+++ b/sys/src/9/pc64/pc64
@@ -85,7 +85,7 @@ link
 	usbuhci		pci
 	usbohci		pci
 	usbehci		pci usbehcipc
-	usbxhci		pci
+	usbxhcipci	pci usbxhci
 
 #	audiosb16	dma
 #	audioac97	pci audioac97mix
diff --git a/sys/src/9/port/portmkfile b/sys/src/9/port/portmkfile
index bdf049e86..53a30e538 100644
--- a/sys/src/9/port/portmkfile
+++ b/sys/src/9/port/portmkfile
@@ -109,8 +109,9 @@ devaoe.$O sdaoe.$O:	/sys/include/fis.h
 sysproc.$O:	/sys/include/a.out.h
 syscallfmt.$O:	/sys/src/libc/9syscall/sys.h
 devusb.$O usbxhci.$O:	../port/usb.h
+usbxhci.$O usbxhcipci.$O:	../port/usbxhci.h
 devether.$O ethersink.$O:	../port/etherif.h ../port/netif.h
 wifi.$O:	../port/etherif.h ../port/netif.h ../port/wifi.h /sys/include/libsec.h
 wifi.$O:	../ip/ip.h ../ip/ipv6.h
 ethermii.$O:	../port/ethermii.h
-pci.$O devpnp.$O devpci.$O usbxhci.$O:	../port/pci.h
+pci.$O devpnp.$O devpci.$O usbxhcipci.$O:	../port/pci.h
diff --git a/sys/src/9/port/usbxhci.c b/sys/src/9/port/usbxhci.c
index 37a86e987..77fad1574 100644
--- a/sys/src/9/port/usbxhci.c
+++ b/sys/src/9/port/usbxhci.c
@@ -4,10 +4,11 @@
 #include	"dat.h"
 #include	"fns.h"
 #include	"io.h"
-#include	"../port/pci.h"
 #include	"../port/error.h"
 #include	"../port/usb.h"
 
+#include	"usbxhci.h"
+
 enum {
 	/* Capability Registers */
 	CAPLENGTH	= 0x00/4,	// 1
@@ -222,9 +223,7 @@ struct Port
 
 struct Ctlr
 {
-	Pcidev	*pcidev;
-
-	u32int	*mmio;
+	Xhci;
 
 	u32int	*opr;	/* operational registers */
 	u32int	*rts;	/* runtime registers */
@@ -255,8 +254,6 @@ struct Ctlr
 	int	nslots;
 
 	Rendez	recover;	
-	void	*active;
-	uvlong	base;
 };
 
 struct Epio
@@ -358,14 +355,14 @@ flushring(Ring *r)
 }
 
 static u64int
-resetring(Ring *r)
+resetring(Ctlr *ctlr, Ring *r)
 {
 	u64int pa;
 
 	ilock(r);
 	flushring(r);
 	r->rp = r->wp;
-	pa = PCIWADDR(&r->base[4*(r->wp & r->mask)]) | ((~r->wp>>r->shift) & 1);
+	pa = (*ctlr->dmaaddr)(&r->base[4*(r->wp & r->mask)]) | ((~r->wp>>r->shift) & 1);
 	iunlock(r);
 
 	return pa;
@@ -376,7 +373,7 @@ xecp(Ctlr *ctlr, uchar id, u32int *p)
 {
 	u32int x, *e;
 
-	e = &ctlr->mmio[ctlr->pcidev->mem[0].size/4];
+	e = &ctlr->mmio[ctlr->size/4];
 	if(p == nil){
 		p = ctlr->mmio;
 		x = ctlr->hccparams>>16;
@@ -417,8 +414,8 @@ handoff(Ctlr *ctlr)
 	r[0] &= ~(1<<16);
 }
 
-static void
-shutdown(Hci *hp)
+void
+xhcishutdown(Hci *hp)
 {
 	Ctlr *ctlr = hp->aux;
 	int i;
@@ -426,8 +423,7 @@ shutdown(Hci *hp)
 	ctlr->opr[USBCMD] = 0;
 	for(i=0; (ctlr->opr[USBSTS] & HCH) == 0 && i < 10; i++)
 		delay(10);
-	intrdisable(ctlr->pcidev->intl, hp->interrupt, hp, ctlr->pcidev->tbdf, hp->type);
-	pcidisable(ctlr->pcidev);
+	intrdisable(hp->irq, hp->interrupt, hp, hp->tbdf, hp->type);
 }
 
 static void
@@ -454,8 +450,8 @@ release(Ctlr *ctlr)
 
 static void recover(void *arg);
 
-static void
-init(Hci *hp)
+void
+xhciinit(Hci *hp)
 {
 	Ctlr *ctlr;
 	Port *pp;
@@ -464,12 +460,6 @@ init(Hci *hp)
 	int i, j;
 
 	ctlr = hp->aux;
-	pcienable(ctlr->pcidev);
-	if(ctlr->mmio[CAPLENGTH] == -1){
-		pcidisable(ctlr->pcidev);
-		error("controller vanished");
-	}
-
 	ctlr->opr = &ctlr->mmio[(ctlr->mmio[CAPLENGTH]&0xFF)/4];
 	ctlr->dba = &ctlr->mmio[ctlr->mmio[DBOFF]/4];
 	ctlr->rts = &ctlr->mmio[ctlr->mmio[RTSOFF]/4];
@@ -481,18 +471,21 @@ init(Hci *hp)
 		tsleep(&up->sleep, return0, nil, 10);
 
 	ctlr->opr[USBCMD] = HCRST;
+
 	/* some intel controllers require 1ms delay after reset */
-	delay(1);
+	tsleep(&up->sleep, return0, nil, 1);
+
 	for(i=0; (ctlr->opr[USBCMD] & HCRST) != 0 && i<100; i++)
 		tsleep(&up->sleep, return0, nil, 10);
 	for(i=0; (ctlr->opr[USBSTS] & (CNR|HCH)) != HCH && i<100; i++)
 		tsleep(&up->sleep, return0, nil, 10);
 
-	pcisetbme(ctlr->pcidev);
-	intrenable(ctlr->pcidev->intl, hp->interrupt, hp, ctlr->pcidev->tbdf, hp->type);
+	if(ctlr->dmaenable != nil)
+		(*ctlr->dmaenable)(ctlr);
+	intrenable(hp->irq, hp->interrupt, hp, hp->tbdf, hp->type);
 
 	if(waserror()){
-		shutdown(hp);
+		(*hp->shutdown)(hp);
 		release(ctlr);
 		nexterror();
 	}
@@ -540,11 +533,11 @@ init(Hci *hp)
 			error(Enomem);
 		for(i=0, p = ctlr->sbp; i<ctlr->nscratch; i++, p += ctlr->pagesize){
 			memset(p, 0, ctlr->pagesize);
-			ctlr->sba[i] = PCIWADDR(p);
+			ctlr->sba[i] = (*ctlr->dmaaddr)(p);
 		}
 		dmaflush(1, ctlr->sbp, ctlr->nscratch*ctlr->pagesize);
 		dmaflush(1, ctlr->sba, ctlr->nscratch*8);
-		ctlr->dcba[0] = PCIWADDR(ctlr->sba);
+		ctlr->dcba[0] = (*ctlr->dmaaddr)(ctlr->sba);
 	} else {
 		ctlr->dcba[0] = 0;
 	}
@@ -554,12 +547,12 @@ init(Hci *hp)
 	ctlr->opr[CONFIG] = (ctlr->opr[CONFIG] & 0xFFFFFC00) | ctlr->nslots;	/* MaxSlotsEn */
 
 	dmaflush(1, ctlr->dcba, (1+ctlr->nslots)*sizeof(ctlr->dcba[0]));
-	setrptr(&ctlr->opr[DCBAAP], PCIWADDR(ctlr->dcba));
+	setrptr(&ctlr->opr[DCBAAP], (*ctlr->dmaaddr)(ctlr->dcba));
 
 	initring(ctlr->cr, 8);		/* 256 entries */
 	ctlr->cr->id = 0;
 	ctlr->cr->doorbell = &ctlr->dba[0];
-	setrptr(&ctlr->opr[CRCR], resetring(ctlr->cr));
+	setrptr(&ctlr->opr[CRCR], resetring(ctlr, ctlr->cr));
 
 	for(i=0; i<ctlr->nintrs; i++){
 		u32int *irs = &ctlr->rts[IR0 + i*8];
@@ -578,7 +571,7 @@ init(Hci *hp)
 		ctlr->erst[i] = mallocalign(4*4, 64, 0, 0);
 		if(ctlr->erst[i] == nil)
 			error(Enomem);
-		*((u64int*)ctlr->erst[i]) = PCIWADDR(ctlr->er[i].base);
+		*((u64int*)ctlr->erst[i]) = (*ctlr->dmaaddr)(ctlr->er[i].base);
 		ctlr->erst[i][2] = ctlr->er[i].mask+1;
 		ctlr->erst[i][3] = 0;
 		dmaflush(1, ctlr->erst[i], 4*4);
@@ -586,8 +579,8 @@ init(Hci *hp)
 		irs[ERSTSZ] = 1;	/* just one segment */
 		irs[IMAN] = 3;
 		irs[IMOD] = 0;
-		setrptr(&irs[ERSTBA], PCIWADDR(ctlr->erst[i]));
-		setrptr(&irs[ERDP], PCIWADDR(ctlr->er[i].base) | (1<<3));
+		setrptr(&irs[ERSTBA], (*ctlr->dmaaddr)(ctlr->erst[i]));
+		setrptr(&irs[ERDP], (*ctlr->dmaaddr)(ctlr->er[i].base) | (1<<3));
 	}
 	poperror();
 
@@ -620,7 +613,7 @@ recover(void *arg)
 		;
 	while(!needrecover(ctlr))
 		tsleep(&ctlr->recover, needrecover, ctlr, 1000);
-	shutdown(hp);
+	(*hp->shutdown)(hp);
 
 	/*
 	 * flush all transactions and wait until all devices have
@@ -665,7 +658,7 @@ recover(void *arg)
 	if(waserror()) {
 		print("xhci recovery failed: %s\n", up->errstr);
 	} else {
-		init(hp);
+		(*hp->init)(hp);
 		poperror();
 	}
 
@@ -687,8 +680,9 @@ queuetd(Ring *r, u32int c, u32int s, u64int p, Wait *w)
 
 	x = r->wp++;
 	if((x & r->mask) == r->mask){
+		Ctlr *ctlr = r->slot->ctlr;
 		td = r->base + 4*(x & r->mask);
-		*(u64int*)td = PCIWADDR(r->base);
+		*(u64int*)td = (*ctlr->dmaaddr)(r->base);
 		td[2] = 0;
 		td[3] = ((~x>>r->shift)&1) | (1<<1) | TR_LINK;
 		dmaflush(1, td, 4*4);
@@ -833,7 +827,7 @@ ctlrcmd(Ctlr *ctlr, u32int c, u32int s, u64int p, u32int *er)
 }
 
 static void
-completering(Ring *r, u32int *er)
+completering(Ctlr *ctlr, Ring *r, u32int *er)
 {
 	Wait *w, **wp;
 	u32int *td, x;
@@ -841,10 +835,9 @@ completering(Ring *r, u32int *er)
 
 	pa = (*(u64int*)er) & ~15ULL;
 	ilock(r);
-
 	for(x = r->rp; (int)(r->wp - x) > 0; x++){
 		td = &r->base[4*(x & r->mask)];
-		if((u64int)PCIWADDR(td) == pa){
+		if((*ctlr->dmaaddr)(td) == pa){
 			if(r->residue != nil)
 				r->residue[x & r->mask] = er[2] & 0xFFFFFF;
 			r->rp = x+1;
@@ -854,7 +847,7 @@ completering(Ring *r, u32int *er)
 
 	wp = &r->pending;
 	while(w = *wp){
-		if((u64int)PCIWADDR(w->td) == pa){
+		if((*ctlr->dmaaddr)(w->td) == pa){
 			Rendez *z = w->z;
 
 			memmove(w->er, er, 4*4);
@@ -899,7 +892,7 @@ interrupt(Ureg*, void *arg)
 
 		switch(td[3] & 0xFC00){
 		case ER_CMDCOMPL:
-			completering(ctlr->cr, td);
+			completering(ctlr, ctlr->cr, td);
 			break;
 		case ER_TRANSFER:
 			x = td[3]>>24;
@@ -908,7 +901,7 @@ interrupt(Ureg*, void *arg)
 			slot = ctlr->slot[x];
 			if(slot == nil)
 				break;
-			completering(&slot->epr[(td[3]>>16)-1&31], td);
+			completering(ctlr, &slot->epr[(td[3]>>16)-1&31], td);
 			break;
 		case ER_MFINDEXWRAP:
 			ctlr->µframe = (ctlr->rts[MFINDEX] & (1<<14)-1) | 
@@ -931,7 +924,7 @@ interrupt(Ureg*, void *arg)
 		}
 	}
 
-	setrptr(&irs[ERDP], PCIWADDR(td) | (1<<3));
+	setrptr(&irs[ERDP], (*ctlr->dmaaddr)(td) | (1<<3));
 }
 
 static void
@@ -1003,7 +996,7 @@ allocslot(Ctlr *ctlr, Udev *dev)
 	poperror();
 
 	dmaflush(1, slot->obase, 32*32 << ctlr->csz);
-	ctlr->dcba[slot->id] = PCIWADDR(slot->obase);
+	ctlr->dcba[slot->id] = (*ctlr->dmaaddr)(slot->obase);
 	dmaflush(1, &ctlr->dcba[slot->id], sizeof(ctlr->dcba[0]));
 
 	ctlr->slot[slot->id] = slot;
@@ -1066,7 +1059,8 @@ epclose(Ep *ep)
 		memset(w, 0, 2*32<<ctlr->csz);
 
 		dmaflush(1, slot->ibase, 32*33 << ctlr->csz);
-		ctlrcmd(ctlr, CR_CONFIGEP | (slot->id<<24), 0, PCIWADDR(slot->ibase), nil);
+		ctlrcmd(ctlr, CR_CONFIGEP | (slot->id<<24), 0,
+			(*ctlr->dmaaddr)(slot->ibase), nil);
 		dmaflush(0, slot->obase, 32*32 << ctlr->csz);
 
 		freering(io[OREAD].ring);
@@ -1078,7 +1072,7 @@ epclose(Ep *ep)
 }
 
 static void
-initepctx(u32int *w, Ring *r, Ep *ep)
+initepctx(Ctlr *ctlr, u32int *w, Ring *r, Ep *ep)
 {
 	int ival;
 
@@ -1093,7 +1087,7 @@ initepctx(u32int *w, Ring *r, Ep *ep)
 	w[1] = ((ep->ttype-Tctl) | (r->id&1)<<2)<<3 | (ep->ntds-1)<<8 | ep->maxpkt<<16;
 	if(ep->ttype != Tiso)
 		w[1] |= 3<<1;
-	*((u64int*)&w[2]) = PCIWADDR(r->base) | 1;
+	*((u64int*)&w[2]) = (*ctlr->dmaaddr)(r->base) | 1;
 	w[4] = 2*ep->maxpkt;
 	if(ep->ttype == Tintr || ep->ttype == Tiso)
 		w[4] |= (ep->maxpkt*ep->ntds)<<16;
@@ -1180,17 +1174,18 @@ initep(Ep *ep)
 	w += (ep->nb&Epmax)*2*8<<ctlr->csz;
 	if(io[OWRITE].ring != nil){
 		memset(w, 0, 5*4);
-		initepctx(w, io[OWRITE].ring, ep);
+		initepctx(ctlr, w, io[OWRITE].ring, ep);
 	}
 
 	w += 8<<ctlr->csz;
 	if(io[OREAD].ring != nil){
 		memset(w, 0, 5*4);
-		initepctx(w, io[OREAD].ring, ep);
+		initepctx(ctlr, w, io[OREAD].ring, ep);
 	}
 
 	dmaflush(1, slot->ibase, 32*33 << ctlr->csz);
-	err = ctlrcmd(ctlr, CR_CONFIGEP | (slot->id<<24), 0, PCIWADDR(slot->ibase), nil);
+	err = ctlrcmd(ctlr, CR_CONFIGEP | (slot->id<<24), 0,
+		(*ctlr->dmaaddr)(slot->ibase), nil);
 	dmaflush(0, slot->obase, 32*32 << ctlr->csz);
 	if(err != nil)
 		error(err);
@@ -1302,10 +1297,11 @@ epopen(Ep *ep)
 
 	/* (input) ep context 0 */
 	w += 8<<ctlr->csz;
-	initepctx(w, io[OWRITE].ring, ep);
+	initepctx(ctlr, w, io[OWRITE].ring, ep);
 
 	dmaflush(1, slot->ibase, 32*33 << ctlr->csz);
-	err = ctlrcmd(ctlr, CR_ADDRESSDEV | (slot->id<<24), 0, PCIWADDR(slot->ibase), nil);
+	err = ctlrcmd(ctlr, CR_ADDRESSDEV | (slot->id<<24), 0,
+		(*ctlr->dmaaddr)(slot->ibase), nil);
 	dmaflush(0, slot->obase, 32*32 << ctlr->csz);
 	if(err != nil)
 		error(err);
@@ -1382,7 +1378,8 @@ isoread(Ep *ep, uchar *p, long n)
 		m = io->tdsz;
 		d = io->b->rp + (i&io->ring->mask)*io->tdsz;
 		dmaflush(1, d, m);
-		queuetd(io->ring, TR_ISOCH | (i*µ/8 & 0x7ff)<<20 | TR_IOC, m, PCIWADDR(d), nil);
+		queuetd(io->ring, TR_ISOCH | (i*µ/8 & 0x7ff)<<20 | TR_IOC, m,
+			(*ctlr->dmaaddr)(d), nil);
 	}
 	io->frame = i;
 
@@ -1439,7 +1436,8 @@ isowrite(Ep *ep, uchar *p, long n)
 		m += io->nleft, d -= io->nleft;
 		io->nleft = 0;
 		dmaflush(1, d, m);
-		queuetd(io->ring, TR_ISOCH | (i*µ/8 & 0x7ff)<<20 | TR_IOC, m, PCIWADDR(d), nil);
+		queuetd(io->ring, TR_ISOCH | (i*µ/8 & 0x7ff)<<20 | TR_IOC, m,
+			(*ctlr->dmaaddr)(d), nil);
 	}
 	io->frame = i;
 
@@ -1464,6 +1462,7 @@ isowrite(Ep *ep, uchar *p, long n)
 static char*
 unstall(Ep *ep, Ring *r)
 {
+	Ctlr *ctlr = r->slot->ctlr;
 	char *err;
 
 	switch(r->ctx[0]&7){
@@ -1473,15 +1472,16 @@ unstall(Ep *ep, Ring *r)
 	}
 	if(ep->clrhalt){
 		ep->clrhalt = 0;
-		err = ctlrcmd(r->slot->ctlr, CR_RESETEP | (r->id<<16) | (r->slot->id<<24), 0, 0, nil);
-		dmaflush(0, r->ctx, 8*4 << r->slot->ctlr->csz);
+		err = ctlrcmd(ctlr, CR_RESETEP | (r->id<<16) | (r->slot->id<<24), 0, 0, nil);
+		dmaflush(0, r->ctx, 8*4 << ctlr->csz);
 		if(err != nil)
 			return err;
 		r->stopped = 1;
 	}
 	if(r->stopped){
-		err = ctlrcmd(r->slot->ctlr, CR_SETTRDQP | (r->id<<16) | (r->slot->id<<24), 0, resetring(r), nil);
-		dmaflush(0, r->ctx, 8*4 << r->slot->ctlr->csz);
+		err = ctlrcmd(ctlr, CR_SETTRDQP | (r->id<<16) | (r->slot->id<<24), 0,
+			resetring(ctlr, r), nil);
+		dmaflush(0, r->ctx, 8*4 << ctlr->csz);
 		if(err != nil)
 			return err;
 		r->stopped = 0;
@@ -1548,7 +1548,7 @@ epread(Ep *ep, void *va, long n)
 		error(err);
 
 	dmaflush(1, p, n);
-	queuetd(io->ring, TR_NORMAL | TR_IOC, n, PCIWADDR(p), w);
+	queuetd(io->ring, TR_NORMAL | TR_IOC, n, (*ctlr->dmaaddr)(p), w);
 	err = waittd(ctlr, w, ep->tmout);
 	dmaflush(0, p, n);
 	if(err != nil)
@@ -1625,21 +1625,21 @@ epwrite(Ep *ep, void *va, long n)
 			w[0] = 0;
 			w[1] = 1<<ring->id;
 			w += (ring->id+1)*8<<ctlr->csz;
-			initepctx(w, ring, ep);
+			initepctx(ctlr, w, ring, ep);
 			dmaflush(1, slot->ibase, 32*33 << ctlr->csz);
-			err = ctlrcmd(ctlr, CR_EVALCTX | (slot->id<<24), 0, PCIWADDR(slot->ibase), nil);
+			err = ctlrcmd(ctlr, CR_EVALCTX | (slot->id<<24), 0,
+				(*ctlr->dmaaddr)(slot->ibase), nil);
 			dmaflush(0, slot->obase, 32*32 << ctlr->csz);
 			if(err != nil)
 				error(err);
 		}
 
 		queuetd(ring, TR_SETUPSTAGE | (len > 0 ? 2+dir : 0)<<16 | TR_IDT | TR_IOC, 8,
-			p[0] | p[1]<<8 | GET2(&p[2])<<16 |
-			(u64int)(GET2(&p[4]) | len<<16)<<32, &w[0]);
+			p[0] | p[1]<<8 | GET2(&p[2])<<16 | (u64int)(GET2(&p[4]) | len<<16)<<32, &w[0]);
 		if(len > 0){
 			dmaflush(1, io->b->rp, len);
 			queuetd(ring, TR_DATASTAGE | dir<<16 | TR_IOC, len,
-				PCIWADDR(io->b->rp), &w[1]);
+				(*ctlr->dmaaddr)(io->b->rp), &w[1]);
 		}
 		queuetd(ring, TR_STATUSSTAGE | (len == 0 || !dir)<<16 | TR_IOC, 0, 0, &w[2]);
 
@@ -1700,7 +1700,7 @@ epwrite(Ep *ep, void *va, long n)
 		error(err);
 
 	dmaflush(1, p, n);
-	queuetd(io->ring, TR_NORMAL | TR_IOC, n, PCIWADDR(p), w);
+	queuetd(io->ring, TR_NORMAL | TR_IOC, n, (*ctlr->dmaaddr)(p), w);
 	if((err = waittd(ctlr, w, ep->tmout)) != nil)
 		error(err);
 
@@ -1783,93 +1783,40 @@ portreset(Hci *hp, int port, int on)
 	return 0;
 }
 
-
-static Ctlr *ctlrs[Nhcis];
-
-static void
-scanpci(void)
+static u64int
+physaddr(void *va)
 {
-	static int already = 0;
-	int i;
-	uvlong io;
-	Ctlr *ctlr;
-	Pcidev *p;
-	u32int *mmio; 
-
-	if(already)
-		return;
-	already = 1;
-	p = nil;
-	while ((p = pcimatch(p, 0, 0)) != nil) {
-		/*
-		 * Find XHCI controllers (Programming Interface = 0x30).
-		 */
-		if(p->ccrb != Pcibcserial || p->ccru != Pciscusb || p->ccrp != 0x30)
-			continue;
-		if(p->mem[0].bar & 1)
-			continue;
-		io = p->mem[0].bar & ~0x0f;
-		if(io == 0)
-			continue;
-		print("usbxhci: %#x %#x: port %llux size %lld irq %d\n",
-			p->vid, p->did, io, p->mem[0].size, p->intl);
-		mmio = vmap(io, p->mem[0].size);
-		if(mmio == nil){
-			print("usbxhci: cannot map registers\n");
-			continue;
-		}
-		ctlr = malloc(sizeof(Ctlr));
-		if(ctlr == nil){
-			print("usbxhci: no memory\n");
-			vunmap(mmio, p->mem[0].size);
-			continue;
-		}
-		ctlr->base = io;
-		ctlr->active = nil;
-		ctlr->pcidev = p;
-		ctlr->mmio = mmio;
-		for(i = 0; i < nelem(ctlrs); i++)
-			if(ctlrs[i] == nil){
-				ctlrs[i] = ctlr;
-				break;
-			}
-		if(i >= nelem(ctlrs))
-			print("xhci: bug: more than %d controllers\n", nelem(ctlrs));
-	}
+	return PADDR(va);
 }
 
-static int
-reset(Hci *hp)
+Xhci*
+xhcialloc(u32int *mmio, u64int base, u64int size)
 {
 	Ctlr *ctlr;
-	int i;
 
-	if(getconf("*nousbxhci"))
-		return -1;
+	ctlr = malloc(sizeof(Ctlr));
+	if(ctlr == nil){
+		print("usbxhci: no memory for controller\n");
+		return nil;
+	}
+	ctlr->base = base;
+	ctlr->size = size;
+	ctlr->mmio = mmio;
 
-	scanpci();
+	ctlr->dmaaddr = physaddr;
 
-	/*
-	 * Any adapter matches if no hp->port is supplied,
-	 * otherwise the ports must match.
-	 */
-	for(i = 0; i < nelem(ctlrs) && ctlrs[i] != nil; i++){
-		ctlr = ctlrs[i];
-		if(ctlr->active == nil)
-		if(hp->port == 0 || hp->port == ctlr->base){
-			ctlr->active = hp;
-			goto Found;
-		}
-	}
-	return -1;
+	return ctlr;
+}
 
-Found:
-	hp->aux = ctlr;
+void
+xhcilinkage(Hci *hp, Xhci *ctlr)
+{
 	hp->port = ctlr->base;
-	hp->irq = ctlr->pcidev->intl;
-	hp->tbdf = ctlr->pcidev->tbdf;
+	hp->aux = ctlr;
+
+	hp->init = xhciinit;
+	hp->shutdown = xhcishutdown;
 
-	hp->init = init;
 	hp->dump = dump;
 	hp->interrupt = interrupt;
 	hp->epopen = epopen;
@@ -1880,15 +1827,14 @@ Found:
 	hp->portenable = portenable;
 	hp->portreset = portreset;
 	hp->portstatus = portstatus;
-	hp->shutdown = shutdown;
+
 	hp->debug = setdebug;
 	hp->type = "xhci";
 
-	return 0;
+	ctlr->active = hp;
 }
 
 void
 usbxhcilink(void)
 {
-	addhcitype("xhci", reset);
 }
diff --git a/sys/src/9/port/usbxhci.h b/sys/src/9/port/usbxhci.h
new file mode 100644
index 000000000..223140d97
--- /dev/null
+++ b/sys/src/9/port/usbxhci.h
@@ -0,0 +1,23 @@
+/*
+ *  I/O interface for usb XHCI controller.
+ */
+
+typedef struct Xhci Xhci;
+struct Xhci
+{
+	u32int	*mmio;
+	u64int	base;
+	u64int	size;
+
+	void	*aux;
+	void	(*dmaenable)(Xhci*);
+	u64int	(*dmaaddr)(void*);
+
+	Hci	*active;
+};
+
+Xhci* xhcialloc(u32int *mmio, u64int base, u64int size);
+void xhcilinkage(Hci *hp, Xhci *ctlr);
+
+void xhciinit(Hci *hp);
+void xhcishutdown(Hci *hp);
diff --git a/sys/src/9/port/usbxhcipci.c b/sys/src/9/port/usbxhcipci.c
new file mode 100644
index 000000000..f3e419817
--- /dev/null
+++ b/sys/src/9/port/usbxhcipci.c
@@ -0,0 +1,146 @@
+#include	"u.h"
+#include	"../port/lib.h"
+#include	"mem.h"
+#include	"dat.h"
+#include	"fns.h"
+#include	"io.h"
+#include	"../port/pci.h"
+#include	"../port/error.h"
+#include	"../port/usb.h"
+
+#include	"usbxhci.h"
+
+static Xhci *ctlrs[Nhcis];
+
+static void
+pcidmaenable(Xhci *ctlr)
+{
+	Pcidev *pcidev = ctlr->aux;
+	pcisetbme(pcidev);
+}
+
+static u64int
+pciwaddr(void *va)
+{
+	return PCIWADDR(va);
+}
+
+static void
+scanpci(void)
+{
+	static int already = 0;
+	int i;
+	u64int io, iosize;
+	Xhci *ctlr;
+	Pcidev *p;
+	u32int *mmio; 
+
+	if(already)
+		return;
+	already = 1;
+	p = nil;
+	while ((p = pcimatch(p, 0, 0)) != nil) {
+		/*
+		 * Find XHCI controllers (Programming Interface = 0x30).
+		 */
+		if(p->ccrb != Pcibcserial || p->ccru != Pciscusb || p->ccrp != 0x30)
+			continue;
+		if(p->mem[0].bar & 1)
+			continue;
+		iosize = p->mem[0].size;
+		if(iosize == 0)
+			continue;
+		io = p->mem[0].bar & ~0x0f;
+		if(io == 0)
+			continue;
+		print("usbxhci: %#x %#x: port %llux size %lld irq %d\n",
+			p->vid, p->did, io, iosize, p->intl);
+		mmio = vmap(io, iosize);
+		if(mmio == nil){
+			print("usbxhci: cannot map registers\n");
+			continue;
+		}
+		ctlr = xhcialloc(mmio, io, iosize);
+		if(ctlr == nil){
+			print("usbxhci: no memory\n");
+			vunmap(mmio, iosize);
+			continue;
+		}
+		ctlr->aux = p;
+		ctlr->dmaenable = pcidmaenable;
+		ctlr->dmaaddr = pciwaddr;
+
+		for(i = 0; i < nelem(ctlrs); i++)
+			if(ctlrs[i] == nil){
+				ctlrs[i] = ctlr;
+				break;
+			}
+		if(i >= nelem(ctlrs))
+			print("xhci: bug: more than %d controllers\n", nelem(ctlrs));
+	}
+}
+
+static void
+init(Hci *hp)
+{
+	Xhci *xhci = hp->aux;
+	Pcidev *pcidev = xhci->aux;
+
+	pcienable(pcidev);
+	if(xhci->mmio[0] == -1){
+		pcidisable(pcidev);
+		error("controller vanished");
+	}
+	xhciinit(hp);
+}
+
+static void
+shutdown(Hci *hp)
+{
+	Xhci *xhci = hp->aux;
+	Pcidev *pcidev = xhci->aux;
+
+	xhcishutdown(hp);
+	pcidisable(pcidev);
+}
+
+static int
+reset(Hci *hp)
+{
+	Xhci *ctlr;
+	Pcidev *pcidev;
+	int i;
+
+	if(getconf("*nousbxhci"))
+		return -1;
+
+	scanpci();
+
+	/*
+	 * Any adapter matches if no hp->port is supplied,
+	 * otherwise the ports must match.
+	 */
+	for(i = 0; i < nelem(ctlrs) && ctlrs[i] != nil; i++){
+		ctlr = ctlrs[i];
+		if(ctlr->active == nil)
+		if(hp->port == 0 || hp->port == ctlr->base)
+			goto Found;
+	}
+	return -1;
+
+Found:
+	pcidev = ctlr->aux;
+	xhcilinkage(hp, ctlr);
+	hp->init = init;
+	hp->shutdown = shutdown;
+	hp->irq = pcidev->intl;
+	hp->tbdf = pcidev->tbdf;
+
+	return 0;
+}
+
+void
+usbxhcipcilink(void)
+{
+	addhcitype("xhci", reset);
+}