sysadmin/ipxe
1
0
Fork 0
mirror of https://github.com/ipxe/ipxe synced 2025-12-07 01:40:28 +03:00
Code Issues Packages Projects Releases Wiki Activity
Files
32d20fdd7e09ba878f9409f8604e14d411c83abc
ipxe/src/drivers/usb/xhci.c
Michael Brown 32d20fdd7e [xhci] Delay after (possibly) forcing port link state to RxDetect 
Some xHCI controllers (observed with a Renesas Electronics PCIe USB3
card) seem to require a delay after forcing the link state of USB3
ports to RxDetect.  Omitting this delay causes strange behaviour
including system lockups.

Add an unconditional 20ms delay after writing the port link states.
This seems to be sufficient to avoid the problem.

Signed-off-by: Michael Brown <mcb30@ipxe.org>
2015-02-11 11:18:35 +00:00

3055 lines
78 KiB
C
Raw Blame History

/*
* Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 2 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
FILE_LICENCE ( GPL2_OR_LATER );
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <strings.h>
#include <errno.h>
#include <byteswap.h>
#include <ipxe/malloc.h>
#include <ipxe/umalloc.h>
#include <ipxe/pci.h>
#include <ipxe/usb.h>
#include <ipxe/profile.h>
#include "xhci.h"
/** @file
*
* USB eXtensible Host Controller Interface (xHCI) driver
*
*/
/** Message transfer profiler */
static struct profiler xhci_message_profiler __profiler =
{ .name = "xhci.message" };
/** Stream transfer profiler */
static struct profiler xhci_stream_profiler __profiler =
{ .name = "xhci.stream" };
/** Event ring profiler */
static struct profiler xhci_event_profiler __profiler =
{ .name = "xhci.event" };
/** Transfer event profiler */
static struct profiler xhci_transfer_profiler __profiler =
{ .name = "xhci.transfer" };
/* Disambiguate the various error causes */
#define EIO_DATA \
__einfo_error ( EINFO_EIO_DATA )
#define EINFO_EIO_DATA \
__einfo_uniqify ( EINFO_EIO, ( 2 - 0 ), \
"Data buffer error" )
#define EIO_BABBLE \
__einfo_error ( EINFO_EIO_BABBLE )
#define EINFO_EIO_BABBLE \
__einfo_uniqify ( EINFO_EIO, ( 3 - 0 ), \
"Babble detected" )
#define EIO_USB \
__einfo_error ( EINFO_EIO_USB )
#define EINFO_EIO_USB \
__einfo_uniqify ( EINFO_EIO, ( 4 - 0 ), \
"USB transaction error" )
#define EIO_TRB \
__einfo_error ( EINFO_EIO_TRB )
#define EINFO_EIO_TRB \
__einfo_uniqify ( EINFO_EIO, ( 5 - 0 ), \
"TRB error" )
#define EIO_STALL \
__einfo_error ( EINFO_EIO_STALL )
#define EINFO_EIO_STALL \
__einfo_uniqify ( EINFO_EIO, ( 6 - 0 ), \
"Stall error" )
#define EIO_RESOURCE \
__einfo_error ( EINFO_EIO_RESOURCE )
#define EINFO_EIO_RESOURCE \
__einfo_uniqify ( EINFO_EIO, ( 7 - 0 ), \
"Resource error" )
#define EIO_BANDWIDTH \
__einfo_error ( EINFO_EIO_BANDWIDTH )
#define EINFO_EIO_BANDWIDTH \
__einfo_uniqify ( EINFO_EIO, ( 8 - 0 ), \
"Bandwidth error" )
#define EIO_NO_SLOTS \
__einfo_error ( EINFO_EIO_NO_SLOTS )
#define EINFO_EIO_NO_SLOTS \
__einfo_uniqify ( EINFO_EIO, ( 9 - 0 ), \
"No slots available" )
#define EIO_STREAM_TYPE \
__einfo_error ( EINFO_EIO_STREAM_TYPE )
#define EINFO_EIO_STREAM_TYPE \
__einfo_uniqify ( EINFO_EIO, ( 10 - 0 ), \
"Invalid stream type" )
#define EIO_SLOT \
__einfo_error ( EINFO_EIO_SLOT )
#define EINFO_EIO_SLOT \
__einfo_uniqify ( EINFO_EIO, ( 11 - 0 ), \
"Slot not enabled" )
#define EIO_ENDPOINT \
__einfo_error ( EINFO_EIO_ENDPOINT )
#define EINFO_EIO_ENDPOINT \
__einfo_uniqify ( EINFO_EIO, ( 12 - 0 ), \
"Endpoint not enabled" )
#define EIO_SHORT \
__einfo_error ( EINFO_EIO_SHORT )
#define EINFO_EIO_SHORT \
__einfo_uniqify ( EINFO_EIO, ( 13 - 0 ), \
"Short packet" )
#define EIO_UNDERRUN \
__einfo_error ( EINFO_EIO_UNDERRUN )
#define EINFO_EIO_UNDERRUN \
__einfo_uniqify ( EINFO_EIO, ( 14 - 0 ), \
"Ring underrun" )
#define EIO_OVERRUN \
__einfo_error ( EINFO_EIO_OVERRUN )
#define EINFO_EIO_OVERRUN \
__einfo_uniqify ( EINFO_EIO, ( 15 - 0 ), \
"Ring overrun" )
#define EIO_VF_RING_FULL \
__einfo_error ( EINFO_EIO_VF_RING_FULL )
#define EINFO_EIO_VF_RING_FULL \
__einfo_uniqify ( EINFO_EIO, ( 16 - 0 ), \
"Virtual function event ring full" )
#define EIO_PARAMETER \
__einfo_error ( EINFO_EIO_PARAMETER )
#define EINFO_EIO_PARAMETER \
__einfo_uniqify ( EINFO_EIO, ( 17 - 0 ), \
"Parameter error" )
#define EIO_BANDWIDTH_OVERRUN \
__einfo_error ( EINFO_EIO_BANDWIDTH_OVERRUN )
#define EINFO_EIO_BANDWIDTH_OVERRUN \
__einfo_uniqify ( EINFO_EIO, ( 18 - 0 ), \
"Bandwidth overrun" )
#define EIO_CONTEXT \
__einfo_error ( EINFO_EIO_CONTEXT )
#define EINFO_EIO_CONTEXT \
__einfo_uniqify ( EINFO_EIO, ( 19 - 0 ), \
"Context state error" )
#define EIO_NO_PING \
__einfo_error ( EINFO_EIO_NO_PING )
#define EINFO_EIO_NO_PING \
__einfo_uniqify ( EINFO_EIO, ( 20 - 0 ), \
"No ping response" )
#define EIO_RING_FULL \
__einfo_error ( EINFO_EIO_RING_FULL )
#define EINFO_EIO_RING_FULL \
__einfo_uniqify ( EINFO_EIO, ( 21 - 0 ), \
"Event ring full" )
#define EIO_INCOMPATIBLE \
__einfo_error ( EINFO_EIO_INCOMPATIBLE )
#define EINFO_EIO_INCOMPATIBLE \
__einfo_uniqify ( EINFO_EIO, ( 22 - 0 ), \
"Incompatible device" )
#define EIO_MISSED \
__einfo_error ( EINFO_EIO_MISSED )
#define EINFO_EIO_MISSED \
__einfo_uniqify ( EINFO_EIO, ( 23 - 0 ), \
"Missed service error" )
#define EIO_CMD_STOPPED \
__einfo_error ( EINFO_EIO_CMD_STOPPED )
#define EINFO_EIO_CMD_STOPPED \
__einfo_uniqify ( EINFO_EIO, ( 24 - 0 ), \
"Command ring stopped" )
#define EIO_CMD_ABORTED \
__einfo_error ( EINFO_EIO_CMD_ABORTED )
#define EINFO_EIO_CMD_ABORTED \
__einfo_uniqify ( EINFO_EIO, ( 25 - 0 ), \
"Command aborted" )
#define EIO_STOP \
__einfo_error ( EINFO_EIO_STOP )
#define EINFO_EIO_STOP \
__einfo_uniqify ( EINFO_EIO, ( 26 - 0 ), \
"Stopped" )
#define EIO_STOP_LEN \
__einfo_error ( EINFO_EIO_STOP_LEN )
#define EINFO_EIO_STOP_LEN \
__einfo_uniqify ( EINFO_EIO, ( 27 - 0 ), \
"Stopped - length invalid" )
#define EIO_STOP_SHORT \
__einfo_error ( EINFO_EIO_STOP_SHORT )
#define EINFO_EIO_STOP_SHORT \
__einfo_uniqify ( EINFO_EIO, ( 28 - 0 ), \
"Stopped - short packet" )
#define EIO_LATENCY \
__einfo_error ( EINFO_EIO_LATENCY )
#define EINFO_EIO_LATENCY \
__einfo_uniqify ( EINFO_EIO, ( 29 - 0 ), \
"Maximum exit latency too large" )
#define EIO_ISOCH \
__einfo_error ( EINFO_EIO_ISOCH )
#define EINFO_EIO_ISOCH \
__einfo_uniqify ( EINFO_EIO, ( 31 - 0 ), \
"Isochronous buffer overrun" )
#define EPROTO_LOST \
__einfo_error ( EINFO_EPROTO_LOST )
#define EINFO_EPROTO_LOST \
__einfo_uniqify ( EINFO_EPROTO, ( 32 - 32 ), \
"Event lost" )
#define EPROTO_UNDEFINED \
__einfo_error ( EINFO_EPROTO_UNDEFINED )
#define EINFO_EPROTO_UNDEFINED \
__einfo_uniqify ( EINFO_EPROTO, ( 33 - 32 ), \
"Undefined error" )
#define EPROTO_STREAM_ID \
__einfo_error ( EINFO_EPROTO_STREAM_ID )
#define EINFO_EPROTO_STREAM_ID \
__einfo_uniqify ( EINFO_EPROTO, ( 34 - 32 ), \
"Invalid stream ID" )
#define EPROTO_SECONDARY \
__einfo_error ( EINFO_EPROTO_SECONDARY )
#define EINFO_EPROTO_SECONDARY \
__einfo_uniqify ( EINFO_EPROTO, ( 35 - 32 ), \
"Secondary bandwidth error" )
#define EPROTO_SPLIT \
__einfo_error ( EINFO_EPROTO_SPLIT )
#define EINFO_EPROTO_SPLIT \
__einfo_uniqify ( EINFO_EPROTO, ( 36 - 32 ), \
"Split transaction error" )
#define ECODE(code) \
( ( (code) < 32 ) ? \
EUNIQ ( EINFO_EIO, ( (code) & 31 ), EIO_DATA, EIO_BABBLE, \
EIO_USB, EIO_TRB, EIO_STALL, EIO_RESOURCE, \
EIO_BANDWIDTH, EIO_NO_SLOTS, EIO_STREAM_TYPE, \
EIO_SLOT, EIO_ENDPOINT, EIO_SHORT, EIO_UNDERRUN, \
EIO_OVERRUN, EIO_VF_RING_FULL, EIO_PARAMETER, \
EIO_BANDWIDTH_OVERRUN, EIO_CONTEXT, EIO_NO_PING, \
EIO_RING_FULL, EIO_INCOMPATIBLE, EIO_MISSED, \
EIO_CMD_STOPPED, EIO_CMD_ABORTED, EIO_STOP, \
EIO_STOP_LEN, EIO_STOP_SHORT, EIO_LATENCY, \
EIO_ISOCH ) : \
( (code) < 64 ) ? \
EUNIQ ( EINFO_EPROTO, ( (code) & 31 ), EPROTO_LOST, \
EPROTO_UNDEFINED, EPROTO_STREAM_ID, \
EPROTO_SECONDARY, EPROTO_SPLIT ) : \
EFAULT )
/******************************************************************************
*
* Register access
*
******************************************************************************
*/
/**
* Initialise device
*
* @v xhci xHCI device
* @v regs MMIO registers
*/
static void xhci_init ( struct xhci_device *xhci, void *regs ) {
uint32_t hcsparams1;
uint32_t hcsparams2;
uint32_t hccparams1;
uint32_t pagesize;
size_t caplength;
size_t rtsoff;
size_t dboff;
/* Locate capability, operational, runtime, and doorbell registers */
xhci->cap = regs;
caplength = readb ( xhci->cap + XHCI_CAP_CAPLENGTH );
rtsoff = readl ( xhci->cap + XHCI_CAP_RTSOFF );
dboff = readl ( xhci->cap + XHCI_CAP_DBOFF );
xhci->op = ( xhci->cap + caplength );
xhci->run = ( xhci->cap + rtsoff );
xhci->db = ( xhci->cap + dboff );
DBGC2 ( xhci, "XHCI %p cap %08lx op %08lx run %08lx db %08lx\n",
xhci, virt_to_phys ( xhci->cap ), virt_to_phys ( xhci->op ),
virt_to_phys ( xhci->run ), virt_to_phys ( xhci->db ) );
/* Read structural parameters 1 */
hcsparams1 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS1 );
xhci->slots = XHCI_HCSPARAMS1_SLOTS ( hcsparams1 );
xhci->intrs = XHCI_HCSPARAMS1_INTRS ( hcsparams1 );
xhci->ports = XHCI_HCSPARAMS1_PORTS ( hcsparams1 );
DBGC ( xhci, "XHCI %p has %d slots %d intrs %d ports\n",
xhci, xhci->slots, xhci->intrs, xhci->ports );
/* Read structural parameters 2 */
hcsparams2 = readl ( xhci->cap + XHCI_CAP_HCSPARAMS2 );
xhci->scratchpads = XHCI_HCSPARAMS2_SCRATCHPADS ( hcsparams2 );
DBGC2 ( xhci, "XHCI %p needs %d scratchpads\n",
xhci, xhci->scratchpads );
/* Read capability parameters 1 */
hccparams1 = readl ( xhci->cap + XHCI_CAP_HCCPARAMS1 );
xhci->addr64 = XHCI_HCCPARAMS1_ADDR64 ( hccparams1 );
xhci->csz_shift = XHCI_HCCPARAMS1_CSZ_SHIFT ( hccparams1 );
xhci->xecp = XHCI_HCCPARAMS1_XECP ( hccparams1 );
/* Read page size */
pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
xhci->pagesize = XHCI_PAGESIZE ( pagesize );
assert ( xhci->pagesize != 0 );
assert ( ( ( xhci->pagesize ) & ( xhci->pagesize - 1 ) ) == 0 );
DBGC2 ( xhci, "XHCI %p page size %zd bytes\n",
xhci, xhci->pagesize );
}
/**
* Find extended capability
*
* @v xhci xHCI device
* @v id Capability ID
* @v offset Offset to previous extended capability instance, or zero
* @ret offset Offset to extended capability, or zero if not found
*/
static unsigned int xhci_extended_capability ( struct xhci_device *xhci,
unsigned int id,
unsigned int offset ) {
uint32_t xecp;
unsigned int next;
/* Locate the extended capability */
while ( 1 ) {
/* Locate first or next capability as applicable */
if ( offset ) {
xecp = readl ( xhci->cap + offset );
next = XHCI_XECP_NEXT ( xecp );
} else {
next = xhci->xecp;
}
if ( ! next )
return 0;
offset += next;
/* Check if this is the requested capability */
xecp = readl ( xhci->cap + offset );
if ( XHCI_XECP_ID ( xecp ) == id )
return offset;
}
}
/**
* Write potentially 64-bit register
*
* @v xhci xHCI device
* @v value Value
* @v reg Register address
* @ret rc Return status code
*/
static inline __attribute__ (( always_inline )) int
xhci_writeq ( struct xhci_device *xhci, physaddr_t value, void *reg ) {
/* If this is a 32-bit build, then this can never fail
* (allowing the compiler to optimise out the error path).
*/
if ( sizeof ( value ) <= sizeof ( uint32_t ) ) {
writel ( value, reg );
writel ( 0, ( reg + sizeof ( uint32_t ) ) );
return 0;
}
/* If the device does not support 64-bit addresses and this
* address is outside the 32-bit address space, then fail.
*/
if ( ( value & ~0xffffffffULL ) && ! xhci->addr64 ) {
DBGC ( xhci, "XHCI %p cannot access address %lx\n",
xhci, value );
return -ENOTSUP;
}
/* If this is a 64-bit build, then writeq() is available */
writeq ( value, reg );
return 0;
}
/**
* Calculate buffer alignment
*
* @v len Length
* @ret align Buffer alignment
*
* Determine alignment required for a buffer which must be aligned to
* at least XHCI_MIN_ALIGN and which must not cross a page boundary.
*/
static inline size_t xhci_align ( size_t len ) {
size_t align;
/* Align to own length (rounded up to a power of two) */
align = ( 1 << fls ( len - 1 ) );
/* Round up to XHCI_MIN_ALIGN if needed */
if ( align < XHCI_MIN_ALIGN )
align = XHCI_MIN_ALIGN;
return align;
}
/**
* Calculate device context offset
*
* @v xhci xHCI device
* @v ctx Context index
*/
static inline size_t xhci_device_context_offset ( struct xhci_device *xhci,
unsigned int ctx ) {
return ( XHCI_DCI ( ctx ) << xhci->csz_shift );
}
/**
* Calculate input context offset
*
* @v xhci xHCI device
* @v ctx Context index
*/
static inline size_t xhci_input_context_offset ( struct xhci_device *xhci,
unsigned int ctx ) {
return ( XHCI_ICI ( ctx ) << xhci->csz_shift );
}
/******************************************************************************
*
* Diagnostics
*
******************************************************************************
*/
/**
* Dump host controller registers
*
* @v xhci xHCI device
*/
static inline void xhci_dump ( struct xhci_device *xhci ) {
uint32_t usbcmd;
uint32_t usbsts;
uint32_t pagesize;
uint32_t dnctrl;
uint32_t config;
/* Do nothing unless debugging is enabled */
if ( ! DBG_LOG )
return;
/* Dump USBCMD */
usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
DBGC ( xhci, "XHCI %p USBCMD %08x%s%s\n", xhci, usbcmd,
( ( usbcmd & XHCI_USBCMD_RUN ) ? " run" : "" ),
( ( usbcmd & XHCI_USBCMD_HCRST ) ? " hcrst" : "" ) );
/* Dump USBSTS */
usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
DBGC ( xhci, "XHCI %p USBSTS %08x%s\n", xhci, usbsts,
( ( usbsts & XHCI_USBSTS_HCH ) ? " hch" : "" ) );
/* Dump PAGESIZE */
pagesize = readl ( xhci->op + XHCI_OP_PAGESIZE );
DBGC ( xhci, "XHCI %p PAGESIZE %08x\n", xhci, pagesize );
/* Dump DNCTRL */
dnctrl = readl ( xhci->op + XHCI_OP_DNCTRL );
DBGC ( xhci, "XHCI %p DNCTRL %08x\n", xhci, dnctrl );
/* Dump CONFIG */
config = readl ( xhci->op + XHCI_OP_CONFIG );
DBGC ( xhci, "XHCI %p CONFIG %08x\n", xhci, config );
}
/**
* Dump port registers
*
* @v xhci xHCI device
* @v port Port number
*/
static inline void xhci_dump_port ( struct xhci_device *xhci,
unsigned int port ) {
uint32_t portsc;
uint32_t portpmsc;
uint32_t portli;
uint32_t porthlpmc;
/* Do nothing unless debugging is enabled */
if ( ! DBG_LOG )
return;
/* Dump PORTSC */
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port ) );
DBGC ( xhci, "XHCI %p port %d PORTSC %08x%s%s%s%s psiv=%d\n",
xhci, port, portsc,
( ( portsc & XHCI_PORTSC_CCS ) ? " ccs" : "" ),
( ( portsc & XHCI_PORTSC_PED ) ? " ped" : "" ),
( ( portsc & XHCI_PORTSC_PR ) ? " pr" : "" ),
( ( portsc & XHCI_PORTSC_PP ) ? " pp" : "" ),
XHCI_PORTSC_PSIV ( portsc ) );
/* Dump PORTPMSC */
portpmsc = readl ( xhci->op + XHCI_OP_PORTPMSC ( port ) );
DBGC ( xhci, "XHCI %p port %d PORTPMSC %08x\n", xhci, port, portpmsc );
/* Dump PORTLI */
portli = readl ( xhci->op + XHCI_OP_PORTLI ( port ) );
DBGC ( xhci, "XHCI %p port %d PORTLI %08x\n", xhci, port, portli );
/* Dump PORTHLPMC */
porthlpmc = readl ( xhci->op + XHCI_OP_PORTHLPMC ( port ) );
DBGC ( xhci, "XHCI %p port %d PORTHLPMC %08x\n",
xhci, port, porthlpmc );
}
/******************************************************************************
*
* USB legacy support
*
******************************************************************************
*/
/**
* Initialise USB legacy support
*
* @v xhci xHCI device
*/
static void xhci_legacy_init ( struct xhci_device *xhci ) {
unsigned int legacy;
uint8_t bios;
/* Locate USB legacy support capability (if present) */
legacy = xhci_extended_capability ( xhci, XHCI_XECP_ID_LEGACY, 0 );
if ( ! legacy ) {
/* Not an error; capability may not be present */
DBGC ( xhci, "XHCI %p has no USB legacy support capability\n",
xhci );
return;
}
/* Check if legacy USB support is enabled */
bios = readb ( xhci->cap + legacy + XHCI_USBLEGSUP_BIOS );
if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
/* Not an error; already owned by OS */
DBGC ( xhci, "XHCI %p USB legacy support already disabled\n",
xhci );
return;
}
/* Record presence of USB legacy support capability */
xhci->legacy = legacy;
}
/**
* Claim ownership from BIOS
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static int xhci_legacy_claim ( struct xhci_device *xhci ) {
uint32_t ctlsts;
uint8_t bios;
unsigned int i;
/* Do nothing unless legacy support capability is present */
if ( ! xhci->legacy )
return 0;
/* Claim ownership */
writeb ( XHCI_USBLEGSUP_OS_OWNED,
xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
/* Wait for BIOS to release ownership */
for ( i = 0 ; i < XHCI_USBLEGSUP_MAX_WAIT_MS ; i++ ) {
/* Check if BIOS has released ownership */
bios = readb ( xhci->cap + xhci->legacy + XHCI_USBLEGSUP_BIOS );
if ( ! ( bios & XHCI_USBLEGSUP_BIOS_OWNED ) ) {
DBGC ( xhci, "XHCI %p claimed ownership from BIOS\n",
xhci );
ctlsts = readl ( xhci->cap + xhci->legacy +
XHCI_USBLEGSUP_CTLSTS );
if ( ctlsts ) {
DBGC ( xhci, "XHCI %p warning: BIOS retained "
"SMIs: %08x\n", xhci, ctlsts );
}
return 0;
}
/* Delay */
mdelay ( 1 );
}
DBGC ( xhci, "XHCI %p timed out waiting for BIOS to release "
"ownership\n", xhci );
return -ETIMEDOUT;
}
/**
* Release ownership back to BIOS
*
* @v xhci xHCI device
*/
static void xhci_legacy_release ( struct xhci_device *xhci ) {
/* Do nothing unless legacy support capability is present */
if ( ! xhci->legacy )
return;
/* Release ownership */
writeb ( 0, xhci->cap + xhci->legacy + XHCI_USBLEGSUP_OS );
DBGC ( xhci, "XHCI %p released ownership to BIOS\n", xhci );
}
/******************************************************************************
*
* Supported protocols
*
******************************************************************************
*/
/**
* Transcribe port speed (for debugging)
*
* @v psi Protocol speed ID
* @ret speed Transcribed speed
*/
static inline const char * xhci_speed_name ( uint32_t psi ) {
static const char *exponents[4] = { "", "k", "M", "G" };
static char buf[ 10 /* "xxxxxXbps" + NUL */ ];
unsigned int mantissa;
unsigned int exponent;
/* Extract mantissa and exponent */
mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
/* Transcribe speed */
snprintf ( buf, sizeof ( buf ), "%d%sbps",
mantissa, exponents[exponent] );
return buf;
}
/**
* Find supported protocol extended capability for a port
*
* @v xhci xHCI device
* @v port Port number
* @ret supported Offset to extended capability, or zero if not found
*/
static unsigned int xhci_supported_protocol ( struct xhci_device *xhci,
unsigned int port ) {
unsigned int supported = 0;
unsigned int offset;
unsigned int count;
uint32_t ports;
/* Iterate over all supported protocol structures */
while ( ( supported = xhci_extended_capability ( xhci,
XHCI_XECP_ID_SUPPORTED,
supported ) ) ) {
/* Determine port range */
ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
offset = XHCI_SUPPORTED_PORTS_OFFSET ( ports );
count = XHCI_SUPPORTED_PORTS_COUNT ( ports );
/* Check if port lies within this range */
if ( ( port - offset ) < count )
return supported;
}
DBGC ( xhci, "XHCI %p port %d has no supported protocol\n",
xhci, port );
return 0;
}
/**
* Find port protocol
*
* @v xhci xHCI device
* @v port Port number
* @ret protocol USB protocol, or zero if not found
*/
static unsigned int xhci_port_protocol ( struct xhci_device *xhci,
unsigned int port ) {
unsigned int supported = xhci_supported_protocol ( xhci, port );
union {
uint32_t raw;
char text[5];
} name;
unsigned int protocol;
unsigned int type;
unsigned int psic;
unsigned int psiv;
unsigned int i;
uint32_t revision;
uint32_t ports;
uint32_t slot;
uint32_t psi;
/* Fail if there is no supported protocol */
if ( ! supported )
return 0;
/* Determine protocol version */
revision = readl ( xhci->cap + supported + XHCI_SUPPORTED_REVISION );
protocol = XHCI_SUPPORTED_REVISION_VER ( revision );
/* Describe port protocol */
if ( DBG_EXTRA ) {
name.raw = cpu_to_le32 ( readl ( xhci->cap + supported +
XHCI_SUPPORTED_NAME ) );
name.text[4] = '\0';
slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
DBGC2 ( xhci, "XHCI %p port %d %sv%04x type %d",
xhci, port, name.text, protocol, type );
ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
if ( psic ) {
DBGC2 ( xhci, " speeds" );
for ( i = 0 ; i < psic ; i++ ) {
psi = readl ( xhci->cap + supported +
XHCI_SUPPORTED_PSI ( i ) );
psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
DBGC2 ( xhci, " %d:%s", psiv,
xhci_speed_name ( psi ) );
}
}
DBGC2 ( xhci, "\n" );
}
return protocol;
}
/**
* Find port slot type
*
* @v xhci xHCI device
* @v port Port number
* @ret type Slot type, or negative error
*/
static int xhci_port_slot_type ( struct xhci_device *xhci, unsigned int port ) {
unsigned int supported = xhci_supported_protocol ( xhci, port );
unsigned int type;
uint32_t slot;
/* Fail if there is no supported protocol */
if ( ! supported )
return -ENOTSUP;
/* Get slot type */
slot = readl ( xhci->cap + supported + XHCI_SUPPORTED_SLOT );
type = XHCI_SUPPORTED_SLOT_TYPE ( slot );
return type;
}
/**
* Find port speed
*
* @v xhci xHCI device
* @v port Port number
* @v psiv Protocol speed ID value
* @ret speed Port speed, or negative error
*/
static int xhci_port_speed ( struct xhci_device *xhci, unsigned int port,
unsigned int psiv ) {
unsigned int supported = xhci_supported_protocol ( xhci, port );
unsigned int psic;
unsigned int mantissa;
unsigned int exponent;
unsigned int speed;
unsigned int i;
uint32_t ports;
uint32_t psi;
/* Fail if there is no supported protocol */
if ( ! supported )
return -ENOTSUP;
/* Get protocol speed ID count */
ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
/* Use the default mappings if applicable */
if ( ! psic ) {
switch ( psiv ) {
case XHCI_SPEED_LOW : return USB_SPEED_LOW;
case XHCI_SPEED_FULL : return USB_SPEED_FULL;
case XHCI_SPEED_HIGH : return USB_SPEED_HIGH;
case XHCI_SPEED_SUPER : return USB_SPEED_SUPER;
default:
DBGC ( xhci, "XHCI %p port %d non-standard PSI value "
"%d\n", xhci, port, psiv );
return -ENOTSUP;
}
}
/* Iterate over PSI dwords looking for a match */
for ( i = 0 ; i < psic ; i++ ) {
psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i ));
if ( psiv == XHCI_SUPPORTED_PSI_VALUE ( psi ) ) {
mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
speed = USB_SPEED ( mantissa, exponent );
return speed;
}
}
DBGC ( xhci, "XHCI %p port %d spurious PSI value %d\n",
xhci, port, psiv );
return -ENOENT;
}
/**
* Find protocol speed ID value
*
* @v xhci xHCI device
* @v port Port number
* @v speed USB speed
* @ret psiv Protocol speed ID value, or negative error
*/
static int xhci_port_psiv ( struct xhci_device *xhci, unsigned int port,
unsigned int speed ) {
unsigned int supported = xhci_supported_protocol ( xhci, port );
unsigned int psic;
unsigned int mantissa;
unsigned int exponent;
unsigned int psiv;
unsigned int i;
uint32_t ports;
uint32_t psi;
/* Fail if there is no supported protocol */
if ( ! supported )
return -ENOTSUP;
/* Get protocol speed ID count */
ports = readl ( xhci->cap + supported + XHCI_SUPPORTED_PORTS );
psic = XHCI_SUPPORTED_PORTS_PSIC ( ports );
/* Use the default mappings if applicable */
if ( ! psic ) {
switch ( speed ) {
case USB_SPEED_LOW : return XHCI_SPEED_LOW;
case USB_SPEED_FULL : return XHCI_SPEED_FULL;
case USB_SPEED_HIGH : return XHCI_SPEED_HIGH;
case USB_SPEED_SUPER : return XHCI_SPEED_SUPER;
default:
DBGC ( xhci, "XHCI %p port %d non-standad speed %d\n",
xhci, port, speed );
return -ENOTSUP;
}
}
/* Iterate over PSI dwords looking for a match */
for ( i = 0 ; i < psic ; i++ ) {
psi = readl ( xhci->cap + supported + XHCI_SUPPORTED_PSI ( i ));
mantissa = XHCI_SUPPORTED_PSI_MANTISSA ( psi );
exponent = XHCI_SUPPORTED_PSI_EXPONENT ( psi );
if ( speed == USB_SPEED ( mantissa, exponent ) ) {
psiv = XHCI_SUPPORTED_PSI_VALUE ( psi );
return psiv;
}
}
DBGC ( xhci, "XHCI %p port %d unrepresentable speed %#x\n",
xhci, port, speed );
return -ENOENT;
}
/******************************************************************************
*
* Device context base address array
*
******************************************************************************
*/
/**
* Allocate device context base address array
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static int xhci_dcbaa_alloc ( struct xhci_device *xhci ) {
size_t len;
physaddr_t dcbaap;
int rc;
/* Allocate and initialise structure. Must be at least
* 64-byte aligned and must not cross a page boundary, so
* align on its own size (rounded up to a power of two and
* with a minimum of 64 bytes).
*/
len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) );
xhci->dcbaa = malloc_dma ( len, xhci_align ( len ) );
if ( ! xhci->dcbaa ) {
DBGC ( xhci, "XHCI %p could not allocate DCBAA\n", xhci );
rc = -ENOMEM;
goto err_alloc;
}
memset ( xhci->dcbaa, 0, len );
/* Program DCBAA pointer */
dcbaap = virt_to_phys ( xhci->dcbaa );
if ( ( rc = xhci_writeq ( xhci, dcbaap,
xhci->op + XHCI_OP_DCBAAP ) ) != 0 )
goto err_writeq;
DBGC2 ( xhci, "XHCI %p DCBAA at [%08lx,%08lx)\n",
xhci, dcbaap, ( dcbaap + len ) );
return 0;
err_writeq:
free_dma ( xhci->dcbaa, len );
err_alloc:
return rc;
}
/**
* Free device context base address array
*
* @v xhci xHCI device
*/
static void xhci_dcbaa_free ( struct xhci_device *xhci ) {
size_t len;
unsigned int i;
/* Sanity check */
for ( i = 0 ; i <= xhci->slots ; i++ )
assert ( xhci->dcbaa[i] == 0 );
/* Clear DCBAA pointer */
xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_DCBAAP );
/* Free DCBAA */
len = ( ( xhci->slots + 1 ) * sizeof ( xhci->dcbaa[0] ) );
free_dma ( xhci->dcbaa, len );
}
/******************************************************************************
*
* Scratchpad buffers
*
******************************************************************************
*/
/**
* Allocate scratchpad buffers
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static int xhci_scratchpad_alloc ( struct xhci_device *xhci ) {
size_t array_len;
size_t len;
physaddr_t phys;
unsigned int i;
int rc;
/* Do nothing if no scratchpad buffers are used */
if ( ! xhci->scratchpads )
return 0;
/* Allocate scratchpads */
len = ( xhci->scratchpads * xhci->pagesize );
xhci->scratchpad = umalloc ( len );
if ( ! xhci->scratchpad ) {
DBGC ( xhci, "XHCI %p could not allocate scratchpad buffers\n",
xhci );
rc = -ENOMEM;
goto err_alloc;
}
memset_user ( xhci->scratchpad, 0, 0, len );
/* Allocate scratchpad array */
array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] ));
xhci->scratchpad_array =
malloc_dma ( array_len, xhci_align ( array_len ) );
if ( ! xhci->scratchpad_array ) {
DBGC ( xhci, "XHCI %p could not allocate scratchpad buffer "
"array\n", xhci );
rc = -ENOMEM;
goto err_alloc_array;
}
/* Populate scratchpad array */
for ( i = 0 ; i < xhci->scratchpads ; i++ ) {
phys = user_to_phys ( xhci->scratchpad, ( i * xhci->pagesize ));
xhci->scratchpad_array[i] = phys;
}
/* Set scratchpad array pointer */
assert ( xhci->dcbaa != NULL );
xhci->dcbaa[0] = cpu_to_le64 ( virt_to_phys ( xhci->scratchpad_array ));
DBGC2 ( xhci, "XHCI %p scratchpad [%08lx,%08lx) array [%08lx,%08lx)\n",
xhci, user_to_phys ( xhci->scratchpad, 0 ),
user_to_phys ( xhci->scratchpad, len ),
virt_to_phys ( xhci->scratchpad_array ),
( virt_to_phys ( xhci->scratchpad_array ) + array_len ) );
return 0;
free_dma ( xhci->scratchpad_array, array_len );
err_alloc_array:
ufree ( xhci->scratchpad );
err_alloc:
return rc;
}
/**
* Free scratchpad buffers
*
* @v xhci xHCI device
*/
static void xhci_scratchpad_free ( struct xhci_device *xhci ) {
size_t array_len;
/* Do nothing if no scratchpad buffers are used */
if ( ! xhci->scratchpads )
return;
/* Clear scratchpad array pointer */
assert ( xhci->dcbaa != NULL );
xhci->dcbaa[0] = 0;
/* Free scratchpad array */
array_len = ( xhci->scratchpads * sizeof ( xhci->scratchpad_array[0] ));
free_dma ( xhci->scratchpad_array, array_len );
/* Free scratchpads */
ufree ( xhci->scratchpad );
}
/******************************************************************************
*
* Run / stop / reset
*
******************************************************************************
*/
/**
* Start xHCI device
*
* @v xhci xHCI device
*/
static void xhci_run ( struct xhci_device *xhci ) {
uint32_t config;
uint32_t usbcmd;
/* Configure number of device slots */
config = readl ( xhci->op + XHCI_OP_CONFIG );
config &= ~XHCI_CONFIG_MAX_SLOTS_EN_MASK;
config |= XHCI_CONFIG_MAX_SLOTS_EN ( xhci->slots );
writel ( config, xhci->op + XHCI_OP_CONFIG );
/* Set run/stop bit */
usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
usbcmd |= XHCI_USBCMD_RUN;
writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
}
/**
* Stop xHCI device
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static int xhci_stop ( struct xhci_device *xhci ) {
uint32_t usbcmd;
uint32_t usbsts;
unsigned int i;
/* Clear run/stop bit */
usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
usbcmd &= ~XHCI_USBCMD_RUN;
writel ( usbcmd, xhci->op + XHCI_OP_USBCMD );
/* Wait for device to stop */
for ( i = 0 ; i < XHCI_STOP_MAX_WAIT_MS ; i++ ) {
/* Check if device is stopped */
usbsts = readl ( xhci->op + XHCI_OP_USBSTS );
if ( usbsts & XHCI_USBSTS_HCH )
return 0;
/* Delay */
mdelay ( 1 );
}
DBGC ( xhci, "XHCI %p timed out waiting for stop\n", xhci );
return -ETIMEDOUT;
}
/**
* Reset xHCI device
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static int xhci_reset ( struct xhci_device *xhci ) {
uint32_t usbcmd;
unsigned int i;
int rc;
/* The xHCI specification states that resetting a running
* device may result in undefined behaviour, so try stopping
* it first.
*/
if ( ( rc = xhci_stop ( xhci ) ) != 0 ) {
/* Ignore errors and attempt to reset the device anyway */
}
/* Reset device */
writel ( XHCI_USBCMD_HCRST, xhci->op + XHCI_OP_USBCMD );
/* Wait for reset to complete */
for ( i = 0 ; i < XHCI_RESET_MAX_WAIT_MS ; i++ ) {
/* Check if reset is complete */
usbcmd = readl ( xhci->op + XHCI_OP_USBCMD );
if ( ! ( usbcmd & XHCI_USBCMD_HCRST ) )
return 0;
/* Delay */
mdelay ( 1 );
}
DBGC ( xhci, "XHCI %p timed out waiting for reset\n", xhci );
return -ETIMEDOUT;
}
/******************************************************************************
*
* Transfer request blocks
*
******************************************************************************
*/
/**
* Allocate transfer request block ring
*
* @v xhci xHCI device
* @v ring TRB ring
* @v shift Ring size (log2)
* @v slot Device slot
* @v target Doorbell target
* @v stream Doorbell stream ID
* @ret rc Return status code
*/
static int xhci_ring_alloc ( struct xhci_device *xhci,
struct xhci_trb_ring *ring,
unsigned int shift, unsigned int slot,
unsigned int target, unsigned int stream ) {
struct xhci_trb_link *link;
unsigned int count;
int rc;
/* Sanity check */
assert ( shift > 0 );
/* Initialise structure */
memset ( ring, 0, sizeof ( *ring ) );
ring->shift = shift;
count = ( 1U << shift );
ring->mask = ( count - 1 );
ring->len = ( ( count + 1 /* Link TRB */ ) * sizeof ( ring->trb[0] ) );
ring->db = ( xhci->db + ( slot * sizeof ( ring->dbval ) ) );
ring->dbval = XHCI_DBVAL ( target, stream );
/* Allocate I/O buffers */
ring->iobuf = zalloc ( count * sizeof ( ring->iobuf[0] ) );
if ( ! ring->iobuf ) {
rc = -ENOMEM;
goto err_alloc_iobuf;
}
/* Allocate TRBs */
ring->trb = malloc_dma ( ring->len, xhci_align ( ring->len ) );
if ( ! ring->trb ) {
rc = -ENOMEM;
goto err_alloc_trb;
}
memset ( ring->trb, 0, ring->len );
/* Initialise Link TRB */
link = &ring->trb[count].link;
link->next = cpu_to_le64 ( virt_to_phys ( ring->trb ) );
link->flags = XHCI_TRB_TC;
link->type = XHCI_TRB_LINK;
ring->link = link;
return 0;
free_dma ( ring->trb, ring->len );
err_alloc_trb:
free ( ring->iobuf );
err_alloc_iobuf:
return rc;
}
/**
* Free transfer request block ring
*
* @v ring TRB ring
*/
static void xhci_ring_free ( struct xhci_trb_ring *ring ) {
unsigned int count = ( 1U << ring->shift );
unsigned int i;
/* Sanity checks */
assert ( ring->cons == ring->prod );
for ( i = 0 ; i < count ; i++ )
assert ( ring->iobuf[i] == NULL );
/* Free TRBs */
free_dma ( ring->trb, ring->len );
/* Free I/O buffers */
free ( ring->iobuf );
}
/**
* Enqueue a transfer request block
*
* @v ring TRB ring
* @v iobuf I/O buffer (if any)
* @v trb Transfer request block (with empty Cycle flag)
* @ret rc Return status code
*
* This operation does not implicitly ring the doorbell register.
*/
static int xhci_enqueue ( struct xhci_trb_ring *ring, struct io_buffer *iobuf,
const union xhci_trb *trb ) {
union xhci_trb *dest;
unsigned int prod;
unsigned int mask;
unsigned int index;
unsigned int cycle;
/* Sanity check */
assert ( ! ( trb->common.flags & XHCI_TRB_C ) );
/* Fail if ring is full */
if ( ! xhci_ring_remaining ( ring ) )
return -ENOBUFS;
/* Update producer counter (and link TRB, if applicable) */
prod = ring->prod++;
mask = ring->mask;
cycle = ( ( ~( prod >> ring->shift ) ) & XHCI_TRB_C );
index = ( prod & mask );
if ( index == 0 )
ring->link->flags = ( XHCI_TRB_TC | ( cycle ^ XHCI_TRB_C ) );
/* Record I/O buffer */
ring->iobuf[index] = iobuf;
/* Enqueue TRB */
dest = &ring->trb[index];
dest->template.parameter = trb->template.parameter;
dest->template.status = trb->template.status;
wmb();
dest->template.control = ( trb->template.control |
cpu_to_le32 ( cycle ) );
return 0;
}
/**
* Dequeue a transfer request block
*
* @v ring TRB ring
* @ret iobuf I/O buffer
*/
static struct io_buffer * xhci_dequeue ( struct xhci_trb_ring *ring ) {
struct io_buffer *iobuf;
unsigned int cons;
unsigned int mask;
unsigned int index;
/* Sanity check */
assert ( xhci_ring_fill ( ring ) != 0 );
/* Update consumer counter */
cons = ring->cons++;
mask = ring->mask;
index = ( cons & mask );
/* Retrieve I/O buffer */
iobuf = ring->iobuf[index];
ring->iobuf[index] = NULL;
return iobuf;
}
/**
* Enqueue multiple transfer request blocks
*
* @v ring TRB ring
* @v iobuf I/O buffer
* @v trbs Transfer request blocks (with empty Cycle flag)
* @v count Number of transfer request blocks
* @ret rc Return status code
*
* This operation does not implicitly ring the doorbell register.
*/
static int xhci_enqueue_multi ( struct xhci_trb_ring *ring,
struct io_buffer *iobuf,
const union xhci_trb *trbs,
unsigned int count ) {
const union xhci_trb *trb = trbs;
int rc;
/* Sanity check */
assert ( iobuf != NULL );
/* Fail if ring does not have sufficient space */
if ( xhci_ring_remaining ( ring ) < count )
return -ENOBUFS;
/* Enqueue each TRB, recording the I/O buffer with the final TRB */
while ( count-- ) {
rc = xhci_enqueue ( ring, ( count ? NULL : iobuf ), trb++ );
assert ( rc == 0 ); /* Should never be able to fail */
}
return 0;
}
/**
* Dequeue multiple transfer request blocks
*
* @v ring TRB ring
* @ret iobuf I/O buffer
*/
static struct io_buffer * xhci_dequeue_multi ( struct xhci_trb_ring *ring ) {
struct io_buffer *iobuf;
/* Dequeue TRBs until we reach the final TRB for an I/O buffer */
do {
iobuf = xhci_dequeue ( ring );
} while ( iobuf == NULL );
return iobuf;
}
/**
* Ring doorbell register
*
* @v ring TRB ring
*/
static inline __attribute__ (( always_inline )) void
xhci_doorbell ( struct xhci_trb_ring *ring ) {
wmb();
writel ( ring->dbval, ring->db );
}
/******************************************************************************
*
* Command and event rings
*
******************************************************************************
*/
/**
* Allocate command ring
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static int xhci_command_alloc ( struct xhci_device *xhci ) {
physaddr_t crp;
int rc;
/* Allocate TRB ring */
if ( ( rc = xhci_ring_alloc ( xhci, &xhci->command, XHCI_CMD_TRBS_LOG2,
0, 0, 0 ) ) != 0 )
goto err_ring_alloc;
/* Program command ring control register */
crp = virt_to_phys ( xhci->command.trb );
if ( ( rc = xhci_writeq ( xhci, ( crp | XHCI_CRCR_RCS ),
xhci->op + XHCI_OP_CRCR ) ) != 0 )
goto err_writeq;
DBGC2 ( xhci, "XHCI %p CRCR at [%08lx,%08lx)\n",
xhci, crp, ( crp + xhci->command.len ) );
return 0;
err_writeq:
xhci_ring_free ( &xhci->command );
err_ring_alloc:
return rc;
}
/**
* Free command ring
*
* @v xhci xHCI device
*/
static void xhci_command_free ( struct xhci_device *xhci ) {
/* Sanity check */
assert ( ( readl ( xhci->op + XHCI_OP_CRCR ) & XHCI_CRCR_CRR ) == 0 );
/* Clear command ring control register */
xhci_writeq ( xhci, 0, xhci->op + XHCI_OP_CRCR );
/* Free TRB ring */
xhci_ring_free ( &xhci->command );
}
/**
* Allocate event ring
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static int xhci_event_alloc ( struct xhci_device *xhci ) {
struct xhci_event_ring *event = &xhci->event;
unsigned int count;
size_t len;
int rc;
/* Allocate event ring */
count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
len = ( count * sizeof ( event->trb[0] ) );
event->trb = malloc_dma ( len, xhci_align ( len ) );
if ( ! event->trb ) {
rc = -ENOMEM;
goto err_alloc_trb;
}
memset ( event->trb, 0, len );
/* Allocate event ring segment table */
event->segment = malloc_dma ( sizeof ( event->segment[0] ),
xhci_align ( sizeof (event->segment[0])));
if ( ! event->segment ) {
rc = -ENOMEM;
goto err_alloc_segment;
}
memset ( event->segment, 0, sizeof ( event->segment[0] ) );
event->segment[0].base = cpu_to_le64 ( virt_to_phys ( event->trb ) );
event->segment[0].count = cpu_to_le32 ( count );
/* Program event ring registers */
writel ( 1, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->trb ),
xhci->run + XHCI_RUN_ERDP ( 0 ) ) ) != 0 )
goto err_writeq_erdp;
if ( ( rc = xhci_writeq ( xhci, virt_to_phys ( event->segment ),
xhci->run + XHCI_RUN_ERSTBA ( 0 ) ) ) != 0 )
goto err_writeq_erstba;
DBGC2 ( xhci, "XHCI %p event ring [%08lx,%08lx) table [%08lx,%08lx)\n",
xhci, virt_to_phys ( event->trb ),
( virt_to_phys ( event->trb ) + len ),
virt_to_phys ( event->segment ),
( virt_to_phys ( event->segment ) +
sizeof (event->segment[0] ) ) );
return 0;
xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
err_writeq_erstba:
xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
err_writeq_erdp:
free_dma ( event->trb, len );
err_alloc_segment:
free_dma ( event->segment, sizeof ( event->segment[0] ) );
err_alloc_trb:
return rc;
}
/**
* Free event ring
*
* @v xhci xHCI device
*/
static void xhci_event_free ( struct xhci_device *xhci ) {
struct xhci_event_ring *event = &xhci->event;
unsigned int count;
size_t len;
/* Clear event ring registers */
writel ( 0, xhci->run + XHCI_RUN_ERSTSZ ( 0 ) );
xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERSTBA ( 0 ) );
xhci_writeq ( xhci, 0, xhci->run + XHCI_RUN_ERDP ( 0 ) );
/* Free event ring segment table */
free_dma ( event->segment, sizeof ( event->segment[0] ) );
/* Free event ring */
count = ( 1 << XHCI_EVENT_TRBS_LOG2 );
len = ( count * sizeof ( event->trb[0] ) );
free_dma ( event->trb, len );
}
/**
* Handle transfer event
*
* @v xhci xHCI device
* @v transfer Transfer event TRB
*/
static void xhci_transfer ( struct xhci_device *xhci,
struct xhci_trb_transfer *transfer ) {
struct xhci_slot *slot;
struct xhci_endpoint *endpoint;
struct io_buffer *iobuf;
int rc;
/* Profile transfer events */
profile_start ( &xhci_transfer_profiler );
/* Identify slot */
if ( ( transfer->slot > xhci->slots ) ||
( ( slot = xhci->slot[transfer->slot] ) == NULL ) ) {
DBGC ( xhci, "XHCI %p transfer event invalid slot %d:\n",
xhci, transfer->slot );
DBGC_HDA ( xhci, 0, transfer, sizeof ( *transfer ) );
return;
}
/* Identify endpoint */
if ( ( transfer->endpoint > XHCI_CTX_END ) ||
( ( endpoint = slot->endpoint[transfer->endpoint] ) == NULL ) ) {
DBGC ( xhci, "XHCI %p slot %d transfer event invalid epid "
"%d:\n", xhci, slot->id, transfer->endpoint );
DBGC_HDA ( xhci, 0, transfer, sizeof ( *transfer ) );
return;
}
/* Dequeue TRB(s) */
iobuf = xhci_dequeue_multi ( &endpoint->ring );
assert ( iobuf != NULL );
/* Check for errors */
if ( ! ( ( transfer->code == XHCI_CMPLT_SUCCESS ) ||
( transfer->code == XHCI_CMPLT_SHORT ) ) ) {
/* Construct error */
rc = -ECODE ( transfer->code );
DBGC ( xhci, "XHCI %p slot %d ctx %d failed (code %d): %s\n",
xhci, slot->id, endpoint->ctx, transfer->code,
strerror ( rc ) );
DBGC_HDA ( xhci, 0, transfer, sizeof ( *transfer ) );
/* Sanity check */
assert ( ( endpoint->context->state & XHCI_ENDPOINT_STATE_MASK )
!= XHCI_ENDPOINT_RUNNING );
/* Report failure to USB core */
usb_complete_err ( endpoint->ep, iobuf, rc );
return;
}
/* Record actual transfer size */
iob_unput ( iobuf, le16_to_cpu ( transfer->residual ) );
/* Sanity check (for successful completions only) */
assert ( xhci_ring_consumed ( &endpoint->ring ) ==
le64_to_cpu ( transfer->transfer ) );
/* Report completion to USB core */
usb_complete ( endpoint->ep, iobuf );
profile_stop ( &xhci_transfer_profiler );
}
/**
* Handle command completion event
*
* @v xhci xHCI device
* @v complete Command completion event
*/
static void xhci_complete ( struct xhci_device *xhci,
struct xhci_trb_complete *complete ) {
/* Dequeue command TRB */
xhci_dequeue ( &xhci->command );
/* Sanity check */
assert ( xhci_ring_consumed ( &xhci->command ) ==
le64_to_cpu ( complete->command ) );
/* Record completion if applicable */
if ( xhci->completion ) {
memcpy ( xhci->completion, complete,
sizeof ( *xhci->completion ) );
xhci->completion = NULL;
} else {
DBGC ( xhci, "XHCI %p unexpected completion:\n", xhci );
DBGC_HDA ( xhci, 0, complete, sizeof ( *complete ) );
}
}
/**
* Handle port status event
*
* @v xhci xHCI device
* @v port Port status event
*/
static void xhci_port_status ( struct xhci_device *xhci,
struct xhci_trb_port_status *port ) {
uint32_t portsc;
/* Sanity check */
assert ( ( port->port > 0 ) && ( port->port <= xhci->ports ) );
/* Clear port status change bits */
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->port ) );
portsc &= ( XHCI_PORTSC_PRESERVE | XHCI_PORTSC_CHANGE );
writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->port ) );
/* Report port status change */
usb_port_changed ( usb_port ( xhci->bus->hub, port->port ) );
}
/**
* Handle host controller event
*
* @v xhci xHCI device
* @v host Host controller event
*/
static void xhci_host_controller ( struct xhci_device *xhci,
struct xhci_trb_host_controller *host ) {
int rc;
/* Construct error */
rc = -ECODE ( host->code );
DBGC ( xhci, "XHCI %p host controller event (code %d): %s\n",
xhci, host->code, strerror ( rc ) );
}
/**
* Poll event ring
*
* @v xhci xHCI device
*/
static void xhci_event_poll ( struct xhci_device *xhci ) {
struct xhci_event_ring *event = &xhci->event;
union xhci_trb *trb;
unsigned int shift = XHCI_EVENT_TRBS_LOG2;
unsigned int count = ( 1 << shift );
unsigned int mask = ( count - 1 );
unsigned int consumed;
unsigned int type;
/* Poll for events */
profile_start ( &xhci_event_profiler );
for ( consumed = 0 ; ; consumed++ ) {
/* Stop if we reach an empty TRB */
rmb();
trb = &event->trb[ event->cons & mask ];
if ( ! ( ( trb->common.flags ^
( event->cons >> shift ) ) & XHCI_TRB_C ) )
break;
/* Handle TRB */
type = ( trb->common.type & XHCI_TRB_TYPE_MASK );
switch ( type ) {
case XHCI_TRB_TRANSFER :
xhci_transfer ( xhci, &trb->transfer );
break;
case XHCI_TRB_COMPLETE :
xhci_complete ( xhci, &trb->complete );
break;
case XHCI_TRB_PORT_STATUS:
xhci_port_status ( xhci, &trb->port );
break;
case XHCI_TRB_HOST_CONTROLLER:
xhci_host_controller ( xhci, &trb->host );
break;
default:
DBGC ( xhci, "XHCI %p unrecognised event %#x\n:",
xhci, event->cons );
DBGC_HDA ( xhci, virt_to_phys ( trb ),
trb, sizeof ( *trb ) );
break;
}
/* Consume this TRB */
event->cons++;
}
/* Update dequeue pointer if applicable */
if ( consumed ) {
xhci_writeq ( xhci, virt_to_phys ( trb ),
xhci->run + XHCI_RUN_ERDP ( 0 ) );
profile_stop ( &xhci_event_profiler );
}
}
/**
* Issue command and wait for completion
*
* @v xhci xHCI device
* @v trb Transfer request block (with empty Cycle flag)
* @ret rc Return status code
*
* On a successful completion, the TRB will be overwritten with the
* completion.
*/
static int xhci_command ( struct xhci_device *xhci, union xhci_trb *trb ) {
struct xhci_trb_complete *complete = &trb->complete;
unsigned int i;
int rc;
/* Record the completion buffer */
xhci->completion = trb;
/* Enqueue the command */
if ( ( rc = xhci_enqueue ( &xhci->command, NULL, trb ) ) != 0 )
goto err_enqueue;
/* Ring the command doorbell */
xhci_doorbell ( &xhci->command );
/* Wait for the command to complete */
for ( i = 0 ; i < XHCI_COMMAND_MAX_WAIT_MS ; i++ ) {
/* Poll event ring */
xhci_event_poll ( xhci );
/* Check for completion */
if ( ! xhci->completion ) {
if ( complete->code != XHCI_CMPLT_SUCCESS ) {
rc = -ECODE ( complete->code );
DBGC ( xhci, "XHCI %p command failed (code "
"%d): %s\n", xhci, complete->code,
strerror ( rc ) );
DBGC_HDA ( xhci, 0, trb, sizeof ( *trb ) );
return rc;
}
return 0;
}
/* Delay */
mdelay ( 1 );
}
/* Timeout */
DBGC ( xhci, "XHCI %p timed out waiting for completion\n", xhci );
rc = -ETIMEDOUT;
err_enqueue:
xhci->completion = NULL;
return rc;
}
/**
* Issue NOP and wait for completion
*
* @v xhci xHCI device
* @ret rc Return status code
*/
static inline int xhci_nop ( struct xhci_device *xhci ) {
union xhci_trb trb;
struct xhci_trb_common *nop = &trb.common;
int rc;
/* Construct command */
memset ( nop, 0, sizeof ( *nop ) );
nop->flags = XHCI_TRB_IOC;
nop->type = XHCI_TRB_NOP_CMD;
/* Issue command and wait for completion */
if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
return rc;
return 0;
}
/**
* Enable slot
*
* @v xhci xHCI device
* @v type Slot type
* @ret slot Device slot ID, or negative error
*/
static inline int xhci_enable_slot ( struct xhci_device *xhci,
unsigned int type ) {
union xhci_trb trb;
struct xhci_trb_enable_slot *enable = &trb.enable;
struct xhci_trb_complete *enabled = &trb.complete;
unsigned int slot;
int rc;
/* Construct command */
memset ( enable, 0, sizeof ( *enable ) );
enable->slot = type;
enable->type = XHCI_TRB_ENABLE_SLOT;
/* Issue command and wait for completion */
if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
DBGC ( xhci, "XHCI %p could not enable new slot: %s\n",
xhci, strerror ( rc ) );
return rc;
}
/* Extract slot number */
slot = enabled->slot;
DBGC2 ( xhci, "XHCI %p slot %d enabled\n", xhci, slot );
return slot;
}
/**
* Disable slot
*
* @v xhci xHCI device
* @v slot Device slot
* @ret rc Return status code
*/
static inline int xhci_disable_slot ( struct xhci_device *xhci,
unsigned int slot ) {
union xhci_trb trb;
struct xhci_trb_disable_slot *disable = &trb.disable;
int rc;
/* Construct command */
memset ( disable, 0, sizeof ( *disable ) );
disable->type = XHCI_TRB_DISABLE_SLOT;
disable->slot = slot;
/* Issue command and wait for completion */
if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
DBGC ( xhci, "XHCI %p could not disable slot %d: %s\n",
xhci, slot, strerror ( rc ) );
return rc;
}
DBGC2 ( xhci, "XHCI %p slot %d disabled\n", xhci, slot );
return 0;
}
/**
* Issue context-based command and wait for completion
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @v type TRB type
* @v populate Input context populater
* @ret rc Return status code
*/
static int xhci_context ( struct xhci_device *xhci, struct xhci_slot *slot,
struct xhci_endpoint *endpoint, unsigned int type,
void ( * populate ) ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint,
void *input ) ) {
union xhci_trb trb;
struct xhci_trb_context *context = &trb.context;
size_t len;
void *input;
int rc;
/* Allocate an input context */
len = xhci_input_context_offset ( xhci, XHCI_CTX_END );
input = malloc_dma ( len, xhci_align ( len ) );
if ( ! input ) {
rc = -ENOMEM;
goto err_alloc;
}
memset ( input, 0, len );
/* Populate input context */
populate ( xhci, slot, endpoint, input );
/* Construct command */
memset ( context, 0, sizeof ( *context ) );
context->type = type;
context->input = cpu_to_le64 ( virt_to_phys ( input ) );
context->slot = slot->id;
/* Issue command and wait for completion */
if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 )
goto err_command;
err_command:
free_dma ( input, len );
err_alloc:
return rc;
}
/**
* Populate address device input context
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @v input Input context
*/
static void xhci_address_device_input ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint,
void *input ) {
struct xhci_control_context *control_ctx;
struct xhci_slot_context *slot_ctx;
struct xhci_endpoint_context *ep_ctx;
/* Sanity checks */
assert ( endpoint->ctx == XHCI_CTX_EP0 );
/* Populate control context */
control_ctx = input;
control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
( 1 << XHCI_CTX_EP0 ) );
/* Populate slot context */
slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( 1, 0, slot->psiv,
slot->route ) );
slot_ctx->port = slot->port;
/* Populate control endpoint context */
ep_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_EP0 ) );
ep_ctx->type = XHCI_EP_TYPE_CONTROL;
ep_ctx->burst = endpoint->ep->burst;
ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) |
XHCI_EP_DCS );
ep_ctx->trb_len = cpu_to_le16 ( XHCI_EP0_TRB_LEN );
}
/**
* Address device
*
* @v xhci xHCI device
* @v slot Device slot
* @ret rc Return status code
*/
static inline int xhci_address_device ( struct xhci_device *xhci,
struct xhci_slot *slot ) {
struct usb_device *usb = slot->usb;
struct xhci_slot_context *slot_ctx;
int rc;
/* Assign device address */
if ( ( rc = xhci_context ( xhci, slot, slot->endpoint[XHCI_CTX_EP0],
XHCI_TRB_ADDRESS_DEVICE,
xhci_address_device_input ) ) != 0 )
return rc;
/* Get assigned address */
slot_ctx = ( slot->context +
xhci_device_context_offset ( xhci, XHCI_CTX_SLOT ) );
usb->address = slot_ctx->address;
DBGC2 ( xhci, "XHCI %p assigned address %d to %s\n",
xhci, usb->address, usb->name );
return 0;
}
/**
* Populate configure endpoint input context
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @v input Input context
*/
static void xhci_configure_endpoint_input ( struct xhci_device *xhci,
struct xhci_slot *slot __unused,
struct xhci_endpoint *endpoint,
void *input ) {
struct xhci_control_context *control_ctx;
struct xhci_slot_context *slot_ctx;
struct xhci_endpoint_context *ep_ctx;
/* Populate control context */
control_ctx = input;
control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
( 1 << endpoint->ctx ) );
/* Populate slot context */
slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
0, 0, 0 ) );
/* Populate endpoint context */
ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
ep_ctx->interval = endpoint->interval;
ep_ctx->type = endpoint->type;
ep_ctx->burst = endpoint->ep->burst;
ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
ep_ctx->dequeue = cpu_to_le64 ( virt_to_phys ( endpoint->ring.trb ) |
XHCI_EP_DCS );
ep_ctx->trb_len = cpu_to_le16 ( endpoint->ep->mtu ); /* best guess */
}
/**
* Configure endpoint
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @ret rc Return status code
*/
static inline int xhci_configure_endpoint ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint ) {
int rc;
/* Configure endpoint */
if ( ( rc = xhci_context ( xhci, slot, endpoint,
XHCI_TRB_CONFIGURE_ENDPOINT,
xhci_configure_endpoint_input ) ) != 0 )
return rc;
DBGC2 ( xhci, "XHCI %p slot %d ctx %d configured\n",
xhci, slot->id, endpoint->ctx );
return 0;
}
/**
* Populate deconfigure endpoint input context
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @v input Input context
*/
static void
xhci_deconfigure_endpoint_input ( struct xhci_device *xhci __unused,
struct xhci_slot *slot __unused,
struct xhci_endpoint *endpoint,
void *input ) {
struct xhci_control_context *control_ctx;
struct xhci_slot_context *slot_ctx;
/* Populate control context */
control_ctx = input;
control_ctx->add = cpu_to_le32 ( 1 << XHCI_CTX_SLOT );
control_ctx->drop = cpu_to_le32 ( 1 << endpoint->ctx );
/* Populate slot context */
slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
0, 0, 0 ) );
}
/**
* Deconfigure endpoint
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @ret rc Return status code
*/
static inline int xhci_deconfigure_endpoint ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint ) {
int rc;
/* Deconfigure endpoint */
if ( ( rc = xhci_context ( xhci, slot, endpoint,
XHCI_TRB_CONFIGURE_ENDPOINT,
xhci_deconfigure_endpoint_input ) ) != 0 )
return rc;
DBGC2 ( xhci, "XHCI %p slot %d ctx %d deconfigured\n",
xhci, slot->id, endpoint->ctx );
return 0;
}
/**
* Populate evaluate context input context
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @v input Input context
*/
static void xhci_evaluate_context_input ( struct xhci_device *xhci,
struct xhci_slot *slot __unused,
struct xhci_endpoint *endpoint,
void *input ) {
struct xhci_control_context *control_ctx;
struct xhci_slot_context *slot_ctx;
struct xhci_endpoint_context *ep_ctx;
/* Populate control context */
control_ctx = input;
control_ctx->add = cpu_to_le32 ( ( 1 << XHCI_CTX_SLOT ) |
( 1 << endpoint->ctx ) );
/* Populate slot context */
slot_ctx = ( input + xhci_input_context_offset ( xhci, XHCI_CTX_SLOT ));
slot_ctx->info = cpu_to_le32 ( XHCI_SLOT_INFO ( ( XHCI_CTX_END - 1 ),
0, 0, 0 ) );
/* Populate endpoint context */
ep_ctx = ( input + xhci_input_context_offset ( xhci, endpoint->ctx ) );
ep_ctx->mtu = cpu_to_le16 ( endpoint->ep->mtu );
}
/**
* Evaluate context
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @ret rc Return status code
*/
static inline int xhci_evaluate_context ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint ) {
int rc;
/* Configure endpoint */
if ( ( rc = xhci_context ( xhci, slot, endpoint,
XHCI_TRB_EVALUATE_CONTEXT,
xhci_evaluate_context_input ) ) != 0 )
return rc;
DBGC2 ( xhci, "XHCI %p slot %d ctx %d (re-)evaluated\n",
xhci, slot->id, endpoint->ctx );
return 0;
}
/**
* Reset endpoint
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @ret rc Return status code
*/
static inline int xhci_reset_endpoint ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint ) {
union xhci_trb trb;
struct xhci_trb_reset_endpoint *reset = &trb.reset;
int rc;
/* Construct command */
memset ( reset, 0, sizeof ( *reset ) );
reset->slot = slot->id;
reset->endpoint = endpoint->ctx;
reset->type = XHCI_TRB_RESET_ENDPOINT;
/* Issue command and wait for completion */
if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
DBGC ( xhci, "XHCI %p slot %d ctx %d could not reset endpoint "
"in state %d: %s\n", xhci, slot->id, endpoint->ctx,
endpoint->context->state, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Stop endpoint
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @ret rc Return status code
*/
static inline int xhci_stop_endpoint ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint ) {
union xhci_trb trb;
struct xhci_trb_stop_endpoint *stop = &trb.stop;
int rc;
/* Construct command */
memset ( stop, 0, sizeof ( *stop ) );
stop->slot = slot->id;
stop->endpoint = endpoint->ctx;
stop->type = XHCI_TRB_STOP_ENDPOINT;
/* Issue command and wait for completion */
if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
DBGC ( xhci, "XHCI %p slot %d ctx %d could not stop endpoint "
"in state %d: %s\n", xhci, slot->id, endpoint->ctx,
endpoint->context->state, strerror ( rc ) );
return rc;
}
return 0;
}
/**
* Set transfer ring dequeue pointer
*
* @v xhci xHCI device
* @v slot Device slot
* @v endpoint Endpoint
* @ret rc Return status code
*/
static inline int
xhci_set_tr_dequeue_pointer ( struct xhci_device *xhci,
struct xhci_slot *slot,
struct xhci_endpoint *endpoint ) {
union xhci_trb trb;
struct xhci_trb_set_tr_dequeue_pointer *dequeue = &trb.dequeue;
struct xhci_trb_ring *ring = &endpoint->ring;
unsigned int cons;
unsigned int mask;
unsigned int index;
unsigned int dcs;
int rc;
/* Construct command */
memset ( dequeue, 0, sizeof ( *dequeue ) );
cons = ring->cons;
mask = ring->mask;
dcs = ( ( ~( cons >> ring->shift ) ) & XHCI_EP_DCS );
index = ( cons & mask );
dequeue->dequeue =
cpu_to_le64 ( virt_to_phys ( &ring->trb[index] ) | dcs );
dequeue->slot = slot->id;
dequeue->endpoint = endpoint->ctx;
dequeue->type = XHCI_TRB_SET_TR_DEQUEUE_POINTER;
/* Issue command and wait for completion */
if ( ( rc = xhci_command ( xhci, &trb ) ) != 0 ) {
DBGC ( xhci, "XHCI %p slot %d ctx %d could not set TR dequeue "
"pointer in state %d: %s\n", xhci, slot->id,
endpoint->ctx, endpoint->context->state, strerror ( rc));
return rc;
}
return 0;
}
/******************************************************************************
*
* Endpoint operations
*
******************************************************************************
*/
/**
* Open endpoint
*
* @v ep USB endpoint
* @ret rc Return status code
*/
static int xhci_endpoint_open ( struct usb_endpoint *ep ) {
struct usb_device *usb = ep->usb;
struct xhci_slot *slot = usb_get_hostdata ( usb );
struct xhci_device *xhci = slot->xhci;
struct xhci_endpoint *endpoint;
unsigned int ctx;
unsigned int type;
unsigned int interval;
int rc;
/* Calculate context index */
ctx = XHCI_CTX ( ep->address );
assert ( slot->endpoint[ctx] == NULL );
/* Calculate endpoint type */
type = XHCI_EP_TYPE ( ep->attributes & USB_ENDPOINT_ATTR_TYPE_MASK );
if ( type == XHCI_EP_TYPE ( USB_ENDPOINT_ATTR_CONTROL ) )
type = XHCI_EP_TYPE_CONTROL;
if ( ep->address & USB_DIR_IN )
type |= XHCI_EP_TYPE_IN;
/* Calculate interval */
if ( type & XHCI_EP_TYPE_PERIODIC ) {
interval = ( fls ( ep->interval ) - 1 );
} else {
interval = ep->interval;
}
/* Allocate and initialise structure */
endpoint = zalloc ( sizeof ( *endpoint ) );
if ( ! endpoint ) {
rc = -ENOMEM;
goto err_alloc;
}
usb_endpoint_set_hostdata ( ep, endpoint );
slot->endpoint[ctx] = endpoint;
endpoint->xhci = xhci;
endpoint->slot = slot;
endpoint->ep = ep;
endpoint->ctx = ctx;
endpoint->type = type;
endpoint->interval = interval;
endpoint->context = ( ( ( void * ) slot->context ) +
xhci_device_context_offset ( xhci, ctx ) );
/* Allocate transfer ring */
if ( ( rc = xhci_ring_alloc ( xhci, &endpoint->ring,
XHCI_TRANSFER_TRBS_LOG2,
slot->id, ctx, 0 ) ) != 0 )
goto err_ring_alloc;
/* Configure endpoint, if applicable */
if ( ( ctx != XHCI_CTX_EP0 ) &&
( ( rc = xhci_configure_endpoint ( xhci, slot, endpoint ) ) != 0 ))
goto err_configure_endpoint;
DBGC2 ( xhci, "XHCI %p slot %d ctx %d ring [%08lx,%08lx)\n",
xhci, slot->id, ctx, virt_to_phys ( endpoint->ring.trb ),
( virt_to_phys ( endpoint->ring.trb ) + endpoint->ring.len ) );
return 0;
xhci_deconfigure_endpoint ( xhci, slot, endpoint );
err_configure_endpoint:
xhci_ring_free ( &endpoint->ring );
err_ring_alloc:
slot->endpoint[ctx] = NULL;
free ( endpoint );
err_alloc:
return rc;
}
/**
* Close endpoint
*
* @v ep USB endpoint
*/
static void xhci_endpoint_close ( struct usb_endpoint *ep ) {
struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
struct xhci_slot *slot = endpoint->slot;
struct xhci_device *xhci = slot->xhci;
struct io_buffer *iobuf;
unsigned int ctx = endpoint->ctx;
/* Deconfigure endpoint, if applicable */
if ( ctx != XHCI_CTX_EP0 )
xhci_deconfigure_endpoint ( xhci, slot, endpoint );
/* Cancel any incomplete transfers */
while ( xhci_ring_fill ( &endpoint->ring ) ) {
iobuf = xhci_dequeue_multi ( &endpoint->ring );
usb_complete_err ( ep, iobuf, -ECANCELED );
}
/* Free endpoint */
xhci_ring_free ( &endpoint->ring );
slot->endpoint[ctx] = NULL;
free ( endpoint );
}
/**
* Reset endpoint
*
* @v ep USB endpoint
* @ret rc Return status code
*/
static int xhci_endpoint_reset ( struct usb_endpoint *ep ) {
struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
struct xhci_slot *slot = endpoint->slot;
struct xhci_device *xhci = slot->xhci;
int rc;
/* Reset endpoint context */
if ( ( rc = xhci_reset_endpoint ( xhci, slot, endpoint ) ) != 0 )
return rc;
/* Set transfer ring dequeue pointer */
if ( ( rc = xhci_set_tr_dequeue_pointer ( xhci, slot, endpoint ) ) != 0)
return rc;
DBGC ( xhci, "XHCI %p slot %d ctx %d reset\n",
xhci, slot->id, endpoint->ctx );
return 0;
}
/**
* Update MTU
*
* @v ep USB endpoint
* @ret rc Return status code
*/
static int xhci_endpoint_mtu ( struct usb_endpoint *ep ) {
struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
struct xhci_slot *slot = endpoint->slot;
struct xhci_device *xhci = slot->xhci;
int rc;
/* Evalulate context */
if ( ( rc = xhci_evaluate_context ( xhci, slot, endpoint ) ) != 0 )
return rc;
return 0;
}
/**
* Enqueue message transfer
*
* @v ep USB endpoint
* @v packet Setup packet
* @v iobuf I/O buffer
* @ret rc Return status code
*/
static int xhci_endpoint_message ( struct usb_endpoint *ep,
struct usb_setup_packet *packet,
struct io_buffer *iobuf ) {
struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
unsigned int input = ( le16_to_cpu ( packet->request ) & USB_DIR_IN );
size_t len = iob_len ( iobuf );
union xhci_trb trbs[ 1 /* setup */ + 1 /* possible data */ +
1 /* status */ ];
union xhci_trb *trb = trbs;
struct xhci_trb_setup *setup;
struct xhci_trb_data *data;
struct xhci_trb_status *status;
int rc;
/* Profile message transfers */
profile_start ( &xhci_message_profiler );
/* Construct setup stage TRB */
memset ( trbs, 0, sizeof ( trbs ) );
setup = &(trb++)->setup;
memcpy ( &setup->packet, packet, sizeof ( setup->packet ) );
setup->len = cpu_to_le32 ( sizeof ( *packet ) );
setup->flags = XHCI_TRB_IDT;
setup->type = XHCI_TRB_SETUP;
if ( len )
setup->direction = ( input ? XHCI_SETUP_IN : XHCI_SETUP_OUT );
/* Construct data stage TRB, if applicable */
if ( len ) {
data = &(trb++)->data;
data->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) );
data->len = cpu_to_le32 ( len );
data->type = XHCI_TRB_DATA;
data->direction = ( input ? XHCI_DATA_IN : XHCI_DATA_OUT );
}
/* Construct status stage TRB */
status = &(trb++)->status;
status->flags = XHCI_TRB_IOC;
status->type = XHCI_TRB_STATUS;
status->direction =
( ( len && input ) ? XHCI_STATUS_OUT : XHCI_STATUS_IN );
/* Enqueue TRBs */
if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
( trb - trbs ) ) ) != 0 )
return rc;
/* Ring the doorbell */
xhci_doorbell ( &endpoint->ring );
profile_stop ( &xhci_message_profiler );
return 0;
}
/**
* Enqueue stream transfer
*
* @v ep USB endpoint
* @v iobuf I/O buffer
* @v terminate Terminate using a short packet
* @ret rc Return status code
*/
static int xhci_endpoint_stream ( struct usb_endpoint *ep,
struct io_buffer *iobuf, int terminate ) {
struct xhci_endpoint *endpoint = usb_endpoint_get_hostdata ( ep );
union xhci_trb trbs[ 1 /* Normal */ + 1 /* Possible zero-length */ ];
union xhci_trb *trb = trbs;
struct xhci_trb_normal *normal;
size_t len = iob_len ( iobuf );
int rc;
/* Profile stream transfers */
profile_start ( &xhci_stream_profiler );
/* Construct normal TRBs */
memset ( &trbs, 0, sizeof ( trbs ) );
normal = &(trb++)->normal;
normal->data = cpu_to_le64 ( virt_to_phys ( iobuf->data ) );
normal->len = cpu_to_le32 ( len );
normal->type = XHCI_TRB_NORMAL;
if ( terminate && ( ( len & ( ep->mtu - 1 ) ) == 0 ) ) {
normal->flags = XHCI_TRB_CH;
normal = &(trb++)->normal;
normal->type = XHCI_TRB_NORMAL;
}
normal->flags = XHCI_TRB_IOC;
/* Enqueue TRBs */
if ( ( rc = xhci_enqueue_multi ( &endpoint->ring, iobuf, trbs,
( trb - trbs ) ) ) != 0 )
return rc;
/* Ring the doorbell */
xhci_doorbell ( &endpoint->ring );
profile_stop ( &xhci_stream_profiler );
return 0;
}
/******************************************************************************
*
* Device operations
*
******************************************************************************
*/
/**
* Open device
*
* @v usb USB device
* @ret rc Return status code
*/
static int xhci_device_open ( struct usb_device *usb ) {
struct xhci_device *xhci = usb_bus_get_hostdata ( usb->port->hub->bus );
struct xhci_slot *slot;
size_t len;
int type;
int id;
int rc;
/* Determine applicable slot type */
type = xhci_port_slot_type ( xhci, usb->port->address );
if ( type < 0 ) {
rc = type;
DBGC ( xhci, "XHCI %p port %d has no slot type\n",
xhci, usb->port->address );
goto err_type;
}
/* Allocate a device slot number */
id = xhci_enable_slot ( xhci, type );
if ( id < 0 ) {
rc = id;
goto err_enable_slot;
}
assert ( xhci->slot[id] == NULL );
/* Allocate and initialise structure */
slot = zalloc ( sizeof ( *slot ) );
if ( ! slot ) {
rc = -ENOMEM;
goto err_alloc;
}
usb_set_hostdata ( usb, slot );
xhci->slot[id] = slot;
slot->xhci = xhci;
slot->usb = usb;
slot->id = id;
/* Allocate a device context */
len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
slot->context = malloc_dma ( len, xhci_align ( len ) );
if ( ! slot->context ) {
rc = -ENOMEM;
goto err_alloc_context;
}
memset ( slot->context, 0, len );
/* Set device context base address */
assert ( xhci->dcbaa[id] == 0 );
xhci->dcbaa[id] = cpu_to_le64 ( virt_to_phys ( slot->context ) );
DBGC2 ( xhci, "XHCI %p slot %d device context [%08lx,%08lx) for %s\n",
xhci, slot->id, virt_to_phys ( slot->context ),
( virt_to_phys ( slot->context ) + len ), usb->name );
return 0;
xhci->dcbaa[id] = 0;
free_dma ( slot->context, len );
err_alloc_context:
xhci->slot[id] = NULL;
free ( slot );
err_alloc:
xhci_disable_slot ( xhci, id );
err_enable_slot:
err_type:
return rc;
}
/**
* Close device
*
* @v usb USB device
*/
static void xhci_device_close ( struct usb_device *usb ) {
struct xhci_slot *slot = usb_get_hostdata ( usb );
struct xhci_device *xhci = slot->xhci;
size_t len = xhci_device_context_offset ( xhci, XHCI_CTX_END );
unsigned int id = slot->id;
/* Disable slot */
xhci_disable_slot ( xhci, id );
/* Free slot */
free_dma ( slot->context, len );
xhci->dcbaa[id] = 0;
xhci->slot[id] = NULL;
free ( slot );
}
/**
* Assign device address
*
* @v usb USB device
* @ret rc Return status code
*/
static int xhci_device_address ( struct usb_device *usb ) {
struct xhci_slot *slot = usb_get_hostdata ( usb );
struct xhci_device *xhci = slot->xhci;
struct usb_port *port = usb->port;
struct usb_port *root_port;
int psiv;
int rc;
/* Calculate route string */
slot->route = usb_route_string ( usb );
/* Calculate root hub port number */
root_port = usb_root_hub_port ( usb );
slot->port = root_port->address;
/* Calculate protocol speed ID */
psiv = xhci_port_psiv ( xhci, slot->port, port->speed );
if ( psiv < 0 ) {
rc = psiv;
return rc;
}
slot->psiv = psiv;
/* Address device */
if ( ( rc = xhci_address_device ( xhci, slot ) ) != 0 )
return rc;
return 0;
}
/******************************************************************************
*
* Bus operations
*
******************************************************************************
*/
/**
* Open USB bus
*
* @v bus USB bus
* @ret rc Return status code
*/
static int xhci_bus_open ( struct usb_bus *bus ) {
struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
int rc;
/* Allocate device slot array */
xhci->slot = zalloc ( xhci->slots * sizeof ( xhci->slot[0] ) );
if ( ! xhci->slot ) {
rc = -ENOMEM;
goto err_slot_alloc;
}
/* Allocate device context base address array */
if ( ( rc = xhci_dcbaa_alloc ( xhci ) ) != 0 )
goto err_dcbaa_alloc;
/* Allocate scratchpad buffers */
if ( ( rc = xhci_scratchpad_alloc ( xhci ) ) != 0 )
goto err_scratchpad_alloc;
/* Allocate command ring */
if ( ( rc = xhci_command_alloc ( xhci ) ) != 0 )
goto err_command_alloc;
/* Allocate event ring */
if ( ( rc = xhci_event_alloc ( xhci ) ) != 0 )
goto err_event_alloc;
/* Start controller */
xhci_run ( xhci );
return 0;
xhci_stop ( xhci );
xhci_event_free ( xhci );
err_event_alloc:
xhci_command_free ( xhci );
err_command_alloc:
xhci_scratchpad_free ( xhci );
err_scratchpad_alloc:
xhci_dcbaa_free ( xhci );
err_dcbaa_alloc:
free ( xhci->slot );
err_slot_alloc:
return rc;
}
/**
* Close USB bus
*
* @v bus USB bus
*/
static void xhci_bus_close ( struct usb_bus *bus ) {
struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
unsigned int i;
/* Sanity checks */
assert ( xhci->slot != NULL );
for ( i = 0 ; i < xhci->slots ; i++ )
assert ( xhci->slot[i] == NULL );
xhci_stop ( xhci );
xhci_event_free ( xhci );
xhci_command_free ( xhci );
xhci_scratchpad_free ( xhci );
xhci_dcbaa_free ( xhci );
free ( xhci->slot );
}
/**
* Poll USB bus
*
* @v bus USB bus
*/
static void xhci_bus_poll ( struct usb_bus *bus ) {
struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
/* Poll event ring */
xhci_event_poll ( xhci );
}
/******************************************************************************
*
* Root hub operations
*
******************************************************************************
*/
/**
* Open root hub
*
* @v hub USB hub
* @ret rc Return status code
*/
static int xhci_hub_open ( struct usb_hub *hub ) {
struct usb_bus *bus = hub->bus;
struct xhci_device *xhci = usb_bus_get_hostdata ( bus );
struct usb_port *port;
uint32_t portsc;
unsigned int i;
/* Enable power to all ports */
for ( i = 1 ; i <= xhci->ports ; i++ ) {
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
portsc &= XHCI_PORTSC_PRESERVE;
portsc |= XHCI_PORTSC_PP;
writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
}
/* xHCI spec requires us to potentially wait 20ms after
* enabling power to a port.
*/
mdelay ( XHCI_PORT_POWER_DELAY_MS );
/* USB3 ports may power up as Disabled */
for ( i = 1 ; i <= xhci->ports ; i++ ) {
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( i ) );
port = usb_port ( hub, i );
if ( ( port->protocol >= USB_PROTO_3_0 ) &&
( ( portsc & XHCI_PORTSC_PLS_MASK ) ==
XHCI_PORTSC_PLS_DISABLED ) ) {
/* Force link state to RxDetect */
portsc &= XHCI_PORTSC_PRESERVE;
portsc |= ( XHCI_PORTSC_PLS_RXDETECT | XHCI_PORTSC_LWS);
writel ( portsc, xhci->op + XHCI_OP_PORTSC ( i ) );
}
}
/* Some xHCI cards seem to require an additional delay after
* setting the link state to RxDetect.
*/
mdelay ( XHCI_LINK_STATE_DELAY_MS );
/* Record hub driver private data */
usb_hub_set_drvdata ( hub, xhci );
return 0;
}
/**
* Close root hub
*
* @v hub USB hub
*/
static void xhci_hub_close ( struct usb_hub *hub ) {
/* Clear hub driver private data */
usb_hub_set_drvdata ( hub, NULL );
}
/**
* Enable port
*
* @v hub USB hub
* @v port USB port
* @ret rc Return status code
*/
static int xhci_hub_enable ( struct usb_hub *hub, struct usb_port *port ) {
struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
uint32_t portsc;
unsigned int i;
/* Reset port if applicable */
if ( port->protocol < USB_PROTO_3_0 ) {
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
portsc &= XHCI_PORTSC_PRESERVE;
portsc |= XHCI_PORTSC_PR;
writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
}
/* Wait for port to become enabled */
for ( i = 0 ; i < XHCI_PORT_RESET_MAX_WAIT_MS ; i++ ) {
/* Check port status */
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
if ( portsc & XHCI_PORTSC_PED )
return 0;
/* Delay */
mdelay ( 1 );
}
DBGC ( xhci, "XHCI %p timed out waiting for port %d to enable\n",
xhci, port->address );
return -ETIMEDOUT;
}
/**
* Disable port
*
* @v hub USB hub
* @v port USB port
* @ret rc Return status code
*/
static int xhci_hub_disable ( struct usb_hub *hub, struct usb_port *port ) {
struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
uint32_t portsc;
/* Disable port */
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
portsc &= XHCI_PORTSC_PRESERVE;
portsc |= XHCI_PORTSC_PED;
writel ( portsc, xhci->op + XHCI_OP_PORTSC ( port->address ) );
return 0;
}
/**
* Update root hub port speed
*
* @v hub USB hub
* @v port USB port
* @ret rc Return status code
*/
static int xhci_hub_speed ( struct usb_hub *hub, struct usb_port *port ) {
struct xhci_device *xhci = usb_hub_get_drvdata ( hub );
uint32_t portsc;
unsigned int psiv;
int ccs;
int ped;
int speed;
int rc;
/* Read port status */
portsc = readl ( xhci->op + XHCI_OP_PORTSC ( port->address ) );
DBGC2 ( xhci, "XHCI %p port %d status is %08x\n",
xhci, port->address, portsc );
/* Check whether or not port is connected */
ccs = ( portsc & XHCI_PORTSC_CCS );
if ( ! ccs ) {
port->speed = USB_SPEED_NONE;
return 0;
}
/* For USB2 ports, the PSIV field is not valid until the port
* completes reset and becomes enabled.
*/
ped = ( portsc & XHCI_PORTSC_PED );
if ( ( port->protocol < USB_PROTO_3_0 ) && ! ped ) {
port->speed = USB_SPEED_FULL;
return 0;
}
/* Get port speed and map to generic USB speed */
psiv = XHCI_PORTSC_PSIV ( portsc );
speed = xhci_port_speed ( xhci, port->address, psiv );
if ( speed < 0 ) {
rc = speed;
return rc;
}
port->speed = speed;
return 0;
}
/******************************************************************************
*
* PCI interface
*
******************************************************************************
*/
/** USB host controller operations */
static struct usb_host_operations xhci_operations = {
.endpoint = {
.open = xhci_endpoint_open,
.close = xhci_endpoint_close,
.reset = xhci_endpoint_reset,
.mtu = xhci_endpoint_mtu,
.message = xhci_endpoint_message,
.stream = xhci_endpoint_stream,
},
.device = {
.open = xhci_device_open,
.close = xhci_device_close,
.address = xhci_device_address,
},
.bus = {
.open = xhci_bus_open,
.close = xhci_bus_close,
.poll = xhci_bus_poll,
},
.hub = {
.open = xhci_hub_open,
.close = xhci_hub_close,
.enable = xhci_hub_enable,
.disable = xhci_hub_disable,
.speed = xhci_hub_speed,
},
};
/**
* Probe PCI device
*
* @v pci PCI device
* @ret rc Return status code
*/
static int xhci_probe ( struct pci_device *pci ) {
struct xhci_device *xhci;
struct usb_port *port;
unsigned long bar_start;
size_t bar_size;
unsigned int i;
int rc;
/* Allocate and initialise structure */
xhci = zalloc ( sizeof ( *xhci ) );
if ( ! xhci ) {
rc = -ENOMEM;
goto err_alloc;
}
/* Fix up PCI device */
adjust_pci_device ( pci );
/* Map registers */
bar_start = pci_bar_start ( pci, XHCI_BAR );
bar_size = pci_bar_size ( pci, XHCI_BAR );
xhci->regs = ioremap ( bar_start, bar_size );
if ( ! xhci->regs ) {
rc = -ENODEV;
goto err_ioremap;
}
/* Initialise xHCI device */
xhci_init ( xhci, xhci->regs );
/* Initialise USB legacy support and claim ownership */
xhci_legacy_init ( xhci );
if ( ( rc = xhci_legacy_claim ( xhci ) ) != 0 )
goto err_legacy_claim;
/* Reset device */
if ( ( rc = xhci_reset ( xhci ) ) != 0 )
goto err_reset;
/* Allocate USB bus */
xhci->bus = alloc_usb_bus ( &pci->dev, xhci->ports,
&xhci_operations );
if ( ! xhci->bus ) {
rc = -ENOMEM;
goto err_alloc_bus;
}
usb_bus_set_hostdata ( xhci->bus, xhci );
usb_hub_set_drvdata ( xhci->bus->hub, xhci );
/* Set port protocols */
for ( i = 1 ; i <= xhci->ports ; i++ ) {
port = usb_port ( xhci->bus->hub, i );
port->protocol = xhci_port_protocol ( xhci, i );
}
/* Register USB bus */
if ( ( rc = register_usb_bus ( xhci->bus ) ) != 0 )
goto err_register;
pci_set_drvdata ( pci, xhci );
return 0;
unregister_usb_bus ( xhci->bus );
err_register:
free_usb_bus ( xhci->bus );
err_alloc_bus:
xhci_reset ( xhci );
err_reset:
xhci_legacy_release ( xhci );
err_legacy_claim:
iounmap ( xhci->regs );
err_ioremap:
free ( xhci );
err_alloc:
return rc;
}
/**
* Remove PCI device
*
* @v pci PCI device
*/
static void xhci_remove ( struct pci_device *pci ) {
struct xhci_device *xhci = pci_get_drvdata ( pci );
struct usb_bus *bus = xhci->bus;
unregister_usb_bus ( bus );
free_usb_bus ( bus );
xhci_reset ( xhci );
xhci_legacy_release ( xhci );
iounmap ( xhci->regs );
free ( xhci );
}
/** XHCI PCI device IDs */
static struct pci_device_id xhci_ids[] = {
PCI_ROM ( 0xffff, 0xffff, "xhci", "xHCI", 0 ),
};
/** XHCI PCI driver */
struct pci_driver xhci_driver __pci_driver = {
.ids = xhci_ids,
.id_count = ( sizeof ( xhci_ids ) / sizeof ( xhci_ids[0] ) ),
.class = PCI_CLASS ( PCI_CLASS_SERIAL, PCI_CLASS_SERIAL_USB,
PCI_CLASS_SERIAL_USB_XHCI ),
.probe = xhci_probe,
.remove = xhci_remove,
};
Reference in New Issue View Git Blame Copy Permalink