Archipelago » qemu

/*

 * USB xHCI controller emulation

 *

 * Copyright (c) 2011 Securiforest

 * Date: 2011-05-11 ;  Author: Hector Martin <hector@marcansoft.com>

 * Based on usb-ohci.c, emulates Renesas NEC USB 3.0

 *

 * This library is free software; you can redistribute it and/or

 * modify it under the terms of the GNU Lesser General Public

 * License as published by the Free Software Foundation; either

 * version 2 of the License, or (at your option) any later version.

 *

 * This library is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 * Lesser General Public License for more details.

 *

 * You should have received a copy of the GNU Lesser General Public

 * License along with this library; if not, see <http://www.gnu.org/licenses/>.

 */

#include "hw/hw.h"

#include "qemu-timer.h"

#include "hw/usb.h"

#include "hw/pci.h"

#include "hw/msi.h"

#include "trace.h"

//#define DEBUG_XHCI

//#define DEBUG_DATA

#ifdef DEBUG_XHCI

#define DPRINTF(...) fprintf(stderr, __VA_ARGS__)

#else

#define DPRINTF(...) do {} while (0)

#endif

#define FIXME() do { fprintf(stderr, "FIXME %s:%d\n", \

                             __func__, __LINE__); abort(); } while (0)

#define MAXSLOTS 8

#define MAXINTRS 1

#define USB2_PORTS 4

#define USB3_PORTS 4

#define MAXPORTS (USB2_PORTS+USB3_PORTS)

#define TD_QUEUE 24

#define BG_XFERS 8

#define BG_PKTS 8

/* Very pessimistic, let's hope it's enough for all cases */

#define EV_QUEUE (((3*TD_QUEUE)+16)*MAXSLOTS)

/* Do not deliver ER Full events. NEC's driver does some things not bound

 * to the specs when it gets them */

#define ER_FULL_HACK

#define LEN_CAP         0x40

#define OFF_OPER        LEN_CAP

#define LEN_OPER        (0x400 + 0x10 * MAXPORTS)

#define OFF_RUNTIME     ((OFF_OPER + LEN_OPER + 0x20) & ~0x1f)

#define LEN_RUNTIME     (0x20 + MAXINTRS * 0x20)

#define OFF_DOORBELL    (OFF_RUNTIME + LEN_RUNTIME)

#define LEN_DOORBELL    ((MAXSLOTS + 1) * 0x20)

/* must be power of 2 */

#define LEN_REGS        0x2000

#if (OFF_DOORBELL + LEN_DOORBELL) > LEN_REGS

# error Increase LEN_REGS

#endif

#if MAXINTRS > 1

# error TODO: only one interrupter supported

#endif

/* bit definitions */

#define USBCMD_RS       (1<<0)

#define USBCMD_HCRST    (1<<1)

#define USBCMD_INTE     (1<<2)

#define USBCMD_HSEE     (1<<3)

#define USBCMD_LHCRST   (1<<7)

#define USBCMD_CSS      (1<<8)

#define USBCMD_CRS      (1<<9)

#define USBCMD_EWE      (1<<10)

#define USBCMD_EU3S     (1<<11)

#define USBSTS_HCH      (1<<0)

#define USBSTS_HSE      (1<<2)

#define USBSTS_EINT     (1<<3)

#define USBSTS_PCD      (1<<4)

#define USBSTS_SSS      (1<<8)

#define USBSTS_RSS      (1<<9)

#define USBSTS_SRE      (1<<10)

#define USBSTS_CNR      (1<<11)

#define USBSTS_HCE      (1<<12)

#define PORTSC_CCS          (1<<0)

#define PORTSC_PED          (1<<1)

#define PORTSC_OCA          (1<<3)

#define PORTSC_PR           (1<<4)

#define PORTSC_PLS_SHIFT        5

#define PORTSC_PLS_MASK     0xf

#define PORTSC_PP           (1<<9)

#define PORTSC_SPEED_SHIFT      10

#define PORTSC_SPEED_MASK   0xf

#define PORTSC_SPEED_FULL   (1<<10)

#define PORTSC_SPEED_LOW    (2<<10)

#define PORTSC_SPEED_HIGH   (3<<10)

#define PORTSC_SPEED_SUPER  (4<<10)

#define PORTSC_PIC_SHIFT        14

#define PORTSC_PIC_MASK     0x3

#define PORTSC_LWS          (1<<16)

#define PORTSC_CSC          (1<<17)

#define PORTSC_PEC          (1<<18)

#define PORTSC_WRC          (1<<19)

#define PORTSC_OCC          (1<<20)

#define PORTSC_PRC          (1<<21)

#define PORTSC_PLC          (1<<22)

#define PORTSC_CEC          (1<<23)

#define PORTSC_CAS          (1<<24)

#define PORTSC_WCE          (1<<25)

#define PORTSC_WDE          (1<<26)

#define PORTSC_WOE          (1<<27)

#define PORTSC_DR           (1<<30)

#define PORTSC_WPR          (1<<31)

#define CRCR_RCS        (1<<0)

#define CRCR_CS         (1<<1)

#define CRCR_CA         (1<<2)

#define CRCR_CRR        (1<<3)

#define IMAN_IP         (1<<0)

#define IMAN_IE         (1<<1)

#define ERDP_EHB        (1<<3)

#define TRB_SIZE 16

typedef struct XHCITRB {

    uint64_t parameter;

    uint32_t status;

    uint32_t control;

    dma_addr_t addr;

    bool ccs;

} XHCITRB;

typedef enum TRBType {

    TRB_RESERVED = 0,

    TR_NORMAL,

    TR_SETUP,

    TR_DATA,

    TR_STATUS,

    TR_ISOCH,

    TR_LINK,

    TR_EVDATA,

    TR_NOOP,

    CR_ENABLE_SLOT,

    CR_DISABLE_SLOT,

    CR_ADDRESS_DEVICE,

    CR_CONFIGURE_ENDPOINT,

    CR_EVALUATE_CONTEXT,

    CR_RESET_ENDPOINT,

    CR_STOP_ENDPOINT,

    CR_SET_TR_DEQUEUE,

    CR_RESET_DEVICE,

    CR_FORCE_EVENT,

    CR_NEGOTIATE_BW,

    CR_SET_LATENCY_TOLERANCE,

    CR_GET_PORT_BANDWIDTH,

    CR_FORCE_HEADER,

    CR_NOOP,

    ER_TRANSFER = 32,

    ER_COMMAND_COMPLETE,

    ER_PORT_STATUS_CHANGE,

    ER_BANDWIDTH_REQUEST,

    ER_DOORBELL,

    ER_HOST_CONTROLLER,

    ER_DEVICE_NOTIFICATION,

    ER_MFINDEX_WRAP,

    /* vendor specific bits */

    CR_VENDOR_VIA_CHALLENGE_RESPONSE = 48,

    CR_VENDOR_NEC_FIRMWARE_REVISION  = 49,

    CR_VENDOR_NEC_CHALLENGE_RESPONSE = 50,

} TRBType;

#define CR_LINK TR_LINK

typedef enum TRBCCode {

    CC_INVALID = 0,

    CC_SUCCESS,

    CC_DATA_BUFFER_ERROR,

    CC_BABBLE_DETECTED,

    CC_USB_TRANSACTION_ERROR,

    CC_TRB_ERROR,

    CC_STALL_ERROR,

    CC_RESOURCE_ERROR,

    CC_BANDWIDTH_ERROR,

    CC_NO_SLOTS_ERROR,

    CC_INVALID_STREAM_TYPE_ERROR,

    CC_SLOT_NOT_ENABLED_ERROR,

    CC_EP_NOT_ENABLED_ERROR,

    CC_SHORT_PACKET,

    CC_RING_UNDERRUN,

    CC_RING_OVERRUN,

    CC_VF_ER_FULL,

    CC_PARAMETER_ERROR,

    CC_BANDWIDTH_OVERRUN,

    CC_CONTEXT_STATE_ERROR,

    CC_NO_PING_RESPONSE_ERROR,

    CC_EVENT_RING_FULL_ERROR,

    CC_INCOMPATIBLE_DEVICE_ERROR,

    CC_MISSED_SERVICE_ERROR,

    CC_COMMAND_RING_STOPPED,

    CC_COMMAND_ABORTED,

    CC_STOPPED,

    CC_STOPPED_LENGTH_INVALID,

    CC_MAX_EXIT_LATENCY_TOO_LARGE_ERROR = 29,

    CC_ISOCH_BUFFER_OVERRUN = 31,

    CC_EVENT_LOST_ERROR,

    CC_UNDEFINED_ERROR,

    CC_INVALID_STREAM_ID_ERROR,

    CC_SECONDARY_BANDWIDTH_ERROR,

    CC_SPLIT_TRANSACTION_ERROR

} TRBCCode;

#define TRB_C               (1<<0)

#define TRB_TYPE_SHIFT          10

#define TRB_TYPE_MASK       0x3f

#define TRB_TYPE(t)         (((t).control >> TRB_TYPE_SHIFT) & TRB_TYPE_MASK)

#define TRB_EV_ED           (1<<2)

#define TRB_TR_ENT          (1<<1)

#define TRB_TR_ISP          (1<<2)

#define TRB_TR_NS           (1<<3)

#define TRB_TR_CH           (1<<4)

#define TRB_TR_IOC          (1<<5)

#define TRB_TR_IDT          (1<<6)

#define TRB_TR_TBC_SHIFT        7

#define TRB_TR_TBC_MASK     0x3

#define TRB_TR_BEI          (1<<9)

#define TRB_TR_TLBPC_SHIFT      16

#define TRB_TR_TLBPC_MASK   0xf

#define TRB_TR_FRAMEID_SHIFT    20

#define TRB_TR_FRAMEID_MASK 0x7ff

#define TRB_TR_SIA          (1<<31)

#define TRB_TR_DIR          (1<<16)

#define TRB_CR_SLOTID_SHIFT     24

#define TRB_CR_SLOTID_MASK  0xff

#define TRB_CR_EPID_SHIFT       16

#define TRB_CR_EPID_MASK    0x1f

#define TRB_CR_BSR          (1<<9)

#define TRB_CR_DC           (1<<9)

#define TRB_LK_TC           (1<<1)

#define EP_TYPE_MASK        0x7

#define EP_TYPE_SHIFT           3

#define EP_STATE_MASK       0x7

#define EP_DISABLED         (0<<0)

#define EP_RUNNING          (1<<0)

#define EP_HALTED           (2<<0)

#define EP_STOPPED          (3<<0)

#define EP_ERROR            (4<<0)

#define SLOT_STATE_MASK     0x1f

#define SLOT_STATE_SHIFT        27

#define SLOT_STATE(s)       (((s)>>SLOT_STATE_SHIFT)&SLOT_STATE_MASK)

#define SLOT_ENABLED        0

#define SLOT_DEFAULT        1

#define SLOT_ADDRESSED      2

#define SLOT_CONFIGURED     3

#define SLOT_CONTEXT_ENTRIES_MASK 0x1f

#define SLOT_CONTEXT_ENTRIES_SHIFT 27

typedef enum EPType {

    ET_INVALID = 0,

    ET_ISO_OUT,

    ET_BULK_OUT,

    ET_INTR_OUT,

    ET_CONTROL,

    ET_ISO_IN,

    ET_BULK_IN,

    ET_INTR_IN,

} EPType;

typedef struct XHCIRing {

    dma_addr_t base;

    dma_addr_t dequeue;

    bool ccs;

} XHCIRing;

typedef struct XHCIPort {

    USBPort port;

    uint32_t portsc;

} XHCIPort;

struct XHCIState;

typedef struct XHCIState XHCIState;

typedef struct XHCITransfer {

    XHCIState *xhci;

    USBPacket packet;

    bool running_async;

    bool running_retry;

    bool cancelled;

    bool complete;

    bool backgrounded;

    unsigned int iso_pkts;

    unsigned int slotid;

    unsigned int epid;

    bool in_xfer;

    bool iso_xfer;

    bool bg_xfer;

    unsigned int trb_count;

    unsigned int trb_alloced;

    XHCITRB *trbs;

    unsigned int data_length;

    unsigned int data_alloced;

    uint8_t *data;

    TRBCCode status;

    unsigned int pkts;

    unsigned int pktsize;

    unsigned int cur_pkt;

} XHCITransfer;

typedef struct XHCIEPContext {

    XHCIRing ring;

    unsigned int next_xfer;

    unsigned int comp_xfer;

    XHCITransfer transfers[TD_QUEUE];

    XHCITransfer *retry;

    bool bg_running;

    bool bg_updating;

    unsigned int next_bg;

    XHCITransfer bg_transfers[BG_XFERS];

    EPType type;

    dma_addr_t pctx;

    unsigned int max_psize;

    bool has_bg;

    uint32_t state;

} XHCIEPContext;

typedef struct XHCISlot {

    bool enabled;

    dma_addr_t ctx;

    unsigned int port;

    unsigned int devaddr;

    XHCIEPContext * eps[31];

} XHCISlot;

typedef struct XHCIEvent {

    TRBType type;

    TRBCCode ccode;

    uint64_t ptr;

    uint32_t length;

    uint32_t flags;

    uint8_t slotid;

    uint8_t epid;

} XHCIEvent;

struct XHCIState {

    PCIDevice pci_dev;

    USBBus bus;

    qemu_irq irq;

    MemoryRegion mem;

    const char *name;

    uint32_t msi;

    unsigned int devaddr;

    /* Operational Registers */

    uint32_t usbcmd;

    uint32_t usbsts;

    uint32_t dnctrl;

    uint32_t crcr_low;

    uint32_t crcr_high;

    uint32_t dcbaap_low;

    uint32_t dcbaap_high;

    uint32_t config;

    XHCIPort ports[MAXPORTS];

    XHCISlot slots[MAXSLOTS];

    /* Runtime Registers */

    uint32_t mfindex;

    /* note: we only support one interrupter */

    uint32_t iman;

    uint32_t imod;

    uint32_t erstsz;

    uint32_t erstba_low;

    uint32_t erstba_high;

    uint32_t erdp_low;

    uint32_t erdp_high;

    dma_addr_t er_start;

    uint32_t er_size;

    bool er_pcs;

    unsigned int er_ep_idx;

    bool er_full;

    XHCIEvent ev_buffer[EV_QUEUE];

    unsigned int ev_buffer_put;

    unsigned int ev_buffer_get;

    XHCIRing cmd_ring;

};

typedef struct XHCIEvRingSeg {

    uint32_t addr_low;

    uint32_t addr_high;

    uint32_t size;

    uint32_t rsvd;

} XHCIEvRingSeg;

static const char *TRBType_names[] = {

    [TRB_RESERVED]                     = "TRB_RESERVED",

    [TR_NORMAL]                        = "TR_NORMAL",

    [TR_SETUP]                         = "TR_SETUP",

    [TR_DATA]                          = "TR_DATA",

    [TR_STATUS]                        = "TR_STATUS",

    [TR_ISOCH]                         = "TR_ISOCH",

    [TR_LINK]                          = "TR_LINK",

    [TR_EVDATA]                        = "TR_EVDATA",

    [TR_NOOP]                          = "TR_NOOP",

    [CR_ENABLE_SLOT]                   = "CR_ENABLE_SLOT",

    [CR_DISABLE_SLOT]                  = "CR_DISABLE_SLOT",

    [CR_ADDRESS_DEVICE]                = "CR_ADDRESS_DEVICE",

    [CR_CONFIGURE_ENDPOINT]            = "CR_CONFIGURE_ENDPOINT",

    [CR_EVALUATE_CONTEXT]              = "CR_EVALUATE_CONTEXT",

    [CR_RESET_ENDPOINT]                = "CR_RESET_ENDPOINT",

    [CR_STOP_ENDPOINT]                 = "CR_STOP_ENDPOINT",

    [CR_SET_TR_DEQUEUE]                = "CR_SET_TR_DEQUEUE",

    [CR_RESET_DEVICE]                  = "CR_RESET_DEVICE",

    [CR_FORCE_EVENT]                   = "CR_FORCE_EVENT",

    [CR_NEGOTIATE_BW]                  = "CR_NEGOTIATE_BW",

    [CR_SET_LATENCY_TOLERANCE]         = "CR_SET_LATENCY_TOLERANCE",

    [CR_GET_PORT_BANDWIDTH]            = "CR_GET_PORT_BANDWIDTH",

    [CR_FORCE_HEADER]                  = "CR_FORCE_HEADER",

    [CR_NOOP]                          = "CR_NOOP",

    [ER_TRANSFER]                      = "ER_TRANSFER",

    [ER_COMMAND_COMPLETE]              = "ER_COMMAND_COMPLETE",

    [ER_PORT_STATUS_CHANGE]            = "ER_PORT_STATUS_CHANGE",

    [ER_BANDWIDTH_REQUEST]             = "ER_BANDWIDTH_REQUEST",

    [ER_DOORBELL]                      = "ER_DOORBELL",

    [ER_HOST_CONTROLLER]               = "ER_HOST_CONTROLLER",

    [ER_DEVICE_NOTIFICATION]           = "ER_DEVICE_NOTIFICATION",

    [ER_MFINDEX_WRAP]                  = "ER_MFINDEX_WRAP",

    [CR_VENDOR_VIA_CHALLENGE_RESPONSE] = "CR_VENDOR_VIA_CHALLENGE_RESPONSE",

    [CR_VENDOR_NEC_FIRMWARE_REVISION]  = "CR_VENDOR_NEC_FIRMWARE_REVISION",

    [CR_VENDOR_NEC_CHALLENGE_RESPONSE] = "CR_VENDOR_NEC_CHALLENGE_RESPONSE",

};

static const char *lookup_name(uint32_t index, const char **list, uint32_t llen)

{

    if (index >= llen || list[index] == NULL) {

        return "???";

    }

    return list[index];

}

static const char *trb_name(XHCITRB *trb)

{

    return lookup_name(TRB_TYPE(*trb), TRBType_names,

                       ARRAY_SIZE(TRBType_names));

}

static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,

                         unsigned int epid);

static inline dma_addr_t xhci_addr64(uint32_t low, uint32_t high)

{

    if (sizeof(dma_addr_t) == 4) {

        return low;

    } else {

        return low | (((dma_addr_t)high << 16) << 16);

    }

}

static inline dma_addr_t xhci_mask64(uint64_t addr)

{

    if (sizeof(dma_addr_t) == 4) {

        return addr & 0xffffffff;

    } else {

        return addr;

    }

}

static void xhci_irq_update(XHCIState *xhci)

{

    int level = 0;

    if (xhci->iman & IMAN_IP && xhci->iman & IMAN_IE &&

        xhci->usbcmd & USBCMD_INTE) {

        level = 1;

    }

    if (xhci->msi && msi_enabled(&xhci->pci_dev)) {

        if (level) {

            trace_usb_xhci_irq_msi(0);

            msi_notify(&xhci->pci_dev, 0);

        }

    } else {

        trace_usb_xhci_irq_intx(level);

        qemu_set_irq(xhci->irq, level);

    }

}

static inline int xhci_running(XHCIState *xhci)

{

    return !(xhci->usbsts & USBSTS_HCH) && !xhci->er_full;

}

static void xhci_die(XHCIState *xhci)

{

    xhci->usbsts |= USBSTS_HCE;

    fprintf(stderr, "xhci: asserted controller error\n");

}

static void xhci_write_event(XHCIState *xhci, XHCIEvent *event)

{

    XHCITRB ev_trb;

    dma_addr_t addr;

    ev_trb.parameter = cpu_to_le64(event->ptr);

    ev_trb.status = cpu_to_le32(event->length | (event->ccode << 24));

    ev_trb.control = (event->slotid << 24) | (event->epid << 16) |

                     event->flags | (event->type << TRB_TYPE_SHIFT);

    if (xhci->er_pcs) {

        ev_trb.control |= TRB_C;

    }

    ev_trb.control = cpu_to_le32(ev_trb.control);

    trace_usb_xhci_queue_event(xhci->er_ep_idx, trb_name(&ev_trb),

                               ev_trb.parameter, ev_trb.status, ev_trb.control);

    addr = xhci->er_start + TRB_SIZE*xhci->er_ep_idx;

    pci_dma_write(&xhci->pci_dev, addr, &ev_trb, TRB_SIZE);

    xhci->er_ep_idx++;

    if (xhci->er_ep_idx >= xhci->er_size) {

        xhci->er_ep_idx = 0;

        xhci->er_pcs = !xhci->er_pcs;

    }

}

static void xhci_events_update(XHCIState *xhci)

{

    dma_addr_t erdp;

    unsigned int dp_idx;

    bool do_irq = 0;

    if (xhci->usbsts & USBSTS_HCH) {

        return;

    }

    erdp = xhci_addr64(xhci->erdp_low, xhci->erdp_high);

    if (erdp < xhci->er_start ||

        erdp >= (xhci->er_start + TRB_SIZE*xhci->er_size)) {

        fprintf(stderr, "xhci: ERDP out of bounds: "DMA_ADDR_FMT"\n", erdp);

        fprintf(stderr, "xhci: ER at "DMA_ADDR_FMT" len %d\n",

                xhci->er_start, xhci->er_size);

        xhci_die(xhci);

        return;

    }

    dp_idx = (erdp - xhci->er_start) / TRB_SIZE;

    assert(dp_idx < xhci->er_size);

    /* NEC didn't read section 4.9.4 of the spec (v1.0 p139 top Note) and thus

     * deadlocks when the ER is full. Hack it by holding off events until

     * the driver decides to free at least half of the ring */

    if (xhci->er_full) {

        int er_free = dp_idx - xhci->er_ep_idx;

        if (er_free <= 0) {

            er_free += xhci->er_size;

        }

        if (er_free < (xhci->er_size/2)) {

            DPRINTF("xhci_events_update(): event ring still "

                    "more than half full (hack)\n");

            return;

        }

    }

    while (xhci->ev_buffer_put != xhci->ev_buffer_get) {

        assert(xhci->er_full);

        if (((xhci->er_ep_idx+1) % xhci->er_size) == dp_idx) {

            DPRINTF("xhci_events_update(): event ring full again\n");

#ifndef ER_FULL_HACK

            XHCIEvent full = {ER_HOST_CONTROLLER, CC_EVENT_RING_FULL_ERROR};

            xhci_write_event(xhci, &full);

#endif

            do_irq = 1;

            break;

        }

        XHCIEvent *event = &xhci->ev_buffer[xhci->ev_buffer_get];

        xhci_write_event(xhci, event);

        xhci->ev_buffer_get++;

        do_irq = 1;

        if (xhci->ev_buffer_get == EV_QUEUE) {

            xhci->ev_buffer_get = 0;

        }

    }

    if (do_irq) {

        xhci->erdp_low |= ERDP_EHB;

        xhci->iman |= IMAN_IP;

        xhci->usbsts |= USBSTS_EINT;

        xhci_irq_update(xhci);

    }

    if (xhci->er_full && xhci->ev_buffer_put == xhci->ev_buffer_get) {

        DPRINTF("xhci_events_update(): event ring no longer full\n");

        xhci->er_full = 0;

    }

    return;

}

static void xhci_event(XHCIState *xhci, XHCIEvent *event)

{

    dma_addr_t erdp;

    unsigned int dp_idx;

    if (xhci->er_full) {

        DPRINTF("xhci_event(): ER full, queueing\n");

        if (((xhci->ev_buffer_put+1) % EV_QUEUE) == xhci->ev_buffer_get) {

            fprintf(stderr, "xhci: event queue full, dropping event!\n");

            return;

        }

        xhci->ev_buffer[xhci->ev_buffer_put++] = *event;

        if (xhci->ev_buffer_put == EV_QUEUE) {

            xhci->ev_buffer_put = 0;

        }

        return;

    }

    erdp = xhci_addr64(xhci->erdp_low, xhci->erdp_high);

    if (erdp < xhci->er_start ||

        erdp >= (xhci->er_start + TRB_SIZE*xhci->er_size)) {

        fprintf(stderr, "xhci: ERDP out of bounds: "DMA_ADDR_FMT"\n", erdp);

        fprintf(stderr, "xhci: ER at "DMA_ADDR_FMT" len %d\n",

                xhci->er_start, xhci->er_size);

        xhci_die(xhci);

        return;

    }

    dp_idx = (erdp - xhci->er_start) / TRB_SIZE;

    assert(dp_idx < xhci->er_size);

    if ((xhci->er_ep_idx+1) % xhci->er_size == dp_idx) {

        DPRINTF("xhci_event(): ER full, queueing\n");

#ifndef ER_FULL_HACK

        XHCIEvent full = {ER_HOST_CONTROLLER, CC_EVENT_RING_FULL_ERROR};

        xhci_write_event(xhci, &full);

#endif

        xhci->er_full = 1;

        if (((xhci->ev_buffer_put+1) % EV_QUEUE) == xhci->ev_buffer_get) {

            fprintf(stderr, "xhci: event queue full, dropping event!\n");

            return;

        }

        xhci->ev_buffer[xhci->ev_buffer_put++] = *event;

        if (xhci->ev_buffer_put == EV_QUEUE) {

            xhci->ev_buffer_put = 0;

        }

    } else {

        xhci_write_event(xhci, event);

    }

    xhci->erdp_low |= ERDP_EHB;

    xhci->iman |= IMAN_IP;

    xhci->usbsts |= USBSTS_EINT;

    xhci_irq_update(xhci);

}

static void xhci_ring_init(XHCIState *xhci, XHCIRing *ring,

                           dma_addr_t base)

{

    ring->base = base;

    ring->dequeue = base;

    ring->ccs = 1;

}

static TRBType xhci_ring_fetch(XHCIState *xhci, XHCIRing *ring, XHCITRB *trb,

                               dma_addr_t *addr)

{

    while (1) {

        TRBType type;

        pci_dma_read(&xhci->pci_dev, ring->dequeue, trb, TRB_SIZE);

        trb->addr = ring->dequeue;

        trb->ccs = ring->ccs;

        le64_to_cpus(&trb->parameter);

        le32_to_cpus(&trb->status);

        le32_to_cpus(&trb->control);

        trace_usb_xhci_fetch_trb(ring->dequeue, trb_name(trb),

                                 trb->parameter, trb->status, trb->control);

        if ((trb->control & TRB_C) != ring->ccs) {

            return 0;

        }

        type = TRB_TYPE(*trb);

        if (type != TR_LINK) {

            if (addr) {

                *addr = ring->dequeue;

            }

            ring->dequeue += TRB_SIZE;

            return type;

        } else {

            ring->dequeue = xhci_mask64(trb->parameter);

            if (trb->control & TRB_LK_TC) {

                ring->ccs = !ring->ccs;

            }

        }

    }

}

static int xhci_ring_chain_length(XHCIState *xhci, const XHCIRing *ring)

{

    XHCITRB trb;

    int length = 0;

    dma_addr_t dequeue = ring->dequeue;

    bool ccs = ring->ccs;

    /* hack to bundle together the two/three TDs that make a setup transfer */

    bool control_td_set = 0;

    while (1) {

        TRBType type;

        pci_dma_read(&xhci->pci_dev, dequeue, &trb, TRB_SIZE);

        le64_to_cpus(&trb.parameter);

        le32_to_cpus(&trb.status);

        le32_to_cpus(&trb.control);

        if ((trb.control & TRB_C) != ccs) {

            return -length;

        }

        type = TRB_TYPE(trb);

        if (type == TR_LINK) {

            dequeue = xhci_mask64(trb.parameter);

            if (trb.control & TRB_LK_TC) {

                ccs = !ccs;

            }

            continue;

        }

        length += 1;

        dequeue += TRB_SIZE;

        if (type == TR_SETUP) {

            control_td_set = 1;

        } else if (type == TR_STATUS) {

            control_td_set = 0;

        }

        if (!control_td_set && !(trb.control & TRB_TR_CH)) {

            return length;

        }

    }

}

static void xhci_er_reset(XHCIState *xhci)

{

    XHCIEvRingSeg seg;

    /* cache the (sole) event ring segment location */

    if (xhci->erstsz != 1) {

        fprintf(stderr, "xhci: invalid value for ERSTSZ: %d\n", xhci->erstsz);

        xhci_die(xhci);

        return;

    }

    dma_addr_t erstba = xhci_addr64(xhci->erstba_low, xhci->erstba_high);

    pci_dma_read(&xhci->pci_dev, erstba, &seg, sizeof(seg));

    le32_to_cpus(&seg.addr_low);

    le32_to_cpus(&seg.addr_high);

    le32_to_cpus(&seg.size);

    if (seg.size < 16 || seg.size > 4096) {

        fprintf(stderr, "xhci: invalid value for segment size: %d\n", seg.size);

        xhci_die(xhci);

        return;

    }

    xhci->er_start = xhci_addr64(seg.addr_low, seg.addr_high);

    xhci->er_size = seg.size;

    xhci->er_ep_idx = 0;

    xhci->er_pcs = 1;

    xhci->er_full = 0;

    DPRINTF("xhci: event ring:" DMA_ADDR_FMT " [%d]\n",

            xhci->er_start, xhci->er_size);

}

static void xhci_run(XHCIState *xhci)

{

    trace_usb_xhci_run();

    xhci->usbsts &= ~USBSTS_HCH;

}

static void xhci_stop(XHCIState *xhci)

{

    trace_usb_xhci_stop();

    xhci->usbsts |= USBSTS_HCH;

    xhci->crcr_low &= ~CRCR_CRR;

}

static void xhci_set_ep_state(XHCIState *xhci, XHCIEPContext *epctx,

                              uint32_t state)

{

    uint32_t ctx[5];

    if (epctx->state == state) {

        return;

    }

    pci_dma_read(&xhci->pci_dev, epctx->pctx, ctx, sizeof(ctx));

    ctx[0] &= ~EP_STATE_MASK;

    ctx[0] |= state;

    ctx[2] = epctx->ring.dequeue | epctx->ring.ccs;

    ctx[3] = (epctx->ring.dequeue >> 16) >> 16;

    DPRINTF("xhci: set epctx: " DMA_ADDR_FMT " state=%d dequeue=%08x%08x\n",

            epctx->pctx, state, ctx[3], ctx[2]);

    pci_dma_write(&xhci->pci_dev, epctx->pctx, ctx, sizeof(ctx));

    epctx->state = state;

}

static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,

                               unsigned int epid, dma_addr_t pctx,

                               uint32_t *ctx)

{

    XHCISlot *slot;

    XHCIEPContext *epctx;

    dma_addr_t dequeue;

    int i;

    trace_usb_xhci_ep_enable(slotid, epid);

    assert(slotid >= 1 && slotid <= MAXSLOTS);

    assert(epid >= 1 && epid <= 31);

    slot = &xhci->slots[slotid-1];

    if (slot->eps[epid-1]) {

        fprintf(stderr, "xhci: slot %d ep %d already enabled!\n", slotid, epid);

        return CC_TRB_ERROR;

    }

    epctx = g_malloc(sizeof(XHCIEPContext));

    memset(epctx, 0, sizeof(XHCIEPContext));

    slot->eps[epid-1] = epctx;

    dequeue = xhci_addr64(ctx[2] & ~0xf, ctx[3]);

    xhci_ring_init(xhci, &epctx->ring, dequeue);

    epctx->ring.ccs = ctx[2] & 1;

    epctx->type = (ctx[1] >> EP_TYPE_SHIFT) & EP_TYPE_MASK;

    DPRINTF("xhci: endpoint %d.%d type is %d\n", epid/2, epid%2, epctx->type);

    epctx->pctx = pctx;

    epctx->max_psize = ctx[1]>>16;

    epctx->max_psize *= 1+((ctx[1]>>8)&0xff);

    epctx->has_bg = false;

    if (epctx->type == ET_ISO_IN) {

        epctx->has_bg = true;

    }

    DPRINTF("xhci: endpoint %d.%d max transaction (burst) size is %d\n",

            epid/2, epid%2, epctx->max_psize);

    for (i = 0; i < ARRAY_SIZE(epctx->transfers); i++) {

        usb_packet_init(&epctx->transfers[i].packet);

    }

    epctx->state = EP_RUNNING;

    ctx[0] &= ~EP_STATE_MASK;

    ctx[0] |= EP_RUNNING;

    return CC_SUCCESS;

}

static int xhci_ep_nuke_xfers(XHCIState *xhci, unsigned int slotid,

                               unsigned int epid)

{

    XHCISlot *slot;

    XHCIEPContext *epctx;

    int i, xferi, killed = 0;

    assert(slotid >= 1 && slotid <= MAXSLOTS);

    assert(epid >= 1 && epid <= 31);

    DPRINTF("xhci_ep_nuke_xfers(%d, %d)\n", slotid, epid);

    slot = &xhci->slots[slotid-1];

    if (!slot->eps[epid-1]) {

        return 0;

    }

    epctx = slot->eps[epid-1];

    xferi = epctx->next_xfer;

    for (i = 0; i < TD_QUEUE; i++) {

        XHCITransfer *t = &epctx->transfers[xferi];

        if (t->running_async) {

            usb_cancel_packet(&t->packet);

            t->running_async = 0;

            t->cancelled = 1;

            DPRINTF("xhci: cancelling transfer %d, waiting for it to complete...\n", i);

            killed++;

        }

        if (t->running_retry) {

            t->running_retry = 0;

            epctx->retry = NULL;

        }

        if (t->backgrounded) {

            t->backgrounded = 0;

        }

        if (t->trbs) {

            g_free(t->trbs);

        }

        if (t->data) {

            g_free(t->data);

        }

        t->trbs = NULL;

        t->data = NULL;

        t->trb_count = t->trb_alloced = 0;

        t->data_length = t->data_alloced = 0;

        xferi = (xferi + 1) % TD_QUEUE;

    }

    if (epctx->has_bg) {

        xferi = epctx->next_bg;

        for (i = 0; i < BG_XFERS; i++) {

            XHCITransfer *t = &epctx->bg_transfers[xferi];

            if (t->running_async) {

                usb_cancel_packet(&t->packet);

                t->running_async = 0;

                t->cancelled = 1;

                DPRINTF("xhci: cancelling bg transfer %d, waiting for it to complete...\n", i);

                killed++;

            }

            if (t->data) {

                g_free(t->data);

            }

            t->data = NULL;

            xferi = (xferi + 1) % BG_XFERS;

        }

    }

    return killed;

}

static TRBCCode xhci_disable_ep(XHCIState *xhci, unsigned int slotid,

                               unsigned int epid)

{

    XHCISlot *slot;

    XHCIEPContext *epctx;

    trace_usb_xhci_ep_disable(slotid, epid);

    assert(slotid >= 1 && slotid <= MAXSLOTS);

    assert(epid >= 1 && epid <= 31);

    slot = &xhci->slots[slotid-1];

    if (!slot->eps[epid-1]) {

        DPRINTF("xhci: slot %d ep %d already disabled\n", slotid, epid);

        return CC_SUCCESS;

    }

    xhci_ep_nuke_xfers(xhci, slotid, epid);

    epctx = slot->eps[epid-1];

    xhci_set_ep_state(xhci, epctx, EP_DISABLED);

    g_free(epctx);

    slot->eps[epid-1] = NULL;

    return CC_SUCCESS;

}

static TRBCCode xhci_stop_ep(XHCIState *xhci, unsigned int slotid,

                             unsigned int epid)

{

    XHCISlot *slot;

    XHCIEPContext *epctx;

    trace_usb_xhci_ep_stop(slotid, epid);

    assert(slotid >= 1 && slotid <= MAXSLOTS);

    if (epid < 1 || epid > 31) {

        fprintf(stderr, "xhci: bad ep %d\n", epid);

        return CC_TRB_ERROR;

    }

    slot = &xhci->slots[slotid-1];

    if (!slot->eps[epid-1]) {

        DPRINTF("xhci: slot %d ep %d not enabled\n", slotid, epid);

        return CC_EP_NOT_ENABLED_ERROR;

    }

    if (xhci_ep_nuke_xfers(xhci, slotid, epid) > 0) {

        fprintf(stderr, "xhci: FIXME: endpoint stopped w/ xfers running, "

                "data might be lost\n");

    }

    epctx = slot->eps[epid-1];

    xhci_set_ep_state(xhci, epctx, EP_STOPPED);

    return CC_SUCCESS;

}

static TRBCCode xhci_reset_ep(XHCIState *xhci, unsigned int slotid,

                              unsigned int epid)

{

    XHCISlot *slot;

    XHCIEPContext *epctx;

    USBDevice *dev;

    trace_usb_xhci_ep_reset(slotid, epid);

    assert(slotid >= 1 && slotid <= MAXSLOTS);

    if (epid < 1 || epid > 31) {

        fprintf(stderr, "xhci: bad ep %d\n", epid);

        return CC_TRB_ERROR;

    }

    slot = &xhci->slots[slotid-1];

    if (!slot->eps[epid-1]) {

        DPRINTF("xhci: slot %d ep %d not enabled\n", slotid, epid);

        return CC_EP_NOT_ENABLED_ERROR;

    }

    epctx = slot->eps[epid-1];

    if (epctx->state != EP_HALTED) {

        fprintf(stderr, "xhci: reset EP while EP %d not halted (%d)\n",

                epid, epctx->state);

        return CC_CONTEXT_STATE_ERROR;

    }

    if (xhci_ep_nuke_xfers(xhci, slotid, epid) > 0) {

        fprintf(stderr, "xhci: FIXME: endpoint reset w/ xfers running, "

                "data might be lost\n");

    }

    uint8_t ep = epid>>1;

    if (epid & 1) {

        ep |= 0x80;

    }

    dev = xhci->ports[xhci->slots[slotid-1].port-1].port.dev;

    if (!dev) {

        return CC_USB_TRANSACTION_ERROR;

    }

    xhci_set_ep_state(xhci, epctx, EP_STOPPED);

    return CC_SUCCESS;

}

static TRBCCode xhci_set_ep_dequeue(XHCIState *xhci, unsigned int slotid,

                                    unsigned int epid, uint64_t pdequeue)

{

    XHCISlot *slot;

    XHCIEPContext *epctx;

    dma_addr_t dequeue;

    assert(slotid >= 1 && slotid <= MAXSLOTS);

    if (epid < 1 || epid > 31) {

        fprintf(stderr, "xhci: bad ep %d\n", epid);

        return CC_TRB_ERROR;

    }

    DPRINTF("xhci_set_ep_dequeue(%d, %d, %016"PRIx64")\n", slotid, epid, pdequeue);

    dequeue = xhci_mask64(pdequeue);

    slot = &xhci->slots[slotid-1];

    if (!slot->eps[epid-1]) {

        DPRINTF("xhci: slot %d ep %d not enabled\n", slotid, epid);

        return CC_EP_NOT_ENABLED_ERROR;

    }

    epctx = slot->eps[epid-1];

    if (epctx->state != EP_STOPPED) {

        fprintf(stderr, "xhci: set EP dequeue pointer while EP %d not stopped\n", epid);

        return CC_CONTEXT_STATE_ERROR;

    }

    xhci_ring_init(xhci, &epctx->ring, dequeue & ~0xF);

    epctx->ring.ccs = dequeue & 1;

    xhci_set_ep_state(xhci, epctx, EP_STOPPED);

    return CC_SUCCESS;

}

static int xhci_xfer_data(XHCITransfer *xfer, uint8_t *data,

                          unsigned int length, bool in_xfer, bool out_xfer,

                          bool report)

{

    int i;

    uint32_t edtla = 0;

    unsigned int transferred = 0;

    unsigned int left = length;

    bool reported = 0;

    bool shortpkt = 0;

    XHCIEvent event = {ER_TRANSFER, CC_SUCCESS};

    XHCIState *xhci = xfer->xhci;

    DPRINTF("xhci_xfer_data(len=%d, in_xfer=%d, out_xfer=%d, report=%d)\n",

            length, in_xfer, out_xfer, report);

    assert(!(in_xfer && out_xfer));

    for (i = 0; i < xfer->trb_count; i++) {

        XHCITRB *trb = &xfer->trbs[i];

        dma_addr_t addr;

        unsigned int chunk = 0;

        switch (TRB_TYPE(*trb)) {

        case TR_DATA:

            if ((!(trb->control & TRB_TR_DIR)) != (!in_xfer)) {

                fprintf(stderr, "xhci: data direction mismatch for TR_DATA\n");

                xhci_die(xhci);

                return transferred;

            }

            /* fallthrough */

        case TR_NORMAL:

        case TR_ISOCH:

            addr = xhci_mask64(trb->parameter);

            chunk = trb->status & 0x1ffff;

            if (chunk > left) {

                chunk = left;

                shortpkt = 1;

            }

            if (in_xfer || out_xfer) {

                if (trb->control & TRB_TR_IDT) {

                    uint64_t idata;

                    if (chunk > 8 || in_xfer) {

                        fprintf(stderr, "xhci: invalid immediate data TRB\n");

                        xhci_die(xhci);

                        return transferred;

                    }

                    idata = le64_to_cpu(trb->parameter);

                    memcpy(data, &idata, chunk);

                } else {

                    DPRINTF("xhci_xfer_data: r/w(%d) %d bytes at "

                            DMA_ADDR_FMT "\n", in_xfer, chunk, addr);

                    if (in_xfer) {

                        pci_dma_write(&xhci->pci_dev, addr, data, chunk);

                    } else {

                        pci_dma_read(&xhci->pci_dev, addr, data, chunk);

                    }

#ifdef DEBUG_DATA

                    unsigned int count = chunk;

                    int i;

                    if (count > 16) {

                        count = 16;

                    }

                    DPRINTF(" ::");

                    for (i = 0; i < count; i++) {

                        DPRINTF(" %02x", data[i]);

                    }

                    DPRINTF("\n");

#endif

                }

            }

            left -= chunk;

            data += chunk;

            edtla += chunk;

            transferred += chunk;

            break;

        case TR_STATUS:

            reported = 0;

            shortpkt = 0;

            break;

        }

        if (report && !reported && (trb->control & TRB_TR_IOC ||

            (shortpkt && (trb->control & TRB_TR_ISP)))) {

            event.slotid = xfer->slotid;

            event.epid = xfer->epid;

            event.length = (trb->status & 0x1ffff) - chunk;

            event.flags = 0;

            event.ptr = trb->addr;

            if (xfer->status == CC_SUCCESS) {

                event.ccode = shortpkt ? CC_SHORT_PACKET : CC_SUCCESS;

            } else {

                event.ccode = xfer->status;

            }

            if (TRB_TYPE(*trb) == TR_EVDATA) {

                event.ptr = trb->parameter;

                event.flags |= TRB_EV_ED;

                event.length = edtla & 0xffffff;

                DPRINTF("xhci_xfer_data: EDTLA=%d\n", event.length);

                edtla = 0;

            }

            xhci_event(xhci, &event);

            reported = 1;

        }

    }

    return transferred;

}

static void xhci_stall_ep(XHCITransfer *xfer)

{

    XHCIState *xhci = xfer->xhci;

    XHCISlot *slot = &xhci->slots[xfer->slotid-1];

    XHCIEPContext *epctx = slot->eps[xfer->epid-1];

    epctx->ring.dequeue = xfer->trbs[0].addr;

    epctx->ring.ccs = xfer->trbs[0].ccs;

    xhci_set_ep_state(xhci, epctx, EP_HALTED);

    DPRINTF("xhci: stalled slot %d ep %d\n", xfer->slotid, xfer->epid);

    DPRINTF("xhci: will continue at "DMA_ADDR_FMT"\n", epctx->ring.dequeue);

}

static int xhci_submit(XHCIState *xhci, XHCITransfer *xfer,

                       XHCIEPContext *epctx);

static void xhci_bg_update(XHCIState *xhci, XHCIEPContext *epctx)

{

    if (epctx->bg_updating) {

        return;

    }

    DPRINTF("xhci_bg_update(%p, %p)\n", xhci, epctx);

    assert(epctx->has_bg);

    DPRINTF("xhci: fg=%d bg=%d\n", epctx->comp_xfer, epctx->next_bg);

    epctx->bg_updating = 1;

    while (epctx->transfers[epctx->comp_xfer].backgrounded &&

           epctx->bg_transfers[epctx->next_bg].complete) {

        XHCITransfer *fg = &epctx->transfers[epctx->comp_xfer];

        XHCITransfer *bg = &epctx->bg_transfers[epctx->next_bg];

#if 0

        DPRINTF("xhci: completing fg %d from bg %d.%d (stat: %d)\n",

                epctx->comp_xfer, epctx->next_bg, bg->cur_pkt,

                bg->usbxfer->iso_packet_desc[bg->cur_pkt].status

               );

#endif

        assert(epctx->type == ET_ISO_IN);

        assert(bg->iso_xfer);

        assert(bg->in_xfer);

        uint8_t *p = bg->data + bg->cur_pkt * bg->pktsize;

#if 0

        int len = bg->usbxfer->iso_packet_desc[bg->cur_pkt].actual_length;

        fg->status = libusb_to_ccode(bg->usbxfer->iso_packet_desc[bg->cur_pkt].status);

#else

        int len = 0;

        FIXME();

#endif

        fg->complete = 1;

        fg->backgrounded = 0;

        if (fg->status == CC_STALL_ERROR) {

            xhci_stall_ep(fg);

        }

        xhci_xfer_data(fg, p, len, 1, 0, 1);

        epctx->comp_xfer++;

        if (epctx->comp_xfer == TD_QUEUE) {

            epctx->comp_xfer = 0;

        }

        DPRINTF("next fg xfer: %d\n", epctx->comp_xfer);

        bg->cur_pkt++;

        if (bg->cur_pkt == bg->pkts) {

            bg->complete = 0;

            if (xhci_submit(xhci, bg, epctx) < 0) {

                fprintf(stderr, "xhci: bg resubmit failed\n");

            }

            epctx->next_bg++;

            if (epctx->next_bg == BG_XFERS) {

                epctx->next_bg = 0;

            }

            DPRINTF("next bg xfer: %d\n", epctx->next_bg);

        xhci_kick_ep(xhci, fg->slotid, fg->epid);

        }

    }

    epctx->bg_updating = 0;

}

#if 0

static void xhci_xfer_cb(struct libusb_transfer *transfer)

{

    XHCIState *xhci;

    XHCITransfer *xfer;

    xfer = (XHCITransfer *)transfer->user_data;

    xhci = xfer->xhci;

    DPRINTF("xhci_xfer_cb(slot=%d, ep=%d, status=%d)\n", xfer->slotid,

            xfer->epid, transfer->status);

    assert(xfer->slotid >= 1 && xfer->slotid <= MAXSLOTS);

    assert(xfer->epid >= 1 && xfer->epid <= 31);