Archipelago » qemu

/*

 * USB xHCI controller emulation

 *

 * Copyright (c) 2011 Securiforest

 * Date: 2011-05-11 ;  Author: Hector Martin <hector@marcansoft.com>

 * Based on usb-ohci.c, emulates Renesas NEC USB 3.0

 *

 * This library is free software; you can redistribute it and/or

 * modify it under the terms of the GNU Lesser General Public

 * License as published by the Free Software Foundation; either

 * version 2 of the License, or (at your option) any later version.

 *

 * This library is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

 * Lesser General Public License for more details.

 *

 * You should have received a copy of the GNU Lesser General Public

 * License along with this library; if not, see <http://www.gnu.org/licenses/>.

 */

#include "hw/hw.h"

#include "qemu/timer.h"

#include "hw/usb.h"

#include "hw/pci/pci.h"

#include "hw/pci/msi.h"

#include "hw/pci/msix.h"

#include "trace.h"

//#define DEBUG_XHCI

//#define DEBUG_DATA

#ifdef DEBUG_XHCI

#define DPRINTF(...) fprintf(stderr, __VA_ARGS__)

#else

#define DPRINTF(...) do {} while (0)

#endif

#define FIXME(_msg) do { fprintf(stderr, "FIXME %s:%d %s\n", \

                                 __func__, __LINE__, _msg); abort(); } while (0)

#define MAXPORTS_2 15

#define MAXPORTS_3 15

#define MAXPORTS (MAXPORTS_2+MAXPORTS_3)

#define MAXSLOTS 64

#define MAXINTRS 16

#define TD_QUEUE 24

/* Very pessimistic, let's hope it's enough for all cases */

#define EV_QUEUE (((3*TD_QUEUE)+16)*MAXSLOTS)

/* Do not deliver ER Full events. NEC's driver does some things not bound

 * to the specs when it gets them */

#define ER_FULL_HACK

#define LEN_CAP         0x40

#define LEN_OPER        (0x400 + 0x10 * MAXPORTS)

#define LEN_RUNTIME     ((MAXINTRS + 1) * 0x20)

#define LEN_DOORBELL    ((MAXSLOTS + 1) * 0x20)

#define OFF_OPER        LEN_CAP

#define OFF_RUNTIME     0x1000

#define OFF_DOORBELL    0x2000

#define OFF_MSIX_TABLE  0x3000

#define OFF_MSIX_PBA    0x3800

/* must be power of 2 */

#define LEN_REGS        0x4000

#if (OFF_OPER + LEN_OPER) > OFF_RUNTIME

#error Increase OFF_RUNTIME

#endif

#if (OFF_RUNTIME + LEN_RUNTIME) > OFF_DOORBELL

#error Increase OFF_DOORBELL

#endif

#if (OFF_DOORBELL + LEN_DOORBELL) > LEN_REGS

# error Increase LEN_REGS

#endif

/* bit definitions */

#define USBCMD_RS       (1<<0)

#define USBCMD_HCRST    (1<<1)

#define USBCMD_INTE     (1<<2)

#define USBCMD_HSEE     (1<<3)

#define USBCMD_LHCRST   (1<<7)

#define USBCMD_CSS      (1<<8)

#define USBCMD_CRS      (1<<9)

#define USBCMD_EWE      (1<<10)

#define USBCMD_EU3S     (1<<11)

#define USBSTS_HCH      (1<<0)

#define USBSTS_HSE      (1<<2)

#define USBSTS_EINT     (1<<3)

#define USBSTS_PCD      (1<<4)

#define USBSTS_SSS      (1<<8)

#define USBSTS_RSS      (1<<9)

#define USBSTS_SRE      (1<<10)

#define USBSTS_CNR      (1<<11)

#define USBSTS_HCE      (1<<12)

#define PORTSC_CCS          (1<<0)

#define PORTSC_PED          (1<<1)

#define PORTSC_OCA          (1<<3)

#define PORTSC_PR           (1<<4)

#define PORTSC_PLS_SHIFT        5

#define PORTSC_PLS_MASK     0xf

#define PORTSC_PP           (1<<9)

#define PORTSC_SPEED_SHIFT      10

#define PORTSC_SPEED_MASK   0xf

#define PORTSC_SPEED_FULL   (1<<10)

#define PORTSC_SPEED_LOW    (2<<10)

#define PORTSC_SPEED_HIGH   (3<<10)

#define PORTSC_SPEED_SUPER  (4<<10)

#define PORTSC_PIC_SHIFT        14

#define PORTSC_PIC_MASK     0x3

#define PORTSC_LWS          (1<<16)

#define PORTSC_CSC          (1<<17)

#define PORTSC_PEC          (1<<18)

#define PORTSC_WRC          (1<<19)

#define PORTSC_OCC          (1<<20)

#define PORTSC_PRC          (1<<21)

#define PORTSC_PLC          (1<<22)

#define PORTSC_CEC          (1<<23)

#define PORTSC_CAS          (1<<24)

#define PORTSC_WCE          (1<<25)

#define PORTSC_WDE          (1<<26)

#define PORTSC_WOE          (1<<27)

#define PORTSC_DR           (1<<30)

#define PORTSC_WPR          (1<<31)

#define CRCR_RCS        (1<<0)

#define CRCR_CS         (1<<1)

#define CRCR_CA         (1<<2)

#define CRCR_CRR        (1<<3)

#define IMAN_IP         (1<<0)

#define IMAN_IE         (1<<1)

#define ERDP_EHB        (1<<3)

#define TRB_SIZE 16

typedef struct XHCITRB {

    uint64_t parameter;

    uint32_t status;

    uint32_t control;

    dma_addr_t addr;

    bool ccs;

} XHCITRB;

enum {

    PLS_U0              =  0,

    PLS_U1              =  1,

    PLS_U2              =  2,

    PLS_U3              =  3,

    PLS_DISABLED        =  4,

    PLS_RX_DETECT       =  5,

    PLS_INACTIVE        =  6,

    PLS_POLLING         =  7,

    PLS_RECOVERY        =  8,

    PLS_HOT_RESET       =  9,

    PLS_COMPILANCE_MODE = 10,

    PLS_TEST_MODE       = 11,

    PLS_RESUME          = 15,

};

typedef enum TRBType {

    TRB_RESERVED = 0,

    TR_NORMAL,

    TR_SETUP,

    TR_DATA,

    TR_STATUS,

    TR_ISOCH,

    TR_LINK,

    TR_EVDATA,

    TR_NOOP,

    CR_ENABLE_SLOT,

    CR_DISABLE_SLOT,

    CR_ADDRESS_DEVICE,

    CR_CONFIGURE_ENDPOINT,

    CR_EVALUATE_CONTEXT,

    CR_RESET_ENDPOINT,

    CR_STOP_ENDPOINT,

    CR_SET_TR_DEQUEUE,

    CR_RESET_DEVICE,

    CR_FORCE_EVENT,

    CR_NEGOTIATE_BW,

    CR_SET_LATENCY_TOLERANCE,

    CR_GET_PORT_BANDWIDTH,

    CR_FORCE_HEADER,

    CR_NOOP,

    ER_TRANSFER = 32,

    ER_COMMAND_COMPLETE,

    ER_PORT_STATUS_CHANGE,

    ER_BANDWIDTH_REQUEST,

    ER_DOORBELL,

    ER_HOST_CONTROLLER,

    ER_DEVICE_NOTIFICATION,

    ER_MFINDEX_WRAP,

    /* vendor specific bits */

    CR_VENDOR_VIA_CHALLENGE_RESPONSE = 48,

    CR_VENDOR_NEC_FIRMWARE_REVISION  = 49,

    CR_VENDOR_NEC_CHALLENGE_RESPONSE = 50,

} TRBType;

#define CR_LINK TR_LINK

typedef enum TRBCCode {

    CC_INVALID = 0,

    CC_SUCCESS,

    CC_DATA_BUFFER_ERROR,

    CC_BABBLE_DETECTED,

    CC_USB_TRANSACTION_ERROR,

    CC_TRB_ERROR,

    CC_STALL_ERROR,

    CC_RESOURCE_ERROR,

    CC_BANDWIDTH_ERROR,

    CC_NO_SLOTS_ERROR,

    CC_INVALID_STREAM_TYPE_ERROR,

    CC_SLOT_NOT_ENABLED_ERROR,

    CC_EP_NOT_ENABLED_ERROR,

    CC_SHORT_PACKET,

    CC_RING_UNDERRUN,

    CC_RING_OVERRUN,

    CC_VF_ER_FULL,

    CC_PARAMETER_ERROR,

    CC_BANDWIDTH_OVERRUN,

    CC_CONTEXT_STATE_ERROR,

    CC_NO_PING_RESPONSE_ERROR,

    CC_EVENT_RING_FULL_ERROR,

    CC_INCOMPATIBLE_DEVICE_ERROR,

    CC_MISSED_SERVICE_ERROR,

    CC_COMMAND_RING_STOPPED,

    CC_COMMAND_ABORTED,

    CC_STOPPED,

    CC_STOPPED_LENGTH_INVALID,

    CC_MAX_EXIT_LATENCY_TOO_LARGE_ERROR = 29,

    CC_ISOCH_BUFFER_OVERRUN = 31,

    CC_EVENT_LOST_ERROR,

    CC_UNDEFINED_ERROR,

    CC_INVALID_STREAM_ID_ERROR,

    CC_SECONDARY_BANDWIDTH_ERROR,

    CC_SPLIT_TRANSACTION_ERROR

} TRBCCode;

#define TRB_C               (1<<0)

#define TRB_TYPE_SHIFT          10

#define TRB_TYPE_MASK       0x3f

#define TRB_TYPE(t)         (((t).control >> TRB_TYPE_SHIFT) & TRB_TYPE_MASK)

#define TRB_EV_ED           (1<<2)

#define TRB_TR_ENT          (1<<1)

#define TRB_TR_ISP          (1<<2)

#define TRB_TR_NS           (1<<3)

#define TRB_TR_CH           (1<<4)

#define TRB_TR_IOC          (1<<5)

#define TRB_TR_IDT          (1<<6)

#define TRB_TR_TBC_SHIFT        7

#define TRB_TR_TBC_MASK     0x3

#define TRB_TR_BEI          (1<<9)

#define TRB_TR_TLBPC_SHIFT      16

#define TRB_TR_TLBPC_MASK   0xf

#define TRB_TR_FRAMEID_SHIFT    20

#define TRB_TR_FRAMEID_MASK 0x7ff

#define TRB_TR_SIA          (1<<31)

#define TRB_TR_DIR          (1<<16)

#define TRB_CR_SLOTID_SHIFT     24

#define TRB_CR_SLOTID_MASK  0xff

#define TRB_CR_EPID_SHIFT       16

#define TRB_CR_EPID_MASK    0x1f

#define TRB_CR_BSR          (1<<9)

#define TRB_CR_DC           (1<<9)

#define TRB_LK_TC           (1<<1)

#define TRB_INTR_SHIFT          22

#define TRB_INTR_MASK       0x3ff

#define TRB_INTR(t)         (((t).status >> TRB_INTR_SHIFT) & TRB_INTR_MASK)

#define EP_TYPE_MASK        0x7

#define EP_TYPE_SHIFT           3

#define EP_STATE_MASK       0x7

#define EP_DISABLED         (0<<0)

#define EP_RUNNING          (1<<0)

#define EP_HALTED           (2<<0)

#define EP_STOPPED          (3<<0)

#define EP_ERROR            (4<<0)

#define SLOT_STATE_MASK     0x1f

#define SLOT_STATE_SHIFT        27

#define SLOT_STATE(s)       (((s)>>SLOT_STATE_SHIFT)&SLOT_STATE_MASK)

#define SLOT_ENABLED        0

#define SLOT_DEFAULT        1

#define SLOT_ADDRESSED      2

#define SLOT_CONFIGURED     3

#define SLOT_CONTEXT_ENTRIES_MASK 0x1f

#define SLOT_CONTEXT_ENTRIES_SHIFT 27

typedef struct XHCIState XHCIState;

typedef struct XHCIStreamContext XHCIStreamContext;

typedef struct XHCIEPContext XHCIEPContext;

#define get_field(data, field)                  \

    (((data) >> field##_SHIFT) & field##_MASK)

#define set_field(data, newval, field) do {                     \

        uint32_t val = *data;                                   \

        val &= ~(field##_MASK << field##_SHIFT);                \

        val |= ((newval) & field##_MASK) << field##_SHIFT;      \

        *data = val;                                            \

    } while (0)

typedef enum EPType {

    ET_INVALID = 0,

    ET_ISO_OUT,

    ET_BULK_OUT,

    ET_INTR_OUT,

    ET_CONTROL,

    ET_ISO_IN,

    ET_BULK_IN,

    ET_INTR_IN,

} EPType;

typedef struct XHCIRing {

    dma_addr_t base;

    dma_addr_t dequeue;

    bool ccs;

} XHCIRing;

typedef struct XHCIPort {

    XHCIState *xhci;

    uint32_t portsc;

    uint32_t portnr;

    USBPort  *uport;

    uint32_t speedmask;

    char name[16];

    MemoryRegion mem;

} XHCIPort;

typedef struct XHCITransfer {

    XHCIState *xhci;

    USBPacket packet;

    QEMUSGList sgl;

    bool running_async;

    bool running_retry;

    bool cancelled;

    bool complete;

    bool int_req;

    unsigned int iso_pkts;

    unsigned int slotid;

    unsigned int epid;

    unsigned int streamid;

    bool in_xfer;

    bool iso_xfer;

    unsigned int trb_count;

    unsigned int trb_alloced;

    XHCITRB *trbs;

    TRBCCode status;

    unsigned int pkts;

    unsigned int pktsize;

    unsigned int cur_pkt;

    uint64_t mfindex_kick;

} XHCITransfer;

struct XHCIStreamContext {

    dma_addr_t pctx;

    unsigned int sct;

    XHCIRing ring;

    XHCIStreamContext *sstreams;

};

struct XHCIEPContext {

    XHCIState *xhci;

    unsigned int slotid;

    unsigned int epid;

    XHCIRing ring;

    unsigned int next_xfer;

    unsigned int comp_xfer;

    XHCITransfer transfers[TD_QUEUE];

    XHCITransfer *retry;

    EPType type;

    dma_addr_t pctx;

    unsigned int max_psize;

    uint32_t state;

    /* streams */

    unsigned int max_pstreams;

    bool         lsa;

    unsigned int nr_pstreams;

    XHCIStreamContext *pstreams;

    /* iso xfer scheduling */

    unsigned int interval;

    int64_t mfindex_last;

    QEMUTimer *kick_timer;

};

typedef struct XHCISlot {

    bool enabled;

    dma_addr_t ctx;

    USBPort *uport;

    unsigned int devaddr;

    XHCIEPContext * eps[31];

} XHCISlot;

typedef struct XHCIEvent {

    TRBType type;

    TRBCCode ccode;

    uint64_t ptr;

    uint32_t length;

    uint32_t flags;

    uint8_t slotid;

    uint8_t epid;

} XHCIEvent;

typedef struct XHCIInterrupter {

    uint32_t iman;

    uint32_t imod;

    uint32_t erstsz;

    uint32_t erstba_low;

    uint32_t erstba_high;

    uint32_t erdp_low;

    uint32_t erdp_high;

    bool msix_used, er_pcs, er_full;

    dma_addr_t er_start;

    uint32_t er_size;

    unsigned int er_ep_idx;

    XHCIEvent ev_buffer[EV_QUEUE];

    unsigned int ev_buffer_put;

    unsigned int ev_buffer_get;

} XHCIInterrupter;

struct XHCIState {

    PCIDevice pci_dev;

    USBBus bus;

    qemu_irq irq;

    MemoryRegion mem;

    MemoryRegion mem_cap;

    MemoryRegion mem_oper;

    MemoryRegion mem_runtime;

    MemoryRegion mem_doorbell;

    const char *name;

    unsigned int devaddr;

    /* properties */

    uint32_t numports_2;

    uint32_t numports_3;

    uint32_t numintrs;

    uint32_t numslots;

    uint32_t flags;

    /* Operational Registers */

    uint32_t usbcmd;

    uint32_t usbsts;

    uint32_t dnctrl;

    uint32_t crcr_low;

    uint32_t crcr_high;

    uint32_t dcbaap_low;

    uint32_t dcbaap_high;

    uint32_t config;

    USBPort  uports[MAX(MAXPORTS_2, MAXPORTS_3)];

    XHCIPort ports[MAXPORTS];

    XHCISlot slots[MAXSLOTS];

    uint32_t numports;

    /* Runtime Registers */

    int64_t mfindex_start;

    QEMUTimer *mfwrap_timer;

    XHCIInterrupter intr[MAXINTRS];

    XHCIRing cmd_ring;

};

typedef struct XHCIEvRingSeg {

    uint32_t addr_low;

    uint32_t addr_high;

    uint32_t size;

    uint32_t rsvd;

} XHCIEvRingSeg;

enum xhci_flags {

    XHCI_FLAG_USE_MSI = 1,

    XHCI_FLAG_USE_MSI_X,

};

static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,

                         unsigned int epid, unsigned int streamid);

static TRBCCode xhci_disable_ep(XHCIState *xhci, unsigned int slotid,

                                unsigned int epid);

static void xhci_event(XHCIState *xhci, XHCIEvent *event, int v);

static void xhci_write_event(XHCIState *xhci, XHCIEvent *event, int v);

static const char *TRBType_names[] = {

    [TRB_RESERVED]                     = "TRB_RESERVED",

    [TR_NORMAL]                        = "TR_NORMAL",

    [TR_SETUP]                         = "TR_SETUP",

    [TR_DATA]                          = "TR_DATA",

    [TR_STATUS]                        = "TR_STATUS",

    [TR_ISOCH]                         = "TR_ISOCH",

    [TR_LINK]                          = "TR_LINK",

    [TR_EVDATA]                        = "TR_EVDATA",

    [TR_NOOP]                          = "TR_NOOP",

    [CR_ENABLE_SLOT]                   = "CR_ENABLE_SLOT",

    [CR_DISABLE_SLOT]                  = "CR_DISABLE_SLOT",

    [CR_ADDRESS_DEVICE]                = "CR_ADDRESS_DEVICE",

    [CR_CONFIGURE_ENDPOINT]            = "CR_CONFIGURE_ENDPOINT",

    [CR_EVALUATE_CONTEXT]              = "CR_EVALUATE_CONTEXT",

    [CR_RESET_ENDPOINT]                = "CR_RESET_ENDPOINT",

    [CR_STOP_ENDPOINT]                 = "CR_STOP_ENDPOINT",

    [CR_SET_TR_DEQUEUE]                = "CR_SET_TR_DEQUEUE",

    [CR_RESET_DEVICE]                  = "CR_RESET_DEVICE",

    [CR_FORCE_EVENT]                   = "CR_FORCE_EVENT",

    [CR_NEGOTIATE_BW]                  = "CR_NEGOTIATE_BW",

    [CR_SET_LATENCY_TOLERANCE]         = "CR_SET_LATENCY_TOLERANCE",

    [CR_GET_PORT_BANDWIDTH]            = "CR_GET_PORT_BANDWIDTH",

    [CR_FORCE_HEADER]                  = "CR_FORCE_HEADER",

    [CR_NOOP]                          = "CR_NOOP",

    [ER_TRANSFER]                      = "ER_TRANSFER",

    [ER_COMMAND_COMPLETE]              = "ER_COMMAND_COMPLETE",

    [ER_PORT_STATUS_CHANGE]            = "ER_PORT_STATUS_CHANGE",

    [ER_BANDWIDTH_REQUEST]             = "ER_BANDWIDTH_REQUEST",

    [ER_DOORBELL]                      = "ER_DOORBELL",

    [ER_HOST_CONTROLLER]               = "ER_HOST_CONTROLLER",

    [ER_DEVICE_NOTIFICATION]           = "ER_DEVICE_NOTIFICATION",

    [ER_MFINDEX_WRAP]                  = "ER_MFINDEX_WRAP",

    [CR_VENDOR_VIA_CHALLENGE_RESPONSE] = "CR_VENDOR_VIA_CHALLENGE_RESPONSE",

    [CR_VENDOR_NEC_FIRMWARE_REVISION]  = "CR_VENDOR_NEC_FIRMWARE_REVISION",

    [CR_VENDOR_NEC_CHALLENGE_RESPONSE] = "CR_VENDOR_NEC_CHALLENGE_RESPONSE",

};

static const char *TRBCCode_names[] = {

    [CC_INVALID]                       = "CC_INVALID",

    [CC_SUCCESS]                       = "CC_SUCCESS",

    [CC_DATA_BUFFER_ERROR]             = "CC_DATA_BUFFER_ERROR",

    [CC_BABBLE_DETECTED]               = "CC_BABBLE_DETECTED",

    [CC_USB_TRANSACTION_ERROR]         = "CC_USB_TRANSACTION_ERROR",

    [CC_TRB_ERROR]                     = "CC_TRB_ERROR",

    [CC_STALL_ERROR]                   = "CC_STALL_ERROR",

    [CC_RESOURCE_ERROR]                = "CC_RESOURCE_ERROR",

    [CC_BANDWIDTH_ERROR]               = "CC_BANDWIDTH_ERROR",

    [CC_NO_SLOTS_ERROR]                = "CC_NO_SLOTS_ERROR",

    [CC_INVALID_STREAM_TYPE_ERROR]     = "CC_INVALID_STREAM_TYPE_ERROR",

    [CC_SLOT_NOT_ENABLED_ERROR]        = "CC_SLOT_NOT_ENABLED_ERROR",

    [CC_EP_NOT_ENABLED_ERROR]          = "CC_EP_NOT_ENABLED_ERROR",

    [CC_SHORT_PACKET]                  = "CC_SHORT_PACKET",

    [CC_RING_UNDERRUN]                 = "CC_RING_UNDERRUN",

    [CC_RING_OVERRUN]                  = "CC_RING_OVERRUN",

    [CC_VF_ER_FULL]                    = "CC_VF_ER_FULL",

    [CC_PARAMETER_ERROR]               = "CC_PARAMETER_ERROR",

    [CC_BANDWIDTH_OVERRUN]             = "CC_BANDWIDTH_OVERRUN",

    [CC_CONTEXT_STATE_ERROR]           = "CC_CONTEXT_STATE_ERROR",

    [CC_NO_PING_RESPONSE_ERROR]        = "CC_NO_PING_RESPONSE_ERROR",

    [CC_EVENT_RING_FULL_ERROR]         = "CC_EVENT_RING_FULL_ERROR",

    [CC_INCOMPATIBLE_DEVICE_ERROR]     = "CC_INCOMPATIBLE_DEVICE_ERROR",

    [CC_MISSED_SERVICE_ERROR]          = "CC_MISSED_SERVICE_ERROR",

    [CC_COMMAND_RING_STOPPED]          = "CC_COMMAND_RING_STOPPED",

    [CC_COMMAND_ABORTED]               = "CC_COMMAND_ABORTED",

    [CC_STOPPED]                       = "CC_STOPPED",

    [CC_STOPPED_LENGTH_INVALID]        = "CC_STOPPED_LENGTH_INVALID",

    [CC_MAX_EXIT_LATENCY_TOO_LARGE_ERROR]

    = "CC_MAX_EXIT_LATENCY_TOO_LARGE_ERROR",

    [CC_ISOCH_BUFFER_OVERRUN]          = "CC_ISOCH_BUFFER_OVERRUN",

    [CC_EVENT_LOST_ERROR]              = "CC_EVENT_LOST_ERROR",

    [CC_UNDEFINED_ERROR]               = "CC_UNDEFINED_ERROR",

    [CC_INVALID_STREAM_ID_ERROR]       = "CC_INVALID_STREAM_ID_ERROR",

    [CC_SECONDARY_BANDWIDTH_ERROR]     = "CC_SECONDARY_BANDWIDTH_ERROR",

    [CC_SPLIT_TRANSACTION_ERROR]       = "CC_SPLIT_TRANSACTION_ERROR",

};

static const char *lookup_name(uint32_t index, const char **list, uint32_t llen)

{

    if (index >= llen || list[index] == NULL) {

        return "???";

    }

    return list[index];

}

static const char *trb_name(XHCITRB *trb)

{

    return lookup_name(TRB_TYPE(*trb), TRBType_names,

                       ARRAY_SIZE(TRBType_names));

}

static const char *event_name(XHCIEvent *event)

{

    return lookup_name(event->ccode, TRBCCode_names,

                       ARRAY_SIZE(TRBCCode_names));

}

static uint64_t xhci_mfindex_get(XHCIState *xhci)

{

    int64_t now = qemu_get_clock_ns(vm_clock);

    return (now - xhci->mfindex_start) / 125000;

}

static void xhci_mfwrap_update(XHCIState *xhci)

{

    const uint32_t bits = USBCMD_RS | USBCMD_EWE;

    uint32_t mfindex, left;

    int64_t now;

    if ((xhci->usbcmd & bits) == bits) {

        now = qemu_get_clock_ns(vm_clock);

        mfindex = ((now - xhci->mfindex_start) / 125000) & 0x3fff;

        left = 0x4000 - mfindex;

        qemu_mod_timer(xhci->mfwrap_timer, now + left * 125000);

    } else {

        qemu_del_timer(xhci->mfwrap_timer);

    }

}

static void xhci_mfwrap_timer(void *opaque)

{

    XHCIState *xhci = opaque;

    XHCIEvent wrap = { ER_MFINDEX_WRAP, CC_SUCCESS };

    xhci_event(xhci, &wrap, 0);

    xhci_mfwrap_update(xhci);

}

static inline dma_addr_t xhci_addr64(uint32_t low, uint32_t high)

{

    if (sizeof(dma_addr_t) == 4) {

        return low;

    } else {

        return low | (((dma_addr_t)high << 16) << 16);

    }

}

static inline dma_addr_t xhci_mask64(uint64_t addr)

{

    if (sizeof(dma_addr_t) == 4) {

        return addr & 0xffffffff;

    } else {

        return addr;

    }

}

static inline void xhci_dma_read_u32s(XHCIState *xhci, dma_addr_t addr,

                                      uint32_t *buf, size_t len)

{

    int i;

    assert((len % sizeof(uint32_t)) == 0);

    pci_dma_read(&xhci->pci_dev, addr, buf, len);

    for (i = 0; i < (len / sizeof(uint32_t)); i++) {

        buf[i] = le32_to_cpu(buf[i]);

    }

}

static inline void xhci_dma_write_u32s(XHCIState *xhci, dma_addr_t addr,

                                       uint32_t *buf, size_t len)

{

    int i;

    uint32_t tmp[len / sizeof(uint32_t)];

    assert((len % sizeof(uint32_t)) == 0);

    for (i = 0; i < (len / sizeof(uint32_t)); i++) {

        tmp[i] = cpu_to_le32(buf[i]);

    }

    pci_dma_write(&xhci->pci_dev, addr, tmp, len);

}

static XHCIPort *xhci_lookup_port(XHCIState *xhci, struct USBPort *uport)

{

    int index;

    if (!uport->dev) {

        return NULL;

    }

    switch (uport->dev->speed) {

    case USB_SPEED_LOW:

    case USB_SPEED_FULL:

    case USB_SPEED_HIGH:

        index = uport->index;

        break;

    case USB_SPEED_SUPER:

        index = uport->index + xhci->numports_2;

        break;

    default:

        return NULL;

    }

    return &xhci->ports[index];

}

static void xhci_intx_update(XHCIState *xhci)

{

    int level = 0;

    if (msix_enabled(&xhci->pci_dev) ||

        msi_enabled(&xhci->pci_dev)) {

        return;

    }

    if (xhci->intr[0].iman & IMAN_IP &&

        xhci->intr[0].iman & IMAN_IE &&

        xhci->usbcmd & USBCMD_INTE) {

        level = 1;

    }

    trace_usb_xhci_irq_intx(level);

    qemu_set_irq(xhci->irq, level);

}

static void xhci_msix_update(XHCIState *xhci, int v)

{

    bool enabled;

    if (!msix_enabled(&xhci->pci_dev)) {

        return;

    }

    enabled = xhci->intr[v].iman & IMAN_IE;

    if (enabled == xhci->intr[v].msix_used) {

        return;

    }

    if (enabled) {

        trace_usb_xhci_irq_msix_use(v);

        msix_vector_use(&xhci->pci_dev, v);

        xhci->intr[v].msix_used = true;

    } else {

        trace_usb_xhci_irq_msix_unuse(v);

        msix_vector_unuse(&xhci->pci_dev, v);

        xhci->intr[v].msix_used = false;

    }

}

static void xhci_intr_raise(XHCIState *xhci, int v)

{

    xhci->intr[v].erdp_low |= ERDP_EHB;

    xhci->intr[v].iman |= IMAN_IP;

    xhci->usbsts |= USBSTS_EINT;

    if (!(xhci->intr[v].iman & IMAN_IE)) {

        return;

    }

    if (!(xhci->usbcmd & USBCMD_INTE)) {

        return;

    }

    if (msix_enabled(&xhci->pci_dev)) {

        trace_usb_xhci_irq_msix(v);

        msix_notify(&xhci->pci_dev, v);

        return;

    }

    if (msi_enabled(&xhci->pci_dev)) {

        trace_usb_xhci_irq_msi(v);

        msi_notify(&xhci->pci_dev, v);

        return;

    }

    if (v == 0) {

        trace_usb_xhci_irq_intx(1);

        qemu_set_irq(xhci->irq, 1);

    }

}

static inline int xhci_running(XHCIState *xhci)

{

    return !(xhci->usbsts & USBSTS_HCH) && !xhci->intr[0].er_full;

}

static void xhci_die(XHCIState *xhci)

{

    xhci->usbsts |= USBSTS_HCE;

    fprintf(stderr, "xhci: asserted controller error\n");

}

static void xhci_write_event(XHCIState *xhci, XHCIEvent *event, int v)

{

    XHCIInterrupter *intr = &xhci->intr[v];

    XHCITRB ev_trb;

    dma_addr_t addr;

    ev_trb.parameter = cpu_to_le64(event->ptr);

    ev_trb.status = cpu_to_le32(event->length | (event->ccode << 24));

    ev_trb.control = (event->slotid << 24) | (event->epid << 16) |

                     event->flags | (event->type << TRB_TYPE_SHIFT);

    if (intr->er_pcs) {

        ev_trb.control |= TRB_C;

    }

    ev_trb.control = cpu_to_le32(ev_trb.control);

    trace_usb_xhci_queue_event(v, intr->er_ep_idx, trb_name(&ev_trb),

                               event_name(event), ev_trb.parameter,

                               ev_trb.status, ev_trb.control);

    addr = intr->er_start + TRB_SIZE*intr->er_ep_idx;

    pci_dma_write(&xhci->pci_dev, addr, &ev_trb, TRB_SIZE);

    intr->er_ep_idx++;

    if (intr->er_ep_idx >= intr->er_size) {

        intr->er_ep_idx = 0;

        intr->er_pcs = !intr->er_pcs;

    }

}

static void xhci_events_update(XHCIState *xhci, int v)

{

    XHCIInterrupter *intr = &xhci->intr[v];

    dma_addr_t erdp;

    unsigned int dp_idx;

    bool do_irq = 0;

    if (xhci->usbsts & USBSTS_HCH) {

        return;

    }

    erdp = xhci_addr64(intr->erdp_low, intr->erdp_high);

    if (erdp < intr->er_start ||

        erdp >= (intr->er_start + TRB_SIZE*intr->er_size)) {

        fprintf(stderr, "xhci: ERDP out of bounds: "DMA_ADDR_FMT"\n", erdp);

        fprintf(stderr, "xhci: ER[%d] at "DMA_ADDR_FMT" len %d\n",

                v, intr->er_start, intr->er_size);

        xhci_die(xhci);

        return;

    }

    dp_idx = (erdp - intr->er_start) / TRB_SIZE;

    assert(dp_idx < intr->er_size);

    /* NEC didn't read section 4.9.4 of the spec (v1.0 p139 top Note) and thus

     * deadlocks when the ER is full. Hack it by holding off events until

     * the driver decides to free at least half of the ring */

    if (intr->er_full) {

        int er_free = dp_idx - intr->er_ep_idx;

        if (er_free <= 0) {

            er_free += intr->er_size;

        }

        if (er_free < (intr->er_size/2)) {

            DPRINTF("xhci_events_update(): event ring still "

                    "more than half full (hack)\n");

            return;

        }

    }

    while (intr->ev_buffer_put != intr->ev_buffer_get) {

        assert(intr->er_full);

        if (((intr->er_ep_idx+1) % intr->er_size) == dp_idx) {

            DPRINTF("xhci_events_update(): event ring full again\n");

#ifndef ER_FULL_HACK

            XHCIEvent full = {ER_HOST_CONTROLLER, CC_EVENT_RING_FULL_ERROR};

            xhci_write_event(xhci, &full, v);

#endif

            do_irq = 1;

            break;

        }

        XHCIEvent *event = &intr->ev_buffer[intr->ev_buffer_get];

        xhci_write_event(xhci, event, v);

        intr->ev_buffer_get++;

        do_irq = 1;

        if (intr->ev_buffer_get == EV_QUEUE) {

            intr->ev_buffer_get = 0;

        }

    }

    if (do_irq) {

        xhci_intr_raise(xhci, v);

    }

    if (intr->er_full && intr->ev_buffer_put == intr->ev_buffer_get) {

        DPRINTF("xhci_events_update(): event ring no longer full\n");

        intr->er_full = 0;

    }

}

static void xhci_event(XHCIState *xhci, XHCIEvent *event, int v)

{

    XHCIInterrupter *intr;

    dma_addr_t erdp;

    unsigned int dp_idx;

    if (v >= xhci->numintrs) {

        DPRINTF("intr nr out of range (%d >= %d)\n", v, xhci->numintrs);

        return;

    }

    intr = &xhci->intr[v];

    if (intr->er_full) {

        DPRINTF("xhci_event(): ER full, queueing\n");

        if (((intr->ev_buffer_put+1) % EV_QUEUE) == intr->ev_buffer_get) {

            fprintf(stderr, "xhci: event queue full, dropping event!\n");

            return;

        }

        intr->ev_buffer[intr->ev_buffer_put++] = *event;

        if (intr->ev_buffer_put == EV_QUEUE) {

            intr->ev_buffer_put = 0;

        }

        return;

    }

    erdp = xhci_addr64(intr->erdp_low, intr->erdp_high);

    if (erdp < intr->er_start ||

        erdp >= (intr->er_start + TRB_SIZE*intr->er_size)) {

        fprintf(stderr, "xhci: ERDP out of bounds: "DMA_ADDR_FMT"\n", erdp);

        fprintf(stderr, "xhci: ER[%d] at "DMA_ADDR_FMT" len %d\n",

                v, intr->er_start, intr->er_size);

        xhci_die(xhci);

        return;

    }

    dp_idx = (erdp - intr->er_start) / TRB_SIZE;

    assert(dp_idx < intr->er_size);

    if ((intr->er_ep_idx+1) % intr->er_size == dp_idx) {

        DPRINTF("xhci_event(): ER full, queueing\n");

#ifndef ER_FULL_HACK

        XHCIEvent full = {ER_HOST_CONTROLLER, CC_EVENT_RING_FULL_ERROR};

        xhci_write_event(xhci, &full);

#endif

        intr->er_full = 1;

        if (((intr->ev_buffer_put+1) % EV_QUEUE) == intr->ev_buffer_get) {

            fprintf(stderr, "xhci: event queue full, dropping event!\n");

            return;

        }

        intr->ev_buffer[intr->ev_buffer_put++] = *event;

        if (intr->ev_buffer_put == EV_QUEUE) {

            intr->ev_buffer_put = 0;

        }

    } else {

        xhci_write_event(xhci, event, v);

    }

    xhci_intr_raise(xhci, v);

}

static void xhci_ring_init(XHCIState *xhci, XHCIRing *ring,

                           dma_addr_t base)

{

    ring->base = base;

    ring->dequeue = base;

    ring->ccs = 1;

}

static TRBType xhci_ring_fetch(XHCIState *xhci, XHCIRing *ring, XHCITRB *trb,

                               dma_addr_t *addr)

{

    while (1) {

        TRBType type;

        pci_dma_read(&xhci->pci_dev, ring->dequeue, trb, TRB_SIZE);

        trb->addr = ring->dequeue;

        trb->ccs = ring->ccs;

        le64_to_cpus(&trb->parameter);

        le32_to_cpus(&trb->status);

        le32_to_cpus(&trb->control);

        trace_usb_xhci_fetch_trb(ring->dequeue, trb_name(trb),

                                 trb->parameter, trb->status, trb->control);

        if ((trb->control & TRB_C) != ring->ccs) {

            return 0;

        }

        type = TRB_TYPE(*trb);

        if (type != TR_LINK) {

            if (addr) {

                *addr = ring->dequeue;

            }

            ring->dequeue += TRB_SIZE;

            return type;

        } else {

            ring->dequeue = xhci_mask64(trb->parameter);

            if (trb->control & TRB_LK_TC) {

                ring->ccs = !ring->ccs;

            }

        }

    }

}

static int xhci_ring_chain_length(XHCIState *xhci, const XHCIRing *ring)

{

    XHCITRB trb;

    int length = 0;

    dma_addr_t dequeue = ring->dequeue;

    bool ccs = ring->ccs;

    /* hack to bundle together the two/three TDs that make a setup transfer */

    bool control_td_set = 0;

    while (1) {

        TRBType type;

        pci_dma_read(&xhci->pci_dev, dequeue, &trb, TRB_SIZE);

        le64_to_cpus(&trb.parameter);

        le32_to_cpus(&trb.status);

        le32_to_cpus(&trb.control);

        if ((trb.control & TRB_C) != ccs) {

            return -length;

        }

        type = TRB_TYPE(trb);

        if (type == TR_LINK) {

            dequeue = xhci_mask64(trb.parameter);

            if (trb.control & TRB_LK_TC) {

                ccs = !ccs;

            }

            continue;

        }

        length += 1;

        dequeue += TRB_SIZE;

        if (type == TR_SETUP) {

            control_td_set = 1;

        } else if (type == TR_STATUS) {

            control_td_set = 0;

        }

        if (!control_td_set && !(trb.control & TRB_TR_CH)) {

            return length;

        }

    }

}

static void xhci_er_reset(XHCIState *xhci, int v)

{

    XHCIInterrupter *intr = &xhci->intr[v];

    XHCIEvRingSeg seg;

    if (intr->erstsz == 0) {

        /* disabled */

        intr->er_start = 0;

        intr->er_size = 0;

        return;

    }

    /* cache the (sole) event ring segment location */

    if (intr->erstsz != 1) {

        fprintf(stderr, "xhci: invalid value for ERSTSZ: %d\n", intr->erstsz);

        xhci_die(xhci);

        return;

    }

    dma_addr_t erstba = xhci_addr64(intr->erstba_low, intr->erstba_high);

    pci_dma_read(&xhci->pci_dev, erstba, &seg, sizeof(seg));

    le32_to_cpus(&seg.addr_low);

    le32_to_cpus(&seg.addr_high);

    le32_to_cpus(&seg.size);

    if (seg.size < 16 || seg.size > 4096) {

        fprintf(stderr, "xhci: invalid value for segment size: %d\n", seg.size);

        xhci_die(xhci);

        return;

    }

    intr->er_start = xhci_addr64(seg.addr_low, seg.addr_high);

    intr->er_size = seg.size;

    intr->er_ep_idx = 0;

    intr->er_pcs = 1;

    intr->er_full = 0;

    DPRINTF("xhci: event ring[%d]:" DMA_ADDR_FMT " [%d]\n",

            v, intr->er_start, intr->er_size);

}

static void xhci_run(XHCIState *xhci)

{

    trace_usb_xhci_run();

    xhci->usbsts &= ~USBSTS_HCH;

    xhci->mfindex_start = qemu_get_clock_ns(vm_clock);

}

static void xhci_stop(XHCIState *xhci)

{

    trace_usb_xhci_stop();

    xhci->usbsts |= USBSTS_HCH;

    xhci->crcr_low &= ~CRCR_CRR;

}

static XHCIStreamContext *xhci_alloc_stream_contexts(unsigned count,

                                                     dma_addr_t base)

{

    XHCIStreamContext *stctx;

    unsigned int i;

    stctx = g_new0(XHCIStreamContext, count);

    for (i = 0; i < count; i++) {

        stctx[i].pctx = base + i * 16;

        stctx[i].sct = -1;

    }

    return stctx;

}

static void xhci_reset_streams(XHCIEPContext *epctx)

{

    unsigned int i;

    for (i = 0; i < epctx->nr_pstreams; i++) {

        epctx->pstreams[i].sct = -1;

        g_free(epctx->pstreams[i].sstreams);

    }

}

static void xhci_alloc_streams(XHCIEPContext *epctx, dma_addr_t base)

{

    assert(epctx->pstreams == NULL);

    epctx->nr_pstreams = 2 << epctx->max_pstreams;

    epctx->pstreams = xhci_alloc_stream_contexts(epctx->nr_pstreams, base);

}

static void xhci_free_streams(XHCIEPContext *epctx)

{

    int i;

    assert(epctx->pstreams != NULL);

    if (!epctx->lsa) {

        for (i = 0; i < epctx->nr_pstreams; i++) {

            g_free(epctx->pstreams[i].sstreams);

        }

    }

    g_free(epctx->pstreams);

    epctx->pstreams = NULL;

    epctx->nr_pstreams = 0;

}

static XHCIStreamContext *xhci_find_stream(XHCIEPContext *epctx,

                                           unsigned int streamid,

                                           uint32_t *cc_error)

{

    XHCIStreamContext *sctx;

    dma_addr_t base;

    uint32_t ctx[2], sct;

    assert(streamid != 0);

    if (epctx->lsa) {

        if (streamid >= epctx->nr_pstreams) {

            *cc_error = CC_INVALID_STREAM_ID_ERROR;

            return NULL;

        }

        sctx = epctx->pstreams + streamid;

    } else {

        FIXME("secondary streams not implemented yet");

    }

    if (sctx->sct == -1) {

        xhci_dma_read_u32s(epctx->xhci, sctx->pctx, ctx, sizeof(ctx));

        fprintf(stderr, "%s: init sctx #%d @ %lx: %08x %08x\n", __func__,

                streamid, sctx->pctx, ctx[0], ctx[1]);

        sct = (ctx[0] >> 1) & 0x07;

        if (epctx->lsa && sct != 1) {

            *cc_error = CC_INVALID_STREAM_TYPE_ERROR;

            return NULL;

        }

        sctx->sct = sct;

        base = xhci_addr64(ctx[0] & ~0xf, ctx[1]);

        xhci_ring_init(epctx->xhci, &sctx->ring, base);

    }

    return sctx;

}

static void xhci_set_ep_state(XHCIState *xhci, XHCIEPContext *epctx,

                              XHCIStreamContext *sctx, uint32_t state)

{

    uint32_t ctx[5];

    uint32_t ctx2[2];

    fprintf(stderr, "%s: epid %d, state %d\n",

            __func__, epctx->epid, state);

    xhci_dma_read_u32s(xhci, epctx->pctx, ctx, sizeof(ctx));

    ctx[0] &= ~EP_STATE_MASK;

    ctx[0] |= state;

    /* update ring dequeue ptr */

    if (epctx->nr_pstreams) {

        if (sctx != NULL) {

            xhci_dma_read_u32s(xhci, sctx->pctx, ctx2, sizeof(ctx2));

            ctx2[0] &= 0xe;

            ctx2[0] |= sctx->ring.dequeue | sctx->ring.ccs;

            ctx2[1] = (sctx->ring.dequeue >> 16) >> 16;

            xhci_dma_write_u32s(xhci, sctx->pctx, ctx2, sizeof(ctx2));

        }

    } else {

        ctx[2] = epctx->ring.dequeue | epctx->ring.ccs;

        ctx[3] = (epctx->ring.dequeue >> 16) >> 16;

        DPRINTF("xhci: set epctx: " DMA_ADDR_FMT " state=%d dequeue=%08x%08x\n",

                epctx->pctx, state, ctx[3], ctx[2]);

    }

    xhci_dma_write_u32s(xhci, epctx->pctx, ctx, sizeof(ctx));

    epctx->state = state;

}

static void xhci_ep_kick_timer(void *opaque)

{

    XHCIEPContext *epctx = opaque;

    xhci_kick_ep(epctx->xhci, epctx->slotid, epctx->epid, 0);

}

static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,

                               unsigned int epid, dma_addr_t pctx,

                               uint32_t *ctx)

{

    XHCISlot *slot;

    XHCIEPContext *epctx;

    dma_addr_t dequeue;

    int i;

    trace_usb_xhci_ep_enable(slotid, epid);

    assert(slotid >= 1 && slotid <= xhci->numslots);

    assert(epid >= 1 && epid <= 31);

    slot = &xhci->slots[slotid-1];

    if (slot->eps[epid-1]) {

        xhci_disable_ep(xhci, slotid, epid);

    }

    epctx = g_malloc(sizeof(XHCIEPContext));

    memset(epctx, 0, sizeof(XHCIEPContext));

    epctx->xhci = xhci;

    epctx->slotid = slotid;

    epctx->epid = epid;

    slot->eps[epid-1] = epctx;

    dequeue = xhci_addr64(ctx[2] & ~0xf, ctx[3]);

    epctx->type = (ctx[1] >> EP_TYPE_SHIFT) & EP_TYPE_MASK;

    DPRINTF("xhci: endpoint %d.%d type is %d\n", epid/2, epid%2, epctx->type);

    epctx->pctx = pctx;

    epctx->max_psize = ctx[1]>>16;

    epctx->max_psize *= 1+((ctx[1]>>8)&0xff);

    epctx->max_pstreams = (ctx[0] >> 10) & 0xf;

    epctx->lsa = (ctx[0] >> 15) & 1;

    DPRINTF("xhci: endpoint %d.%d max transaction (burst) size is %d\n",

            epid/2, epid%2, epctx->max_psize);

    if (epctx->max_pstreams) {

        xhci_alloc_streams(epctx, dequeue);

    } else {

        xhci_ring_init(xhci, &epctx->ring, dequeue);

        epctx->ring.ccs = ctx[2] & 1;

    }

    for (i = 0; i < ARRAY_SIZE(epctx->transfers); i++) {

        usb_packet_init(&epctx->transfers[i].packet);

    }

    epctx->interval = 1 << (ctx[0] >> 16) & 0xff;

    epctx->mfindex_last = 0;

    epctx->kick_timer = qemu_new_timer_ns(vm_clock, xhci_ep_kick_timer, epctx);

    epctx->state = EP_RUNNING;

    ctx[0] &= ~EP_STATE_MASK;

    ctx[0] |= EP_RUNNING;

    return CC_SUCCESS;

}

static int xhci_ep_nuke_one_xfer(XHCITransfer *t)

{

    int killed = 0;

    if (t->running_async) {

        usb_cancel_packet(&t->packet);

        t->running_async = 0;

        t->cancelled = 1;

        DPRINTF("xhci: cancelling transfer, waiting for it to complete\n");

        killed = 1;

    }

    if (t->running_retry) {

        XHCIEPContext *epctx = t->xhci->slots[t->slotid-1].eps[t->epid-1];

        if (epctx) {

            epctx->retry = NULL;

            qemu_del_timer(epctx->kick_timer);

        }

        t->running_retry = 0;

    }

    if (t->trbs) {

        g_free(t->trbs);

    }

    t->trbs = NULL;

    t->trb_count = t->trb_alloced = 0;

    return killed;

}

static int xhci_ep_nuke_xfers(XHCIState *xhci, unsigned int slotid,

                               unsigned int epid)

{

    XHCISlot *slot;

    XHCIEPContext *epctx;

    int i, xferi, killed = 0;

    USBEndpoint *ep = NULL;

    assert(slotid >= 1 && slotid <= xhci->numslots);

    assert(epid >= 1 && epid <= 31);

    DPRINTF("xhci_ep_nuke_xfers(%d, %d)\n", slotid, epid);