drivers/staging/octeon-usb/octeon-hcd.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2008 Cavium Networks
   7  *
   8  * Some parts of the code were originally released under BSD license:
   9  *
  10  * Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
  11  * reserved.
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions are
  15  * met:
  16  *
  17  *   * Redistributions of source code must retain the above copyright
  18  *     notice, this list of conditions and the following disclaimer.
  19  *
  20  *   * Redistributions in binary form must reproduce the above
  21  *     copyright notice, this list of conditions and the following
  22  *     disclaimer in the documentation and/or other materials provided
  23  *     with the distribution.
  24  *
  25  *   * Neither the name of Cavium Networks nor the names of
  26  *     its contributors may be used to endorse or promote products
  27  *     derived from this software without specific prior written
  28  *     permission.
  29  *
  30  * This Software, including technical data, may be subject to U.S. export
  31  * control laws, including the U.S. Export Administration Act and its associated
  32  * regulations, and may be subject to export or import regulations in other
  33  * countries.
  34  *
  35  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
  36  * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
  37  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
  38  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION
  39  * OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  40  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  41  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  42  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  43  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
  44  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
  45  */
  46 #include <linux/kernel.h>
  47 #include <linux/module.h>
  48 #include <linux/init.h>
  49 #include <linux/pci.h>
  50 #include <linux/prefetch.h>
  51 #include <linux/interrupt.h>
  52 #include <linux/platform_device.h>
  53 #include <linux/usb.h>
  54
  55 #include <linux/time.h>
  56 #include <linux/delay.h>
  57
  58 #include <asm/octeon/cvmx.h>
  59 #include <asm/octeon/cvmx-iob-defs.h>
  60
  61 #include <linux/usb/hcd.h>
  62
  63 #include <linux/err.h>
  64
  65 #include <asm/octeon/octeon.h>
  66 #include <asm/octeon/cvmx-helper.h>
  67 #include <asm/octeon/cvmx-sysinfo.h>
  68 #include <asm/octeon/cvmx-helper-board.h>
  69
  70 #include "octeon-hcd.h"
  71
  72 /**
  73  * enum cvmx_usb_speed - the possible USB device speeds
  74  *
  75  * @CVMX_USB_SPEED_HIGH: Device is operation at 480Mbps
  76  * @CVMX_USB_SPEED_FULL: Device is operation at 12Mbps
  77  * @CVMX_USB_SPEED_LOW:  Device is operation at 1.5Mbps
  78  */
  79 enum cvmx_usb_speed {
  80         CVMX_USB_SPEED_HIGH = 0,
  81         CVMX_USB_SPEED_FULL = 1,
  82         CVMX_USB_SPEED_LOW = 2,
  83 };
  84
  85 /**
  86  * enum cvmx_usb_transfer - the possible USB transfer types
  87  *
  88  * @CVMX_USB_TRANSFER_CONTROL:     USB transfer type control for hub and status
  89  *                                 transfers
  90  * @CVMX_USB_TRANSFER_ISOCHRONOUS: USB transfer type isochronous for low
  91  *                                 priority periodic transfers
  92  * @CVMX_USB_TRANSFER_BULK:        USB transfer type bulk for large low priority
  93  *                                 transfers
  94  * @CVMX_USB_TRANSFER_INTERRUPT:   USB transfer type interrupt for high priority
  95  *                                 periodic transfers
  96  */
  97 enum cvmx_usb_transfer {
  98         CVMX_USB_TRANSFER_CONTROL = 0,
  99         CVMX_USB_TRANSFER_ISOCHRONOUS = 1,
 100         CVMX_USB_TRANSFER_BULK = 2,
 101         CVMX_USB_TRANSFER_INTERRUPT = 3,
 102 };
 103
 104 /**
 105  * enum cvmx_usb_direction - the transfer directions
 106  *
 107  * @CVMX_USB_DIRECTION_OUT: Data is transferring from Octeon to the device/host
 108  * @CVMX_USB_DIRECTION_IN:  Data is transferring from the device/host to Octeon
 109  */
 110 enum cvmx_usb_direction {
 111         CVMX_USB_DIRECTION_OUT,
 112         CVMX_USB_DIRECTION_IN,
 113 };
 114
 115 /**
 116  * enum cvmx_usb_complete - possible callback function status codes
 117  *
 118  * @CVMX_USB_COMPLETE_SUCCESS:    The transaction / operation finished without
 119  *                                any errors
 120  * @CVMX_USB_COMPLETE_SHORT:      FIXME: This is currently not implemented
 121  * @CVMX_USB_COMPLETE_CANCEL:     The transaction was canceled while in flight
 122  *                                by a user call to cvmx_usb_cancel
 123  * @CVMX_USB_COMPLETE_ERROR:      The transaction aborted with an unexpected
 124  *                                error status
 125  * @CVMX_USB_COMPLETE_STALL:      The transaction received a USB STALL response
 126  *                                from the device
 127  * @CVMX_USB_COMPLETE_XACTERR:    The transaction failed with an error from the
 128  *                                device even after a number of retries
 129  * @CVMX_USB_COMPLETE_DATATGLERR: The transaction failed with a data toggle
 130  *                                error even after a number of retries
 131  * @CVMX_USB_COMPLETE_BABBLEERR:  The transaction failed with a babble error
 132  * @CVMX_USB_COMPLETE_FRAMEERR:   The transaction failed with a frame error
 133  *                                even after a number of retries
 134  */
 135 enum cvmx_usb_complete {
 136         CVMX_USB_COMPLETE_SUCCESS,
 137         CVMX_USB_COMPLETE_SHORT,
 138         CVMX_USB_COMPLETE_CANCEL,
 139         CVMX_USB_COMPLETE_ERROR,
 140         CVMX_USB_COMPLETE_STALL,
 141         CVMX_USB_COMPLETE_XACTERR,
 142         CVMX_USB_COMPLETE_DATATGLERR,
 143         CVMX_USB_COMPLETE_BABBLEERR,
 144         CVMX_USB_COMPLETE_FRAMEERR,
 145 };
 146
 147 /**
 148  * struct cvmx_usb_port_status - the USB port status information
 149  *
 150  * @port_enabled:       1 = Usb port is enabled, 0 = disabled
 151  * @port_over_current:  1 = Over current detected, 0 = Over current not
 152  *                      detected. Octeon doesn't support over current detection.
 153  * @port_powered:       1 = Port power is being supplied to the device, 0 =
 154  *                      power is off. Octeon doesn't support turning port power
 155  *                      off.
 156  * @port_speed:         Current port speed.
 157  * @connected:          1 = A device is connected to the port, 0 = No device is
 158  *                      connected.
 159  * @connect_change:     1 = Device connected state changed since the last set
 160  *                      status call.
 161  */
 162 struct cvmx_usb_port_status {
 163         uint32_t reserved               : 25;
 164         uint32_t port_enabled           : 1;
 165         uint32_t port_over_current      : 1;
 166         uint32_t port_powered           : 1;
 167         enum cvmx_usb_speed port_speed  : 2;
 168         uint32_t connected              : 1;
 169         uint32_t connect_change         : 1;
 170 };
 171
 172 /**
 173  * struct cvmx_usb_iso_packet - descriptor for Isochronous packets
 174  *
 175  * @offset:     This is the offset in bytes into the main buffer where this data
 176  *              is stored.
 177  * @length:     This is the length in bytes of the data.
 178  * @status:     This is the status of this individual packet transfer.
 179  */
 180 struct cvmx_usb_iso_packet {
 181         int offset;
 182         int length;
 183         enum cvmx_usb_complete status;
 184 };
 185
 186 /**
 187  * enum cvmx_usb_initialize_flags - flags used by the initialization function
 188  *
 189  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI:    The USB port uses a 12MHz crystal
 190  *                                            as clock source at USB_XO and
 191  *                                            USB_XI.
 192  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND:   The USB port uses 12/24/48MHz 2.5V
 193  *                                            board clock source at USB_XO.
 194  *                                            USB_XI should be tied to GND.
 195  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK: Mask for clock speed field
 196  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:    Speed of reference clock or
 197  *                                            crystal
 198  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:    Speed of reference clock
 199  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:    Speed of reference clock
 200  * @CVMX_USB_INITIALIZE_FLAGS_NO_DMA:         Disable DMA and used polled IO for
 201  *                                            data transfer use for the USB
 202  */
 203 enum cvmx_usb_initialize_flags {
 204         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI           = 1 << 0,
 205         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND          = 1 << 1,
 206         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK        = 3 << 3,
 207         CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ           = 1 << 3,
 208         CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ           = 2 << 3,
 209         CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ           = 3 << 3,
 210         /* Bits 3-4 used to encode the clock frequency */
 211         CVMX_USB_INITIALIZE_FLAGS_NO_DMA                = 1 << 5,
 212 };
 213
 214 /**
 215  * enum cvmx_usb_pipe_flags - internal flags for a pipe.
 216  *
 217  * @__CVMX_USB_PIPE_FLAGS_SCHEDULED: Used internally to determine if a pipe is
 218  *                                   actively using hardware. Do not use.
 219  * @__CVMX_USB_PIPE_FLAGS_NEED_PING: Used internally to determine if a high
 220  *                                   speed pipe is in the ping state. Do not
 221  *                                   use.
 222  */
 223 enum cvmx_usb_pipe_flags {
 224         __CVMX_USB_PIPE_FLAGS_SCHEDULED = 1 << 17,
 225         __CVMX_USB_PIPE_FLAGS_NEED_PING = 1 << 18,
 226 };
 227
 228 /* Maximum number of times to retry failed transactions */
 229 #define MAX_RETRIES             3
 230
 231 /* Maximum number of hardware channels supported by the USB block */
 232 #define MAX_CHANNELS            8
 233
 234 /* The highest valid USB device address */
 235 #define MAX_USB_ADDRESS         127
 236
 237 /* The highest valid USB endpoint number */
 238 #define MAX_USB_ENDPOINT        15
 239
 240 /* The highest valid port number on a hub */
 241 #define MAX_USB_HUB_PORT        15
 242
 243 /*
 244  * The low level hardware can transfer a maximum of this number of bytes in each
 245  * transfer. The field is 19 bits wide
 246  */
 247 #define MAX_TRANSFER_BYTES      ((1<<19)-1)
 248
 249 /*
 250  * The low level hardware can transfer a maximum of this number of packets in
 251  * each transfer. The field is 10 bits wide
 252  */
 253 #define MAX_TRANSFER_PACKETS    ((1<<10)-1)
 254
 255 /**
 256  * Logical transactions may take numerous low level
 257  * transactions, especially when splits are concerned. This
 258  * enum represents all of the possible stages a transaction can
 259  * be in. Note that split completes are always even. This is so
 260  * the NAK handler can backup to the previous low level
 261  * transaction with a simple clearing of bit 0.
 262  */
 263 enum cvmx_usb_stage {
 264         CVMX_USB_STAGE_NON_CONTROL,
 265         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
 266         CVMX_USB_STAGE_SETUP,
 267         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
 268         CVMX_USB_STAGE_DATA,
 269         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
 270         CVMX_USB_STAGE_STATUS,
 271         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
 272 };
 273
 274 /**
 275  * struct cvmx_usb_transaction - describes each pending USB transaction
 276  *                               regardless of type. These are linked together
 277  *                               to form a list of pending requests for a pipe.
 278  *
 279  * @node:               List node for transactions in the pipe.
 280  * @type:               Type of transaction, duplicated of the pipe.
 281  * @flags:              State flags for this transaction.
 282  * @buffer:             User's physical buffer address to read/write.
 283  * @buffer_length:      Size of the user's buffer in bytes.
 284  * @control_header:     For control transactions, physical address of the 8
 285  *                      byte standard header.
 286  * @iso_start_frame:    For ISO transactions, the starting frame number.
 287  * @iso_number_packets: For ISO transactions, the number of packets in the
 288  *                      request.
 289  * @iso_packets:        For ISO transactions, the sub packets in the request.
 290  * @actual_bytes:       Actual bytes transfer for this transaction.
 291  * @stage:              For control transactions, the current stage.
 292  * @urb:                URB.
 293  */
 294 struct cvmx_usb_transaction {
 295         struct list_head node;
 296         enum cvmx_usb_transfer type;
 297         uint64_t buffer;
 298         int buffer_length;
 299         uint64_t control_header;
 300         int iso_start_frame;
 301         int iso_number_packets;
 302         struct cvmx_usb_iso_packet *iso_packets;
 303         int xfersize;
 304         int pktcnt;
 305         int retries;
 306         int actual_bytes;
 307         enum cvmx_usb_stage stage;
 308         struct urb *urb;
 309 };
 310
 311 /**
 312  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
 313  *                        and some USB device. It contains a list of pending
 314  *                        request to the device.
 315  *
 316  * @node:               List node for pipe list
 317  * @next:               Pipe after this one in the list
 318  * @transactions:       List of pending transactions
 319  * @interval:           For periodic pipes, the interval between packets in
 320  *                      frames
 321  * @next_tx_frame:      The next frame this pipe is allowed to transmit on
 322  * @flags:              State flags for this pipe
 323  * @device_speed:       Speed of device connected to this pipe
 324  * @transfer_type:      Type of transaction supported by this pipe
 325  * @transfer_dir:       IN or OUT. Ignored for Control
 326  * @multi_count:        Max packet in a row for the device
 327  * @max_packet:         The device's maximum packet size in bytes
 328  * @device_addr:        USB device address at other end of pipe
 329  * @endpoint_num:       USB endpoint number at other end of pipe
 330  * @hub_device_addr:    Hub address this device is connected to
 331  * @hub_port:           Hub port this device is connected to
 332  * @pid_toggle:         This toggles between 0/1 on every packet send to track
 333  *                      the data pid needed
 334  * @channel:            Hardware DMA channel for this pipe
 335  * @split_sc_frame:     The low order bits of the frame number the split
 336  *                      complete should be sent on
 337  */
 338 struct cvmx_usb_pipe {
 339         struct list_head node;
 340         struct list_head transactions;
 341         uint64_t interval;
 342         uint64_t next_tx_frame;
 343         enum cvmx_usb_pipe_flags flags;
 344         enum cvmx_usb_speed device_speed;
 345         enum cvmx_usb_transfer transfer_type;
 346         enum cvmx_usb_direction transfer_dir;
 347         int multi_count;
 348         uint16_t max_packet;
 349         uint8_t device_addr;
 350         uint8_t endpoint_num;
 351         uint8_t hub_device_addr;
 352         uint8_t hub_port;
 353         uint8_t pid_toggle;
 354         uint8_t channel;
 355         int8_t split_sc_frame;
 356 };
 357
 358 struct cvmx_usb_tx_fifo {
 359         struct {
 360                 int channel;
 361                 int size;
 362                 uint64_t address;
 363         } entry[MAX_CHANNELS+1];
 364         int head;
 365         int tail;
 366 };
 367
 368 /**
 369  * struct cvmx_usb_state - the state of the USB block
 370  *
 371  * init_flags:             Flags passed to initialize.
 372  * index:                  Which USB block this is for.
 373  * idle_hardware_channels: Bit set for every idle hardware channel.
 374  * usbcx_hprt:             Stored port status so we don't need to read a CSR to
 375  *                         determine splits.
 376  * pipe_for_channel:       Map channels to pipes.
 377  * pipe:                   Storage for pipes.
 378  * indent:                 Used by debug output to indent functions.
 379  * port_status:            Last port status used for change notification.
 380  * idle_pipes:             List of open pipes that have no transactions.
 381  * active_pipes:           Active pipes indexed by transfer type.
 382  * frame_number:           Increments every SOF interrupt for time keeping.
 383  * active_split:           Points to the current active split, or NULL.
 384  */
 385 struct cvmx_usb_state {
 386         int init_flags;
 387         int index;
 388         int idle_hardware_channels;
 389         union cvmx_usbcx_hprt usbcx_hprt;
 390         struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
 391         int indent;
 392         struct cvmx_usb_port_status port_status;
 393         struct list_head idle_pipes;
 394         struct list_head active_pipes[4];
 395         uint64_t frame_number;
 396         struct cvmx_usb_transaction *active_split;
 397         struct cvmx_usb_tx_fifo periodic;
 398         struct cvmx_usb_tx_fifo nonperiodic;
 399 };
 400
 401 struct octeon_hcd {
 402         spinlock_t lock;
 403         struct cvmx_usb_state usb;
 404 };
 405
 406 /* This macro spins on a field waiting for it to reach a value */
 407 #define CVMX_WAIT_FOR_FIELD32(address, type, field, op, value, timeout_usec)\
 408         ({int result;                                                       \
 409         do {                                                                \
 410                 uint64_t done = cvmx_get_cycle() + (uint64_t)timeout_usec * \
 411                         octeon_get_clock_rate() / 1000000;                  \
 412                 type c;                                                     \
 413                 while (1) {                                                 \
 414                         c.u32 = __cvmx_usb_read_csr32(usb, address);        \
 415                         if (c.s.field op (value)) {                         \
 416                                 result = 0;                                 \
 417                                 break;                                      \
 418                         } else if (cvmx_get_cycle() > done) {               \
 419                                 result = -1;                                \
 420                                 break;                                      \
 421                         } else                                              \
 422                                 cvmx_wait(100);                             \
 423                 }                                                           \
 424         } while (0);                                                        \
 425         result; })
 426
 427 /*
 428  * This macro logically sets a single field in a CSR. It does the sequence
 429  * read, modify, and write
 430  */
 431 #define USB_SET_FIELD32(address, type, field, value)            \
 432         do {                                                    \
 433                 type c;                                         \
 434                 c.u32 = __cvmx_usb_read_csr32(usb, address);    \
 435                 c.s.field = value;                              \
 436                 __cvmx_usb_write_csr32(usb, address, c.u32);    \
 437         } while (0)
 438
 439 /* Returns the IO address to push/pop stuff data from the FIFOs */
 440 #define USB_FIFO_ADDRESS(channel, usb_index) \
 441         (CVMX_USBCX_GOTGCTL(usb_index) + ((channel)+1)*0x1000)
 442
 443 /**
 444  * struct octeon_temp_buffer - a bounce buffer for USB transfers
 445  * @temp_buffer: the newly allocated temporary buffer (including meta-data)
 446  * @orig_buffer: the original buffer passed by the USB stack
 447  * @data:        the newly allocated temporary buffer (excluding meta-data)
 448  *
 449  * Both the DMA engine and FIFO mode will always transfer full 32-bit words. If
 450  * the buffer is too short, we need to allocate a temporary one, and this struct
 451  * represents it.
 452  */
 453 struct octeon_temp_buffer {
 454         void *temp_buffer;
 455         void *orig_buffer;
 456         u8 data[0];
 457 };
 458
 459 /**
 460  * octeon_alloc_temp_buffer - allocate a temporary buffer for USB transfer
 461  *                            (if needed)
 462  * @urb:        URB.
 463  * @mem_flags:  Memory allocation flags.
 464  *
 465  * This function allocates a temporary bounce buffer whenever it's needed
 466  * due to HW limitations.
 467  */
 468 static int octeon_alloc_temp_buffer(struct urb *urb, gfp_t mem_flags)
 469 {
 470         struct octeon_temp_buffer *temp;
 471
 472         if (urb->num_sgs || urb->sg ||
 473             (urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP) ||
 474             !(urb->transfer_buffer_length % sizeof(u32)))
 475                 return 0;
 476
 477         temp = kmalloc(ALIGN(urb->transfer_buffer_length, sizeof(u32)) +
 478                        sizeof(*temp), mem_flags);
 479         if (!temp)
 480                 return -ENOMEM;
 481
 482         temp->temp_buffer = temp;
 483         temp->orig_buffer = urb->transfer_buffer;
 484         if (usb_urb_dir_out(urb))
 485                 memcpy(temp->data, urb->transfer_buffer,
 486                        urb->transfer_buffer_length);
 487         urb->transfer_buffer = temp->data;
 488         urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER;
 489
 490         return 0;
 491 }
 492
 493 /**
 494  * octeon_free_temp_buffer - free a temporary buffer used by USB transfers.
 495  * @urb: URB.
 496  *
 497  * Frees a buffer allocated by octeon_alloc_temp_buffer().
 498  */
 499 static void octeon_free_temp_buffer(struct urb *urb)
 500 {
 501         struct octeon_temp_buffer *temp;
 502
 503         if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER))
 504                 return;
 505
 506         temp = container_of(urb->transfer_buffer, struct octeon_temp_buffer,
 507                             data);
 508         if (usb_urb_dir_in(urb))
 509                 memcpy(temp->orig_buffer, urb->transfer_buffer,
 510                        urb->actual_length);
 511         urb->transfer_buffer = temp->orig_buffer;
 512         urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER;
 513         kfree(temp->temp_buffer);
 514 }
 515
 516 /**
 517  * octeon_map_urb_for_dma - Octeon-specific map_urb_for_dma().
 518  * @hcd:        USB HCD structure.
 519  * @urb:        URB.
 520  * @mem_flags:  Memory allocation flags.
 521  */
 522 static int octeon_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
 523                                   gfp_t mem_flags)
 524 {
 525         int ret;
 526
 527         ret = octeon_alloc_temp_buffer(urb, mem_flags);
 528         if (ret)
 529                 return ret;
 530
 531         ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
 532         if (ret)
 533                 octeon_free_temp_buffer(urb);
 534
 535         return ret;
 536 }
 537
 538 /**
 539  * octeon_unmap_urb_for_dma - Octeon-specific unmap_urb_for_dma()
 540  * @hcd:        USB HCD structure.
 541  * @urb:        URB.
 542  */
 543 static void octeon_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
 544 {
 545         usb_hcd_unmap_urb_for_dma(hcd, urb);
 546         octeon_free_temp_buffer(urb);
 547 }
 548
 549 /**
 550  * Read a USB 32bit CSR. It performs the necessary address swizzle
 551  * for 32bit CSRs and logs the value in a readable format if
 552  * debugging is on.
 553  *
 554  * @usb:     USB block this access is for
 555  * @address: 64bit address to read
 556  *
 557  * Returns: Result of the read
 558  */
 559 static inline uint32_t __cvmx_usb_read_csr32(struct cvmx_usb_state *usb,
 560                                              uint64_t address)
 561 {
 562         uint32_t result = cvmx_read64_uint32(address ^ 4);
 563         return result;
 564 }
 565
 566
 567 /**
 568  * Write a USB 32bit CSR. It performs the necessary address
 569  * swizzle for 32bit CSRs and logs the value in a readable format
 570  * if debugging is on.
 571  *
 572  * @usb:     USB block this access is for
 573  * @address: 64bit address to write
 574  * @value:   Value to write
 575  */
 576 static inline void __cvmx_usb_write_csr32(struct cvmx_usb_state *usb,
 577                                           uint64_t address, uint32_t value)
 578 {
 579         cvmx_write64_uint32(address ^ 4, value);
 580         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
 581 }
 582
 583
 584 /**
 585  * Read a USB 64bit CSR. It logs the value in a readable format if
 586  * debugging is on.
 587  *
 588  * @usb:     USB block this access is for
 589  * @address: 64bit address to read
 590  *
 591  * Returns: Result of the read
 592  */
 593 static inline uint64_t __cvmx_usb_read_csr64(struct cvmx_usb_state *usb,
 594                                              uint64_t address)
 595 {
 596         uint64_t result = cvmx_read64_uint64(address);
 597         return result;
 598 }
 599
 600
 601 /**
 602  * Write a USB 64bit CSR. It logs the value in a readable format
 603  * if debugging is on.
 604  *
 605  * @usb:     USB block this access is for
 606  * @address: 64bit address to write
 607  * @value:   Value to write
 608  */
 609 static inline void __cvmx_usb_write_csr64(struct cvmx_usb_state *usb,
 610                                           uint64_t address, uint64_t value)
 611 {
 612         cvmx_write64_uint64(address, value);
 613 }
 614
 615 /**
 616  * Return non zero if this pipe connects to a non HIGH speed
 617  * device through a high speed hub.
 618  *
 619  * @usb:    USB block this access is for
 620  * @pipe:   Pipe to check
 621  *
 622  * Returns: Non zero if we need to do split transactions
 623  */
 624 static inline int __cvmx_usb_pipe_needs_split(struct cvmx_usb_state *usb,
 625                                               struct cvmx_usb_pipe *pipe)
 626 {
 627         return pipe->device_speed != CVMX_USB_SPEED_HIGH &&
 628                usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH;
 629 }
 630
 631
 632 /**
 633  * Trivial utility function to return the correct PID for a pipe
 634  *
 635  * @pipe:   pipe to check
 636  *
 637  * Returns: PID for pipe
 638  */
 639 static inline int __cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
 640 {
 641         if (pipe->pid_toggle)
 642                 return 2; /* Data1 */
 643         else
 644                 return 0; /* Data0 */
 645 }
 646
 647 /**
 648  * Initialize a USB port for use. This must be called before any
 649  * other access to the Octeon USB port is made. The port starts
 650  * off in the disabled state.
 651  *
 652  * @usb:         Pointer to an empty struct cvmx_usb_state
 653  *               that will be populated by the initialize call.
 654  *               This structure is then passed to all other USB
 655  *               functions.
 656  * @usb_port_number:
 657  *               Which Octeon USB port to initialize.
 658  *
 659  * Returns: 0 or a negative error code.
 660  */
 661 static int cvmx_usb_initialize(struct cvmx_usb_state *usb,
 662                                int usb_port_number,
 663                                enum cvmx_usb_initialize_flags flags)
 664 {
 665         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 666         union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
 667         int i;
 668
 669         /* At first allow 0-1 for the usb port number */
 670         if ((usb_port_number < 0) || (usb_port_number > 1))
 671                 return -EINVAL;
 672
 673         memset(usb, 0, sizeof(*usb));
 674         usb->init_flags = flags;
 675
 676         /* Initialize the USB state structure */
 677         usb->index = usb_port_number;
 678         INIT_LIST_HEAD(&usb->idle_pipes);
 679         for (i = 0; i < ARRAY_SIZE(usb->active_pipes); i++)
 680                 INIT_LIST_HEAD(&usb->active_pipes[i]);
 681
 682         /*
 683          * Power On Reset and PHY Initialization
 684          *
 685          * 1. Wait for DCOK to assert (nothing to do)
 686          *
 687          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
 688          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
 689          */
 690         usbn_clk_ctl.u64 =
 691                 __cvmx_usb_read_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index));
 692         usbn_clk_ctl.s.por = 1;
 693         usbn_clk_ctl.s.hrst = 0;
 694         usbn_clk_ctl.s.prst = 0;
 695         usbn_clk_ctl.s.hclk_rst = 0;
 696         usbn_clk_ctl.s.enable = 0;
 697         /*
 698          * 2b. Select the USB reference clock/crystal parameters by writing
 699          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
 700          */
 701         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
 702                 /*
 703                  * The USB port uses 12/24/48MHz 2.5V board clock
 704                  * source at USB_XO. USB_XI should be tied to GND.
 705                  * Most Octeon evaluation boards require this setting
 706                  */
 707                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
 708                     OCTEON_IS_MODEL(OCTEON_CN56XX) ||
 709                     OCTEON_IS_MODEL(OCTEON_CN50XX))
 710                         /* From CN56XX,CN50XX,CN31XX,CN30XX manuals */
 711                         usbn_clk_ctl.s.p_rtype = 2; /* p_rclk=1 & p_xenbn=0 */
 712                 else
 713                         /* From CN52XX manual */
 714                         usbn_clk_ctl.s.p_rtype = 1;
 715
 716                 switch (flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
 717                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
 718                         usbn_clk_ctl.s.p_c_sel = 0;
 719                         break;
 720                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
 721                         usbn_clk_ctl.s.p_c_sel = 1;
 722                         break;
 723                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
 724                         usbn_clk_ctl.s.p_c_sel = 2;
 725                         break;
 726                 }
 727         } else {
 728                 /*
 729                  * The USB port uses a 12MHz crystal as clock source
 730                  * at USB_XO and USB_XI
 731                  */
 732                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
 733                         /* From CN31XX,CN30XX manual */
 734                         usbn_clk_ctl.s.p_rtype = 3; /* p_rclk=1 & p_xenbn=1 */
 735                 else
 736                         /* From CN56XX,CN52XX,CN50XX manuals. */
 737                         usbn_clk_ctl.s.p_rtype = 0;
 738
 739                 usbn_clk_ctl.s.p_c_sel = 0;
 740         }
 741         /*
 742          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
 743          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
 744          *     such that USB is as close as possible to 125Mhz
 745          */
 746         {
 747                 int divisor = (octeon_get_clock_rate()+125000000-1)/125000000;
 748                 /* Lower than 4 doesn't seem to work properly */
 749                 if (divisor < 4)
 750                         divisor = 4;
 751                 usbn_clk_ctl.s.divide = divisor;
 752                 usbn_clk_ctl.s.divide2 = 0;
 753         }
 754         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 755                                usbn_clk_ctl.u64);
 756         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
 757         usbn_clk_ctl.s.hclk_rst = 1;
 758         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 759                                usbn_clk_ctl.u64);
 760         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
 761         cvmx_wait(64);
 762         /*
 763          * 3. Program the power-on reset field in the USBN clock-control
 764          *    register:
 765          *    USBN_CLK_CTL[POR] = 0
 766          */
 767         usbn_clk_ctl.s.por = 0;
 768         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 769                                usbn_clk_ctl.u64);
 770         /* 4. Wait 1 ms for PHY clock to start */
 771         mdelay(1);
 772         /*
 773          * 5. Program the Reset input from automatic test equipment field in the
 774          *    USBP control and status register:
 775          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
 776          */
 777         usbn_usbp_ctl_status.u64 = __cvmx_usb_read_csr64(usb,
 778                         CVMX_USBNX_USBP_CTL_STATUS(usb->index));
 779         usbn_usbp_ctl_status.s.ate_reset = 1;
 780         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 781                                usbn_usbp_ctl_status.u64);
 782         /* 6. Wait 10 cycles */
 783         cvmx_wait(10);
 784         /*
 785          * 7. Clear ATE_RESET field in the USBN clock-control register:
 786          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
 787          */
 788         usbn_usbp_ctl_status.s.ate_reset = 0;
 789         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 790                                usbn_usbp_ctl_status.u64);
 791         /*
 792          * 8. Program the PHY reset field in the USBN clock-control register:
 793          *    USBN_CLK_CTL[PRST] = 1
 794          */
 795         usbn_clk_ctl.s.prst = 1;
 796         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 797                                usbn_clk_ctl.u64);
 798         /*
 799          * 9. Program the USBP control and status register to select host or
 800          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
 801          *    device
 802          */
 803         usbn_usbp_ctl_status.s.hst_mode = 0;
 804         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
 805                                usbn_usbp_ctl_status.u64);
 806         /* 10. Wait 1 us */
 807         udelay(1);
 808         /*
 809          * 11. Program the hreset_n field in the USBN clock-control register:
 810          *     USBN_CLK_CTL[HRST] = 1
 811          */
 812         usbn_clk_ctl.s.hrst = 1;
 813         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 814                                usbn_clk_ctl.u64);
 815         /* 12. Proceed to USB core initialization */
 816         usbn_clk_ctl.s.enable = 1;
 817         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 818                                usbn_clk_ctl.u64);
 819         udelay(1);
 820
 821         /*
 822          * USB Core Initialization
 823          *
 824          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
 825          *    determine USB core configuration parameters.
 826          *
 827          *    Nothing needed
 828          *
 829          * 2. Program the following fields in the global AHB configuration
 830          *    register (USBC_GAHBCFG)
 831          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
 832          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
 833          *    Nonperiodic TxFIFO empty level (slave mode only),
 834          *    USBC_GAHBCFG[NPTXFEMPLVL]
 835          *    Periodic TxFIFO empty level (slave mode only),
 836          *    USBC_GAHBCFG[PTXFEMPLVL]
 837          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
 838          */
 839         {
 840                 union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
 841                 /* Due to an errata, CN31XX doesn't support DMA */
 842                 if (OCTEON_IS_MODEL(OCTEON_CN31XX))
 843                         usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
 844                 usbcx_gahbcfg.u32 = 0;
 845                 usbcx_gahbcfg.s.dmaen = !(usb->init_flags &
 846                                           CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
 847                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 848                         /* Only use one channel with non DMA */
 849                         usb->idle_hardware_channels = 0x1;
 850                 else if (OCTEON_IS_MODEL(OCTEON_CN5XXX))
 851                         /* CN5XXX have an errata with channel 3 */
 852                         usb->idle_hardware_channels = 0xf7;
 853                 else
 854                         usb->idle_hardware_channels = 0xff;
 855                 usbcx_gahbcfg.s.hbstlen = 0;
 856                 usbcx_gahbcfg.s.nptxfemplvl = 1;
 857                 usbcx_gahbcfg.s.ptxfemplvl = 1;
 858                 usbcx_gahbcfg.s.glblintrmsk = 1;
 859                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
 860                                        usbcx_gahbcfg.u32);
 861         }
 862         /*
 863          * 3. Program the following fields in USBC_GUSBCFG register.
 864          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
 865          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
 866          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
 867          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
 868          */
 869         {
 870                 union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
 871
 872                 usbcx_gusbcfg.u32 = __cvmx_usb_read_csr32(usb,
 873                                 CVMX_USBCX_GUSBCFG(usb->index));
 874                 usbcx_gusbcfg.s.toutcal = 0;
 875                 usbcx_gusbcfg.s.ddrsel = 0;
 876                 usbcx_gusbcfg.s.usbtrdtim = 0x5;
 877                 usbcx_gusbcfg.s.phylpwrclksel = 0;
 878                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
 879                                        usbcx_gusbcfg.u32);
 880         }
 881         /*
 882          * 4. The software must unmask the following bits in the USBC_GINTMSK
 883          *    register.
 884          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
 885          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
 886          */
 887         {
 888                 union cvmx_usbcx_gintmsk usbcx_gintmsk;
 889                 int channel;
 890
 891                 usbcx_gintmsk.u32 = __cvmx_usb_read_csr32(usb,
 892                                 CVMX_USBCX_GINTMSK(usb->index));
 893                 usbcx_gintmsk.s.otgintmsk = 1;
 894                 usbcx_gintmsk.s.modemismsk = 1;
 895                 usbcx_gintmsk.s.hchintmsk = 1;
 896                 usbcx_gintmsk.s.sofmsk = 0;
 897                 /* We need RX FIFO interrupts if we don't have DMA */
 898                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
 899                         usbcx_gintmsk.s.rxflvlmsk = 1;
 900                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
 901                                        usbcx_gintmsk.u32);
 902
 903                 /*
 904                  * Disable all channel interrupts. We'll enable them per channel
 905                  * later.
 906                  */
 907                 for (channel = 0; channel < 8; channel++)
 908                         __cvmx_usb_write_csr32(usb,
 909                                 CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
 910         }
 911
 912         {
 913                 /*
 914                  * Host Port Initialization
 915                  *
 916                  * 1. Program the host-port interrupt-mask field to unmask,
 917                  *    USBC_GINTMSK[PRTINT] = 1
 918                  */
 919                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 920                                 union cvmx_usbcx_gintmsk, prtintmsk, 1);
 921                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
 922                                 union cvmx_usbcx_gintmsk, disconnintmsk, 1);
 923                 /*
 924                  * 2. Program the USBC_HCFG register to select full-speed host
 925                  *    or high-speed host.
 926                  */
 927                 {
 928                         union cvmx_usbcx_hcfg usbcx_hcfg;
 929
 930                         usbcx_hcfg.u32 = __cvmx_usb_read_csr32(usb,
 931                                         CVMX_USBCX_HCFG(usb->index));
 932                         usbcx_hcfg.s.fslssupp = 0;
 933                         usbcx_hcfg.s.fslspclksel = 0;
 934                         __cvmx_usb_write_csr32(usb,
 935                                         CVMX_USBCX_HCFG(usb->index),
 936                                         usbcx_hcfg.u32);
 937                 }
 938                 /*
 939                  * 3. Program the port power bit to drive VBUS on the USB,
 940                  *    USBC_HPRT[PRTPWR] = 1
 941                  */
 942                 USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index),
 943                                 union cvmx_usbcx_hprt, prtpwr, 1);
 944
 945                 /*
 946                  * Steps 4-15 from the manual are done later in the port enable
 947                  */
 948         }
 949
 950         return 0;
 951 }
 952
 953
 954 /**
 955  * Shutdown a USB port after a call to cvmx_usb_initialize().
 956  * The port should be disabled with all pipes closed when this
 957  * function is called.
 958  *
 959  * @usb: USB device state populated by cvmx_usb_initialize().
 960  *
 961  * Returns: 0 or a negative error code.
 962  */
 963 static int cvmx_usb_shutdown(struct cvmx_usb_state *usb)
 964 {
 965         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
 966
 967         /* Make sure all pipes are closed */
 968         if (!list_empty(&usb->idle_pipes) ||
 969             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS]) ||
 970             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT]) ||
 971             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_CONTROL]) ||
 972             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_BULK]))
 973                 return -EBUSY;
 974
 975         /* Disable the clocks and put them in power on reset */
 976         usbn_clk_ctl.u64 = __cvmx_usb_read_csr64(usb,
 977                         CVMX_USBNX_CLK_CTL(usb->index));
 978         usbn_clk_ctl.s.enable = 1;
 979         usbn_clk_ctl.s.por = 1;
 980         usbn_clk_ctl.s.hclk_rst = 1;
 981         usbn_clk_ctl.s.prst = 0;
 982         usbn_clk_ctl.s.hrst = 0;
 983         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
 984                                usbn_clk_ctl.u64);
 985         return 0;
 986 }
 987
 988
 989 /**
 990  * Enable a USB port. After this call succeeds, the USB port is
 991  * online and servicing requests.
 992  *
 993  * @usb: USB device state populated by cvmx_usb_initialize().
 994  *
 995  * Returns: 0 or a negative error code.
 996  */
 997 static int cvmx_usb_enable(struct cvmx_usb_state *usb)
 998 {
 999         union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
1000
1001         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb,
1002                         CVMX_USBCX_HPRT(usb->index));
1003
1004         /*
1005          * If the port is already enabled the just return. We don't need to do
1006          * anything
1007          */
1008         if (usb->usbcx_hprt.s.prtena)
1009                 return 0;
1010
1011         /* If there is nothing plugged into the port then fail immediately */
1012         if (!usb->usbcx_hprt.s.prtconnsts)
1013                 return -ETIMEDOUT;
1014
1015         /* Program the port reset bit to start the reset process */
1016         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1017                         prtrst, 1);
1018
1019         /*
1020          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
1021          * process to complete.
1022          */
1023         mdelay(50);
1024
1025         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
1026         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1027                         prtrst, 0);
1028
1029         /* Wait for the USBC_HPRT[PRTENA]. */
1030         if (CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_HPRT(usb->index),
1031                                 union cvmx_usbcx_hprt, prtena, ==, 1, 100000))
1032                 return -ETIMEDOUT;
1033
1034         /*
1035          * Read the port speed field to get the enumerated speed,
1036          * USBC_HPRT[PRTSPD].
1037          */
1038         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb,
1039                         CVMX_USBCX_HPRT(usb->index));
1040         usbcx_ghwcfg3.u32 = __cvmx_usb_read_csr32(usb,
1041                         CVMX_USBCX_GHWCFG3(usb->index));
1042
1043         /*
1044          * 13. Program the USBC_GRXFSIZ register to select the size of the
1045          *     receive FIFO (25%).
1046          */
1047         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index),
1048                         union cvmx_usbcx_grxfsiz, rxfdep,
1049                         usbcx_ghwcfg3.s.dfifodepth / 4);
1050         /*
1051          * 14. Program the USBC_GNPTXFSIZ register to select the size and the
1052          *     start address of the non- periodic transmit FIFO for nonperiodic
1053          *     transactions (50%).
1054          */
1055         {
1056                 union cvmx_usbcx_gnptxfsiz siz;
1057
1058                 siz.u32 = __cvmx_usb_read_csr32(usb,
1059                                 CVMX_USBCX_GNPTXFSIZ(usb->index));
1060                 siz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
1061                 siz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
1062                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index),
1063                                        siz.u32);
1064         }
1065         /*
1066          * 15. Program the USBC_HPTXFSIZ register to select the size and start
1067          *     address of the periodic transmit FIFO for periodic transactions
1068          *     (25%).
1069          */
1070         {
1071                 union cvmx_usbcx_hptxfsiz siz;
1072
1073                 siz.u32 = __cvmx_usb_read_csr32(usb,
1074                                 CVMX_USBCX_HPTXFSIZ(usb->index));
1075                 siz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
1076                 siz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
1077                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index),
1078                                        siz.u32);
1079         }
1080         /* Flush all FIFOs */
1081         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1082                         union cvmx_usbcx_grstctl, txfnum, 0x10);
1083         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1084                         union cvmx_usbcx_grstctl, txfflsh, 1);
1085         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1086                               union cvmx_usbcx_grstctl,
1087                               txfflsh, ==, 0, 100);
1088         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1089                         union cvmx_usbcx_grstctl, rxfflsh, 1);
1090         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1091                               union cvmx_usbcx_grstctl,
1092                               rxfflsh, ==, 0, 100);
1093
1094         return 0;
1095 }
1096
1097
1098 /**
1099  * Disable a USB port. After this call the USB port will not
1100  * generate data transfers and will not generate events.
1101  * Transactions in process will fail and call their
1102  * associated callbacks.
1103  *
1104  * @usb: USB device state populated by cvmx_usb_initialize().
1105  *
1106  * Returns: 0 or a negative error code.
1107  */
1108 static int cvmx_usb_disable(struct cvmx_usb_state *usb)
1109 {
1110         /* Disable the port */
1111         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1112                         prtena, 1);
1113         return 0;
1114 }
1115
1116
1117 /**
1118  * Get the current state of the USB port. Use this call to
1119  * determine if the usb port has anything connected, is enabled,
1120  * or has some sort of error condition. The return value of this
1121  * call has "changed" bits to signal of the value of some fields
1122  * have changed between calls.
1123  *
1124  * @usb: USB device state populated by cvmx_usb_initialize().
1125  *
1126  * Returns: Port status information
1127  */
1128 static struct cvmx_usb_port_status cvmx_usb_get_status(
1129                 struct cvmx_usb_state *usb)
1130 {
1131         union cvmx_usbcx_hprt usbc_hprt;
1132         struct cvmx_usb_port_status result;
1133
1134         memset(&result, 0, sizeof(result));
1135
1136         usbc_hprt.u32 = __cvmx_usb_read_csr32(usb,
1137                         CVMX_USBCX_HPRT(usb->index));
1138         result.port_enabled = usbc_hprt.s.prtena;
1139         result.port_over_current = usbc_hprt.s.prtovrcurract;
1140         result.port_powered = usbc_hprt.s.prtpwr;
1141         result.port_speed = usbc_hprt.s.prtspd;
1142         result.connected = usbc_hprt.s.prtconnsts;
1143         result.connect_change =
1144                 (result.connected != usb->port_status.connected);
1145
1146         return result;
1147 }
1148
1149 /**
1150  * Open a virtual pipe between the host and a USB device. A pipe
1151  * must be opened before data can be transferred between a device
1152  * and Octeon.
1153  *
1154  * @usb:             USB device state populated by cvmx_usb_initialize().
1155  * @device_addr:
1156  *                   USB device address to open the pipe to
1157  *                   (0-127).
1158  * @endpoint_num:
1159  *                   USB endpoint number to open the pipe to
1160  *                   (0-15).
1161  * @device_speed:
1162  *                   The speed of the device the pipe is going
1163  *                   to. This must match the device's speed,
1164  *                   which may be different than the port speed.
1165  * @max_packet:      The maximum packet length the device can
1166  *                   transmit/receive (low speed=0-8, full
1167  *                   speed=0-1023, high speed=0-1024). This value
1168  *                   comes from the standard endpoint descriptor
1169  *                   field wMaxPacketSize bits <10:0>.
1170  * @transfer_type:
1171  *                   The type of transfer this pipe is for.
1172  * @transfer_dir:
1173  *                   The direction the pipe is in. This is not
1174  *                   used for control pipes.
1175  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
1176  *                   this is how often the transfer is scheduled
1177  *                   for. All other transfers should specify
1178  *                   zero. The units are in frames (8000/sec at
1179  *                   high speed, 1000/sec for full speed).
1180  * @multi_count:
1181  *                   For high speed devices, this is the maximum
1182  *                   allowed number of packet per microframe.
1183  *                   Specify zero for non high speed devices. This
1184  *                   value comes from the standard endpoint descriptor
1185  *                   field wMaxPacketSize bits <12:11>.
1186  * @hub_device_addr:
1187  *                   Hub device address this device is connected
1188  *                   to. Devices connected directly to Octeon
1189  *                   use zero. This is only used when the device
1190  *                   is full/low speed behind a high speed hub.
1191  *                   The address will be of the high speed hub,
1192  *                   not and full speed hubs after it.
1193  * @hub_port:        Which port on the hub the device is
1194  *                   connected. Use zero for devices connected
1195  *                   directly to Octeon. Like hub_device_addr,
1196  *                   this is only used for full/low speed
1197  *                   devices behind a high speed hub.
1198  *
1199  * Returns: A non-NULL value is a pipe. NULL means an error.
1200  */
1201 static struct cvmx_usb_pipe *cvmx_usb_open_pipe(struct cvmx_usb_state *usb,
1202                                                 int device_addr,
1203                                                 int endpoint_num,
1204                                                 enum cvmx_usb_speed
1205                                                         device_speed,
1206                                                 int max_packet,
1207                                                 enum cvmx_usb_transfer
1208                                                         transfer_type,
1209                                                 enum cvmx_usb_direction
1210                                                         transfer_dir,
1211                                                 int interval, int multi_count,
1212                                                 int hub_device_addr,
1213                                                 int hub_port)
1214 {
1215         struct cvmx_usb_pipe *pipe;
1216
1217         if (unlikely((device_addr < 0) || (device_addr > MAX_USB_ADDRESS)))
1218                 return NULL;
1219         if (unlikely((endpoint_num < 0) || (endpoint_num > MAX_USB_ENDPOINT)))
1220                 return NULL;
1221         if (unlikely(device_speed > CVMX_USB_SPEED_LOW))
1222                 return NULL;
1223         if (unlikely((max_packet <= 0) || (max_packet > 1024)))
1224                 return NULL;
1225         if (unlikely(transfer_type > CVMX_USB_TRANSFER_INTERRUPT))
1226                 return NULL;
1227         if (unlikely((transfer_dir != CVMX_USB_DIRECTION_OUT) &&
1228                 (transfer_dir != CVMX_USB_DIRECTION_IN)))
1229                 return NULL;
1230         if (unlikely(interval < 0))
1231                 return NULL;
1232         if (unlikely((transfer_type == CVMX_USB_TRANSFER_CONTROL) && interval))
1233                 return NULL;
1234         if (unlikely(multi_count < 0))
1235                 return NULL;
1236         if (unlikely((device_speed != CVMX_USB_SPEED_HIGH) &&
1237                 (multi_count != 0)))
1238                 return NULL;
1239         if (unlikely((hub_device_addr < 0) ||
1240                 (hub_device_addr > MAX_USB_ADDRESS)))
1241                 return NULL;
1242         if (unlikely((hub_port < 0) || (hub_port > MAX_USB_HUB_PORT)))
1243                 return NULL;
1244
1245         pipe = kzalloc(sizeof(*pipe), GFP_ATOMIC);
1246         if (!pipe)
1247                 return NULL;
1248         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1249                 (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1250                 (transfer_type == CVMX_USB_TRANSFER_BULK))
1251                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
1252         pipe->device_addr = device_addr;
1253         pipe->endpoint_num = endpoint_num;
1254         pipe->device_speed = device_speed;
1255         pipe->max_packet = max_packet;
1256         pipe->transfer_type = transfer_type;
1257         pipe->transfer_dir = transfer_dir;
1258         INIT_LIST_HEAD(&pipe->transactions);
1259
1260         /*
1261          * All pipes use interval to rate limit NAK processing. Force an
1262          * interval if one wasn't supplied
1263          */
1264         if (!interval)
1265                 interval = 1;
1266         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1267                 pipe->interval = interval*8;
1268                 /* Force start splits to be schedule on uFrame 0 */
1269                 pipe->next_tx_frame = ((usb->frame_number+7)&~7) +
1270                                         pipe->interval;
1271         } else {
1272                 pipe->interval = interval;
1273                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1274         }
1275         pipe->multi_count = multi_count;
1276         pipe->hub_device_addr = hub_device_addr;
1277         pipe->hub_port = hub_port;
1278         pipe->pid_toggle = 0;
1279         pipe->split_sc_frame = -1;
1280         list_add_tail(&pipe->node, &usb->idle_pipes);
1281
1282         /*
1283          * We don't need to tell the hardware about this pipe yet since
1284          * it doesn't have any submitted requests
1285          */
1286
1287         return pipe;
1288 }
1289
1290
1291 /**
1292  * Poll the RX FIFOs and remove data as needed. This function is only used
1293  * in non DMA mode. It is very important that this function be called quickly
1294  * enough to prevent FIFO overflow.
1295  *
1296  * @usb:        USB device state populated by cvmx_usb_initialize().
1297  */
1298 static void __cvmx_usb_poll_rx_fifo(struct cvmx_usb_state *usb)
1299 {
1300         union cvmx_usbcx_grxstsph rx_status;
1301         int channel;
1302         int bytes;
1303         uint64_t address;
1304         uint32_t *ptr;
1305
1306         rx_status.u32 = __cvmx_usb_read_csr32(usb,
1307                         CVMX_USBCX_GRXSTSPH(usb->index));
1308         /* Only read data if IN data is there */
1309         if (rx_status.s.pktsts != 2)
1310                 return;
1311         /* Check if no data is available */
1312         if (!rx_status.s.bcnt)
1313                 return;
1314
1315         channel = rx_status.s.chnum;
1316         bytes = rx_status.s.bcnt;
1317         if (!bytes)
1318                 return;
1319
1320         /* Get where the DMA engine would have written this data */
1321         address = __cvmx_usb_read_csr64(usb,
1322                         CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8);
1323
1324         ptr = cvmx_phys_to_ptr(address);
1325         __cvmx_usb_write_csr64(usb,
1326                                CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8,
1327                                address + bytes);
1328
1329         /* Loop writing the FIFO data for this packet into memory */
1330         while (bytes > 0) {
1331                 *ptr++ = __cvmx_usb_read_csr32(usb, USB_FIFO_ADDRESS(channel, usb->index));
1332                 bytes -= 4;
1333         }
1334         CVMX_SYNCW;
1335 }
1336
1337
1338 /**
1339  * Fill the TX hardware fifo with data out of the software
1340  * fifos
1341  *
1342  * @usb:            USB device state populated by cvmx_usb_initialize().
1343  * @fifo:           Software fifo to use
1344  * @available:      Amount of space in the hardware fifo
1345  *
1346  * Returns: Non zero if the hardware fifo was too small and needs
1347  *          to be serviced again.
1348  */
1349 static int __cvmx_usb_fill_tx_hw(struct cvmx_usb_state *usb,
1350                                  struct cvmx_usb_tx_fifo *fifo, int available)
1351 {
1352         /*
1353          * We're done either when there isn't anymore space or the software FIFO
1354          * is empty
1355          */
1356         while (available && (fifo->head != fifo->tail)) {
1357                 int i = fifo->tail;
1358                 const uint32_t *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1359                 uint64_t csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel,
1360                                                         usb->index) ^ 4;
1361                 int words = available;
1362
1363                 /* Limit the amount of data to waht the SW fifo has */
1364                 if (fifo->entry[i].size <= available) {
1365                         words = fifo->entry[i].size;
1366                         fifo->tail++;
1367                         if (fifo->tail > MAX_CHANNELS)
1368                                 fifo->tail = 0;
1369                 }
1370
1371                 /* Update the next locations and counts */
1372                 available -= words;
1373                 fifo->entry[i].address += words * 4;
1374                 fifo->entry[i].size -= words;
1375
1376                 /*
1377                  * Write the HW fifo data. The read every three writes is due
1378                  * to an errata on CN3XXX chips
1379                  */
1380                 while (words > 3) {
1381                         cvmx_write64_uint32(csr_address, *ptr++);
1382                         cvmx_write64_uint32(csr_address, *ptr++);
1383                         cvmx_write64_uint32(csr_address, *ptr++);
1384                         cvmx_read64_uint64(
1385                                         CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1386                         words -= 3;
1387                 }
1388                 cvmx_write64_uint32(csr_address, *ptr++);
1389                 if (--words) {
1390                         cvmx_write64_uint32(csr_address, *ptr++);
1391                         if (--words)
1392                                 cvmx_write64_uint32(csr_address, *ptr++);
1393                 }
1394                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1395         }
1396         return fifo->head != fifo->tail;
1397 }
1398
1399
1400 /**
1401  * Check the hardware FIFOs and fill them as needed
1402  *
1403  * @usb:        USB device state populated by cvmx_usb_initialize().
1404  */
1405 static void __cvmx_usb_poll_tx_fifo(struct cvmx_usb_state *usb)
1406 {
1407         if (usb->periodic.head != usb->periodic.tail) {
1408                 union cvmx_usbcx_hptxsts tx_status;
1409
1410                 tx_status.u32 = __cvmx_usb_read_csr32(usb,
1411                                 CVMX_USBCX_HPTXSTS(usb->index));
1412                 if (__cvmx_usb_fill_tx_hw(usb, &usb->periodic,
1413                                           tx_status.s.ptxfspcavail))
1414                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1415                                         union cvmx_usbcx_gintmsk,
1416                                         ptxfempmsk, 1);
1417                 else
1418                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1419                                         union cvmx_usbcx_gintmsk,
1420                                         ptxfempmsk, 0);
1421         }
1422
1423         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1424                 union cvmx_usbcx_gnptxsts tx_status;
1425
1426                 tx_status.u32 = __cvmx_usb_read_csr32(usb,
1427                                 CVMX_USBCX_GNPTXSTS(usb->index));
1428                 if (__cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic,
1429                                           tx_status.s.nptxfspcavail))
1430                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1431                                         union cvmx_usbcx_gintmsk,
1432                                         nptxfempmsk, 1);
1433                 else
1434                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1435                                         union cvmx_usbcx_gintmsk,
1436                                         nptxfempmsk, 0);
1437         }
1438 }
1439
1440
1441 /**
1442  * Fill the TX FIFO with an outgoing packet
1443  *
1444  * @usb:          USB device state populated by cvmx_usb_initialize().
1445  * @channel:      Channel number to get packet from
1446  */
1447 static void __cvmx_usb_fill_tx_fifo(struct cvmx_usb_state *usb, int channel)
1448 {
1449         union cvmx_usbcx_hccharx hcchar;
1450         union cvmx_usbcx_hcspltx usbc_hcsplt;
1451         union cvmx_usbcx_hctsizx usbc_hctsiz;
1452         struct cvmx_usb_tx_fifo *fifo;
1453
1454         /* We only need to fill data on outbound channels */
1455         hcchar.u32 = __cvmx_usb_read_csr32(usb,
1456                         CVMX_USBCX_HCCHARX(channel, usb->index));
1457         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1458                 return;
1459
1460         /* OUT Splits only have data on the start and not the complete */
1461         usbc_hcsplt.u32 = __cvmx_usb_read_csr32(usb,
1462                         CVMX_USBCX_HCSPLTX(channel, usb->index));
1463         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1464                 return;
1465
1466         /*
1467          * Find out how many bytes we need to fill and convert it into 32bit
1468          * words.
1469          */
1470         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
1471                         CVMX_USBCX_HCTSIZX(channel, usb->index));
1472         if (!usbc_hctsiz.s.xfersize)
1473                 return;
1474
1475         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1476                 (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1477                 fifo = &usb->periodic;
1478         else
1479                 fifo = &usb->nonperiodic;
1480
1481         fifo->entry[fifo->head].channel = channel;
1482         fifo->entry[fifo->head].address = __cvmx_usb_read_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8);
1483         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize+3)>>2;
1484         fifo->head++;
1485         if (fifo->head > MAX_CHANNELS)
1486                 fifo->head = 0;
1487
1488         __cvmx_usb_poll_tx_fifo(usb);
1489 }
1490
1491 /**
1492  * Perform channel specific setup for Control transactions. All
1493  * the generic stuff will already have been done in
1494  * __cvmx_usb_start_channel()
1495  *
1496  * @usb:          USB device state populated by cvmx_usb_initialize().
1497  * @channel:      Channel to setup
1498  * @pipe:         Pipe for control transaction
1499  */
1500 static void __cvmx_usb_start_channel_control(struct cvmx_usb_state *usb,
1501                                              int channel,
1502                                              struct cvmx_usb_pipe *pipe)
1503 {
1504         struct cvmx_usb_transaction *transaction =
1505                 list_first_entry(&pipe->transactions, typeof(*transaction),
1506                                  node);
1507         struct usb_ctrlrequest *header =
1508                 cvmx_phys_to_ptr(transaction->control_header);
1509         int bytes_to_transfer = transaction->buffer_length -
1510                 transaction->actual_bytes;
1511         int packets_to_transfer;
1512         union cvmx_usbcx_hctsizx usbc_hctsiz;
1513
1514         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
1515                         CVMX_USBCX_HCTSIZX(channel, usb->index));
1516
1517         switch (transaction->stage) {
1518         case CVMX_USB_STAGE_NON_CONTROL:
1519         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1520                 cvmx_dprintf("%s: ERROR - Non control stage\n", __func__);
1521                 break;
1522         case CVMX_USB_STAGE_SETUP:
1523                 usbc_hctsiz.s.pid = 3; /* Setup */
1524                 bytes_to_transfer = sizeof(*header);
1525                 /* All Control operations start with a setup going OUT */
1526                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1527                                 union cvmx_usbcx_hccharx, epdir,
1528                                 CVMX_USB_DIRECTION_OUT);
1529                 /*
1530                  * Setup send the control header instead of the buffer data. The
1531                  * buffer data will be used in the next stage
1532                  */
1533                 __cvmx_usb_write_csr64(usb,
1534                         CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8,
1535                         transaction->control_header);
1536                 break;
1537         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1538                 usbc_hctsiz.s.pid = 3; /* Setup */
1539                 bytes_to_transfer = 0;
1540                 /* All Control operations start with a setup going OUT */
1541                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1542                                 union cvmx_usbcx_hccharx, epdir,
1543                                 CVMX_USB_DIRECTION_OUT);
1544
1545                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1546                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1547                 break;
1548         case CVMX_USB_STAGE_DATA:
1549                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1550                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1551                         if (header->bRequestType & USB_DIR_IN)
1552                                 bytes_to_transfer = 0;
1553                         else if (bytes_to_transfer > pipe->max_packet)
1554                                 bytes_to_transfer = pipe->max_packet;
1555                 }
1556                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1557                                 union cvmx_usbcx_hccharx, epdir,
1558                                 ((header->bRequestType & USB_DIR_IN) ?
1559                                         CVMX_USB_DIRECTION_IN :
1560                                         CVMX_USB_DIRECTION_OUT));
1561                 break;
1562         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1563                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1564                 if (!(header->bRequestType & USB_DIR_IN))
1565                         bytes_to_transfer = 0;
1566                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1567                                 union cvmx_usbcx_hccharx, epdir,
1568                                 ((header->bRequestType & USB_DIR_IN) ?
1569                                         CVMX_USB_DIRECTION_IN :
1570                                         CVMX_USB_DIRECTION_OUT));
1571                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1572                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1573                 break;
1574         case CVMX_USB_STAGE_STATUS:
1575                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1576                 bytes_to_transfer = 0;
1577                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1578                                 union cvmx_usbcx_hccharx, epdir,
1579                                 ((header->bRequestType & USB_DIR_IN) ?
1580                                         CVMX_USB_DIRECTION_OUT :
1581                                         CVMX_USB_DIRECTION_IN));
1582                 break;
1583         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1584                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1585                 bytes_to_transfer = 0;
1586                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1587                                 union cvmx_usbcx_hccharx, epdir,
1588                                 ((header->bRequestType & USB_DIR_IN) ?
1589                                         CVMX_USB_DIRECTION_OUT :
1590                                         CVMX_USB_DIRECTION_IN));
1591                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1592                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1593                 break;
1594         }
1595
1596         /*
1597          * Make sure the transfer never exceeds the byte limit of the hardware.
1598          * Further bytes will be sent as continued transactions
1599          */
1600         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1601                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1602                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1603                 bytes_to_transfer *= pipe->max_packet;
1604         }
1605
1606         /*
1607          * Calculate the number of packets to transfer. If the length is zero
1608          * we still need to transfer one packet
1609          */
1610         packets_to_transfer = (bytes_to_transfer + pipe->max_packet - 1) /
1611                 pipe->max_packet;
1612         if (packets_to_transfer == 0)
1613                 packets_to_transfer = 1;
1614         else if ((packets_to_transfer > 1) &&
1615                         (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1616                 /*
1617                  * Limit to one packet when not using DMA. Channels must be
1618                  * restarted between every packet for IN transactions, so there
1619                  * is no reason to do multiple packets in a row
1620                  */
1621                 packets_to_transfer = 1;
1622                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1623         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1624                 /*
1625                  * Limit the number of packet and data transferred to what the
1626                  * hardware can handle
1627                  */
1628                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1629                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1630         }
1631
1632         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1633         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1634
1635         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index),
1636                                usbc_hctsiz.u32);
1637 }
1638
1639
1640 /**
1641  * Start a channel to perform the pipe's head transaction
1642  *
1643  * @usb:          USB device state populated by cvmx_usb_initialize().
1644  * @channel:      Channel to setup
1645  * @pipe:         Pipe to start
1646  */
1647 static void __cvmx_usb_start_channel(struct cvmx_usb_state *usb,
1648                                      int channel,
1649                                      struct cvmx_usb_pipe *pipe)
1650 {
1651         struct cvmx_usb_transaction *transaction =
1652                 list_first_entry(&pipe->transactions, typeof(*transaction),
1653                                  node);
1654
1655         /* Make sure all writes to the DMA region get flushed */
1656         CVMX_SYNCW;
1657
1658         /* Attach the channel to the pipe */
1659         usb->pipe_for_channel[channel] = pipe;
1660         pipe->channel = channel;
1661         pipe->flags |= __CVMX_USB_PIPE_FLAGS_SCHEDULED;
1662
1663         /* Mark this channel as in use */
1664         usb->idle_hardware_channels &= ~(1<<channel);
1665
1666         /* Enable the channel interrupt bits */
1667         {
1668                 union cvmx_usbcx_hcintx usbc_hcint;
1669                 union cvmx_usbcx_hcintmskx usbc_hcintmsk;
1670                 union cvmx_usbcx_haintmsk usbc_haintmsk;
1671
1672                 /* Clear all channel status bits */
1673                 usbc_hcint.u32 = __cvmx_usb_read_csr32(usb,
1674                                 CVMX_USBCX_HCINTX(channel, usb->index));
1675
1676                 __cvmx_usb_write_csr32(usb,
1677                                        CVMX_USBCX_HCINTX(channel, usb->index),
1678                                        usbc_hcint.u32);
1679
1680                 usbc_hcintmsk.u32 = 0;
1681                 usbc_hcintmsk.s.chhltdmsk = 1;
1682                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1683                         /*
1684                          * Channels need these extra interrupts when we aren't
1685                          * in DMA mode.
1686                          */
1687                         usbc_hcintmsk.s.datatglerrmsk = 1;
1688                         usbc_hcintmsk.s.frmovrunmsk = 1;
1689                         usbc_hcintmsk.s.bblerrmsk = 1;
1690                         usbc_hcintmsk.s.xacterrmsk = 1;
1691                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1692                                 /*
1693                                  * Splits don't generate xfercompl, so we need
1694                                  * ACK and NYET.
1695                                  */
1696                                 usbc_hcintmsk.s.nyetmsk = 1;
1697                                 usbc_hcintmsk.s.ackmsk = 1;
1698                         }
1699                         usbc_hcintmsk.s.nakmsk = 1;
1700                         usbc_hcintmsk.s.stallmsk = 1;
1701                         usbc_hcintmsk.s.xfercomplmsk = 1;
1702                 }
1703                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), usbc_hcintmsk.u32);
1704
1705                 /* Enable the channel interrupt to propagate */
1706                 usbc_haintmsk.u32 = __cvmx_usb_read_csr32(usb,
1707                                         CVMX_USBCX_HAINTMSK(usb->index));
1708                 usbc_haintmsk.s.haintmsk |= 1<<channel;
1709                 __cvmx_usb_write_csr32(usb,
1710                                         CVMX_USBCX_HAINTMSK(usb->index),
1711                                         usbc_haintmsk.u32);
1712         }
1713
1714         /* Setup the locations the DMA engines use  */
1715         {
1716                 uint64_t dma_address = transaction->buffer +
1717                                         transaction->actual_bytes;
1718
1719                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1720                         dma_address = transaction->buffer +
1721                                         transaction->iso_packets[0].offset +
1722                                         transaction->actual_bytes;
1723
1724                 __cvmx_usb_write_csr64(usb,
1725                         CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8,
1726                         dma_address);
1727
1728                 __cvmx_usb_write_csr64(usb,
1729                         CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8,
1730                         dma_address);
1731         }
1732
1733         /* Setup both the size of the transfer and the SPLIT characteristics */
1734         {
1735                 union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
1736                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
1737                 int packets_to_transfer;
1738                 int bytes_to_transfer = transaction->buffer_length -
1739                         transaction->actual_bytes;
1740
1741                 /*
1742                  * ISOCHRONOUS transactions store each individual transfer size
1743                  * in the packet structure, not the global buffer_length
1744                  */
1745                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1746                         bytes_to_transfer =
1747                                 transaction->iso_packets[0].length -
1748                                 transaction->actual_bytes;
1749
1750                 /*
1751                  * We need to do split transactions when we are talking to non
1752                  * high speed devices that are behind a high speed hub
1753                  */
1754                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1755                         /*
1756                          * On the start split phase (stage is even) record the
1757                          * frame number we will need to send the split complete.
1758                          * We only store the lower two bits since the time ahead
1759                          * can only be two frames
1760                          */
1761                         if ((transaction->stage&1) == 0) {
1762                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1763                                         pipe->split_sc_frame =
1764                                                 (usb->frame_number + 1) & 0x7f;
1765                                 else
1766                                         pipe->split_sc_frame =
1767                                                 (usb->frame_number + 2) & 0x7f;
1768                         } else
1769                                 pipe->split_sc_frame = -1;
1770
1771                         usbc_hcsplt.s.spltena = 1;
1772                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1773                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1774                         usbc_hcsplt.s.compsplt = (transaction->stage ==
1775                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1776
1777                         /*
1778                          * SPLIT transactions can only ever transmit one data
1779                          * packet so limit the transfer size to the max packet
1780                          * size
1781                          */
1782                         if (bytes_to_transfer > pipe->max_packet)
1783                                 bytes_to_transfer = pipe->max_packet;
1784
1785                         /*
1786                          * ISOCHRONOUS OUT splits are unique in that they limit
1787                          * data transfers to 188 byte chunks representing the
1788                          * begin/middle/end of the data or all
1789                          */
1790                         if (!usbc_hcsplt.s.compsplt &&
1791                                 (pipe->transfer_dir ==
1792                                  CVMX_USB_DIRECTION_OUT) &&
1793                                 (pipe->transfer_type ==
1794                                  CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1795                                 /*
1796                                  * Clear the split complete frame number as
1797                                  * there isn't going to be a split complete
1798                                  */
1799                                 pipe->split_sc_frame = -1;
1800                                 /*
1801                                  * See if we've started this transfer and sent
1802                                  * data
1803                                  */
1804                                 if (transaction->actual_bytes == 0) {
1805                                         /*
1806                                          * Nothing sent yet, this is either a
1807                                          * begin or the entire payload
1808                                          */
1809                                         if (bytes_to_transfer <= 188)
1810                                                 /* Entire payload in one go */
1811                                                 usbc_hcsplt.s.xactpos = 3;
1812                                         else
1813                                                 /* First part of payload */
1814                                                 usbc_hcsplt.s.xactpos = 2;
1815                                 } else {
1816                                         /*
1817                                          * Continuing the previous data, we must
1818                                          * either be in the middle or at the end
1819                                          */
1820                                         if (bytes_to_transfer <= 188)
1821                                                 /* End of payload */
1822                                                 usbc_hcsplt.s.xactpos = 1;
1823                                         else
1824                                                 /* Middle of payload */
1825                                                 usbc_hcsplt.s.xactpos = 0;
1826                                 }
1827                                 /*
1828                                  * Again, the transfer size is limited to 188
1829                                  * bytes
1830                                  */
1831                                 if (bytes_to_transfer > 188)
1832                                         bytes_to_transfer = 188;
1833                         }
1834                 }
1835
1836                 /*
1837                  * Make sure the transfer never exceeds the byte limit of the
1838                  * hardware. Further bytes will be sent as continued
1839                  * transactions
1840                  */
1841                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1842                         /*
1843                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1844                          * size
1845                          */
1846                         bytes_to_transfer = MAX_TRANSFER_BYTES /
1847                                 pipe->max_packet;
1848                         bytes_to_transfer *= pipe->max_packet;
1849                 }
1850
1851                 /*
1852                  * Calculate the number of packets to transfer. If the length is
1853                  * zero we still need to transfer one packet
1854                  */
1855                 packets_to_transfer =
1856                         (bytes_to_transfer + pipe->max_packet - 1) /
1857                         pipe->max_packet;
1858                 if (packets_to_transfer == 0)
1859                         packets_to_transfer = 1;
1860                 else if ((packets_to_transfer > 1) &&
1861                                 (usb->init_flags &
1862                                  CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1863                         /*
1864                          * Limit to one packet when not using DMA. Channels must
1865                          * be restarted between every packet for IN
1866                          * transactions, so there is no reason to do multiple
1867                          * packets in a row
1868                          */
1869                         packets_to_transfer = 1;
1870                         bytes_to_transfer = packets_to_transfer *
1871                                 pipe->max_packet;
1872                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1873                         /*
1874                          * Limit the number of packet and data transferred to
1875                          * what the hardware can handle
1876                          */
1877                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1878                         bytes_to_transfer = packets_to_transfer *
1879                                 pipe->max_packet;
1880                 }
1881
1882                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1883                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1884
1885                 /* Update the DATA0/DATA1 toggle */
1886                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1887                 /*
1888                  * High speed pipes may need a hardware ping before they start
1889                  */
1890                 if (pipe->flags & __CVMX_USB_PIPE_FLAGS_NEED_PING)
1891                         usbc_hctsiz.s.dopng = 1;
1892
1893                 __cvmx_usb_write_csr32(usb,
1894                                        CVMX_USBCX_HCSPLTX(channel, usb->index),
1895                                        usbc_hcsplt.u32);
1896                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel,
1897                                         usb->index), usbc_hctsiz.u32);
1898         }
1899
1900         /* Setup the Host Channel Characteristics Register */
1901         {
1902                 union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
1903
1904                 /*
1905                  * Set the startframe odd/even properly. This is only used for
1906                  * periodic
1907                  */
1908                 usbc_hcchar.s.oddfrm = usb->frame_number&1;
1909
1910                 /*
1911                  * Set the number of back to back packets allowed by this
1912                  * endpoint. Split transactions interpret "ec" as the number of
1913                  * immediate retries of failure. These retries happen too
1914                  * quickly, so we disable these entirely for splits
1915                  */
1916                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
1917                         usbc_hcchar.s.ec = 1;
1918                 else if (pipe->multi_count < 1)
1919                         usbc_hcchar.s.ec = 1;
1920                 else if (pipe->multi_count > 3)
1921                         usbc_hcchar.s.ec = 3;
1922                 else
1923                         usbc_hcchar.s.ec = pipe->multi_count;
1924
1925                 /* Set the rest of the endpoint specific settings */
1926                 usbc_hcchar.s.devaddr = pipe->device_addr;
1927                 usbc_hcchar.s.eptype = transaction->type;
1928                 usbc_hcchar.s.lspddev =
1929                         (pipe->device_speed == CVMX_USB_SPEED_LOW);
1930                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1931                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1932                 usbc_hcchar.s.mps = pipe->max_packet;
1933                 __cvmx_usb_write_csr32(usb,
1934                                        CVMX_USBCX_HCCHARX(channel, usb->index),
1935                                        usbc_hcchar.u32);
1936         }
1937
1938         /* Do transaction type specific fixups as needed */
1939         switch (transaction->type) {
1940         case CVMX_USB_TRANSFER_CONTROL:
1941                 __cvmx_usb_start_channel_control(usb, channel, pipe);
1942                 break;
1943         case CVMX_USB_TRANSFER_BULK:
1944         case CVMX_USB_TRANSFER_INTERRUPT:
1945                 break;
1946         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1947                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
1948                         /*
1949                          * ISO transactions require different PIDs depending on
1950                          * direction and how many packets are needed
1951                          */
1952                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1953                                 if (pipe->multi_count < 2) /* Need DATA0 */
1954                                         USB_SET_FIELD32(
1955                                                 CVMX_USBCX_HCTSIZX(channel,
1956                                                                    usb->index),
1957                                                 union cvmx_usbcx_hctsizx,
1958                                                 pid, 0);
1959                                 else /* Need MDATA */
1960                                         USB_SET_FIELD32(
1961                                                 CVMX_USBCX_HCTSIZX(channel,
1962                                                                    usb->index),
1963                                                 union cvmx_usbcx_hctsizx,
1964                                                 pid, 3);
1965                         }
1966                 }
1967                 break;
1968         }
1969         {
1970                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 =
1971                         __cvmx_usb_read_csr32(usb,
1972                                 CVMX_USBCX_HCTSIZX(channel, usb->index))};
1973                 transaction->xfersize = usbc_hctsiz.s.xfersize;
1974                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
1975         }
1976         /* Remeber when we start a split transaction */
1977         if (__cvmx_usb_pipe_needs_split(usb, pipe))
1978                 usb->active_split = transaction;
1979         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1980                         union cvmx_usbcx_hccharx, chena, 1);
1981         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
1982                 __cvmx_usb_fill_tx_fifo(usb, channel);
1983 }
1984
1985
1986 /**
1987  * Find a pipe that is ready to be scheduled to hardware.
1988  * @usb:         USB device state populated by cvmx_usb_initialize().
1989  * @list:        Pipe list to search
1990  * @current_frame:
1991  *               Frame counter to use as a time reference.
1992  *
1993  * Returns: Pipe or NULL if none are ready
1994  */
1995 static struct cvmx_usb_pipe *__cvmx_usb_find_ready_pipe(
1996                 struct cvmx_usb_state *usb,
1997                 struct list_head *list,
1998                 uint64_t current_frame)
1999 {
2000         struct cvmx_usb_pipe *pipe;
2001
2002         list_for_each_entry(pipe, list, node) {
2003                 struct cvmx_usb_transaction *t =
2004                         list_first_entry(&pipe->transactions, typeof(*t),
2005                                          node);
2006                 if (!(pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED) && t &&
2007                         (pipe->next_tx_frame <= current_frame) &&
2008                         ((pipe->split_sc_frame == -1) ||
2009                          ((((int)current_frame - (int)pipe->split_sc_frame)
2010                            & 0x7f) < 0x40)) &&
2011                         (!usb->active_split || (usb->active_split == t))) {
2012                         prefetch(t);
2013                         return pipe;
2014                 }
2015         }
2016         return NULL;
2017 }
2018
2019
2020 /**
2021  * Called whenever a pipe might need to be scheduled to the
2022  * hardware.
2023  *
2024  * @usb:         USB device state populated by cvmx_usb_initialize().
2025  * @is_sof:      True if this schedule was called on a SOF interrupt.
2026  */
2027 static void __cvmx_usb_schedule(struct cvmx_usb_state *usb, int is_sof)
2028 {
2029         int channel;
2030         struct cvmx_usb_pipe *pipe;
2031         int need_sof;
2032         enum cvmx_usb_transfer ttype;
2033
2034         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2035                 /*
2036                  * Without DMA we need to be careful to not schedule something
2037                  * at the end of a frame and cause an overrun.
2038                  */
2039                 union cvmx_usbcx_hfnum hfnum = {
2040                         .u32 = __cvmx_usb_read_csr32(usb,
2041                                                 CVMX_USBCX_HFNUM(usb->index))
2042                 };
2043
2044                 union cvmx_usbcx_hfir hfir = {
2045                         .u32 = __cvmx_usb_read_csr32(usb,
2046                                                 CVMX_USBCX_HFIR(usb->index))
2047                 };
2048
2049                 if (hfnum.s.frrem < hfir.s.frint/4)
2050                         goto done;
2051         }
2052
2053         while (usb->idle_hardware_channels) {
2054                 /* Find an idle channel */
2055                 channel = __fls(usb->idle_hardware_channels);
2056                 if (unlikely(channel > 7))
2057                         break;
2058
2059                 /* Find a pipe needing service */
2060                 pipe = NULL;
2061                 if (is_sof) {
2062                         /*
2063                          * Only process periodic pipes on SOF interrupts. This
2064                          * way we are sure that the periodic data is sent in the
2065                          * beginning of the frame
2066                          */
2067                         pipe = __cvmx_usb_find_ready_pipe(usb,
2068                                         usb->active_pipes +
2069                                         CVMX_USB_TRANSFER_ISOCHRONOUS,
2070                                         usb->frame_number);
2071                         if (likely(!pipe))
2072                                 pipe = __cvmx_usb_find_ready_pipe(usb,
2073                                                 usb->active_pipes +
2074                                                 CVMX_USB_TRANSFER_INTERRUPT,
2075                                                 usb->frame_number);
2076                 }
2077                 if (likely(!pipe)) {
2078                         pipe = __cvmx_usb_find_ready_pipe(usb,
2079                                         usb->active_pipes +
2080                                         CVMX_USB_TRANSFER_CONTROL,
2081                                         usb->frame_number);
2082                         if (likely(!pipe))
2083                                 pipe = __cvmx_usb_find_ready_pipe(usb,
2084                                                 usb->active_pipes +
2085                                                 CVMX_USB_TRANSFER_BULK,
2086                                                 usb->frame_number);
2087                 }
2088                 if (!pipe)
2089                         break;
2090
2091                 __cvmx_usb_start_channel(usb, channel, pipe);
2092         }
2093
2094 done:
2095         /*
2096          * Only enable SOF interrupts when we have transactions pending in the
2097          * future that might need to be scheduled
2098          */
2099         need_sof = 0;
2100         for (ttype = CVMX_USB_TRANSFER_CONTROL;
2101                         ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
2102                 list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
2103                         if (pipe->next_tx_frame > usb->frame_number) {
2104                                 need_sof = 1;
2105                                 break;
2106                         }
2107                 }
2108         }
2109         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
2110                         union cvmx_usbcx_gintmsk, sofmsk, need_sof);
2111 }
2112
2113 static inline struct octeon_hcd *cvmx_usb_to_octeon(struct cvmx_usb_state *p)
2114 {
2115         return container_of(p, struct octeon_hcd, usb);
2116 }
2117
2118 static inline struct usb_hcd *octeon_to_hcd(struct octeon_hcd *p)
2119 {
2120         return container_of((void *)p, struct usb_hcd, hcd_priv);
2121 }
2122
2123 static void octeon_usb_urb_complete_callback(struct cvmx_usb_state *usb,
2124                                              enum cvmx_usb_complete status,
2125                                              struct cvmx_usb_pipe *pipe,
2126                                              struct cvmx_usb_transaction
2127                                                 *transaction,
2128                                              int bytes_transferred,
2129                                              struct urb *urb)
2130 {
2131         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
2132         struct usb_hcd *hcd = octeon_to_hcd(priv);
2133         struct device *dev = hcd->self.controller;
2134
2135         if (likely(status == CVMX_USB_COMPLETE_SUCCESS))
2136                 urb->actual_length = bytes_transferred;
2137         else
2138                 urb->actual_length = 0;
2139
2140         urb->hcpriv = NULL;
2141
2142         /* For Isochronous transactions we need to update the URB packet status
2143            list from data in our private copy */
2144         if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
2145                 int i;
2146                 /*
2147                  * The pointer to the private list is stored in the setup_packet
2148                  * field.
2149                  */
2150                 struct cvmx_usb_iso_packet *iso_packet =
2151                         (struct cvmx_usb_iso_packet *) urb->setup_packet;
2152                 /* Recalculate the transfer size by adding up each packet */
2153                 urb->actual_length = 0;
2154                 for (i = 0; i < urb->number_of_packets; i++) {
2155                         if (iso_packet[i].status ==
2156                                         CVMX_USB_COMPLETE_SUCCESS) {
2157                                 urb->iso_frame_desc[i].status = 0;
2158                                 urb->iso_frame_desc[i].actual_length =
2159                                         iso_packet[i].length;
2160                                 urb->actual_length +=
2161                                         urb->iso_frame_desc[i].actual_length;
2162                         } else {
2163                                 dev_dbg(dev, "ISOCHRONOUS packet=%d of %d status=%d pipe=%p transaction=%p size=%d\n",
2164                                         i, urb->number_of_packets,
2165                                         iso_packet[i].status, pipe,
2166                                         transaction, iso_packet[i].length);
2167                                 urb->iso_frame_desc[i].status = -EREMOTEIO;
2168                         }
2169                 }
2170                 /* Free the private list now that we don't need it anymore */
2171                 kfree(iso_packet);
2172                 urb->setup_packet = NULL;
2173         }
2174
2175         switch (status) {
2176         case CVMX_USB_COMPLETE_SUCCESS:
2177                 urb->status = 0;
2178                 break;
2179         case CVMX_USB_COMPLETE_CANCEL:
2180                 if (urb->status == 0)
2181                         urb->status = -ENOENT;
2182                 break;
2183         case CVMX_USB_COMPLETE_STALL:
2184                 dev_dbg(dev, "status=stall pipe=%p transaction=%p size=%d\n",
2185                         pipe, transaction, bytes_transferred);
2186                 urb->status = -EPIPE;
2187                 break;
2188         case CVMX_USB_COMPLETE_BABBLEERR:
2189                 dev_dbg(dev, "status=babble pipe=%p transaction=%p size=%d\n",
2190                         pipe, transaction, bytes_transferred);
2191                 urb->status = -EPIPE;
2192                 break;
2193         case CVMX_USB_COMPLETE_SHORT:
2194                 dev_dbg(dev, "status=short pipe=%p transaction=%p size=%d\n",
2195                         pipe, transaction, bytes_transferred);
2196                 urb->status = -EREMOTEIO;
2197                 break;
2198         case CVMX_USB_COMPLETE_ERROR:
2199         case CVMX_USB_COMPLETE_XACTERR:
2200         case CVMX_USB_COMPLETE_DATATGLERR:
2201         case CVMX_USB_COMPLETE_FRAMEERR:
2202                 dev_dbg(dev, "status=%d pipe=%p transaction=%p size=%d\n",
2203                         status, pipe, transaction, bytes_transferred);
2204                 urb->status = -EPROTO;
2205                 break;
2206         }
2207         usb_hcd_unlink_urb_from_ep(octeon_to_hcd(priv), urb);
2208         spin_unlock(&priv->lock);
2209         usb_hcd_giveback_urb(octeon_to_hcd(priv), urb, urb->status);
2210         spin_lock(&priv->lock);
2211 }
2212
2213 /**
2214  * Signal the completion of a transaction and free it. The
2215  * transaction will be removed from the pipe transaction list.
2216  *
2217  * @usb:         USB device state populated by cvmx_usb_initialize().
2218  * @pipe:        Pipe the transaction is on
2219  * @transaction:
2220  *               Transaction that completed
2221  * @complete_code:
2222  *               Completion code
2223  */
2224 static void __cvmx_usb_perform_complete(
2225                                 struct cvmx_usb_state *usb,
2226                                 struct cvmx_usb_pipe *pipe,
2227                                 struct cvmx_usb_transaction *transaction,
2228                                 enum cvmx_usb_complete complete_code)
2229 {
2230         /* If this was a split then clear our split in progress marker */
2231         if (usb->active_split == transaction)
2232                 usb->active_split = NULL;
2233
2234         /*
2235          * Isochronous transactions need extra processing as they might not be
2236          * done after a single data transfer
2237          */
2238         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
2239                 /* Update the number of bytes transferred in this ISO packet */
2240                 transaction->iso_packets[0].length = transaction->actual_bytes;
2241                 transaction->iso_packets[0].status = complete_code;
2242
2243                 /*
2244                  * If there are more ISOs pending and we succeeded, schedule the
2245                  * next one
2246                  */
2247                 if ((transaction->iso_number_packets > 1) &&
2248                         (complete_code == CVMX_USB_COMPLETE_SUCCESS)) {
2249                         /* No bytes transferred for this packet as of yet */
2250                         transaction->actual_bytes = 0;
2251                         /* One less ISO waiting to transfer */
2252                         transaction->iso_number_packets--;
2253                         /* Increment to the next location in our packet array */
2254                         transaction->iso_packets++;
2255                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2256                         return;
2257                 }
2258         }
2259
2260         /* Remove the transaction from the pipe list */
2261         list_del(&transaction->node);
2262         if (list_empty(&pipe->transactions))
2263                 list_move_tail(&pipe->node, &usb->idle_pipes);
2264         octeon_usb_urb_complete_callback(usb, complete_code, pipe,
2265                                          transaction,
2266                                          transaction->actual_bytes,
2267                                          transaction->urb);
2268         kfree(transaction);
2269 }
2270
2271
2272 /**
2273  * Submit a usb transaction to a pipe. Called for all types
2274  * of transactions.
2275  *
2276  * @usb:
2277  * @pipe:           Which pipe to submit to.
2278  * @type:           Transaction type
2279  * @buffer:         User buffer for the transaction
2280  * @buffer_length:
2281  *                  User buffer's length in bytes
2282  * @control_header:
2283  *                  For control transactions, the 8 byte standard header
2284  * @iso_start_frame:
2285  *                  For ISO transactions, the start frame
2286  * @iso_number_packets:
2287  *                  For ISO, the number of packet in the transaction.
2288  * @iso_packets:
2289  *                  A description of each ISO packet
2290  * @urb:            URB for the callback
2291  *
2292  * Returns: Transaction or NULL on failure.
2293  */
2294 static struct cvmx_usb_transaction *__cvmx_usb_submit_transaction(
2295                                 struct cvmx_usb_state *usb,
2296                                 struct cvmx_usb_pipe *pipe,
2297                                 enum cvmx_usb_transfer type,
2298                                 uint64_t buffer,
2299                                 int buffer_length,
2300                                 uint64_t control_header,
2301                                 int iso_start_frame,
2302                                 int iso_number_packets,
2303                                 struct cvmx_usb_iso_packet *iso_packets,
2304                                 struct urb *urb)
2305 {
2306         struct cvmx_usb_transaction *transaction;
2307
2308         if (unlikely(pipe->transfer_type != type))
2309                 return NULL;
2310
2311         transaction = kzalloc(sizeof(*transaction), GFP_ATOMIC);
2312         if (unlikely(!transaction))
2313                 return NULL;
2314
2315         transaction->type = type;
2316         transaction->buffer = buffer;
2317         transaction->buffer_length = buffer_length;
2318         transaction->control_header = control_header;
2319         /* FIXME: This is not used, implement it. */
2320         transaction->iso_start_frame = iso_start_frame;
2321         transaction->iso_number_packets = iso_number_packets;
2322         transaction->iso_packets = iso_packets;
2323         transaction->urb = urb;
2324         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2325                 transaction->stage = CVMX_USB_STAGE_SETUP;
2326         else
2327                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2328
2329         if (!list_empty(&pipe->transactions)) {
2330                 list_add_tail(&transaction->node, &pipe->transactions);
2331         } else {
2332                 list_add_tail(&transaction->node, &pipe->transactions);
2333                 list_move_tail(&pipe->node,
2334                                &usb->active_pipes[pipe->transfer_type]);
2335
2336                 /*
2337                  * We may need to schedule the pipe if this was the head of the
2338                  * pipe.
2339                  */
2340                 __cvmx_usb_schedule(usb, 0);
2341         }
2342
2343         return transaction;
2344 }
2345
2346
2347 /**
2348  * Call to submit a USB Bulk transfer to a pipe.
2349  *
2350  * @usb:            USB device state populated by cvmx_usb_initialize().
2351  * @pipe:           Handle to the pipe for the transfer.
2352  * @urb:            URB.
2353  *
2354  * Returns: A submitted transaction or NULL on failure.
2355  */
2356 static struct cvmx_usb_transaction *cvmx_usb_submit_bulk(
2357                                                 struct cvmx_usb_state *usb,
2358                                                 struct cvmx_usb_pipe *pipe,
2359                                                 struct urb *urb)
2360 {
2361         return __cvmx_usb_submit_transaction(usb, pipe, CVMX_USB_TRANSFER_BULK,
2362                                              urb->transfer_dma,
2363                                              urb->transfer_buffer_length,
2364                                              0, /* control_header */
2365                                              0, /* iso_start_frame */
2366                                              0, /* iso_number_packets */
2367                                              NULL, /* iso_packets */
2368                                              urb);
2369 }
2370
2371
2372 /**
2373  * Call to submit a USB Interrupt transfer to a pipe.
2374  *
2375  * @usb:            USB device state populated by cvmx_usb_initialize().
2376  * @pipe:           Handle to the pipe for the transfer.
2377  * @urb:            URB returned when the callback is called.
2378  *
2379  * Returns: A submitted transaction or NULL on failure.
2380  */
2381 static struct cvmx_usb_transaction *cvmx_usb_submit_interrupt(
2382                                                 struct cvmx_usb_state *usb,
2383                                                 struct cvmx_usb_pipe *pipe,
2384                                                 struct urb *urb)
2385 {
2386         return __cvmx_usb_submit_transaction(usb, pipe,
2387                                              CVMX_USB_TRANSFER_INTERRUPT,
2388                                              urb->transfer_dma,
2389                                              urb->transfer_buffer_length,
2390                                              0, /* control_header */
2391                                              0, /* iso_start_frame */
2392                                              0, /* iso_number_packets */
2393                                              NULL, /* iso_packets */
2394                                              urb);
2395 }
2396
2397
2398 /**
2399  * Call to submit a USB Control transfer to a pipe.
2400  *
2401  * @usb:            USB device state populated by cvmx_usb_initialize().
2402  * @pipe:           Handle to the pipe for the transfer.
2403  * @urb:            URB.
2404  *
2405  * Returns: A submitted transaction or NULL on failure.
2406  */
2407 static struct cvmx_usb_transaction *cvmx_usb_submit_control(
2408                                                 struct cvmx_usb_state *usb,
2409                                                 struct cvmx_usb_pipe *pipe,
2410                                                 struct urb *urb)
2411 {
2412         int buffer_length = urb->transfer_buffer_length;
2413         uint64_t control_header = urb->setup_dma;
2414         struct usb_ctrlrequest *header = cvmx_phys_to_ptr(control_header);
2415
2416         if ((header->bRequestType & USB_DIR_IN) == 0)
2417                 buffer_length = le16_to_cpu(header->wLength);
2418
2419         return __cvmx_usb_submit_transaction(usb, pipe,
2420                                              CVMX_USB_TRANSFER_CONTROL,
2421                                              urb->transfer_dma, buffer_length,
2422                                              control_header,
2423                                              0, /* iso_start_frame */
2424                                              0, /* iso_number_packets */
2425                                              NULL, /* iso_packets */
2426                                              urb);
2427 }
2428
2429
2430 /**
2431  * Call to submit a USB Isochronous transfer to a pipe.
2432  *
2433  * @usb:            USB device state populated by cvmx_usb_initialize().
2434  * @pipe:           Handle to the pipe for the transfer.
2435  * @urb:            URB returned when the callback is called.
2436  *
2437  * Returns: A submitted transaction or NULL on failure.
2438  */
2439 static struct cvmx_usb_transaction *cvmx_usb_submit_isochronous(
2440                                                 struct cvmx_usb_state *usb,
2441                                                 struct cvmx_usb_pipe *pipe,
2442                                                 struct urb *urb)
2443 {
2444         struct cvmx_usb_iso_packet *packets;
2445
2446         packets = (struct cvmx_usb_iso_packet *) urb->setup_packet;
2447         return __cvmx_usb_submit_transaction(usb, pipe,
2448                                              CVMX_USB_TRANSFER_ISOCHRONOUS,
2449                                              urb->transfer_dma,
2450                                              urb->transfer_buffer_length,
2451                                              0, /* control_header */
2452                                              urb->start_frame,
2453                                              urb->number_of_packets,
2454                                              packets, urb);
2455 }
2456
2457
2458 /**
2459  * Cancel one outstanding request in a pipe. Canceling a request
2460  * can fail if the transaction has already completed before cancel
2461  * is called. Even after a successful cancel call, it may take
2462  * a frame or two for the cvmx_usb_poll() function to call the
2463  * associated callback.
2464  *
2465  * @usb:         USB device state populated by cvmx_usb_initialize().
2466  * @pipe:        Pipe to cancel requests in.
2467  * @transaction: Transaction to cancel, returned by the submit function.
2468  *
2469  * Returns: 0 or a negative error code.
2470  */
2471 static int cvmx_usb_cancel(struct cvmx_usb_state *usb,
2472                            struct cvmx_usb_pipe *pipe,
2473                            struct cvmx_usb_transaction *transaction)
2474 {
2475         /*
2476          * If the transaction is the HEAD of the queue and scheduled. We need to
2477          * treat it special
2478          */
2479         if (list_first_entry(&pipe->transactions, typeof(*transaction), node) ==
2480             transaction && (pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2481                 union cvmx_usbcx_hccharx usbc_hcchar;
2482
2483                 usb->pipe_for_channel[pipe->channel] = NULL;
2484                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2485
2486                 CVMX_SYNCW;
2487
2488                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2489                                 CVMX_USBCX_HCCHARX(pipe->channel, usb->index));
2490                 /*
2491                  * If the channel isn't enabled then the transaction already
2492                  * completed.
2493                  */
2494                 if (usbc_hcchar.s.chena) {
2495                         usbc_hcchar.s.chdis = 1;
2496                         __cvmx_usb_write_csr32(usb,
2497                                         CVMX_USBCX_HCCHARX(pipe->channel,
2498                                                 usb->index),
2499                                         usbc_hcchar.u32);
2500                 }
2501         }
2502         __cvmx_usb_perform_complete(usb, pipe, transaction,
2503                                     CVMX_USB_COMPLETE_CANCEL);
2504         return 0;
2505 }
2506
2507
2508 /**
2509  * Cancel all outstanding requests in a pipe. Logically all this
2510  * does is call cvmx_usb_cancel() in a loop.
2511  *
2512  * @usb:         USB device state populated by cvmx_usb_initialize().
2513  * @pipe:        Pipe to cancel requests in.
2514  *
2515  * Returns: 0 or a negative error code.
2516  */
2517 static int cvmx_usb_cancel_all(struct cvmx_usb_state *usb,
2518                                struct cvmx_usb_pipe *pipe)
2519 {
2520         struct cvmx_usb_transaction *transaction, *next;
2521
2522         /* Simply loop through and attempt to cancel each transaction */
2523         list_for_each_entry_safe(transaction, next, &pipe->transactions, node) {
2524                 int result = cvmx_usb_cancel(usb, pipe, transaction);
2525
2526                 if (unlikely(result != 0))
2527                         return result;
2528         }
2529         return 0;
2530 }
2531
2532
2533 /**
2534  * Close a pipe created with cvmx_usb_open_pipe().
2535  *
2536  * @usb:         USB device state populated by cvmx_usb_initialize().
2537  * @pipe:        Pipe to close.
2538  *
2539  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
2540  *          outstanding transfers.
2541  */
2542 static int cvmx_usb_close_pipe(struct cvmx_usb_state *usb,
2543                                struct cvmx_usb_pipe *pipe)
2544 {
2545         /* Fail if the pipe has pending transactions */
2546         if (!list_empty(&pipe->transactions))
2547                 return -EBUSY;
2548
2549         list_del(&pipe->node);
2550         kfree(pipe);
2551
2552         return 0;
2553 }
2554
2555 /**
2556  * Get the current USB protocol level frame number. The frame
2557  * number is always in the range of 0-0x7ff.
2558  *
2559  * @usb: USB device state populated by cvmx_usb_initialize().
2560  *
2561  * Returns: USB frame number
2562  */
2563 static int cvmx_usb_get_frame_number(struct cvmx_usb_state *usb)
2564 {
2565         int frame_number;
2566         union cvmx_usbcx_hfnum usbc_hfnum;
2567
2568         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb,
2569                         CVMX_USBCX_HFNUM(usb->index));
2570         frame_number = usbc_hfnum.s.frnum;
2571
2572         return frame_number;
2573 }
2574
2575
2576 /**
2577  * Poll a channel for status
2578  *
2579  * @usb:     USB device
2580  * @channel: Channel to poll
2581  *
2582  * Returns: Zero on success
2583  */
2584 static int __cvmx_usb_poll_channel(struct cvmx_usb_state *usb, int channel)
2585 {
2586         union cvmx_usbcx_hcintx usbc_hcint;
2587         union cvmx_usbcx_hctsizx usbc_hctsiz;
2588         union cvmx_usbcx_hccharx usbc_hcchar;
2589         struct cvmx_usb_pipe *pipe;
2590         struct cvmx_usb_transaction *transaction;
2591         int bytes_this_transfer;
2592         int bytes_in_last_packet;
2593         int packets_processed;
2594         int buffer_space_left;
2595
2596         /* Read the interrupt status bits for the channel */
2597         usbc_hcint.u32 = __cvmx_usb_read_csr32(usb,
2598                         CVMX_USBCX_HCINTX(channel, usb->index));
2599
2600         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2601                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2602                                 CVMX_USBCX_HCCHARX(channel, usb->index));
2603
2604                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2605                         /*
2606                          * There seems to be a bug in CN31XX which can cause
2607                          * interrupt IN transfers to get stuck until we do a
2608                          * write of HCCHARX without changing things
2609                          */
2610                         __cvmx_usb_write_csr32(usb,
2611                                         CVMX_USBCX_HCCHARX(channel,
2612                                                            usb->index),
2613                                         usbc_hcchar.u32);
2614                         return 0;
2615                 }
2616
2617                 /*
2618                  * In non DMA mode the channels don't halt themselves. We need
2619                  * to manually disable channels that are left running
2620                  */
2621                 if (!usbc_hcint.s.chhltd) {
2622                         if (usbc_hcchar.s.chena) {
2623                                 union cvmx_usbcx_hcintmskx hcintmsk;
2624                                 /* Disable all interrupts except CHHLTD */
2625                                 hcintmsk.u32 = 0;
2626                                 hcintmsk.s.chhltdmsk = 1;
2627                                 __cvmx_usb_write_csr32(usb,
2628                                                 CVMX_USBCX_HCINTMSKX(channel,
2629                                                         usb->index),
2630                                                 hcintmsk.u32);
2631                                 usbc_hcchar.s.chdis = 1;
2632                                 __cvmx_usb_write_csr32(usb,
2633                                                 CVMX_USBCX_HCCHARX(channel,
2634                                                         usb->index),
2635                                                 usbc_hcchar.u32);
2636                                 return 0;
2637                         } else if (usbc_hcint.s.xfercompl) {
2638                                 /*
2639                                  * Successful IN/OUT with transfer complete.
2640                                  * Channel halt isn't needed.
2641                                  */
2642                         } else {
2643                                 cvmx_dprintf("USB%d: Channel %d interrupt without halt\n",
2644                                                 usb->index, channel);
2645                                 return 0;
2646                         }
2647                 }
2648         } else {
2649                 /*
2650                  * There is are no interrupts that we need to process when the
2651                  * channel is still running
2652                  */
2653                 if (!usbc_hcint.s.chhltd)
2654                         return 0;
2655         }
2656
2657         /* Disable the channel interrupts now that it is done */
2658         __cvmx_usb_write_csr32(usb,
2659                                 CVMX_USBCX_HCINTMSKX(channel, usb->index),
2660                                 0);
2661         usb->idle_hardware_channels |= (1<<channel);
2662
2663         /* Make sure this channel is tied to a valid pipe */
2664         pipe = usb->pipe_for_channel[channel];
2665         prefetch(pipe);
2666         if (!pipe)
2667                 return 0;
2668         transaction = list_first_entry(&pipe->transactions,
2669                                        typeof(*transaction),
2670                                        node);
2671         prefetch(transaction);
2672
2673         /*
2674          * Disconnect this pipe from the HW channel. Later the schedule
2675          * function will figure out which pipe needs to go
2676          */
2677         usb->pipe_for_channel[channel] = NULL;
2678         pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2679
2680         /*
2681          * Read the channel config info so we can figure out how much data
2682          * transfered
2683          */
2684         usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2685                         CVMX_USBCX_HCCHARX(channel, usb->index));
2686         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
2687                         CVMX_USBCX_HCTSIZX(channel, usb->index));
2688
2689         /*
2690          * Calculating the number of bytes successfully transferred is dependent
2691          * on the transfer direction
2692          */
2693         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2694         if (usbc_hcchar.s.epdir) {
2695                 /*
2696                  * IN transactions are easy. For every byte received the
2697                  * hardware decrements xfersize. All we need to do is subtract
2698                  * the current value of xfersize from its starting value and we
2699                  * know how many bytes were written to the buffer
2700                  */
2701                 bytes_this_transfer = transaction->xfersize -
2702                         usbc_hctsiz.s.xfersize;
2703         } else {
2704                 /*
2705                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2706                  * decremented on every successful packet send. The hardware
2707                  * does this when it receives an ACK, or NYET. If it doesn't
2708                  * receive one of these responses pktcnt doesn't change
2709                  */
2710                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2711                 /*
2712                  * The last packet may not be a full transfer if we didn't have
2713                  * enough data
2714                  */
2715                 if (bytes_this_transfer > transaction->xfersize)
2716                         bytes_this_transfer = transaction->xfersize;
2717         }
2718         /* Figure out how many bytes were in the last packet of the transfer */
2719         if (packets_processed)
2720                 bytes_in_last_packet = bytes_this_transfer -
2721                         (packets_processed - 1) * usbc_hcchar.s.mps;
2722         else
2723                 bytes_in_last_packet = bytes_this_transfer;
2724
2725         /*
2726          * As a special case, setup transactions output the setup header, not
2727          * the user's data. For this reason we don't count setup data as bytes
2728          * transferred
2729          */
2730         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2731                 (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2732                 bytes_this_transfer = 0;
2733
2734         /*
2735          * Add the bytes transferred to the running total. It is important that
2736          * bytes_this_transfer doesn't count any data that needs to be
2737          * retransmitted
2738          */
2739         transaction->actual_bytes += bytes_this_transfer;
2740         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2741                 buffer_space_left = transaction->iso_packets[0].length -
2742                         transaction->actual_bytes;
2743         else
2744                 buffer_space_left = transaction->buffer_length -
2745                         transaction->actual_bytes;
2746
2747         /*
2748          * We need to remember the PID toggle state for the next transaction.
2749          * The hardware already updated it for the next transaction
2750          */
2751         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2752
2753         /*
2754          * For high speed bulk out, assume the next transaction will need to do
2755          * a ping before proceeding. If this isn't true the ACK processing below
2756          * will clear this flag
2757          */
2758         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2759                 (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2760                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2761                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2762
2763         if (usbc_hcint.s.stall) {
2764                 /*
2765                  * STALL as a response means this transaction cannot be
2766                  * completed because the device can't process transactions. Tell
2767                  * the user. Any data that was transferred will be counted on
2768                  * the actual bytes transferred
2769                  */
2770                 pipe->pid_toggle = 0;
2771                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2772                                             CVMX_USB_COMPLETE_STALL);
2773         } else if (usbc_hcint.s.xacterr) {
2774                 /*
2775                  * We know at least one packet worked if we get a ACK or NAK.
2776                  * Reset the retry counter
2777                  */
2778                 if (usbc_hcint.s.nak || usbc_hcint.s.ack)
2779                         transaction->retries = 0;
2780                 transaction->retries++;
2781                 if (transaction->retries > MAX_RETRIES) {
2782                         /*
2783                          * XactErr as a response means the device signaled
2784                          * something wrong with the transfer. For example, PID
2785                          * toggle errors cause these
2786                          */
2787                         __cvmx_usb_perform_complete(usb, pipe, transaction,
2788                                                     CVMX_USB_COMPLETE_XACTERR);
2789                 } else {
2790                         /*
2791                          * If this was a split then clear our split in progress
2792                          * marker
2793                          */
2794                         if (usb->active_split == transaction)
2795                                 usb->active_split = NULL;
2796                         /*
2797                          * Rewind to the beginning of the transaction by anding
2798                          * off the split complete bit
2799                          */
2800                         transaction->stage &= ~1;
2801                         pipe->split_sc_frame = -1;
2802                         pipe->next_tx_frame += pipe->interval;
2803                         if (pipe->next_tx_frame < usb->frame_number)
2804                                 pipe->next_tx_frame =
2805                                         usb->frame_number + pipe->interval -
2806                                         (usb->frame_number -
2807                                          pipe->next_tx_frame) % pipe->interval;
2808                 }
2809         } else if (usbc_hcint.s.bblerr) {
2810                 /* Babble Error (BblErr) */
2811                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2812                                             CVMX_USB_COMPLETE_BABBLEERR);
2813         } else if (usbc_hcint.s.datatglerr) {
2814                 /* Data toggle error */
2815                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2816                                             CVMX_USB_COMPLETE_DATATGLERR);
2817         } else if (usbc_hcint.s.nyet) {
2818                 /*
2819                  * NYET as a response is only allowed in three cases: as a
2820                  * response to a ping, as a response to a split transaction, and
2821                  * as a response to a bulk out. The ping case is handled by
2822                  * hardware, so we only have splits and bulk out
2823                  */
2824                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
2825                         transaction->retries = 0;
2826                         /*
2827                          * If there is more data to go then we need to try
2828                          * again. Otherwise this transaction is complete
2829                          */
2830                         if ((buffer_space_left == 0) ||
2831                                 (bytes_in_last_packet < pipe->max_packet))
2832                                 __cvmx_usb_perform_complete(usb, pipe,
2833                                                 transaction,
2834                                                 CVMX_USB_COMPLETE_SUCCESS);
2835                 } else {
2836                         /*
2837                          * Split transactions retry the split complete 4 times
2838                          * then rewind to the start split and do the entire
2839                          * transactions again
2840                          */
2841                         transaction->retries++;
2842                         if ((transaction->retries & 0x3) == 0) {
2843                                 /*
2844                                  * Rewind to the beginning of the transaction by
2845                                  * anding off the split complete bit
2846                                  */
2847                                 transaction->stage &= ~1;
2848                                 pipe->split_sc_frame = -1;
2849                         }
2850                 }
2851         } else if (usbc_hcint.s.ack) {
2852                 transaction->retries = 0;
2853                 /*
2854                  * The ACK bit can only be checked after the other error bits.
2855                  * This is because a multi packet transfer may succeed in a
2856                  * number of packets and then get a different response on the
2857                  * last packet. In this case both ACK and the last response bit
2858                  * will be set. If none of the other response bits is set, then
2859                  * the last packet must have been an ACK
2860                  *
2861                  * Since we got an ACK, we know we don't need to do a ping on
2862                  * this pipe
2863                  */
2864                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_NEED_PING;
2865
2866                 switch (transaction->type) {
2867                 case CVMX_USB_TRANSFER_CONTROL:
2868                         switch (transaction->stage) {
2869                         case CVMX_USB_STAGE_NON_CONTROL:
2870                         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2871                                 /* This should be impossible */
2872                                 __cvmx_usb_perform_complete(usb, pipe,
2873                                         transaction, CVMX_USB_COMPLETE_ERROR);
2874                                 break;
2875                         case CVMX_USB_STAGE_SETUP:
2876                                 pipe->pid_toggle = 1;
2877                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2878                                         transaction->stage =
2879                                                 CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2880                                 else {
2881                                         struct usb_ctrlrequest *header =
2882                                                 cvmx_phys_to_ptr(transaction->control_header);
2883                                         if (header->wLength)
2884                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2885                                         else
2886                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2887                                 }
2888                                 break;
2889                         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2890                                 {
2891                                         struct usb_ctrlrequest *header =
2892                                                 cvmx_phys_to_ptr(transaction->control_header);
2893                                         if (header->wLength)
2894                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2895                                         else
2896                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2897                                 }
2898                                 break;
2899                         case CVMX_USB_STAGE_DATA:
2900                                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2901                                         transaction->stage =
2902                                                 CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2903                                         /*
2904                                          * For setup OUT data that are splits,
2905                                          * the hardware doesn't appear to count
2906                                          * transferred data. Here we manually
2907                                          * update the data transferred
2908                                          */
2909                                         if (!usbc_hcchar.s.epdir) {
2910                                                 if (buffer_space_left < pipe->max_packet)
2911                                                         transaction->actual_bytes +=
2912                                                                 buffer_space_left;
2913                                                 else
2914                                                         transaction->actual_bytes +=
2915                                                                 pipe->max_packet;
2916                                         }
2917                                 } else if ((buffer_space_left == 0) ||
2918                                                 (bytes_in_last_packet <
2919                                                  pipe->max_packet)) {
2920                                         pipe->pid_toggle = 1;
2921                                         transaction->stage =
2922                                                 CVMX_USB_STAGE_STATUS;
2923                                 }
2924                                 break;
2925                         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2926                                 if ((buffer_space_left == 0) ||
2927                                                 (bytes_in_last_packet <
2928                                                  pipe->max_packet)) {
2929                                         pipe->pid_toggle = 1;
2930                                         transaction->stage =
2931                                                 CVMX_USB_STAGE_STATUS;
2932                                 } else {
2933                                         transaction->stage =
2934                                                 CVMX_USB_STAGE_DATA;
2935                                 }
2936                                 break;
2937                         case CVMX_USB_STAGE_STATUS:
2938                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2939                                         transaction->stage =
2940                                                 CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2941                                 else
2942                                         __cvmx_usb_perform_complete(usb, pipe,
2943                                                 transaction,
2944                                                 CVMX_USB_COMPLETE_SUCCESS);
2945                                 break;
2946                         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2947                                 __cvmx_usb_perform_complete(usb, pipe,
2948                                                 transaction,
2949                                                 CVMX_USB_COMPLETE_SUCCESS);
2950                                 break;
2951                         }
2952                         break;
2953                 case CVMX_USB_TRANSFER_BULK:
2954                 case CVMX_USB_TRANSFER_INTERRUPT:
2955                         /*
2956                          * The only time a bulk transfer isn't complete when it
2957                          * finishes with an ACK is during a split transaction.
2958                          * For splits we need to continue the transfer if more
2959                          * data is needed
2960                          */
2961                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2962                                 if (transaction->stage ==
2963                                                 CVMX_USB_STAGE_NON_CONTROL)
2964                                         transaction->stage =
2965                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2966                                 else {
2967                                         if (buffer_space_left &&
2968                                                 (bytes_in_last_packet ==
2969                                                  pipe->max_packet))
2970                                                 transaction->stage =
2971                                                         CVMX_USB_STAGE_NON_CONTROL;
2972                                         else {
2973                                                 if (transaction->type ==
2974                                                         CVMX_USB_TRANSFER_INTERRUPT)
2975                                                         pipe->next_tx_frame +=
2976                                                                 pipe->interval;
2977                                                         __cvmx_usb_perform_complete(
2978                                                                 usb,
2979                                                                 pipe,
2980                                                                 transaction,
2981                                                                 CVMX_USB_COMPLETE_SUCCESS);
2982                                         }
2983                                 }
2984                         } else {
2985                                 if ((pipe->device_speed ==
2986                                         CVMX_USB_SPEED_HIGH) &&
2987                                     (pipe->transfer_type ==
2988                                      CVMX_USB_TRANSFER_BULK) &&
2989                                     (pipe->transfer_dir ==
2990                                      CVMX_USB_DIRECTION_OUT) &&
2991                                     (usbc_hcint.s.nak))
2992                                         pipe->flags |=
2993                                                 __CVMX_USB_PIPE_FLAGS_NEED_PING;
2994                                 if (!buffer_space_left ||
2995                                         (bytes_in_last_packet <
2996                                          pipe->max_packet)) {
2997                                         if (transaction->type ==
2998                                                 CVMX_USB_TRANSFER_INTERRUPT)
2999                                                 pipe->next_tx_frame +=
3000                                                         pipe->interval;
3001                                         __cvmx_usb_perform_complete(usb,
3002                                                 pipe,
3003                                                 transaction,
3004                                                 CVMX_USB_COMPLETE_SUCCESS);
3005                                 }
3006                         }
3007                         break;
3008                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
3009                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
3010                                 /*
3011                                  * ISOCHRONOUS OUT splits don't require a
3012                                  * complete split stage. Instead they use a
3013                                  * sequence of begin OUT splits to transfer the
3014                                  * data 188 bytes at a time. Once the transfer
3015                                  * is complete, the pipe sleeps until the next
3016                                  * schedule interval
3017                                  */
3018                                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
3019                                         /*
3020                                          * If no space left or this wasn't a max
3021                                          * size packet then this transfer is
3022                                          * complete. Otherwise start it again to
3023                                          * send the next 188 bytes
3024                                          */
3025                                         if (!buffer_space_left ||
3026                                                 (bytes_this_transfer < 188)) {
3027                                                 pipe->next_tx_frame +=
3028                                                         pipe->interval;
3029                                                 __cvmx_usb_perform_complete(
3030                                                         usb,
3031                                                         pipe,
3032                                                         transaction,
3033                                                         CVMX_USB_COMPLETE_SUCCESS);
3034                                         }
3035                                 } else {
3036                                         if (transaction->stage ==
3037                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
3038                                                 /*
3039                                                  * We are in the incoming data
3040                                                  * phase. Keep getting data
3041                                                  * until we run out of space or
3042                                                  * get a small packet
3043                                                  */
3044                                                 if ((buffer_space_left == 0) ||
3045                                                         (bytes_in_last_packet <
3046                                                          pipe->max_packet)) {
3047                                                         pipe->next_tx_frame +=
3048                                                                 pipe->interval;
3049                                                         __cvmx_usb_perform_complete(
3050                                                                 usb,
3051                                                                 pipe,
3052                                                                 transaction,
3053                                                                 CVMX_USB_COMPLETE_SUCCESS);
3054                                                 }
3055                                         } else
3056                                                 transaction->stage =
3057                                                         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
3058                                 }
3059                         } else {
3060                                 pipe->next_tx_frame += pipe->interval;
3061                                 __cvmx_usb_perform_complete(usb,
3062                                                 pipe,
3063                                                 transaction,
3064                                                 CVMX_USB_COMPLETE_SUCCESS);
3065                         }
3066                         break;
3067                 }
3068         } else if (usbc_hcint.s.nak) {
3069                 /*
3070                  * If this was a split then clear our split in progress marker.
3071                  */
3072                 if (usb->active_split == transaction)
3073                         usb->active_split = NULL;
3074                 /*
3075                  * NAK as a response means the device couldn't accept the
3076                  * transaction, but it should be retried in the future. Rewind
3077                  * to the beginning of the transaction by anding off the split
3078                  * complete bit. Retry in the next interval
3079                  */
3080                 transaction->retries = 0;
3081                 transaction->stage &= ~1;
3082                 pipe->next_tx_frame += pipe->interval;
3083                 if (pipe->next_tx_frame < usb->frame_number)
3084                         pipe->next_tx_frame = usb->frame_number +
3085                                 pipe->interval -
3086                                 (usb->frame_number - pipe->next_tx_frame) %
3087                                 pipe->interval;
3088         } else {
3089                 struct cvmx_usb_port_status port;
3090
3091                 port = cvmx_usb_get_status(usb);
3092                 if (port.port_enabled) {
3093                         /* We'll retry the exact same transaction again */
3094                         transaction->retries++;
3095                 } else {
3096                         /*
3097                          * We get channel halted interrupts with no result bits
3098                          * sets when the cable is unplugged
3099                          */
3100                         __cvmx_usb_perform_complete(usb, pipe, transaction,
3101                                         CVMX_USB_COMPLETE_ERROR);
3102                 }
3103         }
3104         return 0;
3105 }
3106
3107 static void octeon_usb_port_callback(struct cvmx_usb_state *usb)
3108 {
3109         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
3110
3111         spin_unlock(&priv->lock);
3112         usb_hcd_poll_rh_status(octeon_to_hcd(priv));
3113         spin_lock(&priv->lock);
3114 }
3115
3116 /**
3117  * Poll the USB block for status and call all needed callback
3118  * handlers. This function is meant to be called in the interrupt
3119  * handler for the USB controller. It can also be called
3120  * periodically in a loop for non-interrupt based operation.
3121  *
3122  * @usb: USB device state populated by cvmx_usb_initialize().
3123  *
3124  * Returns: 0 or a negative error code.
3125  */
3126 static int cvmx_usb_poll(struct cvmx_usb_state *usb)
3127 {
3128         union cvmx_usbcx_hfnum usbc_hfnum;
3129         union cvmx_usbcx_gintsts usbc_gintsts;
3130
3131         prefetch_range(usb, sizeof(*usb));
3132
3133         /* Update the frame counter */
3134         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb,
3135                                                 CVMX_USBCX_HFNUM(usb->index));
3136         if ((usb->frame_number&0x3fff) > usbc_hfnum.s.frnum)
3137                 usb->frame_number += 0x4000;
3138         usb->frame_number &= ~0x3fffull;
3139         usb->frame_number |= usbc_hfnum.s.frnum;
3140
3141         /* Read the pending interrupts */
3142         usbc_gintsts.u32 = __cvmx_usb_read_csr32(usb,
3143                                                 CVMX_USBCX_GINTSTS(usb->index));
3144
3145         /* Clear the interrupts now that we know about them */
3146         __cvmx_usb_write_csr32(usb,
3147                                 CVMX_USBCX_GINTSTS(usb->index),
3148                                 usbc_gintsts.u32);
3149
3150         if (usbc_gintsts.s.rxflvl) {
3151                 /*
3152                  * RxFIFO Non-Empty (RxFLvl)
3153                  * Indicates that there is at least one packet pending to be
3154                  * read from the RxFIFO.
3155                  *
3156                  * In DMA mode this is handled by hardware
3157                  */
3158                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3159                         __cvmx_usb_poll_rx_fifo(usb);
3160         }
3161         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
3162                 /* Fill the Tx FIFOs when not in DMA mode */
3163                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3164                         __cvmx_usb_poll_tx_fifo(usb);
3165         }
3166         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
3167                 union cvmx_usbcx_hprt usbc_hprt;
3168                 /*
3169                  * Disconnect Detected Interrupt (DisconnInt)
3170                  * Asserted when a device disconnect is detected.
3171                  *
3172                  * Host Port Interrupt (PrtInt)
3173                  * The core sets this bit to indicate a change in port status of
3174                  * one of the O2P USB core ports in Host mode. The application
3175                  * must read the Host Port Control and Status (HPRT) register to
3176                  * determine the exact event that caused this interrupt. The
3177                  * application must clear the appropriate status bit in the Host
3178                  * Port Control and Status register to clear this bit.
3179                  *
3180                  * Call the user's port callback
3181                  */
3182                 octeon_usb_port_callback(usb);
3183                 /* Clear the port change bits */
3184                 usbc_hprt.u32 = __cvmx_usb_read_csr32(usb,
3185                                 CVMX_USBCX_HPRT(usb->index));
3186                 usbc_hprt.s.prtena = 0;
3187                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index),
3188                                        usbc_hprt.u32);
3189         }
3190         if (usbc_gintsts.s.hchint) {
3191                 /*
3192                  * Host Channels Interrupt (HChInt)
3193                  * The core sets this bit to indicate that an interrupt is
3194                  * pending on one of the channels of the core (in Host mode).
3195                  * The application must read the Host All Channels Interrupt
3196                  * (HAINT) register to determine the exact number of the channel
3197                  * on which the interrupt occurred, and then read the
3198                  * corresponding Host Channel-n Interrupt (HCINTn) register to
3199                  * determine the exact cause of the interrupt. The application
3200                  * must clear the appropriate status bit in the HCINTn register
3201                  * to clear this bit.
3202                  */
3203                 union cvmx_usbcx_haint usbc_haint;
3204
3205                 usbc_haint.u32 = __cvmx_usb_read_csr32(usb,
3206                                         CVMX_USBCX_HAINT(usb->index));
3207                 while (usbc_haint.u32) {
3208                         int channel;
3209
3210                         channel = __fls(usbc_haint.u32);
3211                         __cvmx_usb_poll_channel(usb, channel);
3212                         usbc_haint.u32 ^= 1<<channel;
3213                 }
3214         }
3215
3216         __cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
3217
3218         return 0;
3219 }
3220
3221 /* convert between an HCD pointer and the corresponding struct octeon_hcd */
3222 static inline struct octeon_hcd *hcd_to_octeon(struct usb_hcd *hcd)
3223 {
3224         return (struct octeon_hcd *)(hcd->hcd_priv);
3225 }
3226
3227 static irqreturn_t octeon_usb_irq(struct usb_hcd *hcd)
3228 {
3229         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3230         unsigned long flags;
3231
3232         spin_lock_irqsave(&priv->lock, flags);
3233         cvmx_usb_poll(&priv->usb);
3234         spin_unlock_irqrestore(&priv->lock, flags);
3235         return IRQ_HANDLED;
3236 }
3237
3238 static int octeon_usb_start(struct usb_hcd *hcd)
3239 {
3240         hcd->state = HC_STATE_RUNNING;
3241         return 0;
3242 }
3243
3244 static void octeon_usb_stop(struct usb_hcd *hcd)
3245 {
3246         hcd->state = HC_STATE_HALT;
3247 }
3248
3249 static int octeon_usb_get_frame_number(struct usb_hcd *hcd)
3250 {
3251         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3252
3253         return cvmx_usb_get_frame_number(&priv->usb);
3254 }
3255
3256 static int octeon_usb_urb_enqueue(struct usb_hcd *hcd,
3257                                   struct urb *urb,
3258                                   gfp_t mem_flags)
3259 {
3260         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3261         struct device *dev = hcd->self.controller;
3262         struct cvmx_usb_transaction *transaction = NULL;
3263         struct cvmx_usb_pipe *pipe;
3264         unsigned long flags;
3265         struct cvmx_usb_iso_packet *iso_packet;
3266         struct usb_host_endpoint *ep = urb->ep;
3267         int rc;
3268
3269         urb->status = 0;
3270         spin_lock_irqsave(&priv->lock, flags);
3271
3272         rc = usb_hcd_link_urb_to_ep(hcd, urb);
3273         if (rc) {
3274                 spin_unlock_irqrestore(&priv->lock, flags);
3275                 return rc;
3276         }
3277
3278         if (!ep->hcpriv) {
3279                 enum cvmx_usb_transfer transfer_type;
3280                 enum cvmx_usb_speed speed;
3281                 int split_device = 0;
3282                 int split_port = 0;
3283
3284                 switch (usb_pipetype(urb->pipe)) {
3285                 case PIPE_ISOCHRONOUS:
3286                         transfer_type = CVMX_USB_TRANSFER_ISOCHRONOUS;
3287                         break;
3288                 case PIPE_INTERRUPT:
3289                         transfer_type = CVMX_USB_TRANSFER_INTERRUPT;
3290                         break;
3291                 case PIPE_CONTROL:
3292                         transfer_type = CVMX_USB_TRANSFER_CONTROL;
3293                         break;
3294                 default:
3295                         transfer_type = CVMX_USB_TRANSFER_BULK;
3296                         break;
3297                 }
3298                 switch (urb->dev->speed) {
3299                 case USB_SPEED_LOW:
3300                         speed = CVMX_USB_SPEED_LOW;
3301                         break;
3302                 case USB_SPEED_FULL:
3303                         speed = CVMX_USB_SPEED_FULL;
3304                         break;
3305                 default:
3306                         speed = CVMX_USB_SPEED_HIGH;
3307                         break;
3308                 }
3309                 /*
3310                  * For slow devices on high speed ports we need to find the hub
3311                  * that does the speed translation so we know where to send the
3312                  * split transactions.
3313                  */
3314                 if (speed != CVMX_USB_SPEED_HIGH) {
3315                         /*
3316                          * Start at this device and work our way up the usb
3317                          * tree.
3318                          */
3319                         struct usb_device *dev = urb->dev;
3320
3321                         while (dev->parent) {
3322                                 /*
3323                                  * If our parent is high speed then he'll
3324                                  * receive the splits.
3325                                  */
3326                                 if (dev->parent->speed == USB_SPEED_HIGH) {
3327                                         split_device = dev->parent->devnum;
3328                                         split_port = dev->portnum;
3329                                         break;
3330                                 }
3331                                 /*
3332                                  * Move up the tree one level. If we make it all
3333                                  * the way up the tree, then the port must not
3334                                  * be in high speed mode and we don't need a
3335                                  * split.
3336                                  */
3337                                 dev = dev->parent;
3338                         }
3339                 }
3340                 pipe = cvmx_usb_open_pipe(&priv->usb, usb_pipedevice(urb->pipe),
3341                                           usb_pipeendpoint(urb->pipe), speed,
3342                                           le16_to_cpu(ep->desc.wMaxPacketSize)
3343                                           & 0x7ff,
3344                                           transfer_type,
3345                                           usb_pipein(urb->pipe) ?
3346                                                 CVMX_USB_DIRECTION_IN :
3347                                                 CVMX_USB_DIRECTION_OUT,
3348                                           urb->interval,
3349                                           (le16_to_cpu(ep->desc.wMaxPacketSize)
3350                                            >> 11) & 0x3,
3351                                           split_device, split_port);
3352                 if (!pipe) {
3353                         usb_hcd_unlink_urb_from_ep(hcd, urb);
3354                         spin_unlock_irqrestore(&priv->lock, flags);
3355                         dev_dbg(dev, "Failed to create pipe\n");
3356                         return -ENOMEM;
3357                 }
3358                 ep->hcpriv = pipe;
3359         } else {
3360                 pipe = ep->hcpriv;
3361         }
3362
3363         switch (usb_pipetype(urb->pipe)) {
3364         case PIPE_ISOCHRONOUS:
3365                 dev_dbg(dev, "Submit isochronous to %d.%d\n",
3366                         usb_pipedevice(urb->pipe),
3367                         usb_pipeendpoint(urb->pipe));
3368                 /*
3369                  * Allocate a structure to use for our private list of
3370                  * isochronous packets.
3371                  */
3372                 iso_packet = kmalloc(urb->number_of_packets *
3373                                      sizeof(struct cvmx_usb_iso_packet),
3374                                      GFP_ATOMIC);
3375                 if (iso_packet) {
3376                         int i;
3377                         /* Fill the list with the data from the URB */
3378                         for (i = 0; i < urb->number_of_packets; i++) {
3379                                 iso_packet[i].offset =
3380                                         urb->iso_frame_desc[i].offset;
3381                                 iso_packet[i].length =
3382                                         urb->iso_frame_desc[i].length;
3383                                 iso_packet[i].status =
3384                                         CVMX_USB_COMPLETE_ERROR;
3385                         }
3386                         /*
3387                          * Store a pointer to the list in the URB setup_packet
3388                          * field. We know this currently isn't being used and
3389                          * this saves us a bunch of logic.
3390                          */
3391                         urb->setup_packet = (char *)iso_packet;
3392                         transaction = cvmx_usb_submit_isochronous(&priv->usb,
3393                                                                   pipe, urb);
3394                         /*
3395                          * If submit failed we need to free our private packet
3396                          * list.
3397                          */
3398                         if (!transaction) {
3399                                 urb->setup_packet = NULL;
3400                                 kfree(iso_packet);
3401                         }
3402                 }
3403                 break;
3404         case PIPE_INTERRUPT:
3405                 dev_dbg(dev, "Submit interrupt to %d.%d\n",
3406                         usb_pipedevice(urb->pipe),
3407                         usb_pipeendpoint(urb->pipe));
3408                 transaction = cvmx_usb_submit_interrupt(&priv->usb, pipe, urb);
3409                 break;
3410         case PIPE_CONTROL:
3411                 dev_dbg(dev, "Submit control to %d.%d\n",
3412                         usb_pipedevice(urb->pipe),
3413                         usb_pipeendpoint(urb->pipe));
3414                 transaction = cvmx_usb_submit_control(&priv->usb, pipe, urb);
3415                 break;
3416         case PIPE_BULK:
3417                 dev_dbg(dev, "Submit bulk to %d.%d\n",
3418                         usb_pipedevice(urb->pipe),
3419                         usb_pipeendpoint(urb->pipe));
3420                 transaction = cvmx_usb_submit_bulk(&priv->usb, pipe, urb);
3421                 break;
3422         }
3423         if (!transaction) {
3424                 usb_hcd_unlink_urb_from_ep(hcd, urb);
3425                 spin_unlock_irqrestore(&priv->lock, flags);
3426                 dev_dbg(dev, "Failed to submit\n");
3427                 return -ENOMEM;
3428         }
3429         urb->hcpriv = transaction;
3430         spin_unlock_irqrestore(&priv->lock, flags);
3431         return 0;
3432 }
3433
3434 static int octeon_usb_urb_dequeue(struct usb_hcd *hcd,
3435                                   struct urb *urb,
3436                                   int status)
3437 {
3438         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3439         unsigned long flags;
3440         int rc;
3441
3442         if (!urb->dev)
3443                 return -EINVAL;
3444
3445         spin_lock_irqsave(&priv->lock, flags);
3446
3447         rc = usb_hcd_check_unlink_urb(hcd, urb, status);
3448         if (rc)
3449                 goto out;
3450
3451         urb->status = status;
3452         cvmx_usb_cancel(&priv->usb, urb->ep->hcpriv, urb->hcpriv);
3453
3454 out:
3455         spin_unlock_irqrestore(&priv->lock, flags);
3456
3457         return rc;
3458 }
3459
3460 static void octeon_usb_endpoint_disable(struct usb_hcd *hcd,
3461                                         struct usb_host_endpoint *ep)
3462 {
3463         struct device *dev = hcd->self.controller;
3464
3465         if (ep->hcpriv) {
3466                 struct octeon_hcd *priv = hcd_to_octeon(hcd);
3467                 struct cvmx_usb_pipe *pipe = ep->hcpriv;
3468                 unsigned long flags;
3469
3470                 spin_lock_irqsave(&priv->lock, flags);
3471                 cvmx_usb_cancel_all(&priv->usb, pipe);
3472                 if (cvmx_usb_close_pipe(&priv->usb, pipe))
3473                         dev_dbg(dev, "Closing pipe %p failed\n", pipe);
3474                 spin_unlock_irqrestore(&priv->lock, flags);
3475                 ep->hcpriv = NULL;
3476         }
3477 }
3478
3479 static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
3480 {
3481         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3482         struct cvmx_usb_port_status port_status;
3483         unsigned long flags;
3484
3485         spin_lock_irqsave(&priv->lock, flags);
3486         port_status = cvmx_usb_get_status(&priv->usb);
3487         spin_unlock_irqrestore(&priv->lock, flags);
3488         buf[0] = 0;
3489         buf[0] = port_status.connect_change << 1;
3490
3491         return buf[0] != 0;
3492 }
3493
3494 static int octeon_usb_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
3495                                 u16 wIndex, char *buf, u16 wLength)
3496 {
3497         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3498         struct device *dev = hcd->self.controller;
3499         struct cvmx_usb_port_status usb_port_status;
3500         int port_status;
3501         struct usb_hub_descriptor *desc;
3502         unsigned long flags;
3503
3504         switch (typeReq) {
3505         case ClearHubFeature:
3506                 dev_dbg(dev, "ClearHubFeature\n");
3507                 switch (wValue) {
3508                 case C_HUB_LOCAL_POWER:
3509                 case C_HUB_OVER_CURRENT:
3510                         /* Nothing required here */
3511                         break;
3512                 default:
3513                         return -EINVAL;
3514                 }
3515                 break;
3516         case ClearPortFeature:
3517                 dev_dbg(dev, "ClearPortFeature\n");
3518                 if (wIndex != 1) {
3519                         dev_dbg(dev, " INVALID\n");
3520                         return -EINVAL;
3521                 }
3522
3523                 switch (wValue) {
3524                 case USB_PORT_FEAT_ENABLE:
3525                         dev_dbg(dev, " ENABLE\n");
3526                         spin_lock_irqsave(&priv->lock, flags);
3527                         cvmx_usb_disable(&priv->usb);
3528                         spin_unlock_irqrestore(&priv->lock, flags);
3529                         break;
3530                 case USB_PORT_FEAT_SUSPEND:
3531                         dev_dbg(dev, " SUSPEND\n");
3532                         /* Not supported on Octeon */
3533                         break;
3534                 case USB_PORT_FEAT_POWER:
3535                         dev_dbg(dev, " POWER\n");
3536                         /* Not supported on Octeon */
3537                         break;
3538                 case USB_PORT_FEAT_INDICATOR:
3539                         dev_dbg(dev, " INDICATOR\n");
3540                         /* Port inidicator not supported */
3541                         break;
3542                 case USB_PORT_FEAT_C_CONNECTION:
3543                         dev_dbg(dev, " C_CONNECTION\n");
3544                         /* Clears drivers internal connect status change flag */
3545                         spin_lock_irqsave(&priv->lock, flags);
3546                         priv->usb.port_status =
3547                                 cvmx_usb_get_status(&priv->usb);
3548                         spin_unlock_irqrestore(&priv->lock, flags);
3549                         break;
3550                 case USB_PORT_FEAT_C_RESET:
3551                         dev_dbg(dev, " C_RESET\n");
3552                         /*
3553                          * Clears the driver's internal Port Reset Change flag.
3554                          */
3555                         spin_lock_irqsave(&priv->lock, flags);
3556                         priv->usb.port_status =
3557                                 cvmx_usb_get_status(&priv->usb);
3558                         spin_unlock_irqrestore(&priv->lock, flags);
3559                         break;
3560                 case USB_PORT_FEAT_C_ENABLE:
3561                         dev_dbg(dev, " C_ENABLE\n");
3562                         /*
3563                          * Clears the driver's internal Port Enable/Disable
3564                          * Change flag.
3565                          */
3566                         spin_lock_irqsave(&priv->lock, flags);
3567                         priv->usb.port_status =
3568                                 cvmx_usb_get_status(&priv->usb);
3569                         spin_unlock_irqrestore(&priv->lock, flags);
3570                         break;
3571                 case USB_PORT_FEAT_C_SUSPEND:
3572                         dev_dbg(dev, " C_SUSPEND\n");
3573                         /*
3574                          * Clears the driver's internal Port Suspend Change
3575                          * flag, which is set when resume signaling on the host
3576                          * port is complete.
3577                          */
3578                         break;
3579                 case USB_PORT_FEAT_C_OVER_CURRENT:
3580                         dev_dbg(dev, " C_OVER_CURRENT\n");
3581                         /* Clears the driver's overcurrent Change flag */
3582                         spin_lock_irqsave(&priv->lock, flags);
3583                         priv->usb.port_status =
3584                                 cvmx_usb_get_status(&priv->usb);
3585                         spin_unlock_irqrestore(&priv->lock, flags);
3586                         break;
3587                 default:
3588                         dev_dbg(dev, " UNKNOWN\n");
3589                         return -EINVAL;
3590                 }
3591                 break;
3592         case GetHubDescriptor:
3593                 dev_dbg(dev, "GetHubDescriptor\n");
3594                 desc = (struct usb_hub_descriptor *)buf;
3595                 desc->bDescLength = 9;
3596                 desc->bDescriptorType = 0x29;
3597                 desc->bNbrPorts = 1;
3598                 desc->wHubCharacteristics = cpu_to_le16(0x08);
3599                 desc->bPwrOn2PwrGood = 1;
3600                 desc->bHubContrCurrent = 0;
3601                 desc->u.hs.DeviceRemovable[0] = 0;
3602                 desc->u.hs.DeviceRemovable[1] = 0xff;
3603                 break;
3604         case GetHubStatus:
3605                 dev_dbg(dev, "GetHubStatus\n");
3606                 *(__le32 *) buf = 0;
3607                 break;
3608         case GetPortStatus:
3609                 dev_dbg(dev, "GetPortStatus\n");
3610                 if (wIndex != 1) {
3611                         dev_dbg(dev, " INVALID\n");
3612                         return -EINVAL;
3613                 }
3614
3615                 spin_lock_irqsave(&priv->lock, flags);
3616                 usb_port_status = cvmx_usb_get_status(&priv->usb);
3617                 spin_unlock_irqrestore(&priv->lock, flags);
3618                 port_status = 0;
3619
3620                 if (usb_port_status.connect_change) {
3621                         port_status |= (1 << USB_PORT_FEAT_C_CONNECTION);
3622                         dev_dbg(dev, " C_CONNECTION\n");
3623                 }
3624
3625                 if (usb_port_status.port_enabled) {
3626                         port_status |= (1 << USB_PORT_FEAT_C_ENABLE);
3627                         dev_dbg(dev, " C_ENABLE\n");
3628                 }
3629
3630                 if (usb_port_status.connected) {
3631                         port_status |= (1 << USB_PORT_FEAT_CONNECTION);
3632                         dev_dbg(dev, " CONNECTION\n");
3633                 }
3634
3635                 if (usb_port_status.port_enabled) {
3636                         port_status |= (1 << USB_PORT_FEAT_ENABLE);
3637                         dev_dbg(dev, " ENABLE\n");
3638                 }
3639
3640                 if (usb_port_status.port_over_current) {
3641                         port_status |= (1 << USB_PORT_FEAT_OVER_CURRENT);
3642                         dev_dbg(dev, " OVER_CURRENT\n");
3643                 }
3644
3645                 if (usb_port_status.port_powered) {
3646                         port_status |= (1 << USB_PORT_FEAT_POWER);
3647                         dev_dbg(dev, " POWER\n");
3648                 }
3649
3650                 if (usb_port_status.port_speed == CVMX_USB_SPEED_HIGH) {
3651                         port_status |= USB_PORT_STAT_HIGH_SPEED;
3652                         dev_dbg(dev, " HIGHSPEED\n");
3653                 } else if (usb_port_status.port_speed == CVMX_USB_SPEED_LOW) {
3654                         port_status |= (1 << USB_PORT_FEAT_LOWSPEED);
3655                         dev_dbg(dev, " LOWSPEED\n");
3656                 }
3657
3658                 *((__le32 *) buf) = cpu_to_le32(port_status);
3659                 break;
3660         case SetHubFeature:
3661                 dev_dbg(dev, "SetHubFeature\n");
3662                 /* No HUB features supported */
3663                 break;
3664         case SetPortFeature:
3665                 dev_dbg(dev, "SetPortFeature\n");
3666                 if (wIndex != 1) {
3667                         dev_dbg(dev, " INVALID\n");
3668                         return -EINVAL;
3669                 }
3670
3671                 switch (wValue) {
3672                 case USB_PORT_FEAT_SUSPEND:
3673                         dev_dbg(dev, " SUSPEND\n");
3674                         return -EINVAL;
3675                 case USB_PORT_FEAT_POWER:
3676                         dev_dbg(dev, " POWER\n");
3677                         return -EINVAL;
3678                 case USB_PORT_FEAT_RESET:
3679                         dev_dbg(dev, " RESET\n");
3680                         spin_lock_irqsave(&priv->lock, flags);
3681                         cvmx_usb_disable(&priv->usb);
3682                         if (cvmx_usb_enable(&priv->usb))
3683                                 dev_dbg(dev, "Failed to enable the port\n");
3684                         spin_unlock_irqrestore(&priv->lock, flags);
3685                         return 0;
3686                 case USB_PORT_FEAT_INDICATOR:
3687                         dev_dbg(dev, " INDICATOR\n");
3688                         /* Not supported */
3689                         break;
3690                 default:
3691                         dev_dbg(dev, " UNKNOWN\n");
3692                         return -EINVAL;
3693                 }
3694                 break;
3695         default:
3696                 dev_dbg(dev, "Unknown root hub request\n");
3697                 return -EINVAL;
3698         }
3699         return 0;
3700 }
3701
3702 static const struct hc_driver octeon_hc_driver = {
3703         .description            = "Octeon USB",
3704         .product_desc           = "Octeon Host Controller",
3705         .hcd_priv_size          = sizeof(struct octeon_hcd),
3706         .irq                    = octeon_usb_irq,
3707         .flags                  = HCD_MEMORY | HCD_USB2,
3708         .start                  = octeon_usb_start,
3709         .stop                   = octeon_usb_stop,
3710         .urb_enqueue            = octeon_usb_urb_enqueue,
3711         .urb_dequeue            = octeon_usb_urb_dequeue,
3712         .endpoint_disable       = octeon_usb_endpoint_disable,
3713         .get_frame_number       = octeon_usb_get_frame_number,
3714         .hub_status_data        = octeon_usb_hub_status_data,
3715         .hub_control            = octeon_usb_hub_control,
3716         .map_urb_for_dma        = octeon_map_urb_for_dma,
3717         .unmap_urb_for_dma      = octeon_unmap_urb_for_dma,
3718 };
3719
3720 static int octeon_usb_probe(struct platform_device *pdev)
3721 {
3722         int status;
3723         int initialize_flags;
3724         int usb_num;
3725         struct resource *res_mem;
3726         struct device_node *usbn_node;
3727         int irq = platform_get_irq(pdev, 0);
3728         struct device *dev = &pdev->dev;
3729         struct octeon_hcd *priv;
3730         struct usb_hcd *hcd;
3731         unsigned long flags;
3732         u32 clock_rate = 48000000;
3733         bool is_crystal_clock = false;
3734         const char *clock_type;
3735         int i;
3736
3737         if (dev->of_node == NULL) {
3738                 dev_err(dev, "Error: empty of_node\n");
3739                 return -ENXIO;
3740         }
3741         usbn_node = dev->of_node->parent;
3742
3743         i = of_property_read_u32(usbn_node,
3744                                  "refclk-frequency", &clock_rate);
3745         if (i) {
3746                 dev_err(dev, "No USBN \"refclk-frequency\"\n");
3747                 return -ENXIO;
3748         }
3749         switch (clock_rate) {
3750         case 12000000:
3751                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
3752                 break;
3753         case 24000000:
3754                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
3755                 break;
3756         case 48000000:
3757                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
3758                 break;
3759         default:
3760                 dev_err(dev, "Illebal USBN \"refclk-frequency\" %u\n",
3761                                 clock_rate);
3762                 return -ENXIO;
3763
3764         }
3765
3766         i = of_property_read_string(usbn_node,
3767                                     "refclk-type", &clock_type);
3768
3769         if (!i && strcmp("crystal", clock_type) == 0)
3770                 is_crystal_clock = true;
3771
3772         if (is_crystal_clock)
3773                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
3774         else
3775                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
3776
3777         res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3778         if (res_mem == NULL) {
3779                 dev_err(dev, "found no memory resource\n");
3780                 return -ENXIO;
3781         }
3782         usb_num = (res_mem->start >> 44) & 1;
3783
3784         if (irq < 0) {
3785                 /* Defective device tree, but we know how to fix it. */
3786                 irq_hw_number_t hwirq = usb_num ? (1 << 6) + 17 : 56;
3787
3788                 irq = irq_create_mapping(NULL, hwirq);
3789         }
3790
3791         /*
3792          * Set the DMA mask to 64bits so we get buffers already translated for
3793          * DMA.
3794          */
3795         dev->coherent_dma_mask = ~0;
3796         dev->dma_mask = &dev->coherent_dma_mask;
3797
3798         /*
3799          * Only cn52XX and cn56XX have DWC_OTG USB hardware and the
3800          * IOB priority registers.  Under heavy network load USB
3801          * hardware can be starved by the IOB causing a crash.  Give
3802          * it a priority boost if it has been waiting more than 400
3803          * cycles to avoid this situation.
3804          *
3805          * Testing indicates that a cnt_val of 8192 is not sufficient,
3806          * but no failures are seen with 4096.  We choose a value of
3807          * 400 to give a safety factor of 10.
3808          */
3809         if (OCTEON_IS_MODEL(OCTEON_CN52XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
3810                 union cvmx_iob_n2c_l2c_pri_cnt pri_cnt;
3811
3812                 pri_cnt.u64 = 0;
3813                 pri_cnt.s.cnt_enb = 1;
3814                 pri_cnt.s.cnt_val = 400;
3815                 cvmx_write_csr(CVMX_IOB_N2C_L2C_PRI_CNT, pri_cnt.u64);
3816         }
3817
3818         hcd = usb_create_hcd(&octeon_hc_driver, dev, dev_name(dev));
3819         if (!hcd) {
3820                 dev_dbg(dev, "Failed to allocate memory for HCD\n");
3821                 return -1;
3822         }
3823         hcd->uses_new_polling = 1;
3824         priv = (struct octeon_hcd *)hcd->hcd_priv;
3825
3826         spin_lock_init(&priv->lock);
3827
3828         status = cvmx_usb_initialize(&priv->usb, usb_num, initialize_flags);
3829         if (status) {
3830                 dev_dbg(dev, "USB initialization failed with %d\n", status);
3831                 kfree(hcd);
3832                 return -1;
3833         }
3834
3835         /* This delay is needed for CN3010, but I don't know why... */
3836         mdelay(10);
3837
3838         spin_lock_irqsave(&priv->lock, flags);
3839         cvmx_usb_poll(&priv->usb);
3840         spin_unlock_irqrestore(&priv->lock, flags);
3841
3842         status = usb_add_hcd(hcd, irq, 0);
3843         if (status) {
3844                 dev_dbg(dev, "USB add HCD failed with %d\n", status);
3845                 kfree(hcd);
3846                 return -1;
3847         }
3848         device_wakeup_enable(hcd->self.controller);
3849
3850         dev_info(dev, "Registered HCD for port %d on irq %d\n", usb_num, irq);
3851
3852         return 0;
3853 }
3854
3855 static int octeon_usb_remove(struct platform_device *pdev)
3856 {
3857         int status;
3858         struct device *dev = &pdev->dev;
3859         struct usb_hcd *hcd = dev_get_drvdata(dev);
3860         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3861         unsigned long flags;
3862
3863         usb_remove_hcd(hcd);
3864         spin_lock_irqsave(&priv->lock, flags);
3865         status = cvmx_usb_shutdown(&priv->usb);
3866         spin_unlock_irqrestore(&priv->lock, flags);
3867         if (status)
3868                 dev_dbg(dev, "USB shutdown failed with %d\n", status);
3869
3870         kfree(hcd);
3871
3872         return 0;
3873 }
3874
3875 static struct of_device_id octeon_usb_match[] = {
3876         {
3877                 .compatible = "cavium,octeon-5750-usbc",
3878         },
3879         {},
3880 };
3881
3882 static struct platform_driver octeon_usb_driver = {
3883         .driver = {
3884                 .name       = "OcteonUSB",
3885                 .owner          = THIS_MODULE,
3886                 .of_match_table = octeon_usb_match,
3887         },
3888         .probe      = octeon_usb_probe,
3889         .remove     = octeon_usb_remove,
3890 };
3891
3892 static int __init octeon_usb_driver_init(void)
3893 {
3894         if (usb_disabled())
3895                 return 0;
3896
3897         return platform_driver_register(&octeon_usb_driver);
3898 }
3899 module_init(octeon_usb_driver_init);
3900
3901 static void __exit octeon_usb_driver_exit(void)
3902 {
3903         if (usb_disabled())
3904                 return;
3905
3906         platform_driver_unregister(&octeon_usb_driver);
3907 }
3908 module_exit(octeon_usb_driver_exit);
3909
3910 MODULE_LICENSE("GPL");
3911 MODULE_AUTHOR("Cavium, Inc. <support@cavium.com>");
3912 MODULE_DESCRIPTION("Cavium Inc. OCTEON USB Host driver.");