Version:  2.0.40 2.2.26 2.4.37 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16

Linux/drivers/staging/octeon-usb/octeon-hcd.c

  1 /*
  2  * This file is subject to the terms and conditions of the GNU General Public
  3  * License.  See the file "COPYING" in the main directory of this archive
  4  * for more details.
  5  *
  6  * Copyright (C) 2008 Cavium Networks
  7  *
  8  * Some parts of the code were originally released under BSD license:
  9  *
 10  * Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
 11  * reserved.
 12  *
 13  * Redistribution and use in source and binary forms, with or without
 14  * modification, are permitted provided that the following conditions are
 15  * met:
 16  *
 17  *   * Redistributions of source code must retain the above copyright
 18  *     notice, this list of conditions and the following disclaimer.
 19  *
 20  *   * Redistributions in binary form must reproduce the above
 21  *     copyright notice, this list of conditions and the following
 22  *     disclaimer in the documentation and/or other materials provided
 23  *     with the distribution.
 24  *
 25  *   * Neither the name of Cavium Networks nor the names of
 26  *     its contributors may be used to endorse or promote products
 27  *     derived from this software without specific prior written
 28  *     permission.
 29  *
 30  * This Software, including technical data, may be subject to U.S. export
 31  * control laws, including the U.S. Export Administration Act and its associated
 32  * regulations, and may be subject to export or import regulations in other
 33  * countries.
 34  *
 35  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
 36  * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
 37  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
 38  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION
 39  * OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
 40  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
 41  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
 42  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
 43  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
 44  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
 45  */
 46 #include <linux/kernel.h>
 47 #include <linux/module.h>
 48 #include <linux/init.h>
 49 #include <linux/pci.h>
 50 #include <linux/prefetch.h>
 51 #include <linux/interrupt.h>
 52 #include <linux/platform_device.h>
 53 #include <linux/usb.h>
 54 
 55 #include <linux/time.h>
 56 #include <linux/delay.h>
 57 
 58 #include <asm/octeon/cvmx.h>
 59 #include <asm/octeon/cvmx-iob-defs.h>
 60 
 61 #include <linux/usb/hcd.h>
 62 
 63 #include <linux/err.h>
 64 
 65 #include <asm/octeon/octeon.h>
 66 #include <asm/octeon/cvmx-helper.h>
 67 #include <asm/octeon/cvmx-sysinfo.h>
 68 #include <asm/octeon/cvmx-helper-board.h>
 69 
 70 #include "octeon-hcd.h"
 71 
 72 /**
 73  * enum cvmx_usb_speed - the possible USB device speeds
 74  *
 75  * @CVMX_USB_SPEED_HIGH: Device is operation at 480Mbps
 76  * @CVMX_USB_SPEED_FULL: Device is operation at 12Mbps
 77  * @CVMX_USB_SPEED_LOW:  Device is operation at 1.5Mbps
 78  */
 79 enum cvmx_usb_speed {
 80         CVMX_USB_SPEED_HIGH = 0,
 81         CVMX_USB_SPEED_FULL = 1,
 82         CVMX_USB_SPEED_LOW = 2,
 83 };
 84 
 85 /**
 86  * enum cvmx_usb_transfer - the possible USB transfer types
 87  *
 88  * @CVMX_USB_TRANSFER_CONTROL:     USB transfer type control for hub and status
 89  *                                 transfers
 90  * @CVMX_USB_TRANSFER_ISOCHRONOUS: USB transfer type isochronous for low
 91  *                                 priority periodic transfers
 92  * @CVMX_USB_TRANSFER_BULK:        USB transfer type bulk for large low priority
 93  *                                 transfers
 94  * @CVMX_USB_TRANSFER_INTERRUPT:   USB transfer type interrupt for high priority
 95  *                                 periodic transfers
 96  */
 97 enum cvmx_usb_transfer {
 98         CVMX_USB_TRANSFER_CONTROL = 0,
 99         CVMX_USB_TRANSFER_ISOCHRONOUS = 1,
100         CVMX_USB_TRANSFER_BULK = 2,
101         CVMX_USB_TRANSFER_INTERRUPT = 3,
102 };
103 
104 /**
105  * enum cvmx_usb_direction - the transfer directions
106  *
107  * @CVMX_USB_DIRECTION_OUT: Data is transferring from Octeon to the device/host
108  * @CVMX_USB_DIRECTION_IN:  Data is transferring from the device/host to Octeon
109  */
110 enum cvmx_usb_direction {
111         CVMX_USB_DIRECTION_OUT,
112         CVMX_USB_DIRECTION_IN,
113 };
114 
115 /**
116  * enum cvmx_usb_complete - possible callback function status codes
117  *
118  * @CVMX_USB_COMPLETE_SUCCESS:    The transaction / operation finished without
119  *                                any errors
120  * @CVMX_USB_COMPLETE_SHORT:      FIXME: This is currently not implemented
121  * @CVMX_USB_COMPLETE_CANCEL:     The transaction was canceled while in flight
122  *                                by a user call to cvmx_usb_cancel
123  * @CVMX_USB_COMPLETE_ERROR:      The transaction aborted with an unexpected
124  *                                error status
125  * @CVMX_USB_COMPLETE_STALL:      The transaction received a USB STALL response
126  *                                from the device
127  * @CVMX_USB_COMPLETE_XACTERR:    The transaction failed with an error from the
128  *                                device even after a number of retries
129  * @CVMX_USB_COMPLETE_DATATGLERR: The transaction failed with a data toggle
130  *                                error even after a number of retries
131  * @CVMX_USB_COMPLETE_BABBLEERR:  The transaction failed with a babble error
132  * @CVMX_USB_COMPLETE_FRAMEERR:   The transaction failed with a frame error
133  *                                even after a number of retries
134  */
135 enum cvmx_usb_complete {
136         CVMX_USB_COMPLETE_SUCCESS,
137         CVMX_USB_COMPLETE_SHORT,
138         CVMX_USB_COMPLETE_CANCEL,
139         CVMX_USB_COMPLETE_ERROR,
140         CVMX_USB_COMPLETE_STALL,
141         CVMX_USB_COMPLETE_XACTERR,
142         CVMX_USB_COMPLETE_DATATGLERR,
143         CVMX_USB_COMPLETE_BABBLEERR,
144         CVMX_USB_COMPLETE_FRAMEERR,
145 };
146 
147 /**
148  * struct cvmx_usb_port_status - the USB port status information
149  *
150  * @port_enabled:       1 = Usb port is enabled, 0 = disabled
151  * @port_over_current:  1 = Over current detected, 0 = Over current not
152  *                      detected. Octeon doesn't support over current detection.
153  * @port_powered:       1 = Port power is being supplied to the device, 0 =
154  *                      power is off. Octeon doesn't support turning port power
155  *                      off.
156  * @port_speed:         Current port speed.
157  * @connected:          1 = A device is connected to the port, 0 = No device is
158  *                      connected.
159  * @connect_change:     1 = Device connected state changed since the last set
160  *                      status call.
161  */
162 struct cvmx_usb_port_status {
163         uint32_t reserved               : 25;
164         uint32_t port_enabled           : 1;
165         uint32_t port_over_current      : 1;
166         uint32_t port_powered           : 1;
167         enum cvmx_usb_speed port_speed  : 2;
168         uint32_t connected              : 1;
169         uint32_t connect_change         : 1;
170 };
171 
172 /**
173  * union cvmx_usb_control_header - the structure of a Control packet header
174  *
175  * @s.request_type:     Bit 7 tells the direction: 1=IN, 0=OUT
176  * @s.request           The standard usb request to make
177  * @s.value             Value parameter for the request in little endian format
178  * @s.index             Index for the request in little endian format
179  * @s.length            Length of the data associated with this request in
180  *                      little endian format
181  */
182 union cvmx_usb_control_header {
183         uint64_t u64;
184         struct {
185                 uint64_t request_type   : 8;
186                 uint64_t request        : 8;
187                 uint64_t value          : 16;
188                 uint64_t index          : 16;
189                 uint64_t length         : 16;
190         } s;
191 };
192 
193 /**
194  * struct cvmx_usb_iso_packet - descriptor for Isochronous packets
195  *
196  * @offset:     This is the offset in bytes into the main buffer where this data
197  *              is stored.
198  * @length:     This is the length in bytes of the data.
199  * @status:     This is the status of this individual packet transfer.
200  */
201 struct cvmx_usb_iso_packet {
202         int offset;
203         int length;
204         enum cvmx_usb_complete status;
205 };
206 
207 /**
208  * enum cvmx_usb_initialize_flags - flags used by the initialization function
209  *
210  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI:    The USB port uses a 12MHz crystal
211  *                                            as clock source at USB_XO and
212  *                                            USB_XI.
213  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND:   The USB port uses 12/24/48MHz 2.5V
214  *                                            board clock source at USB_XO.
215  *                                            USB_XI should be tied to GND.
216  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK: Mask for clock speed field
217  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:    Speed of reference clock or
218  *                                            crystal
219  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:    Speed of reference clock
220  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:    Speed of reference clock
221  * @CVMX_USB_INITIALIZE_FLAGS_NO_DMA:         Disable DMA and used polled IO for
222  *                                            data transfer use for the USB
223  */
224 enum cvmx_usb_initialize_flags {
225         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI           = 1 << 0,
226         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND          = 1 << 1,
227         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK        = 3 << 3,
228         CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ           = 1 << 3,
229         CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ           = 2 << 3,
230         CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ           = 3 << 3,
231         /* Bits 3-4 used to encode the clock frequency */
232         CVMX_USB_INITIALIZE_FLAGS_NO_DMA                = 1 << 5,
233 };
234 
235 /**
236  * enum cvmx_usb_pipe_flags - internal flags for a pipe.
237  *
238  * @__CVMX_USB_PIPE_FLAGS_SCHEDULED: Used internally to determine if a pipe is
239  *                                   actively using hardware. Do not use.
240  * @__CVMX_USB_PIPE_FLAGS_NEED_PING: Used internally to determine if a high
241  *                                   speed pipe is in the ping state. Do not
242  *                                   use.
243  */
244 enum cvmx_usb_pipe_flags {
245         __CVMX_USB_PIPE_FLAGS_SCHEDULED = 1 << 17,
246         __CVMX_USB_PIPE_FLAGS_NEED_PING = 1 << 18,
247 };
248 
249 /* Maximum number of times to retry failed transactions */
250 #define MAX_RETRIES             3
251 
252 /* Maximum number of hardware channels supported by the USB block */
253 #define MAX_CHANNELS            8
254 
255 /* The highest valid USB device address */
256 #define MAX_USB_ADDRESS         127
257 
258 /* The highest valid USB endpoint number */
259 #define MAX_USB_ENDPOINT        15
260 
261 /* The highest valid port number on a hub */
262 #define MAX_USB_HUB_PORT        15
263 
264 /*
265  * The low level hardware can transfer a maximum of this number of bytes in each
266  * transfer. The field is 19 bits wide
267  */
268 #define MAX_TRANSFER_BYTES      ((1<<19)-1)
269 
270 /*
271  * The low level hardware can transfer a maximum of this number of packets in
272  * each transfer. The field is 10 bits wide
273  */
274 #define MAX_TRANSFER_PACKETS    ((1<<10)-1)
275 
276 /**
277  * Logical transactions may take numerous low level
278  * transactions, especially when splits are concerned. This
279  * enum represents all of the possible stages a transaction can
280  * be in. Note that split completes are always even. This is so
281  * the NAK handler can backup to the previous low level
282  * transaction with a simple clearing of bit 0.
283  */
284 enum cvmx_usb_stage {
285         CVMX_USB_STAGE_NON_CONTROL,
286         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
287         CVMX_USB_STAGE_SETUP,
288         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
289         CVMX_USB_STAGE_DATA,
290         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
291         CVMX_USB_STAGE_STATUS,
292         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
293 };
294 
295 /**
296  * struct cvmx_usb_transaction - describes each pending USB transaction
297  *                               regardless of type. These are linked together
298  *                               to form a list of pending requests for a pipe.
299  *
300  * @node:               List node for transactions in the pipe.
301  * @type:               Type of transaction, duplicated of the pipe.
302  * @flags:              State flags for this transaction.
303  * @buffer:             User's physical buffer address to read/write.
304  * @buffer_length:      Size of the user's buffer in bytes.
305  * @control_header:     For control transactions, physical address of the 8
306  *                      byte standard header.
307  * @iso_start_frame:    For ISO transactions, the starting frame number.
308  * @iso_number_packets: For ISO transactions, the number of packets in the
309  *                      request.
310  * @iso_packets:        For ISO transactions, the sub packets in the request.
311  * @actual_bytes:       Actual bytes transfer for this transaction.
312  * @stage:              For control transactions, the current stage.
313  * @urb:                URB.
314  */
315 struct cvmx_usb_transaction {
316         struct list_head node;
317         enum cvmx_usb_transfer type;
318         uint64_t buffer;
319         int buffer_length;
320         uint64_t control_header;
321         int iso_start_frame;
322         int iso_number_packets;
323         struct cvmx_usb_iso_packet *iso_packets;
324         int xfersize;
325         int pktcnt;
326         int retries;
327         int actual_bytes;
328         enum cvmx_usb_stage stage;
329         struct urb *urb;
330 };
331 
332 /**
333  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
334  *                        and some USB device. It contains a list of pending
335  *                        request to the device.
336  *
337  * @node:               List node for pipe list
338  * @next:               Pipe after this one in the list
339  * @transactions:       List of pending transactions
340  * @interval:           For periodic pipes, the interval between packets in
341  *                      frames
342  * @next_tx_frame:      The next frame this pipe is allowed to transmit on
343  * @flags:              State flags for this pipe
344  * @device_speed:       Speed of device connected to this pipe
345  * @transfer_type:      Type of transaction supported by this pipe
346  * @transfer_dir:       IN or OUT. Ignored for Control
347  * @multi_count:        Max packet in a row for the device
348  * @max_packet:         The device's maximum packet size in bytes
349  * @device_addr:        USB device address at other end of pipe
350  * @endpoint_num:       USB endpoint number at other end of pipe
351  * @hub_device_addr:    Hub address this device is connected to
352  * @hub_port:           Hub port this device is connected to
353  * @pid_toggle:         This toggles between 0/1 on every packet send to track
354  *                      the data pid needed
355  * @channel:            Hardware DMA channel for this pipe
356  * @split_sc_frame:     The low order bits of the frame number the split
357  *                      complete should be sent on
358  */
359 struct cvmx_usb_pipe {
360         struct list_head node;
361         struct list_head transactions;
362         uint64_t interval;
363         uint64_t next_tx_frame;
364         enum cvmx_usb_pipe_flags flags;
365         enum cvmx_usb_speed device_speed;
366         enum cvmx_usb_transfer transfer_type;
367         enum cvmx_usb_direction transfer_dir;
368         int multi_count;
369         uint16_t max_packet;
370         uint8_t device_addr;
371         uint8_t endpoint_num;
372         uint8_t hub_device_addr;
373         uint8_t hub_port;
374         uint8_t pid_toggle;
375         uint8_t channel;
376         int8_t split_sc_frame;
377 };
378 
379 struct cvmx_usb_tx_fifo {
380         struct {
381                 int channel;
382                 int size;
383                 uint64_t address;
384         } entry[MAX_CHANNELS+1];
385         int head;
386         int tail;
387 };
388 
389 /**
390  * struct cvmx_usb_state - the state of the USB block
391  *
392  * init_flags:             Flags passed to initialize.
393  * index:                  Which USB block this is for.
394  * idle_hardware_channels: Bit set for every idle hardware channel.
395  * usbcx_hprt:             Stored port status so we don't need to read a CSR to
396  *                         determine splits.
397  * pipe_for_channel:       Map channels to pipes.
398  * pipe:                   Storage for pipes.
399  * indent:                 Used by debug output to indent functions.
400  * port_status:            Last port status used for change notification.
401  * idle_pipes:             List of open pipes that have no transactions.
402  * active_pipes:           Active pipes indexed by transfer type.
403  * frame_number:           Increments every SOF interrupt for time keeping.
404  * active_split:           Points to the current active split, or NULL.
405  */
406 struct cvmx_usb_state {
407         int init_flags;
408         int index;
409         int idle_hardware_channels;
410         union cvmx_usbcx_hprt usbcx_hprt;
411         struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
412         int indent;
413         struct cvmx_usb_port_status port_status;
414         struct list_head idle_pipes;
415         struct list_head active_pipes[4];
416         uint64_t frame_number;
417         struct cvmx_usb_transaction *active_split;
418         struct cvmx_usb_tx_fifo periodic;
419         struct cvmx_usb_tx_fifo nonperiodic;
420 };
421 
422 struct octeon_hcd {
423         spinlock_t lock;
424         struct cvmx_usb_state usb;
425         struct tasklet_struct dequeue_tasklet;
426         struct list_head dequeue_list;
427 };
428 
429 /* This macro spins on a field waiting for it to reach a value */
430 #define CVMX_WAIT_FOR_FIELD32(address, type, field, op, value, timeout_usec)\
431         ({int result;                                                       \
432         do {                                                                \
433                 uint64_t done = cvmx_get_cycle() + (uint64_t)timeout_usec * \
434                         octeon_get_clock_rate() / 1000000;                  \
435                 type c;                                                     \
436                 while (1) {                                                 \
437                         c.u32 = __cvmx_usb_read_csr32(usb, address);        \
438                         if (c.s.field op (value)) {                         \
439                                 result = 0;                                 \
440                                 break;                                      \
441                         } else if (cvmx_get_cycle() > done) {               \
442                                 result = -1;                                \
443                                 break;                                      \
444                         } else                                              \
445                                 cvmx_wait(100);                             \
446                 }                                                           \
447         } while (0);                                                        \
448         result; })
449 
450 /*
451  * This macro logically sets a single field in a CSR. It does the sequence
452  * read, modify, and write
453  */
454 #define USB_SET_FIELD32(address, type, field, value)            \
455         do {                                                    \
456                 type c;                                         \
457                 c.u32 = __cvmx_usb_read_csr32(usb, address);    \
458                 c.s.field = value;                              \
459                 __cvmx_usb_write_csr32(usb, address, c.u32);    \
460         } while (0)
461 
462 /* Returns the IO address to push/pop stuff data from the FIFOs */
463 #define USB_FIFO_ADDRESS(channel, usb_index) \
464         (CVMX_USBCX_GOTGCTL(usb_index) + ((channel)+1)*0x1000)
465 
466 /**
467  * struct octeon_temp_buffer - a bounce buffer for USB transfers
468  * @temp_buffer: the newly allocated temporary buffer (including meta-data)
469  * @orig_buffer: the original buffer passed by the USB stack
470  * @data:        the newly allocated temporary buffer (excluding meta-data)
471  *
472  * Both the DMA engine and FIFO mode will always transfer full 32-bit words. If
473  * the buffer is too short, we need to allocate a temporary one, and this struct
474  * represents it.
475  */
476 struct octeon_temp_buffer {
477         void *temp_buffer;
478         void *orig_buffer;
479         u8 data[0];
480 };
481 
482 /**
483  * octeon_alloc_temp_buffer - allocate a temporary buffer for USB transfer
484  *                            (if needed)
485  * @urb:        URB.
486  * @mem_flags:  Memory allocation flags.
487  *
488  * This function allocates a temporary bounce buffer whenever it's needed
489  * due to HW limitations.
490  */
491 static int octeon_alloc_temp_buffer(struct urb *urb, gfp_t mem_flags)
492 {
493         struct octeon_temp_buffer *temp;
494 
495         if (urb->num_sgs || urb->sg ||
496             (urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP) ||
497             !(urb->transfer_buffer_length % sizeof(u32)))
498                 return 0;
499 
500         temp = kmalloc(ALIGN(urb->transfer_buffer_length, sizeof(u32)) +
501                        sizeof(*temp), mem_flags);
502         if (!temp)
503                 return -ENOMEM;
504 
505         temp->temp_buffer = temp;
506         temp->orig_buffer = urb->transfer_buffer;
507         if (usb_urb_dir_out(urb))
508                 memcpy(temp->data, urb->transfer_buffer,
509                        urb->transfer_buffer_length);
510         urb->transfer_buffer = temp->data;
511         urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER;
512 
513         return 0;
514 }
515 
516 /**
517  * octeon_free_temp_buffer - free a temporary buffer used by USB transfers.
518  * @urb: URB.
519  *
520  * Frees a buffer allocated by octeon_alloc_temp_buffer().
521  */
522 static void octeon_free_temp_buffer(struct urb *urb)
523 {
524         struct octeon_temp_buffer *temp;
525 
526         if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER))
527                 return;
528 
529         temp = container_of(urb->transfer_buffer, struct octeon_temp_buffer,
530                             data);
531         if (usb_urb_dir_in(urb))
532                 memcpy(temp->orig_buffer, urb->transfer_buffer,
533                        urb->actual_length);
534         urb->transfer_buffer = temp->orig_buffer;
535         urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER;
536         kfree(temp->temp_buffer);
537 }
538 
539 /**
540  * octeon_map_urb_for_dma - Octeon-specific map_urb_for_dma().
541  * @hcd:        USB HCD structure.
542  * @urb:        URB.
543  * @mem_flags:  Memory allocation flags.
544  */
545 static int octeon_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
546                                   gfp_t mem_flags)
547 {
548         int ret;
549 
550         ret = octeon_alloc_temp_buffer(urb, mem_flags);
551         if (ret)
552                 return ret;
553 
554         ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
555         if (ret)
556                 octeon_free_temp_buffer(urb);
557 
558         return ret;
559 }
560 
561 /**
562  * octeon_unmap_urb_for_dma - Octeon-specific unmap_urb_for_dma()
563  * @hcd:        USB HCD structure.
564  * @urb:        URB.
565  */
566 static void octeon_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
567 {
568         usb_hcd_unmap_urb_for_dma(hcd, urb);
569         octeon_free_temp_buffer(urb);
570 }
571 
572 /**
573  * Read a USB 32bit CSR. It performs the necessary address swizzle
574  * for 32bit CSRs and logs the value in a readable format if
575  * debugging is on.
576  *
577  * @usb:     USB block this access is for
578  * @address: 64bit address to read
579  *
580  * Returns: Result of the read
581  */
582 static inline uint32_t __cvmx_usb_read_csr32(struct cvmx_usb_state *usb,
583                                              uint64_t address)
584 {
585         uint32_t result = cvmx_read64_uint32(address ^ 4);
586         return result;
587 }
588 
589 
590 /**
591  * Write a USB 32bit CSR. It performs the necessary address
592  * swizzle for 32bit CSRs and logs the value in a readable format
593  * if debugging is on.
594  *
595  * @usb:     USB block this access is for
596  * @address: 64bit address to write
597  * @value:   Value to write
598  */
599 static inline void __cvmx_usb_write_csr32(struct cvmx_usb_state *usb,
600                                           uint64_t address, uint32_t value)
601 {
602         cvmx_write64_uint32(address ^ 4, value);
603         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
604 }
605 
606 
607 /**
608  * Read a USB 64bit CSR. It logs the value in a readable format if
609  * debugging is on.
610  *
611  * @usb:     USB block this access is for
612  * @address: 64bit address to read
613  *
614  * Returns: Result of the read
615  */
616 static inline uint64_t __cvmx_usb_read_csr64(struct cvmx_usb_state *usb,
617                                              uint64_t address)
618 {
619         uint64_t result = cvmx_read64_uint64(address);
620         return result;
621 }
622 
623 
624 /**
625  * Write a USB 64bit CSR. It logs the value in a readable format
626  * if debugging is on.
627  *
628  * @usb:     USB block this access is for
629  * @address: 64bit address to write
630  * @value:   Value to write
631  */
632 static inline void __cvmx_usb_write_csr64(struct cvmx_usb_state *usb,
633                                           uint64_t address, uint64_t value)
634 {
635         cvmx_write64_uint64(address, value);
636 }
637 
638 /**
639  * Return non zero if this pipe connects to a non HIGH speed
640  * device through a high speed hub.
641  *
642  * @usb:    USB block this access is for
643  * @pipe:   Pipe to check
644  *
645  * Returns: Non zero if we need to do split transactions
646  */
647 static inline int __cvmx_usb_pipe_needs_split(struct cvmx_usb_state *usb,
648                                               struct cvmx_usb_pipe *pipe)
649 {
650         return pipe->device_speed != CVMX_USB_SPEED_HIGH &&
651                usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH;
652 }
653 
654 
655 /**
656  * Trivial utility function to return the correct PID for a pipe
657  *
658  * @pipe:   pipe to check
659  *
660  * Returns: PID for pipe
661  */
662 static inline int __cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
663 {
664         if (pipe->pid_toggle)
665                 return 2; /* Data1 */
666         else
667                 return 0; /* Data0 */
668 }
669 
670 /**
671  * Initialize a USB port for use. This must be called before any
672  * other access to the Octeon USB port is made. The port starts
673  * off in the disabled state.
674  *
675  * @usb:         Pointer to an empty struct cvmx_usb_state
676  *               that will be populated by the initialize call.
677  *               This structure is then passed to all other USB
678  *               functions.
679  * @usb_port_number:
680  *               Which Octeon USB port to initialize.
681  *
682  * Returns: 0 or a negative error code.
683  */
684 static int cvmx_usb_initialize(struct cvmx_usb_state *usb,
685                                int usb_port_number,
686                                enum cvmx_usb_initialize_flags flags)
687 {
688         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
689         union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
690         int i;
691 
692         /* At first allow 0-1 for the usb port number */
693         if ((usb_port_number < 0) || (usb_port_number > 1))
694                 return -EINVAL;
695 
696         memset(usb, 0, sizeof(*usb));
697         usb->init_flags = flags;
698 
699         /* Initialize the USB state structure */
700         usb->index = usb_port_number;
701         INIT_LIST_HEAD(&usb->idle_pipes);
702         for (i = 0; i < ARRAY_SIZE(usb->active_pipes); i++)
703                 INIT_LIST_HEAD(&usb->active_pipes[i]);
704 
705         /*
706          * Power On Reset and PHY Initialization
707          *
708          * 1. Wait for DCOK to assert (nothing to do)
709          *
710          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
711          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
712          */
713         usbn_clk_ctl.u64 =
714                 __cvmx_usb_read_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index));
715         usbn_clk_ctl.s.por = 1;
716         usbn_clk_ctl.s.hrst = 0;
717         usbn_clk_ctl.s.prst = 0;
718         usbn_clk_ctl.s.hclk_rst = 0;
719         usbn_clk_ctl.s.enable = 0;
720         /*
721          * 2b. Select the USB reference clock/crystal parameters by writing
722          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
723          */
724         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
725                 /*
726                  * The USB port uses 12/24/48MHz 2.5V board clock
727                  * source at USB_XO. USB_XI should be tied to GND.
728                  * Most Octeon evaluation boards require this setting
729                  */
730                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
731                     OCTEON_IS_MODEL(OCTEON_CN56XX) ||
732                     OCTEON_IS_MODEL(OCTEON_CN50XX))
733                         /* From CN56XX,CN50XX,CN31XX,CN30XX manuals */
734                         usbn_clk_ctl.s.p_rtype = 2; /* p_rclk=1 & p_xenbn=0 */
735                 else
736                         /* From CN52XX manual */
737                         usbn_clk_ctl.s.p_rtype = 1;
738 
739                 switch (flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
740                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
741                         usbn_clk_ctl.s.p_c_sel = 0;
742                         break;
743                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
744                         usbn_clk_ctl.s.p_c_sel = 1;
745                         break;
746                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
747                         usbn_clk_ctl.s.p_c_sel = 2;
748                         break;
749                 }
750         } else {
751                 /*
752                  * The USB port uses a 12MHz crystal as clock source
753                  * at USB_XO and USB_XI
754                  */
755                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
756                         /* From CN31XX,CN30XX manual */
757                         usbn_clk_ctl.s.p_rtype = 3; /* p_rclk=1 & p_xenbn=1 */
758                 else
759                         /* From CN56XX,CN52XX,CN50XX manuals. */
760                         usbn_clk_ctl.s.p_rtype = 0;
761 
762                 usbn_clk_ctl.s.p_c_sel = 0;
763         }
764         /*
765          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
766          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
767          *     such that USB is as close as possible to 125Mhz
768          */
769         {
770                 int divisor = (octeon_get_clock_rate()+125000000-1)/125000000;
771                 /* Lower than 4 doesn't seem to work properly */
772                 if (divisor < 4)
773                         divisor = 4;
774                 usbn_clk_ctl.s.divide = divisor;
775                 usbn_clk_ctl.s.divide2 = 0;
776         }
777         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
778                                usbn_clk_ctl.u64);
779         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
780         usbn_clk_ctl.s.hclk_rst = 1;
781         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
782                                usbn_clk_ctl.u64);
783         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
784         cvmx_wait(64);
785         /*
786          * 3. Program the power-on reset field in the USBN clock-control
787          *    register:
788          *    USBN_CLK_CTL[POR] = 0
789          */
790         usbn_clk_ctl.s.por = 0;
791         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
792                                usbn_clk_ctl.u64);
793         /* 4. Wait 1 ms for PHY clock to start */
794         mdelay(1);
795         /*
796          * 5. Program the Reset input from automatic test equipment field in the
797          *    USBP control and status register:
798          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
799          */
800         usbn_usbp_ctl_status.u64 = __cvmx_usb_read_csr64(usb,
801                         CVMX_USBNX_USBP_CTL_STATUS(usb->index));
802         usbn_usbp_ctl_status.s.ate_reset = 1;
803         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
804                                usbn_usbp_ctl_status.u64);
805         /* 6. Wait 10 cycles */
806         cvmx_wait(10);
807         /*
808          * 7. Clear ATE_RESET field in the USBN clock-control register:
809          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
810          */
811         usbn_usbp_ctl_status.s.ate_reset = 0;
812         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
813                                usbn_usbp_ctl_status.u64);
814         /*
815          * 8. Program the PHY reset field in the USBN clock-control register:
816          *    USBN_CLK_CTL[PRST] = 1
817          */
818         usbn_clk_ctl.s.prst = 1;
819         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
820                                usbn_clk_ctl.u64);
821         /*
822          * 9. Program the USBP control and status register to select host or
823          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
824          *    device
825          */
826         usbn_usbp_ctl_status.s.hst_mode = 0;
827         __cvmx_usb_write_csr64(usb, CVMX_USBNX_USBP_CTL_STATUS(usb->index),
828                                usbn_usbp_ctl_status.u64);
829         /* 10. Wait 1 us */
830         udelay(1);
831         /*
832          * 11. Program the hreset_n field in the USBN clock-control register:
833          *     USBN_CLK_CTL[HRST] = 1
834          */
835         usbn_clk_ctl.s.hrst = 1;
836         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
837                                usbn_clk_ctl.u64);
838         /* 12. Proceed to USB core initialization */
839         usbn_clk_ctl.s.enable = 1;
840         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
841                                usbn_clk_ctl.u64);
842         udelay(1);
843 
844         /*
845          * USB Core Initialization
846          *
847          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
848          *    determine USB core configuration parameters.
849          *
850          *    Nothing needed
851          *
852          * 2. Program the following fields in the global AHB configuration
853          *    register (USBC_GAHBCFG)
854          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
855          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
856          *    Nonperiodic TxFIFO empty level (slave mode only),
857          *    USBC_GAHBCFG[NPTXFEMPLVL]
858          *    Periodic TxFIFO empty level (slave mode only),
859          *    USBC_GAHBCFG[PTXFEMPLVL]
860          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
861          */
862         {
863                 union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
864                 /* Due to an errata, CN31XX doesn't support DMA */
865                 if (OCTEON_IS_MODEL(OCTEON_CN31XX))
866                         usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
867                 usbcx_gahbcfg.u32 = 0;
868                 usbcx_gahbcfg.s.dmaen = !(usb->init_flags &
869                                           CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
870                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
871                         /* Only use one channel with non DMA */
872                         usb->idle_hardware_channels = 0x1;
873                 else if (OCTEON_IS_MODEL(OCTEON_CN5XXX))
874                         /* CN5XXX have an errata with channel 3 */
875                         usb->idle_hardware_channels = 0xf7;
876                 else
877                         usb->idle_hardware_channels = 0xff;
878                 usbcx_gahbcfg.s.hbstlen = 0;
879                 usbcx_gahbcfg.s.nptxfemplvl = 1;
880                 usbcx_gahbcfg.s.ptxfemplvl = 1;
881                 usbcx_gahbcfg.s.glblintrmsk = 1;
882                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
883                                        usbcx_gahbcfg.u32);
884         }
885         /*
886          * 3. Program the following fields in USBC_GUSBCFG register.
887          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
888          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
889          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
890          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
891          */
892         {
893                 union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
894 
895                 usbcx_gusbcfg.u32 = __cvmx_usb_read_csr32(usb,
896                                 CVMX_USBCX_GUSBCFG(usb->index));
897                 usbcx_gusbcfg.s.toutcal = 0;
898                 usbcx_gusbcfg.s.ddrsel = 0;
899                 usbcx_gusbcfg.s.usbtrdtim = 0x5;
900                 usbcx_gusbcfg.s.phylpwrclksel = 0;
901                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
902                                        usbcx_gusbcfg.u32);
903         }
904         /*
905          * 4. The software must unmask the following bits in the USBC_GINTMSK
906          *    register.
907          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
908          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
909          */
910         {
911                 union cvmx_usbcx_gintmsk usbcx_gintmsk;
912                 int channel;
913 
914                 usbcx_gintmsk.u32 = __cvmx_usb_read_csr32(usb,
915                                 CVMX_USBCX_GINTMSK(usb->index));
916                 usbcx_gintmsk.s.otgintmsk = 1;
917                 usbcx_gintmsk.s.modemismsk = 1;
918                 usbcx_gintmsk.s.hchintmsk = 1;
919                 usbcx_gintmsk.s.sofmsk = 0;
920                 /* We need RX FIFO interrupts if we don't have DMA */
921                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
922                         usbcx_gintmsk.s.rxflvlmsk = 1;
923                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
924                                        usbcx_gintmsk.u32);
925 
926                 /*
927                  * Disable all channel interrupts. We'll enable them per channel
928                  * later.
929                  */
930                 for (channel = 0; channel < 8; channel++)
931                         __cvmx_usb_write_csr32(usb,
932                                 CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
933         }
934 
935         {
936                 /*
937                  * Host Port Initialization
938                  *
939                  * 1. Program the host-port interrupt-mask field to unmask,
940                  *    USBC_GINTMSK[PRTINT] = 1
941                  */
942                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
943                                 union cvmx_usbcx_gintmsk, prtintmsk, 1);
944                 USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
945                                 union cvmx_usbcx_gintmsk, disconnintmsk, 1);
946                 /*
947                  * 2. Program the USBC_HCFG register to select full-speed host
948                  *    or high-speed host.
949                  */
950                 {
951                         union cvmx_usbcx_hcfg usbcx_hcfg;
952 
953                         usbcx_hcfg.u32 = __cvmx_usb_read_csr32(usb,
954                                         CVMX_USBCX_HCFG(usb->index));
955                         usbcx_hcfg.s.fslssupp = 0;
956                         usbcx_hcfg.s.fslspclksel = 0;
957                         __cvmx_usb_write_csr32(usb,
958                                         CVMX_USBCX_HCFG(usb->index),
959                                         usbcx_hcfg.u32);
960                 }
961                 /*
962                  * 3. Program the port power bit to drive VBUS on the USB,
963                  *    USBC_HPRT[PRTPWR] = 1
964                  */
965                 USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index),
966                                 union cvmx_usbcx_hprt, prtpwr, 1);
967 
968                 /*
969                  * Steps 4-15 from the manual are done later in the port enable
970                  */
971         }
972 
973         return 0;
974 }
975 
976 
977 /**
978  * Shutdown a USB port after a call to cvmx_usb_initialize().
979  * The port should be disabled with all pipes closed when this
980  * function is called.
981  *
982  * @usb: USB device state populated by cvmx_usb_initialize().
983  *
984  * Returns: 0 or a negative error code.
985  */
986 static int cvmx_usb_shutdown(struct cvmx_usb_state *usb)
987 {
988         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
989 
990         /* Make sure all pipes are closed */
991         if (!list_empty(&usb->idle_pipes) ||
992             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS]) ||
993             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT]) ||
994             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_CONTROL]) ||
995             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_BULK]))
996                 return -EBUSY;
997 
998         /* Disable the clocks and put them in power on reset */
999         usbn_clk_ctl.u64 = __cvmx_usb_read_csr64(usb,
1000                         CVMX_USBNX_CLK_CTL(usb->index));
1001         usbn_clk_ctl.s.enable = 1;
1002         usbn_clk_ctl.s.por = 1;
1003         usbn_clk_ctl.s.hclk_rst = 1;
1004         usbn_clk_ctl.s.prst = 0;
1005         usbn_clk_ctl.s.hrst = 0;
1006         __cvmx_usb_write_csr64(usb, CVMX_USBNX_CLK_CTL(usb->index),
1007                                usbn_clk_ctl.u64);
1008         return 0;
1009 }
1010 
1011 
1012 /**
1013  * Enable a USB port. After this call succeeds, the USB port is
1014  * online and servicing requests.
1015  *
1016  * @usb: USB device state populated by cvmx_usb_initialize().
1017  *
1018  * Returns: 0 or a negative error code.
1019  */
1020 static int cvmx_usb_enable(struct cvmx_usb_state *usb)
1021 {
1022         union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
1023 
1024         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb,
1025                         CVMX_USBCX_HPRT(usb->index));
1026 
1027         /*
1028          * If the port is already enabled the just return. We don't need to do
1029          * anything
1030          */
1031         if (usb->usbcx_hprt.s.prtena)
1032                 return 0;
1033 
1034         /* If there is nothing plugged into the port then fail immediately */
1035         if (!usb->usbcx_hprt.s.prtconnsts)
1036                 return -ETIMEDOUT;
1037 
1038         /* Program the port reset bit to start the reset process */
1039         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1040                         prtrst, 1);
1041 
1042         /*
1043          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
1044          * process to complete.
1045          */
1046         mdelay(50);
1047 
1048         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
1049         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1050                         prtrst, 0);
1051 
1052         /* Wait for the USBC_HPRT[PRTENA]. */
1053         if (CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_HPRT(usb->index),
1054                                 union cvmx_usbcx_hprt, prtena, ==, 1, 100000))
1055                 return -ETIMEDOUT;
1056 
1057         /*
1058          * Read the port speed field to get the enumerated speed,
1059          * USBC_HPRT[PRTSPD].
1060          */
1061         usb->usbcx_hprt.u32 = __cvmx_usb_read_csr32(usb,
1062                         CVMX_USBCX_HPRT(usb->index));
1063         usbcx_ghwcfg3.u32 = __cvmx_usb_read_csr32(usb,
1064                         CVMX_USBCX_GHWCFG3(usb->index));
1065 
1066         /*
1067          * 13. Program the USBC_GRXFSIZ register to select the size of the
1068          *     receive FIFO (25%).
1069          */
1070         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index),
1071                         union cvmx_usbcx_grxfsiz, rxfdep,
1072                         usbcx_ghwcfg3.s.dfifodepth / 4);
1073         /*
1074          * 14. Program the USBC_GNPTXFSIZ register to select the size and the
1075          *     start address of the non- periodic transmit FIFO for nonperiodic
1076          *     transactions (50%).
1077          */
1078         {
1079                 union cvmx_usbcx_gnptxfsiz siz;
1080 
1081                 siz.u32 = __cvmx_usb_read_csr32(usb,
1082                                 CVMX_USBCX_GNPTXFSIZ(usb->index));
1083                 siz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
1084                 siz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
1085                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index),
1086                                        siz.u32);
1087         }
1088         /*
1089          * 15. Program the USBC_HPTXFSIZ register to select the size and start
1090          *     address of the periodic transmit FIFO for periodic transactions
1091          *     (25%).
1092          */
1093         {
1094                 union cvmx_usbcx_hptxfsiz siz;
1095 
1096                 siz.u32 = __cvmx_usb_read_csr32(usb,
1097                                 CVMX_USBCX_HPTXFSIZ(usb->index));
1098                 siz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
1099                 siz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
1100                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index),
1101                                        siz.u32);
1102         }
1103         /* Flush all FIFOs */
1104         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1105                         union cvmx_usbcx_grstctl, txfnum, 0x10);
1106         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1107                         union cvmx_usbcx_grstctl, txfflsh, 1);
1108         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1109                               union cvmx_usbcx_grstctl,
1110                               txfflsh, ==, 0, 100);
1111         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1112                         union cvmx_usbcx_grstctl, rxfflsh, 1);
1113         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
1114                               union cvmx_usbcx_grstctl,
1115                               rxfflsh, ==, 0, 100);
1116 
1117         return 0;
1118 }
1119 
1120 
1121 /**
1122  * Disable a USB port. After this call the USB port will not
1123  * generate data transfers and will not generate events.
1124  * Transactions in process will fail and call their
1125  * associated callbacks.
1126  *
1127  * @usb: USB device state populated by cvmx_usb_initialize().
1128  *
1129  * Returns: 0 or a negative error code.
1130  */
1131 static int cvmx_usb_disable(struct cvmx_usb_state *usb)
1132 {
1133         /* Disable the port */
1134         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), union cvmx_usbcx_hprt,
1135                         prtena, 1);
1136         return 0;
1137 }
1138 
1139 
1140 /**
1141  * Get the current state of the USB port. Use this call to
1142  * determine if the usb port has anything connected, is enabled,
1143  * or has some sort of error condition. The return value of this
1144  * call has "changed" bits to signal of the value of some fields
1145  * have changed between calls.
1146  *
1147  * @usb: USB device state populated by cvmx_usb_initialize().
1148  *
1149  * Returns: Port status information
1150  */
1151 static struct cvmx_usb_port_status cvmx_usb_get_status(
1152                 struct cvmx_usb_state *usb)
1153 {
1154         union cvmx_usbcx_hprt usbc_hprt;
1155         struct cvmx_usb_port_status result;
1156 
1157         memset(&result, 0, sizeof(result));
1158 
1159         usbc_hprt.u32 = __cvmx_usb_read_csr32(usb,
1160                         CVMX_USBCX_HPRT(usb->index));
1161         result.port_enabled = usbc_hprt.s.prtena;
1162         result.port_over_current = usbc_hprt.s.prtovrcurract;
1163         result.port_powered = usbc_hprt.s.prtpwr;
1164         result.port_speed = usbc_hprt.s.prtspd;
1165         result.connected = usbc_hprt.s.prtconnsts;
1166         result.connect_change =
1167                 (result.connected != usb->port_status.connected);
1168 
1169         return result;
1170 }
1171 
1172 /**
1173  * Open a virtual pipe between the host and a USB device. A pipe
1174  * must be opened before data can be transferred between a device
1175  * and Octeon.
1176  *
1177  * @usb:             USB device state populated by cvmx_usb_initialize().
1178  * @device_addr:
1179  *                   USB device address to open the pipe to
1180  *                   (0-127).
1181  * @endpoint_num:
1182  *                   USB endpoint number to open the pipe to
1183  *                   (0-15).
1184  * @device_speed:
1185  *                   The speed of the device the pipe is going
1186  *                   to. This must match the device's speed,
1187  *                   which may be different than the port speed.
1188  * @max_packet:      The maximum packet length the device can
1189  *                   transmit/receive (low speed=0-8, full
1190  *                   speed=0-1023, high speed=0-1024). This value
1191  *                   comes from the standard endpoint descriptor
1192  *                   field wMaxPacketSize bits <10:0>.
1193  * @transfer_type:
1194  *                   The type of transfer this pipe is for.
1195  * @transfer_dir:
1196  *                   The direction the pipe is in. This is not
1197  *                   used for control pipes.
1198  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
1199  *                   this is how often the transfer is scheduled
1200  *                   for. All other transfers should specify
1201  *                   zero. The units are in frames (8000/sec at
1202  *                   high speed, 1000/sec for full speed).
1203  * @multi_count:
1204  *                   For high speed devices, this is the maximum
1205  *                   allowed number of packet per microframe.
1206  *                   Specify zero for non high speed devices. This
1207  *                   value comes from the standard endpoint descriptor
1208  *                   field wMaxPacketSize bits <12:11>.
1209  * @hub_device_addr:
1210  *                   Hub device address this device is connected
1211  *                   to. Devices connected directly to Octeon
1212  *                   use zero. This is only used when the device
1213  *                   is full/low speed behind a high speed hub.
1214  *                   The address will be of the high speed hub,
1215  *                   not and full speed hubs after it.
1216  * @hub_port:        Which port on the hub the device is
1217  *                   connected. Use zero for devices connected
1218  *                   directly to Octeon. Like hub_device_addr,
1219  *                   this is only used for full/low speed
1220  *                   devices behind a high speed hub.
1221  *
1222  * Returns: A non-NULL value is a pipe. NULL means an error.
1223  */
1224 static struct cvmx_usb_pipe *cvmx_usb_open_pipe(struct cvmx_usb_state *usb,
1225                                                 int device_addr,
1226                                                 int endpoint_num,
1227                                                 enum cvmx_usb_speed
1228                                                         device_speed,
1229                                                 int max_packet,
1230                                                 enum cvmx_usb_transfer
1231                                                         transfer_type,
1232                                                 enum cvmx_usb_direction
1233                                                         transfer_dir,
1234                                                 int interval, int multi_count,
1235                                                 int hub_device_addr,
1236                                                 int hub_port)
1237 {
1238         struct cvmx_usb_pipe *pipe;
1239 
1240         if (unlikely((device_addr < 0) || (device_addr > MAX_USB_ADDRESS)))
1241                 return NULL;
1242         if (unlikely((endpoint_num < 0) || (endpoint_num > MAX_USB_ENDPOINT)))
1243                 return NULL;
1244         if (unlikely(device_speed > CVMX_USB_SPEED_LOW))
1245                 return NULL;
1246         if (unlikely((max_packet <= 0) || (max_packet > 1024)))
1247                 return NULL;
1248         if (unlikely(transfer_type > CVMX_USB_TRANSFER_INTERRUPT))
1249                 return NULL;
1250         if (unlikely((transfer_dir != CVMX_USB_DIRECTION_OUT) &&
1251                 (transfer_dir != CVMX_USB_DIRECTION_IN)))
1252                 return NULL;
1253         if (unlikely(interval < 0))
1254                 return NULL;
1255         if (unlikely((transfer_type == CVMX_USB_TRANSFER_CONTROL) && interval))
1256                 return NULL;
1257         if (unlikely(multi_count < 0))
1258                 return NULL;
1259         if (unlikely((device_speed != CVMX_USB_SPEED_HIGH) &&
1260                 (multi_count != 0)))
1261                 return NULL;
1262         if (unlikely((hub_device_addr < 0) ||
1263                 (hub_device_addr > MAX_USB_ADDRESS)))
1264                 return NULL;
1265         if (unlikely((hub_port < 0) || (hub_port > MAX_USB_HUB_PORT)))
1266                 return NULL;
1267 
1268         pipe = kzalloc(sizeof(*pipe), GFP_ATOMIC);
1269         if (!pipe)
1270                 return NULL;
1271         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1272                 (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1273                 (transfer_type == CVMX_USB_TRANSFER_BULK))
1274                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
1275         pipe->device_addr = device_addr;
1276         pipe->endpoint_num = endpoint_num;
1277         pipe->device_speed = device_speed;
1278         pipe->max_packet = max_packet;
1279         pipe->transfer_type = transfer_type;
1280         pipe->transfer_dir = transfer_dir;
1281         INIT_LIST_HEAD(&pipe->transactions);
1282 
1283         /*
1284          * All pipes use interval to rate limit NAK processing. Force an
1285          * interval if one wasn't supplied
1286          */
1287         if (!interval)
1288                 interval = 1;
1289         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1290                 pipe->interval = interval*8;
1291                 /* Force start splits to be schedule on uFrame 0 */
1292                 pipe->next_tx_frame = ((usb->frame_number+7)&~7) +
1293                                         pipe->interval;
1294         } else {
1295                 pipe->interval = interval;
1296                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1297         }
1298         pipe->multi_count = multi_count;
1299         pipe->hub_device_addr = hub_device_addr;
1300         pipe->hub_port = hub_port;
1301         pipe->pid_toggle = 0;
1302         pipe->split_sc_frame = -1;
1303         list_add_tail(&pipe->node, &usb->idle_pipes);
1304 
1305         /*
1306          * We don't need to tell the hardware about this pipe yet since
1307          * it doesn't have any submitted requests
1308          */
1309 
1310         return pipe;
1311 }
1312 
1313 
1314 /**
1315  * Poll the RX FIFOs and remove data as needed. This function is only used
1316  * in non DMA mode. It is very important that this function be called quickly
1317  * enough to prevent FIFO overflow.
1318  *
1319  * @usb:        USB device state populated by cvmx_usb_initialize().
1320  */
1321 static void __cvmx_usb_poll_rx_fifo(struct cvmx_usb_state *usb)
1322 {
1323         union cvmx_usbcx_grxstsph rx_status;
1324         int channel;
1325         int bytes;
1326         uint64_t address;
1327         uint32_t *ptr;
1328 
1329         rx_status.u32 = __cvmx_usb_read_csr32(usb,
1330                         CVMX_USBCX_GRXSTSPH(usb->index));
1331         /* Only read data if IN data is there */
1332         if (rx_status.s.pktsts != 2)
1333                 return;
1334         /* Check if no data is available */
1335         if (!rx_status.s.bcnt)
1336                 return;
1337 
1338         channel = rx_status.s.chnum;
1339         bytes = rx_status.s.bcnt;
1340         if (!bytes)
1341                 return;
1342 
1343         /* Get where the DMA engine would have written this data */
1344         address = __cvmx_usb_read_csr64(usb,
1345                         CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8);
1346 
1347         ptr = cvmx_phys_to_ptr(address);
1348         __cvmx_usb_write_csr64(usb,
1349                                CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8,
1350                                address + bytes);
1351 
1352         /* Loop writing the FIFO data for this packet into memory */
1353         while (bytes > 0) {
1354                 *ptr++ = __cvmx_usb_read_csr32(usb, USB_FIFO_ADDRESS(channel, usb->index));
1355                 bytes -= 4;
1356         }
1357         CVMX_SYNCW;
1358 
1359         return;
1360 }
1361 
1362 
1363 /**
1364  * Fill the TX hardware fifo with data out of the software
1365  * fifos
1366  *
1367  * @usb:            USB device state populated by cvmx_usb_initialize().
1368  * @fifo:           Software fifo to use
1369  * @available:      Amount of space in the hardware fifo
1370  *
1371  * Returns: Non zero if the hardware fifo was too small and needs
1372  *          to be serviced again.
1373  */
1374 static int __cvmx_usb_fill_tx_hw(struct cvmx_usb_state *usb,
1375                                  struct cvmx_usb_tx_fifo *fifo, int available)
1376 {
1377         /*
1378          * We're done either when there isn't anymore space or the software FIFO
1379          * is empty
1380          */
1381         while (available && (fifo->head != fifo->tail)) {
1382                 int i = fifo->tail;
1383                 const uint32_t *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1384                 uint64_t csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel,
1385                                                         usb->index) ^ 4;
1386                 int words = available;
1387 
1388                 /* Limit the amount of data to waht the SW fifo has */
1389                 if (fifo->entry[i].size <= available) {
1390                         words = fifo->entry[i].size;
1391                         fifo->tail++;
1392                         if (fifo->tail > MAX_CHANNELS)
1393                                 fifo->tail = 0;
1394                 }
1395 
1396                 /* Update the next locations and counts */
1397                 available -= words;
1398                 fifo->entry[i].address += words * 4;
1399                 fifo->entry[i].size -= words;
1400 
1401                 /*
1402                  * Write the HW fifo data. The read every three writes is due
1403                  * to an errata on CN3XXX chips
1404                  */
1405                 while (words > 3) {
1406                         cvmx_write64_uint32(csr_address, *ptr++);
1407                         cvmx_write64_uint32(csr_address, *ptr++);
1408                         cvmx_write64_uint32(csr_address, *ptr++);
1409                         cvmx_read64_uint64(
1410                                         CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1411                         words -= 3;
1412                 }
1413                 cvmx_write64_uint32(csr_address, *ptr++);
1414                 if (--words) {
1415                         cvmx_write64_uint32(csr_address, *ptr++);
1416                         if (--words)
1417                                 cvmx_write64_uint32(csr_address, *ptr++);
1418                 }
1419                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1420         }
1421         return fifo->head != fifo->tail;
1422 }
1423 
1424 
1425 /**
1426  * Check the hardware FIFOs and fill them as needed
1427  *
1428  * @usb:        USB device state populated by cvmx_usb_initialize().
1429  */
1430 static void __cvmx_usb_poll_tx_fifo(struct cvmx_usb_state *usb)
1431 {
1432         if (usb->periodic.head != usb->periodic.tail) {
1433                 union cvmx_usbcx_hptxsts tx_status;
1434 
1435                 tx_status.u32 = __cvmx_usb_read_csr32(usb,
1436                                 CVMX_USBCX_HPTXSTS(usb->index));
1437                 if (__cvmx_usb_fill_tx_hw(usb, &usb->periodic,
1438                                           tx_status.s.ptxfspcavail))
1439                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1440                                         union cvmx_usbcx_gintmsk,
1441                                         ptxfempmsk, 1);
1442                 else
1443                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1444                                         union cvmx_usbcx_gintmsk,
1445                                         ptxfempmsk, 0);
1446         }
1447 
1448         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1449                 union cvmx_usbcx_gnptxsts tx_status;
1450 
1451                 tx_status.u32 = __cvmx_usb_read_csr32(usb,
1452                                 CVMX_USBCX_GNPTXSTS(usb->index));
1453                 if (__cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic,
1454                                           tx_status.s.nptxfspcavail))
1455                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1456                                         union cvmx_usbcx_gintmsk,
1457                                         nptxfempmsk, 1);
1458                 else
1459                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1460                                         union cvmx_usbcx_gintmsk,
1461                                         nptxfempmsk, 0);
1462         }
1463 
1464         return;
1465 }
1466 
1467 
1468 /**
1469  * Fill the TX FIFO with an outgoing packet
1470  *
1471  * @usb:          USB device state populated by cvmx_usb_initialize().
1472  * @channel:      Channel number to get packet from
1473  */
1474 static void __cvmx_usb_fill_tx_fifo(struct cvmx_usb_state *usb, int channel)
1475 {
1476         union cvmx_usbcx_hccharx hcchar;
1477         union cvmx_usbcx_hcspltx usbc_hcsplt;
1478         union cvmx_usbcx_hctsizx usbc_hctsiz;
1479         struct cvmx_usb_tx_fifo *fifo;
1480 
1481         /* We only need to fill data on outbound channels */
1482         hcchar.u32 = __cvmx_usb_read_csr32(usb,
1483                         CVMX_USBCX_HCCHARX(channel, usb->index));
1484         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1485                 return;
1486 
1487         /* OUT Splits only have data on the start and not the complete */
1488         usbc_hcsplt.u32 = __cvmx_usb_read_csr32(usb,
1489                         CVMX_USBCX_HCSPLTX(channel, usb->index));
1490         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1491                 return;
1492 
1493         /*
1494          * Find out how many bytes we need to fill and convert it into 32bit
1495          * words.
1496          */
1497         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
1498                         CVMX_USBCX_HCTSIZX(channel, usb->index));
1499         if (!usbc_hctsiz.s.xfersize)
1500                 return;
1501 
1502         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1503                 (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1504                 fifo = &usb->periodic;
1505         else
1506                 fifo = &usb->nonperiodic;
1507 
1508         fifo->entry[fifo->head].channel = channel;
1509         fifo->entry[fifo->head].address = __cvmx_usb_read_csr64(usb, CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8);
1510         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize+3)>>2;
1511         fifo->head++;
1512         if (fifo->head > MAX_CHANNELS)
1513                 fifo->head = 0;
1514 
1515         __cvmx_usb_poll_tx_fifo(usb);
1516 
1517         return;
1518 }
1519 
1520 /**
1521  * Perform channel specific setup for Control transactions. All
1522  * the generic stuff will already have been done in
1523  * __cvmx_usb_start_channel()
1524  *
1525  * @usb:          USB device state populated by cvmx_usb_initialize().
1526  * @channel:      Channel to setup
1527  * @pipe:         Pipe for control transaction
1528  */
1529 static void __cvmx_usb_start_channel_control(struct cvmx_usb_state *usb,
1530                                              int channel,
1531                                              struct cvmx_usb_pipe *pipe)
1532 {
1533         struct cvmx_usb_transaction *transaction =
1534                 list_first_entry(&pipe->transactions, typeof(*transaction),
1535                                  node);
1536         union cvmx_usb_control_header *header =
1537                 cvmx_phys_to_ptr(transaction->control_header);
1538         int bytes_to_transfer = transaction->buffer_length -
1539                 transaction->actual_bytes;
1540         int packets_to_transfer;
1541         union cvmx_usbcx_hctsizx usbc_hctsiz;
1542 
1543         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
1544                         CVMX_USBCX_HCTSIZX(channel, usb->index));
1545 
1546         switch (transaction->stage) {
1547         case CVMX_USB_STAGE_NON_CONTROL:
1548         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1549                 cvmx_dprintf("%s: ERROR - Non control stage\n", __func__);
1550                 break;
1551         case CVMX_USB_STAGE_SETUP:
1552                 usbc_hctsiz.s.pid = 3; /* Setup */
1553                 bytes_to_transfer = sizeof(*header);
1554                 /* All Control operations start with a setup going OUT */
1555                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1556                                 union cvmx_usbcx_hccharx, epdir,
1557                                 CVMX_USB_DIRECTION_OUT);
1558                 /*
1559                  * Setup send the control header instead of the buffer data. The
1560                  * buffer data will be used in the next stage
1561                  */
1562                 __cvmx_usb_write_csr64(usb,
1563                         CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8,
1564                         transaction->control_header);
1565                 break;
1566         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1567                 usbc_hctsiz.s.pid = 3; /* Setup */
1568                 bytes_to_transfer = 0;
1569                 /* All Control operations start with a setup going OUT */
1570                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1571                                 union cvmx_usbcx_hccharx, epdir,
1572                                 CVMX_USB_DIRECTION_OUT);
1573 
1574                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1575                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1576                 break;
1577         case CVMX_USB_STAGE_DATA:
1578                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1579                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1580                         if (header->s.request_type & 0x80)
1581                                 bytes_to_transfer = 0;
1582                         else if (bytes_to_transfer > pipe->max_packet)
1583                                 bytes_to_transfer = pipe->max_packet;
1584                 }
1585                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1586                                 union cvmx_usbcx_hccharx, epdir,
1587                                 ((header->s.request_type & 0x80) ?
1588                                         CVMX_USB_DIRECTION_IN :
1589                                         CVMX_USB_DIRECTION_OUT));
1590                 break;
1591         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1592                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1593                 if (!(header->s.request_type & 0x80))
1594                         bytes_to_transfer = 0;
1595                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1596                                 union cvmx_usbcx_hccharx, epdir,
1597                                 ((header->s.request_type & 0x80) ?
1598                                         CVMX_USB_DIRECTION_IN :
1599                                         CVMX_USB_DIRECTION_OUT));
1600                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1601                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1602                 break;
1603         case CVMX_USB_STAGE_STATUS:
1604                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1605                 bytes_to_transfer = 0;
1606                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1607                                 union cvmx_usbcx_hccharx, epdir,
1608                                 ((header->s.request_type & 0x80) ?
1609                                         CVMX_USB_DIRECTION_OUT :
1610                                         CVMX_USB_DIRECTION_IN));
1611                 break;
1612         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1613                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1614                 bytes_to_transfer = 0;
1615                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1616                                 union cvmx_usbcx_hccharx, epdir,
1617                                 ((header->s.request_type & 0x80) ?
1618                                         CVMX_USB_DIRECTION_OUT :
1619                                         CVMX_USB_DIRECTION_IN));
1620                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1621                                 union cvmx_usbcx_hcspltx, compsplt, 1);
1622                 break;
1623         }
1624 
1625         /*
1626          * Make sure the transfer never exceeds the byte limit of the hardware.
1627          * Further bytes will be sent as continued transactions
1628          */
1629         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1630                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1631                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1632                 bytes_to_transfer *= pipe->max_packet;
1633         }
1634 
1635         /*
1636          * Calculate the number of packets to transfer. If the length is zero
1637          * we still need to transfer one packet
1638          */
1639         packets_to_transfer = (bytes_to_transfer + pipe->max_packet - 1) /
1640                 pipe->max_packet;
1641         if (packets_to_transfer == 0)
1642                 packets_to_transfer = 1;
1643         else if ((packets_to_transfer > 1) &&
1644                         (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1645                 /*
1646                  * Limit to one packet when not using DMA. Channels must be
1647                  * restarted between every packet for IN transactions, so there
1648                  * is no reason to do multiple packets in a row
1649                  */
1650                 packets_to_transfer = 1;
1651                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1652         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1653                 /*
1654                  * Limit the number of packet and data transferred to what the
1655                  * hardware can handle
1656                  */
1657                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1658                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1659         }
1660 
1661         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1662         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1663 
1664         __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index),
1665                                usbc_hctsiz.u32);
1666         return;
1667 }
1668 
1669 
1670 /**
1671  * Start a channel to perform the pipe's head transaction
1672  *
1673  * @usb:          USB device state populated by cvmx_usb_initialize().
1674  * @channel:      Channel to setup
1675  * @pipe:         Pipe to start
1676  */
1677 static void __cvmx_usb_start_channel(struct cvmx_usb_state *usb,
1678                                      int channel,
1679                                      struct cvmx_usb_pipe *pipe)
1680 {
1681         struct cvmx_usb_transaction *transaction =
1682                 list_first_entry(&pipe->transactions, typeof(*transaction),
1683                                  node);
1684 
1685         /* Make sure all writes to the DMA region get flushed */
1686         CVMX_SYNCW;
1687 
1688         /* Attach the channel to the pipe */
1689         usb->pipe_for_channel[channel] = pipe;
1690         pipe->channel = channel;
1691         pipe->flags |= __CVMX_USB_PIPE_FLAGS_SCHEDULED;
1692 
1693         /* Mark this channel as in use */
1694         usb->idle_hardware_channels &= ~(1<<channel);
1695 
1696         /* Enable the channel interrupt bits */
1697         {
1698                 union cvmx_usbcx_hcintx usbc_hcint;
1699                 union cvmx_usbcx_hcintmskx usbc_hcintmsk;
1700                 union cvmx_usbcx_haintmsk usbc_haintmsk;
1701 
1702                 /* Clear all channel status bits */
1703                 usbc_hcint.u32 = __cvmx_usb_read_csr32(usb,
1704                                 CVMX_USBCX_HCINTX(channel, usb->index));
1705 
1706                 __cvmx_usb_write_csr32(usb,
1707                                        CVMX_USBCX_HCINTX(channel, usb->index),
1708                                        usbc_hcint.u32);
1709 
1710                 usbc_hcintmsk.u32 = 0;
1711                 usbc_hcintmsk.s.chhltdmsk = 1;
1712                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1713                         /*
1714                          * Channels need these extra interrupts when we aren't
1715                          * in DMA mode.
1716                          */
1717                         usbc_hcintmsk.s.datatglerrmsk = 1;
1718                         usbc_hcintmsk.s.frmovrunmsk = 1;
1719                         usbc_hcintmsk.s.bblerrmsk = 1;
1720                         usbc_hcintmsk.s.xacterrmsk = 1;
1721                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1722                                 /*
1723                                  * Splits don't generate xfercompl, so we need
1724                                  * ACK and NYET.
1725                                  */
1726                                 usbc_hcintmsk.s.nyetmsk = 1;
1727                                 usbc_hcintmsk.s.ackmsk = 1;
1728                         }
1729                         usbc_hcintmsk.s.nakmsk = 1;
1730                         usbc_hcintmsk.s.stallmsk = 1;
1731                         usbc_hcintmsk.s.xfercomplmsk = 1;
1732                 }
1733                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), usbc_hcintmsk.u32);
1734 
1735                 /* Enable the channel interrupt to propagate */
1736                 usbc_haintmsk.u32 = __cvmx_usb_read_csr32(usb,
1737                                         CVMX_USBCX_HAINTMSK(usb->index));
1738                 usbc_haintmsk.s.haintmsk |= 1<<channel;
1739                 __cvmx_usb_write_csr32(usb,
1740                                         CVMX_USBCX_HAINTMSK(usb->index),
1741                                         usbc_haintmsk.u32);
1742         }
1743 
1744         /* Setup the locations the DMA engines use  */
1745         {
1746                 uint64_t dma_address = transaction->buffer +
1747                                         transaction->actual_bytes;
1748 
1749                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1750                         dma_address = transaction->buffer +
1751                                         transaction->iso_packets[0].offset +
1752                                         transaction->actual_bytes;
1753 
1754                 __cvmx_usb_write_csr64(usb,
1755                         CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) + channel*8,
1756                         dma_address);
1757 
1758                 __cvmx_usb_write_csr64(usb,
1759                         CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel*8,
1760                         dma_address);
1761         }
1762 
1763         /* Setup both the size of the transfer and the SPLIT characteristics */
1764         {
1765                 union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
1766                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
1767                 int packets_to_transfer;
1768                 int bytes_to_transfer = transaction->buffer_length -
1769                         transaction->actual_bytes;
1770 
1771                 /*
1772                  * ISOCHRONOUS transactions store each individual transfer size
1773                  * in the packet structure, not the global buffer_length
1774                  */
1775                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1776                         bytes_to_transfer =
1777                                 transaction->iso_packets[0].length -
1778                                 transaction->actual_bytes;
1779 
1780                 /*
1781                  * We need to do split transactions when we are talking to non
1782                  * high speed devices that are behind a high speed hub
1783                  */
1784                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
1785                         /*
1786                          * On the start split phase (stage is even) record the
1787                          * frame number we will need to send the split complete.
1788                          * We only store the lower two bits since the time ahead
1789                          * can only be two frames
1790                          */
1791                         if ((transaction->stage&1) == 0) {
1792                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1793                                         pipe->split_sc_frame =
1794                                                 (usb->frame_number + 1) & 0x7f;
1795                                 else
1796                                         pipe->split_sc_frame =
1797                                                 (usb->frame_number + 2) & 0x7f;
1798                         } else
1799                                 pipe->split_sc_frame = -1;
1800 
1801                         usbc_hcsplt.s.spltena = 1;
1802                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1803                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1804                         usbc_hcsplt.s.compsplt = (transaction->stage ==
1805                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1806 
1807                         /*
1808                          * SPLIT transactions can only ever transmit one data
1809                          * packet so limit the transfer size to the max packet
1810                          * size
1811                          */
1812                         if (bytes_to_transfer > pipe->max_packet)
1813                                 bytes_to_transfer = pipe->max_packet;
1814 
1815                         /*
1816                          * ISOCHRONOUS OUT splits are unique in that they limit
1817                          * data transfers to 188 byte chunks representing the
1818                          * begin/middle/end of the data or all
1819                          */
1820                         if (!usbc_hcsplt.s.compsplt &&
1821                                 (pipe->transfer_dir ==
1822                                  CVMX_USB_DIRECTION_OUT) &&
1823                                 (pipe->transfer_type ==
1824                                  CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1825                                 /*
1826                                  * Clear the split complete frame number as
1827                                  * there isn't going to be a split complete
1828                                  */
1829                                 pipe->split_sc_frame = -1;
1830                                 /*
1831                                  * See if we've started this transfer and sent
1832                                  * data
1833                                  */
1834                                 if (transaction->actual_bytes == 0) {
1835                                         /*
1836                                          * Nothing sent yet, this is either a
1837                                          * begin or the entire payload
1838                                          */
1839                                         if (bytes_to_transfer <= 188)
1840                                                 /* Entire payload in one go */
1841                                                 usbc_hcsplt.s.xactpos = 3;
1842                                         else
1843                                                 /* First part of payload */
1844                                                 usbc_hcsplt.s.xactpos = 2;
1845                                 } else {
1846                                         /*
1847                                          * Continuing the previous data, we must
1848                                          * either be in the middle or at the end
1849                                          */
1850                                         if (bytes_to_transfer <= 188)
1851                                                 /* End of payload */
1852                                                 usbc_hcsplt.s.xactpos = 1;
1853                                         else
1854                                                 /* Middle of payload */
1855                                                 usbc_hcsplt.s.xactpos = 0;
1856                                 }
1857                                 /*
1858                                  * Again, the transfer size is limited to 188
1859                                  * bytes
1860                                  */
1861                                 if (bytes_to_transfer > 188)
1862                                         bytes_to_transfer = 188;
1863                         }
1864                 }
1865 
1866                 /*
1867                  * Make sure the transfer never exceeds the byte limit of the
1868                  * hardware. Further bytes will be sent as continued
1869                  * transactions
1870                  */
1871                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1872                         /*
1873                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1874                          * size
1875                          */
1876                         bytes_to_transfer = MAX_TRANSFER_BYTES /
1877                                 pipe->max_packet;
1878                         bytes_to_transfer *= pipe->max_packet;
1879                 }
1880 
1881                 /*
1882                  * Calculate the number of packets to transfer. If the length is
1883                  * zero we still need to transfer one packet
1884                  */
1885                 packets_to_transfer =
1886                         (bytes_to_transfer + pipe->max_packet - 1) /
1887                         pipe->max_packet;
1888                 if (packets_to_transfer == 0)
1889                         packets_to_transfer = 1;
1890                 else if ((packets_to_transfer > 1) &&
1891                                 (usb->init_flags &
1892                                  CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1893                         /*
1894                          * Limit to one packet when not using DMA. Channels must
1895                          * be restarted between every packet for IN
1896                          * transactions, so there is no reason to do multiple
1897                          * packets in a row
1898                          */
1899                         packets_to_transfer = 1;
1900                         bytes_to_transfer = packets_to_transfer *
1901                                 pipe->max_packet;
1902                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1903                         /*
1904                          * Limit the number of packet and data transferred to
1905                          * what the hardware can handle
1906                          */
1907                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1908                         bytes_to_transfer = packets_to_transfer *
1909                                 pipe->max_packet;
1910                 }
1911 
1912                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1913                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1914 
1915                 /* Update the DATA0/DATA1 toggle */
1916                 usbc_hctsiz.s.pid = __cvmx_usb_get_data_pid(pipe);
1917                 /*
1918                  * High speed pipes may need a hardware ping before they start
1919                  */
1920                 if (pipe->flags & __CVMX_USB_PIPE_FLAGS_NEED_PING)
1921                         usbc_hctsiz.s.dopng = 1;
1922 
1923                 __cvmx_usb_write_csr32(usb,
1924                                        CVMX_USBCX_HCSPLTX(channel, usb->index),
1925                                        usbc_hcsplt.u32);
1926                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel,
1927                                         usb->index), usbc_hctsiz.u32);
1928         }
1929 
1930         /* Setup the Host Channel Characteristics Register */
1931         {
1932                 union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
1933 
1934                 /*
1935                  * Set the startframe odd/even properly. This is only used for
1936                  * periodic
1937                  */
1938                 usbc_hcchar.s.oddfrm = usb->frame_number&1;
1939 
1940                 /*
1941                  * Set the number of back to back packets allowed by this
1942                  * endpoint. Split transactions interpret "ec" as the number of
1943                  * immediate retries of failure. These retries happen too
1944                  * quickly, so we disable these entirely for splits
1945                  */
1946                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
1947                         usbc_hcchar.s.ec = 1;
1948                 else if (pipe->multi_count < 1)
1949                         usbc_hcchar.s.ec = 1;
1950                 else if (pipe->multi_count > 3)
1951                         usbc_hcchar.s.ec = 3;
1952                 else
1953                         usbc_hcchar.s.ec = pipe->multi_count;
1954 
1955                 /* Set the rest of the endpoint specific settings */
1956                 usbc_hcchar.s.devaddr = pipe->device_addr;
1957                 usbc_hcchar.s.eptype = transaction->type;
1958                 usbc_hcchar.s.lspddev =
1959                         (pipe->device_speed == CVMX_USB_SPEED_LOW);
1960                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1961                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1962                 usbc_hcchar.s.mps = pipe->max_packet;
1963                 __cvmx_usb_write_csr32(usb,
1964                                        CVMX_USBCX_HCCHARX(channel, usb->index),
1965                                        usbc_hcchar.u32);
1966         }
1967 
1968         /* Do transaction type specific fixups as needed */
1969         switch (transaction->type) {
1970         case CVMX_USB_TRANSFER_CONTROL:
1971                 __cvmx_usb_start_channel_control(usb, channel, pipe);
1972                 break;
1973         case CVMX_USB_TRANSFER_BULK:
1974         case CVMX_USB_TRANSFER_INTERRUPT:
1975                 break;
1976         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1977                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
1978                         /*
1979                          * ISO transactions require different PIDs depending on
1980                          * direction and how many packets are needed
1981                          */
1982                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1983                                 if (pipe->multi_count < 2) /* Need DATA0 */
1984                                         USB_SET_FIELD32(
1985                                                 CVMX_USBCX_HCTSIZX(channel,
1986                                                                    usb->index),
1987                                                 union cvmx_usbcx_hctsizx,
1988                                                 pid, 0);
1989                                 else /* Need MDATA */
1990                                         USB_SET_FIELD32(
1991                                                 CVMX_USBCX_HCTSIZX(channel,
1992                                                                    usb->index),
1993                                                 union cvmx_usbcx_hctsizx,
1994                                                 pid, 3);
1995                         }
1996                 }
1997                 break;
1998         }
1999         {
2000                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 =
2001                         __cvmx_usb_read_csr32(usb,
2002                                 CVMX_USBCX_HCTSIZX(channel, usb->index))};
2003                 transaction->xfersize = usbc_hctsiz.s.xfersize;
2004                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
2005         }
2006         /* Remeber when we start a split transaction */
2007         if (__cvmx_usb_pipe_needs_split(usb, pipe))
2008                 usb->active_split = transaction;
2009         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
2010                         union cvmx_usbcx_hccharx, chena, 1);
2011         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
2012                 __cvmx_usb_fill_tx_fifo(usb, channel);
2013         return;
2014 }
2015 
2016 
2017 /**
2018  * Find a pipe that is ready to be scheduled to hardware.
2019  * @usb:         USB device state populated by cvmx_usb_initialize().
2020  * @list:        Pipe list to search
2021  * @current_frame:
2022  *               Frame counter to use as a time reference.
2023  *
2024  * Returns: Pipe or NULL if none are ready
2025  */
2026 static struct cvmx_usb_pipe *__cvmx_usb_find_ready_pipe(
2027                 struct cvmx_usb_state *usb,
2028                 struct list_head *list,
2029                 uint64_t current_frame)
2030 {
2031         struct cvmx_usb_pipe *pipe;
2032 
2033         list_for_each_entry(pipe, list, node) {
2034                 struct cvmx_usb_transaction *t =
2035                         list_first_entry(&pipe->transactions, typeof(*t),
2036                                          node);
2037                 if (!(pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED) && t &&
2038                         (pipe->next_tx_frame <= current_frame) &&
2039                         ((pipe->split_sc_frame == -1) ||
2040                          ((((int)current_frame - (int)pipe->split_sc_frame)
2041                            & 0x7f) < 0x40)) &&
2042                         (!usb->active_split || (usb->active_split == t))) {
2043                         prefetch(t);
2044                         return pipe;
2045                 }
2046         }
2047         return NULL;
2048 }
2049 
2050 
2051 /**
2052  * Called whenever a pipe might need to be scheduled to the
2053  * hardware.
2054  *
2055  * @usb:         USB device state populated by cvmx_usb_initialize().
2056  * @is_sof:      True if this schedule was called on a SOF interrupt.
2057  */
2058 static void __cvmx_usb_schedule(struct cvmx_usb_state *usb, int is_sof)
2059 {
2060         int channel;
2061         struct cvmx_usb_pipe *pipe;
2062         int need_sof;
2063         enum cvmx_usb_transfer ttype;
2064 
2065         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2066                 /*
2067                  * Without DMA we need to be careful to not schedule something
2068                  * at the end of a frame and cause an overrun.
2069                  */
2070                 union cvmx_usbcx_hfnum hfnum = {
2071                         .u32 = __cvmx_usb_read_csr32(usb,
2072                                                 CVMX_USBCX_HFNUM(usb->index))
2073                 };
2074 
2075                 union cvmx_usbcx_hfir hfir = {
2076                         .u32 = __cvmx_usb_read_csr32(usb,
2077                                                 CVMX_USBCX_HFIR(usb->index))
2078                 };
2079 
2080                 if (hfnum.s.frrem < hfir.s.frint/4)
2081                         goto done;
2082         }
2083 
2084         while (usb->idle_hardware_channels) {
2085                 /* Find an idle channel */
2086                 channel = __fls(usb->idle_hardware_channels);
2087                 if (unlikely(channel > 7))
2088                         break;
2089 
2090                 /* Find a pipe needing service */
2091                 pipe = NULL;
2092                 if (is_sof) {
2093                         /*
2094                          * Only process periodic pipes on SOF interrupts. This
2095                          * way we are sure that the periodic data is sent in the
2096                          * beginning of the frame
2097                          */
2098                         pipe = __cvmx_usb_find_ready_pipe(usb,
2099                                         usb->active_pipes +
2100                                         CVMX_USB_TRANSFER_ISOCHRONOUS,
2101                                         usb->frame_number);
2102                         if (likely(!pipe))
2103                                 pipe = __cvmx_usb_find_ready_pipe(usb,
2104                                                 usb->active_pipes +
2105                                                 CVMX_USB_TRANSFER_INTERRUPT,
2106                                                 usb->frame_number);
2107                 }
2108                 if (likely(!pipe)) {
2109                         pipe = __cvmx_usb_find_ready_pipe(usb,
2110                                         usb->active_pipes +
2111                                         CVMX_USB_TRANSFER_CONTROL,
2112                                         usb->frame_number);
2113                         if (likely(!pipe))
2114                                 pipe = __cvmx_usb_find_ready_pipe(usb,
2115                                                 usb->active_pipes +
2116                                                 CVMX_USB_TRANSFER_BULK,
2117                                                 usb->frame_number);
2118                 }
2119                 if (!pipe)
2120                         break;
2121 
2122                 __cvmx_usb_start_channel(usb, channel, pipe);
2123         }
2124 
2125 done:
2126         /*
2127          * Only enable SOF interrupts when we have transactions pending in the
2128          * future that might need to be scheduled
2129          */
2130         need_sof = 0;
2131         for (ttype = CVMX_USB_TRANSFER_CONTROL;
2132                         ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
2133                 list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
2134                         if (pipe->next_tx_frame > usb->frame_number) {
2135                                 need_sof = 1;
2136                                 break;
2137                         }
2138                 }
2139         }
2140         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
2141                         union cvmx_usbcx_gintmsk, sofmsk, need_sof);
2142         return;
2143 }
2144 
2145 static inline struct octeon_hcd *cvmx_usb_to_octeon(struct cvmx_usb_state *p)
2146 {
2147         return container_of(p, struct octeon_hcd, usb);
2148 }
2149 
2150 static inline struct usb_hcd *octeon_to_hcd(struct octeon_hcd *p)
2151 {
2152         return container_of((void *)p, struct usb_hcd, hcd_priv);
2153 }
2154 
2155 static void octeon_usb_urb_complete_callback(struct cvmx_usb_state *usb,
2156                                              enum cvmx_usb_complete status,
2157                                              struct cvmx_usb_pipe *pipe,
2158                                              struct cvmx_usb_transaction
2159                                                 *transaction,
2160                                              int bytes_transferred,
2161                                              struct urb *urb)
2162 {
2163         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
2164         struct usb_hcd *hcd = octeon_to_hcd(priv);
2165         struct device *dev = hcd->self.controller;
2166 
2167         urb->actual_length = bytes_transferred;
2168         urb->hcpriv = NULL;
2169 
2170         if (!list_empty(&urb->urb_list))
2171                 /*
2172                  * It is on the dequeue_list, but we are going to call
2173                  * usb_hcd_giveback_urb(), so we must clear it from
2174                  * the list.  We got to it before the
2175                  * octeon_usb_urb_dequeue_work() tasklet did.
2176                  */
2177                 list_del_init(&urb->urb_list);
2178 
2179         /* For Isochronous transactions we need to update the URB packet status
2180            list from data in our private copy */
2181         if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
2182                 int i;
2183                 /*
2184                  * The pointer to the private list is stored in the setup_packet
2185                  * field.
2186                  */
2187                 struct cvmx_usb_iso_packet *iso_packet =
2188                         (struct cvmx_usb_iso_packet *) urb->setup_packet;
2189                 /* Recalculate the transfer size by adding up each packet */
2190                 urb->actual_length = 0;
2191                 for (i = 0; i < urb->number_of_packets; i++) {
2192                         if (iso_packet[i].status ==
2193                                         CVMX_USB_COMPLETE_SUCCESS) {
2194                                 urb->iso_frame_desc[i].status = 0;
2195                                 urb->iso_frame_desc[i].actual_length =
2196                                         iso_packet[i].length;
2197                                 urb->actual_length +=
2198                                         urb->iso_frame_desc[i].actual_length;
2199                         } else {
2200                                 dev_dbg(dev, "ISOCHRONOUS packet=%d of %d status=%d pipe=%p transaction=%p size=%d\n",
2201                                         i, urb->number_of_packets,
2202                                         iso_packet[i].status, pipe,
2203                                         transaction, iso_packet[i].length);
2204                                 urb->iso_frame_desc[i].status = -EREMOTEIO;
2205                         }
2206                 }
2207                 /* Free the private list now that we don't need it anymore */
2208                 kfree(iso_packet);
2209                 urb->setup_packet = NULL;
2210         }
2211 
2212         switch (status) {
2213         case CVMX_USB_COMPLETE_SUCCESS:
2214                 urb->status = 0;
2215                 break;
2216         case CVMX_USB_COMPLETE_CANCEL:
2217                 if (urb->status == 0)
2218                         urb->status = -ENOENT;
2219                 break;
2220         case CVMX_USB_COMPLETE_STALL:
2221                 dev_dbg(dev, "status=stall pipe=%p transaction=%p size=%d\n",
2222                         pipe, transaction, bytes_transferred);
2223                 urb->status = -EPIPE;
2224                 break;
2225         case CVMX_USB_COMPLETE_BABBLEERR:
2226                 dev_dbg(dev, "status=babble pipe=%p transaction=%p size=%d\n",
2227                         pipe, transaction, bytes_transferred);
2228                 urb->status = -EPIPE;
2229                 break;
2230         case CVMX_USB_COMPLETE_SHORT:
2231                 dev_dbg(dev, "status=short pipe=%p transaction=%p size=%d\n",
2232                         pipe, transaction, bytes_transferred);
2233                 urb->status = -EREMOTEIO;
2234                 break;
2235         case CVMX_USB_COMPLETE_ERROR:
2236         case CVMX_USB_COMPLETE_XACTERR:
2237         case CVMX_USB_COMPLETE_DATATGLERR:
2238         case CVMX_USB_COMPLETE_FRAMEERR:
2239                 dev_dbg(dev, "status=%d pipe=%p transaction=%p size=%d\n",
2240                         status, pipe, transaction, bytes_transferred);
2241                 urb->status = -EPROTO;
2242                 break;
2243         }
2244         spin_unlock(&priv->lock);
2245         usb_hcd_giveback_urb(octeon_to_hcd(priv), urb, urb->status);
2246         spin_lock(&priv->lock);
2247 }
2248 
2249 /**
2250  * Signal the completion of a transaction and free it. The
2251  * transaction will be removed from the pipe transaction list.
2252  *
2253  * @usb:         USB device state populated by cvmx_usb_initialize().
2254  * @pipe:        Pipe the transaction is on
2255  * @transaction:
2256  *               Transaction that completed
2257  * @complete_code:
2258  *               Completion code
2259  */
2260 static void __cvmx_usb_perform_complete(
2261                                 struct cvmx_usb_state *usb,
2262                                 struct cvmx_usb_pipe *pipe,
2263                                 struct cvmx_usb_transaction *transaction,
2264                                 enum cvmx_usb_complete complete_code)
2265 {
2266         /* If this was a split then clear our split in progress marker */
2267         if (usb->active_split == transaction)
2268                 usb->active_split = NULL;
2269 
2270         /*
2271          * Isochronous transactions need extra processing as they might not be
2272          * done after a single data transfer
2273          */
2274         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
2275                 /* Update the number of bytes transferred in this ISO packet */
2276                 transaction->iso_packets[0].length = transaction->actual_bytes;
2277                 transaction->iso_packets[0].status = complete_code;
2278 
2279                 /*
2280                  * If there are more ISOs pending and we succeeded, schedule the
2281                  * next one
2282                  */
2283                 if ((transaction->iso_number_packets > 1) &&
2284                         (complete_code == CVMX_USB_COMPLETE_SUCCESS)) {
2285                         /* No bytes transferred for this packet as of yet */
2286                         transaction->actual_bytes = 0;
2287                         /* One less ISO waiting to transfer */
2288                         transaction->iso_number_packets--;
2289                         /* Increment to the next location in our packet array */
2290                         transaction->iso_packets++;
2291                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2292                         goto done;
2293                 }
2294         }
2295 
2296         /* Remove the transaction from the pipe list */
2297         list_del(&transaction->node);
2298         if (list_empty(&pipe->transactions))
2299                 list_move_tail(&pipe->node, &usb->idle_pipes);
2300         octeon_usb_urb_complete_callback(usb, complete_code, pipe,
2301                                          transaction,
2302                                          transaction->actual_bytes,
2303                                          transaction->urb);
2304         kfree(transaction);
2305 done:
2306         return;
2307 }
2308 
2309 
2310 /**
2311  * Submit a usb transaction to a pipe. Called for all types
2312  * of transactions.
2313  *
2314  * @usb:
2315  * @pipe:           Which pipe to submit to.
2316  * @type:           Transaction type
2317  * @buffer:         User buffer for the transaction
2318  * @buffer_length:
2319  *                  User buffer's length in bytes
2320  * @control_header:
2321  *                  For control transactions, the 8 byte standard header
2322  * @iso_start_frame:
2323  *                  For ISO transactions, the start frame
2324  * @iso_number_packets:
2325  *                  For ISO, the number of packet in the transaction.
2326  * @iso_packets:
2327  *                  A description of each ISO packet
2328  * @urb:            URB for the callback
2329  *
2330  * Returns: Transaction or NULL on failure.
2331  */
2332 static struct cvmx_usb_transaction *__cvmx_usb_submit_transaction(
2333                                 struct cvmx_usb_state *usb,
2334                                 struct cvmx_usb_pipe *pipe,
2335                                 enum cvmx_usb_transfer type,
2336                                 uint64_t buffer,
2337                                 int buffer_length,
2338                                 uint64_t control_header,
2339                                 int iso_start_frame,
2340                                 int iso_number_packets,
2341                                 struct cvmx_usb_iso_packet *iso_packets,
2342                                 struct urb *urb)
2343 {
2344         struct cvmx_usb_transaction *transaction;
2345 
2346         if (unlikely(pipe->transfer_type != type))
2347                 return NULL;
2348 
2349         transaction = kzalloc(sizeof(*transaction), GFP_ATOMIC);
2350         if (unlikely(!transaction))
2351                 return NULL;
2352 
2353         transaction->type = type;
2354         transaction->buffer = buffer;
2355         transaction->buffer_length = buffer_length;
2356         transaction->control_header = control_header;
2357         /* FIXME: This is not used, implement it. */
2358         transaction->iso_start_frame = iso_start_frame;
2359         transaction->iso_number_packets = iso_number_packets;
2360         transaction->iso_packets = iso_packets;
2361         transaction->urb = urb;
2362         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2363                 transaction->stage = CVMX_USB_STAGE_SETUP;
2364         else
2365                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2366 
2367         if (!list_empty(&pipe->transactions)) {
2368                 list_add_tail(&transaction->node, &pipe->transactions);
2369         } else {
2370                 list_add_tail(&transaction->node, &pipe->transactions);
2371                 list_move_tail(&pipe->node,
2372                                &usb->active_pipes[pipe->transfer_type]);
2373 
2374                 /*
2375                  * We may need to schedule the pipe if this was the head of the
2376                  * pipe.
2377                  */
2378                 __cvmx_usb_schedule(usb, 0);
2379         }
2380 
2381         return transaction;
2382 }
2383 
2384 
2385 /**
2386  * Call to submit a USB Bulk transfer to a pipe.
2387  *
2388  * @usb:            USB device state populated by cvmx_usb_initialize().
2389  * @pipe:           Handle to the pipe for the transfer.
2390  * @urb:            URB.
2391  *
2392  * Returns: A submitted transaction or NULL on failure.
2393  */
2394 static struct cvmx_usb_transaction *cvmx_usb_submit_bulk(
2395                                                 struct cvmx_usb_state *usb,
2396                                                 struct cvmx_usb_pipe *pipe,
2397                                                 struct urb *urb)
2398 {
2399         return __cvmx_usb_submit_transaction(usb, pipe, CVMX_USB_TRANSFER_BULK,
2400                                              urb->transfer_dma,
2401                                              urb->transfer_buffer_length,
2402                                              0, /* control_header */
2403                                              0, /* iso_start_frame */
2404                                              0, /* iso_number_packets */
2405                                              NULL, /* iso_packets */
2406                                              urb);
2407 }
2408 
2409 
2410 /**
2411  * Call to submit a USB Interrupt transfer to a pipe.
2412  *
2413  * @usb:            USB device state populated by cvmx_usb_initialize().
2414  * @pipe:           Handle to the pipe for the transfer.
2415  * @urb:            URB returned when the callback is called.
2416  *
2417  * Returns: A submitted transaction or NULL on failure.
2418  */
2419 static struct cvmx_usb_transaction *cvmx_usb_submit_interrupt(
2420                                                 struct cvmx_usb_state *usb,
2421                                                 struct cvmx_usb_pipe *pipe,
2422                                                 struct urb *urb)
2423 {
2424         return __cvmx_usb_submit_transaction(usb, pipe,
2425                                              CVMX_USB_TRANSFER_INTERRUPT,
2426                                              urb->transfer_dma,
2427                                              urb->transfer_buffer_length,
2428                                              0, /* control_header */
2429                                              0, /* iso_start_frame */
2430                                              0, /* iso_number_packets */
2431                                              NULL, /* iso_packets */
2432                                              urb);
2433 }
2434 
2435 
2436 /**
2437  * Call to submit a USB Control transfer to a pipe.
2438  *
2439  * @usb:            USB device state populated by cvmx_usb_initialize().
2440  * @pipe:           Handle to the pipe for the transfer.
2441  * @urb:            URB.
2442  *
2443  * Returns: A submitted transaction or NULL on failure.
2444  */
2445 static struct cvmx_usb_transaction *cvmx_usb_submit_control(
2446                                                 struct cvmx_usb_state *usb,
2447                                                 struct cvmx_usb_pipe *pipe,
2448                                                 struct urb *urb)
2449 {
2450         int buffer_length = urb->transfer_buffer_length;
2451         uint64_t control_header = urb->setup_dma;
2452         union cvmx_usb_control_header *header =
2453                 cvmx_phys_to_ptr(control_header);
2454 
2455         if ((header->s.request_type & 0x80) == 0)
2456                 buffer_length = le16_to_cpu(header->s.length);
2457 
2458         return __cvmx_usb_submit_transaction(usb, pipe,
2459                                              CVMX_USB_TRANSFER_CONTROL,
2460                                              urb->transfer_dma, buffer_length,
2461                                              control_header,
2462                                              0, /* iso_start_frame */
2463                                              0, /* iso_number_packets */
2464                                              NULL, /* iso_packets */
2465                                              urb);
2466 }
2467 
2468 
2469 /**
2470  * Call to submit a USB Isochronous transfer to a pipe.
2471  *
2472  * @usb:            USB device state populated by cvmx_usb_initialize().
2473  * @pipe:           Handle to the pipe for the transfer.
2474  * @urb:            URB returned when the callback is called.
2475  *
2476  * Returns: A submitted transaction or NULL on failure.
2477  */
2478 static struct cvmx_usb_transaction *cvmx_usb_submit_isochronous(
2479                                                 struct cvmx_usb_state *usb,
2480                                                 struct cvmx_usb_pipe *pipe,
2481                                                 struct urb *urb)
2482 {
2483         struct cvmx_usb_iso_packet *packets;
2484 
2485         packets = (struct cvmx_usb_iso_packet *) urb->setup_packet;
2486         return __cvmx_usb_submit_transaction(usb, pipe,
2487                                              CVMX_USB_TRANSFER_ISOCHRONOUS,
2488                                              urb->transfer_dma,
2489                                              urb->transfer_buffer_length,
2490                                              0, /* control_header */
2491                                              urb->start_frame,
2492                                              urb->number_of_packets,
2493                                              packets, urb);
2494 }
2495 
2496 
2497 /**
2498  * Cancel one outstanding request in a pipe. Canceling a request
2499  * can fail if the transaction has already completed before cancel
2500  * is called. Even after a successful cancel call, it may take
2501  * a frame or two for the cvmx_usb_poll() function to call the
2502  * associated callback.
2503  *
2504  * @usb:         USB device state populated by cvmx_usb_initialize().
2505  * @pipe:        Pipe to cancel requests in.
2506  * @transaction: Transaction to cancel, returned by the submit function.
2507  *
2508  * Returns: 0 or a negative error code.
2509  */
2510 static int cvmx_usb_cancel(struct cvmx_usb_state *usb,
2511                            struct cvmx_usb_pipe *pipe,
2512                            struct cvmx_usb_transaction *transaction)
2513 {
2514         /*
2515          * If the transaction is the HEAD of the queue and scheduled. We need to
2516          * treat it special
2517          */
2518         if (list_first_entry(&pipe->transactions, typeof(*transaction), node) ==
2519             transaction && (pipe->flags & __CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2520                 union cvmx_usbcx_hccharx usbc_hcchar;
2521 
2522                 usb->pipe_for_channel[pipe->channel] = NULL;
2523                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2524 
2525                 CVMX_SYNCW;
2526 
2527                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2528                                 CVMX_USBCX_HCCHARX(pipe->channel, usb->index));
2529                 /*
2530                  * If the channel isn't enabled then the transaction already
2531                  * completed.
2532                  */
2533                 if (usbc_hcchar.s.chena) {
2534                         usbc_hcchar.s.chdis = 1;
2535                         __cvmx_usb_write_csr32(usb,
2536                                         CVMX_USBCX_HCCHARX(pipe->channel,
2537                                                 usb->index),
2538                                         usbc_hcchar.u32);
2539                 }
2540         }
2541         __cvmx_usb_perform_complete(usb, pipe, transaction,
2542                                     CVMX_USB_COMPLETE_CANCEL);
2543         return 0;
2544 }
2545 
2546 
2547 /**
2548  * Cancel all outstanding requests in a pipe. Logically all this
2549  * does is call cvmx_usb_cancel() in a loop.
2550  *
2551  * @usb:         USB device state populated by cvmx_usb_initialize().
2552  * @pipe:        Pipe to cancel requests in.
2553  *
2554  * Returns: 0 or a negative error code.
2555  */
2556 static int cvmx_usb_cancel_all(struct cvmx_usb_state *usb,
2557                                struct cvmx_usb_pipe *pipe)
2558 {
2559         struct cvmx_usb_transaction *transaction, *next;
2560 
2561         /* Simply loop through and attempt to cancel each transaction */
2562         list_for_each_entry_safe(transaction, next, &pipe->transactions, node) {
2563                 int result = cvmx_usb_cancel(usb, pipe, transaction);
2564 
2565                 if (unlikely(result != 0))
2566                         return result;
2567         }
2568         return 0;
2569 }
2570 
2571 
2572 /**
2573  * Close a pipe created with cvmx_usb_open_pipe().
2574  *
2575  * @usb:         USB device state populated by cvmx_usb_initialize().
2576  * @pipe:        Pipe to close.
2577  *
2578  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
2579  *          outstanding transfers.
2580  */
2581 static int cvmx_usb_close_pipe(struct cvmx_usb_state *usb,
2582                                struct cvmx_usb_pipe *pipe)
2583 {
2584         /* Fail if the pipe has pending transactions */
2585         if (!list_empty(&pipe->transactions))
2586                 return -EBUSY;
2587 
2588         list_del(&pipe->node);
2589         kfree(pipe);
2590 
2591         return 0;
2592 }
2593 
2594 /**
2595  * Get the current USB protocol level frame number. The frame
2596  * number is always in the range of 0-0x7ff.
2597  *
2598  * @usb: USB device state populated by cvmx_usb_initialize().
2599  *
2600  * Returns: USB frame number
2601  */
2602 static int cvmx_usb_get_frame_number(struct cvmx_usb_state *usb)
2603 {
2604         int frame_number;
2605         union cvmx_usbcx_hfnum usbc_hfnum;
2606 
2607         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb,
2608                         CVMX_USBCX_HFNUM(usb->index));
2609         frame_number = usbc_hfnum.s.frnum;
2610 
2611         return frame_number;
2612 }
2613 
2614 
2615 /**
2616  * Poll a channel for status
2617  *
2618  * @usb:     USB device
2619  * @channel: Channel to poll
2620  *
2621  * Returns: Zero on success
2622  */
2623 static int __cvmx_usb_poll_channel(struct cvmx_usb_state *usb, int channel)
2624 {
2625         union cvmx_usbcx_hcintx usbc_hcint;
2626         union cvmx_usbcx_hctsizx usbc_hctsiz;
2627         union cvmx_usbcx_hccharx usbc_hcchar;
2628         struct cvmx_usb_pipe *pipe;
2629         struct cvmx_usb_transaction *transaction;
2630         int bytes_this_transfer;
2631         int bytes_in_last_packet;
2632         int packets_processed;
2633         int buffer_space_left;
2634 
2635         /* Read the interrupt status bits for the channel */
2636         usbc_hcint.u32 = __cvmx_usb_read_csr32(usb,
2637                         CVMX_USBCX_HCINTX(channel, usb->index));
2638 
2639         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2640                 usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2641                                 CVMX_USBCX_HCCHARX(channel, usb->index));
2642 
2643                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2644                         /*
2645                          * There seems to be a bug in CN31XX which can cause
2646                          * interrupt IN transfers to get stuck until we do a
2647                          * write of HCCHARX without changing things
2648                          */
2649                         __cvmx_usb_write_csr32(usb,
2650                                         CVMX_USBCX_HCCHARX(channel,
2651                                                            usb->index),
2652                                         usbc_hcchar.u32);
2653                         return 0;
2654                 }
2655 
2656                 /*
2657                  * In non DMA mode the channels don't halt themselves. We need
2658                  * to manually disable channels that are left running
2659                  */
2660                 if (!usbc_hcint.s.chhltd) {
2661                         if (usbc_hcchar.s.chena) {
2662                                 union cvmx_usbcx_hcintmskx hcintmsk;
2663                                 /* Disable all interrupts except CHHLTD */
2664                                 hcintmsk.u32 = 0;
2665                                 hcintmsk.s.chhltdmsk = 1;
2666                                 __cvmx_usb_write_csr32(usb,
2667                                                 CVMX_USBCX_HCINTMSKX(channel,
2668                                                         usb->index),
2669                                                 hcintmsk.u32);
2670                                 usbc_hcchar.s.chdis = 1;
2671                                 __cvmx_usb_write_csr32(usb,
2672                                                 CVMX_USBCX_HCCHARX(channel,
2673                                                         usb->index),
2674                                                 usbc_hcchar.u32);
2675                                 return 0;
2676                         } else if (usbc_hcint.s.xfercompl) {
2677                                 /*
2678                                  * Successful IN/OUT with transfer complete.
2679                                  * Channel halt isn't needed.
2680                                  */
2681                         } else {
2682                                 cvmx_dprintf("USB%d: Channel %d interrupt without halt\n",
2683                                                 usb->index, channel);
2684                                 return 0;
2685                         }
2686                 }
2687         } else {
2688                 /*
2689                  * There is are no interrupts that we need to process when the
2690                  * channel is still running
2691                  */
2692                 if (!usbc_hcint.s.chhltd)
2693                         return 0;
2694         }
2695 
2696         /* Disable the channel interrupts now that it is done */
2697         __cvmx_usb_write_csr32(usb,
2698                                 CVMX_USBCX_HCINTMSKX(channel, usb->index),
2699                                 0);
2700         usb->idle_hardware_channels |= (1<<channel);
2701 
2702         /* Make sure this channel is tied to a valid pipe */
2703         pipe = usb->pipe_for_channel[channel];
2704         prefetch(pipe);
2705         if (!pipe)
2706                 return 0;
2707         transaction = list_first_entry(&pipe->transactions,
2708                                        typeof(*transaction),
2709                                        node);
2710         prefetch(transaction);
2711 
2712         /*
2713          * Disconnect this pipe from the HW channel. Later the schedule
2714          * function will figure out which pipe needs to go
2715          */
2716         usb->pipe_for_channel[channel] = NULL;
2717         pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_SCHEDULED;
2718 
2719         /*
2720          * Read the channel config info so we can figure out how much data
2721          * transfered
2722          */
2723         usbc_hcchar.u32 = __cvmx_usb_read_csr32(usb,
2724                         CVMX_USBCX_HCCHARX(channel, usb->index));
2725         usbc_hctsiz.u32 = __cvmx_usb_read_csr32(usb,
2726                         CVMX_USBCX_HCTSIZX(channel, usb->index));
2727 
2728         /*
2729          * Calculating the number of bytes successfully transferred is dependent
2730          * on the transfer direction
2731          */
2732         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2733         if (usbc_hcchar.s.epdir) {
2734                 /*
2735                  * IN transactions are easy. For every byte received the
2736                  * hardware decrements xfersize. All we need to do is subtract
2737                  * the current value of xfersize from its starting value and we
2738                  * know how many bytes were written to the buffer
2739                  */
2740                 bytes_this_transfer = transaction->xfersize -
2741                         usbc_hctsiz.s.xfersize;
2742         } else {
2743                 /*
2744                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2745                  * decremented on every successful packet send. The hardware
2746                  * does this when it receives an ACK, or NYET. If it doesn't
2747                  * receive one of these responses pktcnt doesn't change
2748                  */
2749                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2750                 /*
2751                  * The last packet may not be a full transfer if we didn't have
2752                  * enough data
2753                  */
2754                 if (bytes_this_transfer > transaction->xfersize)
2755                         bytes_this_transfer = transaction->xfersize;
2756         }
2757         /* Figure out how many bytes were in the last packet of the transfer */
2758         if (packets_processed)
2759                 bytes_in_last_packet = bytes_this_transfer -
2760                         (packets_processed - 1) * usbc_hcchar.s.mps;
2761         else
2762                 bytes_in_last_packet = bytes_this_transfer;
2763 
2764         /*
2765          * As a special case, setup transactions output the setup header, not
2766          * the user's data. For this reason we don't count setup data as bytes
2767          * transferred
2768          */
2769         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2770                 (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2771                 bytes_this_transfer = 0;
2772 
2773         /*
2774          * Add the bytes transferred to the running total. It is important that
2775          * bytes_this_transfer doesn't count any data that needs to be
2776          * retransmitted
2777          */
2778         transaction->actual_bytes += bytes_this_transfer;
2779         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2780                 buffer_space_left = transaction->iso_packets[0].length -
2781                         transaction->actual_bytes;
2782         else
2783                 buffer_space_left = transaction->buffer_length -
2784                         transaction->actual_bytes;
2785 
2786         /*
2787          * We need to remember the PID toggle state for the next transaction.
2788          * The hardware already updated it for the next transaction
2789          */
2790         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2791 
2792         /*
2793          * For high speed bulk out, assume the next transaction will need to do
2794          * a ping before proceeding. If this isn't true the ACK processing below
2795          * will clear this flag
2796          */
2797         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2798                 (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2799                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2800                 pipe->flags |= __CVMX_USB_PIPE_FLAGS_NEED_PING;
2801 
2802         if (usbc_hcint.s.stall) {
2803                 /*
2804                  * STALL as a response means this transaction cannot be
2805                  * completed because the device can't process transactions. Tell
2806                  * the user. Any data that was transferred will be counted on
2807                  * the actual bytes transferred
2808                  */
2809                 pipe->pid_toggle = 0;
2810                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2811                                             CVMX_USB_COMPLETE_STALL);
2812         } else if (usbc_hcint.s.xacterr) {
2813                 /*
2814                  * We know at least one packet worked if we get a ACK or NAK.
2815                  * Reset the retry counter
2816                  */
2817                 if (usbc_hcint.s.nak || usbc_hcint.s.ack)
2818                         transaction->retries = 0;
2819                 transaction->retries++;
2820                 if (transaction->retries > MAX_RETRIES) {
2821                         /*
2822                          * XactErr as a response means the device signaled
2823                          * something wrong with the transfer. For example, PID
2824                          * toggle errors cause these
2825                          */
2826                         __cvmx_usb_perform_complete(usb, pipe, transaction,
2827                                                     CVMX_USB_COMPLETE_XACTERR);
2828                 } else {
2829                         /*
2830                          * If this was a split then clear our split in progress
2831                          * marker
2832                          */
2833                         if (usb->active_split == transaction)
2834                                 usb->active_split = NULL;
2835                         /*
2836                          * Rewind to the beginning of the transaction by anding
2837                          * off the split complete bit
2838                          */
2839                         transaction->stage &= ~1;
2840                         pipe->split_sc_frame = -1;
2841                         pipe->next_tx_frame += pipe->interval;
2842                         if (pipe->next_tx_frame < usb->frame_number)
2843                                 pipe->next_tx_frame =
2844                                         usb->frame_number + pipe->interval -
2845                                         (usb->frame_number -
2846                                          pipe->next_tx_frame) % pipe->interval;
2847                 }
2848         } else if (usbc_hcint.s.bblerr) {
2849                 /* Babble Error (BblErr) */
2850                 __cvmx_usb_perform_complete(usb, pipe, transaction,
2851                                             CVMX_USB_COMPLETE_BABBLEERR);
2852         } else if (usbc_hcint.s.datatglerr) {
2853                 /* We'll retry the exact same transaction again */
2854                 transaction->retries++;
2855         } else if (usbc_hcint.s.nyet) {
2856                 /*
2857                  * NYET as a response is only allowed in three cases: as a
2858                  * response to a ping, as a response to a split transaction, and
2859                  * as a response to a bulk out. The ping case is handled by
2860                  * hardware, so we only have splits and bulk out
2861                  */
2862                 if (!__cvmx_usb_pipe_needs_split(usb, pipe)) {
2863                         transaction->retries = 0;
2864                         /*
2865                          * If there is more data to go then we need to try
2866                          * again. Otherwise this transaction is complete
2867                          */
2868                         if ((buffer_space_left == 0) ||
2869                                 (bytes_in_last_packet < pipe->max_packet))
2870                                 __cvmx_usb_perform_complete(usb, pipe,
2871                                                 transaction,
2872                                                 CVMX_USB_COMPLETE_SUCCESS);
2873                 } else {
2874                         /*
2875                          * Split transactions retry the split complete 4 times
2876                          * then rewind to the start split and do the entire
2877                          * transactions again
2878                          */
2879                         transaction->retries++;
2880                         if ((transaction->retries & 0x3) == 0) {
2881                                 /*
2882                                  * Rewind to the beginning of the transaction by
2883                                  * anding off the split complete bit
2884                                  */
2885                                 transaction->stage &= ~1;
2886                                 pipe->split_sc_frame = -1;
2887                         }
2888                 }
2889         } else if (usbc_hcint.s.ack) {
2890                 transaction->retries = 0;
2891                 /*
2892                  * The ACK bit can only be checked after the other error bits.
2893                  * This is because a multi packet transfer may succeed in a
2894                  * number of packets and then get a different response on the
2895                  * last packet. In this case both ACK and the last response bit
2896                  * will be set. If none of the other response bits is set, then
2897                  * the last packet must have been an ACK
2898                  *
2899                  * Since we got an ACK, we know we don't need to do a ping on
2900                  * this pipe
2901                  */
2902                 pipe->flags &= ~__CVMX_USB_PIPE_FLAGS_NEED_PING;
2903 
2904                 switch (transaction->type) {
2905                 case CVMX_USB_TRANSFER_CONTROL:
2906                         switch (transaction->stage) {
2907                         case CVMX_USB_STAGE_NON_CONTROL:
2908                         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2909                                 /* This should be impossible */
2910                                 __cvmx_usb_perform_complete(usb, pipe,
2911                                         transaction, CVMX_USB_COMPLETE_ERROR);
2912                                 break;
2913                         case CVMX_USB_STAGE_SETUP:
2914                                 pipe->pid_toggle = 1;
2915                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2916                                         transaction->stage =
2917                                                 CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2918                                 else {
2919                                         union cvmx_usb_control_header *header =
2920                                                 cvmx_phys_to_ptr(transaction->control_header);
2921                                         if (header->s.length)
2922                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2923                                         else
2924                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2925                                 }
2926                                 break;
2927                         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2928                                 {
2929                                         union cvmx_usb_control_header *header =
2930                                                 cvmx_phys_to_ptr(transaction->control_header);
2931                                         if (header->s.length)
2932                                                 transaction->stage = CVMX_USB_STAGE_DATA;
2933                                         else
2934                                                 transaction->stage = CVMX_USB_STAGE_STATUS;
2935                                 }
2936                                 break;
2937                         case CVMX_USB_STAGE_DATA:
2938                                 if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
2939                                         transaction->stage =
2940                                                 CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2941                                         /*
2942                                          * For setup OUT data that are splits,
2943                                          * the hardware doesn't appear to count
2944                                          * transferred data. Here we manually
2945                                          * update the data transferred
2946                                          */
2947                                         if (!usbc_hcchar.s.epdir) {
2948                                                 if (buffer_space_left < pipe->max_packet)
2949                                                         transaction->actual_bytes +=
2950                                                                 buffer_space_left;
2951                                                 else
2952                                                         transaction->actual_bytes +=
2953                                                                 pipe->max_packet;
2954                                         }
2955                                 } else if ((buffer_space_left == 0) ||
2956                                                 (bytes_in_last_packet <
2957                                                  pipe->max_packet)) {
2958                                         pipe->pid_toggle = 1;
2959                                         transaction->stage =
2960                                                 CVMX_USB_STAGE_STATUS;
2961                                 }
2962                                 break;
2963                         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2964                                 if ((buffer_space_left == 0) ||
2965                                                 (bytes_in_last_packet <
2966                                                  pipe->max_packet)) {
2967                                         pipe->pid_toggle = 1;
2968                                         transaction->stage =
2969                                                 CVMX_USB_STAGE_STATUS;
2970                                 } else {
2971                                         transaction->stage =
2972                                                 CVMX_USB_STAGE_DATA;
2973                                 }
2974                                 break;
2975                         case CVMX_USB_STAGE_STATUS:
2976                                 if (__cvmx_usb_pipe_needs_split(usb, pipe))
2977                                         transaction->stage =
2978                                                 CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2979                                 else
2980                                         __cvmx_usb_perform_complete(usb, pipe,
2981                                                 transaction,
2982                                                 CVMX_USB_COMPLETE_SUCCESS);
2983                                 break;
2984                         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2985                                 __cvmx_usb_perform_complete(usb, pipe,
2986                                                 transaction,
2987                                                 CVMX_USB_COMPLETE_SUCCESS);
2988                                 break;
2989                         }
2990                         break;
2991                 case CVMX_USB_TRANSFER_BULK:
2992                 case CVMX_USB_TRANSFER_INTERRUPT:
2993                         /*
2994                          * The only time a bulk transfer isn't complete when it
2995                          * finishes with an ACK is during a split transaction.
2996                          * For splits we need to continue the transfer if more
2997                          * data is needed
2998                          */
2999                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
3000                                 if (transaction->stage ==
3001                                                 CVMX_USB_STAGE_NON_CONTROL)
3002                                         transaction->stage =
3003                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
3004                                 else {
3005                                         if (buffer_space_left &&
3006                                                 (bytes_in_last_packet ==
3007                                                  pipe->max_packet))
3008                                                 transaction->stage =
3009                                                         CVMX_USB_STAGE_NON_CONTROL;
3010                                         else {
3011                                                 if (transaction->type ==
3012                                                         CVMX_USB_TRANSFER_INTERRUPT)
3013                                                         pipe->next_tx_frame +=
3014                                                                 pipe->interval;
3015                                                         __cvmx_usb_perform_complete(
3016                                                                 usb,
3017                                                                 pipe,
3018                                                                 transaction,
3019                                                                 CVMX_USB_COMPLETE_SUCCESS);
3020                                         }
3021                                 }
3022                         } else {
3023                                 if ((pipe->device_speed ==
3024                                         CVMX_USB_SPEED_HIGH) &&
3025                                     (pipe->transfer_type ==
3026                                      CVMX_USB_TRANSFER_BULK) &&
3027                                     (pipe->transfer_dir ==
3028                                      CVMX_USB_DIRECTION_OUT) &&
3029                                     (usbc_hcint.s.nak))
3030                                         pipe->flags |=
3031                                                 __CVMX_USB_PIPE_FLAGS_NEED_PING;
3032                                 if (!buffer_space_left ||
3033                                         (bytes_in_last_packet <
3034                                          pipe->max_packet)) {
3035                                         if (transaction->type ==
3036                                                 CVMX_USB_TRANSFER_INTERRUPT)
3037                                                 pipe->next_tx_frame +=
3038                                                         pipe->interval;
3039                                         __cvmx_usb_perform_complete(usb,
3040                                                 pipe,
3041                                                 transaction,
3042                                                 CVMX_USB_COMPLETE_SUCCESS);
3043                                 }
3044                         }
3045                         break;
3046                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
3047                         if (__cvmx_usb_pipe_needs_split(usb, pipe)) {
3048                                 /*
3049                                  * ISOCHRONOUS OUT splits don't require a
3050                                  * complete split stage. Instead they use a
3051                                  * sequence of begin OUT splits to transfer the
3052                                  * data 188 bytes at a time. Once the transfer
3053                                  * is complete, the pipe sleeps until the next
3054                                  * schedule interval
3055                                  */
3056                                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
3057                                         /*
3058                                          * If no space left or this wasn't a max
3059                                          * size packet then this transfer is
3060                                          * complete. Otherwise start it again to
3061                                          * send the next 188 bytes
3062                                          */
3063                                         if (!buffer_space_left ||
3064                                                 (bytes_this_transfer < 188)) {
3065                                                 pipe->next_tx_frame +=
3066                                                         pipe->interval;
3067                                                 __cvmx_usb_perform_complete(
3068                                                         usb,
3069                                                         pipe,
3070                                                         transaction,
3071                                                         CVMX_USB_COMPLETE_SUCCESS);
3072                                         }
3073                                 } else {
3074                                         if (transaction->stage ==
3075                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
3076                                                 /*
3077                                                  * We are in the incoming data
3078                                                  * phase. Keep getting data
3079                                                  * until we run out of space or
3080                                                  * get a small packet
3081                                                  */
3082                                                 if ((buffer_space_left == 0) ||
3083                                                         (bytes_in_last_packet <
3084                                                          pipe->max_packet)) {
3085                                                         pipe->next_tx_frame +=
3086                                                                 pipe->interval;
3087                                                         __cvmx_usb_perform_complete(
3088                                                                 usb,
3089                                                                 pipe,
3090                                                                 transaction,
3091                                                                 CVMX_USB_COMPLETE_SUCCESS);
3092                                                 }
3093                                         } else
3094                                                 transaction->stage =
3095                                                         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
3096                                 }
3097                         } else {
3098                                 pipe->next_tx_frame += pipe->interval;
3099                                 __cvmx_usb_perform_complete(usb,
3100                                                 pipe,
3101                                                 transaction,
3102                                                 CVMX_USB_COMPLETE_SUCCESS);
3103                         }
3104                         break;
3105                 }
3106         } else if (usbc_hcint.s.nak) {
3107                 /*
3108                  * If this was a split then clear our split in progress marker.
3109                  */
3110                 if (usb->active_split == transaction)
3111                         usb->active_split = NULL;
3112                 /*
3113                  * NAK as a response means the device couldn't accept the
3114                  * transaction, but it should be retried in the future. Rewind
3115                  * to the beginning of the transaction by anding off the split
3116                  * complete bit. Retry in the next interval
3117                  */
3118                 transaction->retries = 0;
3119                 transaction->stage &= ~1;
3120                 pipe->next_tx_frame += pipe->interval;
3121                 if (pipe->next_tx_frame < usb->frame_number)
3122                         pipe->next_tx_frame = usb->frame_number +
3123                                 pipe->interval -
3124                                 (usb->frame_number - pipe->next_tx_frame) %
3125                                 pipe->interval;
3126         } else {
3127                 struct cvmx_usb_port_status port;
3128 
3129                 port = cvmx_usb_get_status(usb);
3130                 if (port.port_enabled) {
3131                         /* We'll retry the exact same transaction again */
3132                         transaction->retries++;
3133                 } else {
3134                         /*
3135                          * We get channel halted interrupts with no result bits
3136                          * sets when the cable is unplugged
3137                          */
3138                         __cvmx_usb_perform_complete(usb, pipe, transaction,
3139                                         CVMX_USB_COMPLETE_ERROR);
3140                 }
3141         }
3142         return 0;
3143 }
3144 
3145 static void octeon_usb_port_callback(struct cvmx_usb_state *usb)
3146 {
3147         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
3148 
3149         spin_unlock(&priv->lock);
3150         usb_hcd_poll_rh_status(octeon_to_hcd(priv));
3151         spin_lock(&priv->lock);
3152 }
3153 
3154 /**
3155  * Poll the USB block for status and call all needed callback
3156  * handlers. This function is meant to be called in the interrupt
3157  * handler for the USB controller. It can also be called
3158  * periodically in a loop for non-interrupt based operation.
3159  *
3160  * @usb: USB device state populated by cvmx_usb_initialize().
3161  *
3162  * Returns: 0 or a negative error code.
3163  */
3164 static int cvmx_usb_poll(struct cvmx_usb_state *usb)
3165 {
3166         union cvmx_usbcx_hfnum usbc_hfnum;
3167         union cvmx_usbcx_gintsts usbc_gintsts;
3168 
3169         prefetch_range(usb, sizeof(*usb));
3170 
3171         /* Update the frame counter */
3172         usbc_hfnum.u32 = __cvmx_usb_read_csr32(usb,
3173                                                 CVMX_USBCX_HFNUM(usb->index));
3174         if ((usb->frame_number&0x3fff) > usbc_hfnum.s.frnum)
3175                 usb->frame_number += 0x4000;
3176         usb->frame_number &= ~0x3fffull;
3177         usb->frame_number |= usbc_hfnum.s.frnum;
3178 
3179         /* Read the pending interrupts */
3180         usbc_gintsts.u32 = __cvmx_usb_read_csr32(usb,
3181                                                 CVMX_USBCX_GINTSTS(usb->index));
3182 
3183         /* Clear the interrupts now that we know about them */
3184         __cvmx_usb_write_csr32(usb,
3185                                 CVMX_USBCX_GINTSTS(usb->index),
3186                                 usbc_gintsts.u32);
3187 
3188         if (usbc_gintsts.s.rxflvl) {
3189                 /*
3190                  * RxFIFO Non-Empty (RxFLvl)
3191                  * Indicates that there is at least one packet pending to be
3192                  * read from the RxFIFO.
3193                  *
3194                  * In DMA mode this is handled by hardware
3195                  */
3196                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3197                         __cvmx_usb_poll_rx_fifo(usb);
3198         }
3199         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
3200                 /* Fill the Tx FIFOs when not in DMA mode */
3201                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3202                         __cvmx_usb_poll_tx_fifo(usb);
3203         }
3204         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
3205                 union cvmx_usbcx_hprt usbc_hprt;
3206                 /*
3207                  * Disconnect Detected Interrupt (DisconnInt)
3208                  * Asserted when a device disconnect is detected.
3209                  *
3210                  * Host Port Interrupt (PrtInt)
3211                  * The core sets this bit to indicate a change in port status of
3212                  * one of the O2P USB core ports in Host mode. The application
3213                  * must read the Host Port Control and Status (HPRT) register to
3214                  * determine the exact event that caused this interrupt. The
3215                  * application must clear the appropriate status bit in the Host
3216                  * Port Control and Status register to clear this bit.
3217                  *
3218                  * Call the user's port callback
3219                  */
3220                 octeon_usb_port_callback(usb);
3221                 /* Clear the port change bits */
3222                 usbc_hprt.u32 = __cvmx_usb_read_csr32(usb,
3223                                 CVMX_USBCX_HPRT(usb->index));
3224                 usbc_hprt.s.prtena = 0;
3225                 __cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index),
3226                                        usbc_hprt.u32);
3227         }
3228         if (usbc_gintsts.s.hchint) {
3229                 /*
3230                  * Host Channels Interrupt (HChInt)
3231                  * The core sets this bit to indicate that an interrupt is
3232                  * pending on one of the channels of the core (in Host mode).
3233                  * The application must read the Host All Channels Interrupt
3234                  * (HAINT) register to determine the exact number of the channel
3235                  * on which the interrupt occurred, and then read the
3236                  * corresponding Host Channel-n Interrupt (HCINTn) register to
3237                  * determine the exact cause of the interrupt. The application
3238                  * must clear the appropriate status bit in the HCINTn register
3239                  * to clear this bit.
3240                  */
3241                 union cvmx_usbcx_haint usbc_haint;
3242 
3243                 usbc_haint.u32 = __cvmx_usb_read_csr32(usb,
3244                                         CVMX_USBCX_HAINT(usb->index));
3245                 while (usbc_haint.u32) {
3246                         int channel;
3247 
3248                         channel = __fls(usbc_haint.u32);
3249                         __cvmx_usb_poll_channel(usb, channel);
3250                         usbc_haint.u32 ^= 1<<channel;
3251                 }
3252         }
3253 
3254         __cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
3255 
3256         return 0;
3257 }
3258 
3259 /* convert between an HCD pointer and the corresponding struct octeon_hcd */
3260 static inline struct octeon_hcd *hcd_to_octeon(struct usb_hcd *hcd)
3261 {
3262         return (struct octeon_hcd *)(hcd->hcd_priv);
3263 }
3264 
3265 static irqreturn_t octeon_usb_irq(struct usb_hcd *hcd)
3266 {
3267         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3268         unsigned long flags;
3269 
3270         spin_lock_irqsave(&priv->lock, flags);
3271         cvmx_usb_poll(&priv->usb);
3272         spin_unlock_irqrestore(&priv->lock, flags);
3273         return IRQ_HANDLED;
3274 }
3275 
3276 static int octeon_usb_start(struct usb_hcd *hcd)
3277 {
3278         hcd->state = HC_STATE_RUNNING;
3279         return 0;
3280 }
3281 
3282 static void octeon_usb_stop(struct usb_hcd *hcd)
3283 {
3284         hcd->state = HC_STATE_HALT;
3285 }
3286 
3287 static int octeon_usb_get_frame_number(struct usb_hcd *hcd)
3288 {
3289         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3290 
3291         return cvmx_usb_get_frame_number(&priv->usb);
3292 }
3293 
3294 static int octeon_usb_urb_enqueue(struct usb_hcd *hcd,
3295                                   struct urb *urb,
3296                                   gfp_t mem_flags)
3297 {
3298         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3299         struct device *dev = hcd->self.controller;
3300         struct cvmx_usb_transaction *transaction = NULL;
3301         struct cvmx_usb_pipe *pipe;
3302         unsigned long flags;
3303         struct cvmx_usb_iso_packet *iso_packet;
3304         struct usb_host_endpoint *ep = urb->ep;
3305 
3306         urb->status = 0;
3307         INIT_LIST_HEAD(&urb->urb_list); /* not enqueued on dequeue_list */
3308         spin_lock_irqsave(&priv->lock, flags);
3309 
3310         if (!ep->hcpriv) {
3311                 enum cvmx_usb_transfer transfer_type;
3312                 enum cvmx_usb_speed speed;
3313                 int split_device = 0;
3314                 int split_port = 0;
3315 
3316                 switch (usb_pipetype(urb->pipe)) {
3317                 case PIPE_ISOCHRONOUS:
3318                         transfer_type = CVMX_USB_TRANSFER_ISOCHRONOUS;
3319                         break;
3320                 case PIPE_INTERRUPT:
3321                         transfer_type = CVMX_USB_TRANSFER_INTERRUPT;
3322                         break;
3323                 case PIPE_CONTROL:
3324                         transfer_type = CVMX_USB_TRANSFER_CONTROL;
3325                         break;
3326                 default:
3327                         transfer_type = CVMX_USB_TRANSFER_BULK;
3328                         break;
3329                 }
3330                 switch (urb->dev->speed) {
3331                 case USB_SPEED_LOW:
3332                         speed = CVMX_USB_SPEED_LOW;
3333                         break;
3334                 case USB_SPEED_FULL:
3335                         speed = CVMX_USB_SPEED_FULL;
3336                         break;
3337                 default:
3338                         speed = CVMX_USB_SPEED_HIGH;
3339                         break;
3340                 }
3341                 /*
3342                  * For slow devices on high speed ports we need to find the hub
3343                  * that does the speed translation so we know where to send the
3344                  * split transactions.
3345                  */
3346                 if (speed != CVMX_USB_SPEED_HIGH) {
3347                         /*
3348                          * Start at this device and work our way up the usb
3349                          * tree.
3350                          */
3351                         struct usb_device *dev = urb->dev;
3352 
3353                         while (dev->parent) {
3354                                 /*
3355                                  * If our parent is high speed then he'll
3356                                  * receive the splits.
3357                                  */
3358                                 if (dev->parent->speed == USB_SPEED_HIGH) {
3359                                         split_device = dev->parent->devnum;
3360                                         split_port = dev->portnum;
3361                                         break;
3362                                 }
3363                                 /*
3364                                  * Move up the tree one level. If we make it all
3365                                  * the way up the tree, then the port must not
3366                                  * be in high speed mode and we don't need a
3367                                  * split.
3368                                  */
3369                                 dev = dev->parent;
3370                         }
3371                 }
3372                 pipe = cvmx_usb_open_pipe(&priv->usb, usb_pipedevice(urb->pipe),
3373                                           usb_pipeendpoint(urb->pipe), speed,
3374                                           le16_to_cpu(ep->desc.wMaxPacketSize)
3375                                           & 0x7ff,
3376                                           transfer_type,
3377                                           usb_pipein(urb->pipe) ?
3378                                                 CVMX_USB_DIRECTION_IN :
3379                                                 CVMX_USB_DIRECTION_OUT,
3380                                           urb->interval,
3381                                           (le16_to_cpu(ep->desc.wMaxPacketSize)
3382                                            >> 11) & 0x3,
3383                                           split_device, split_port);
3384                 if (!pipe) {
3385                         spin_unlock_irqrestore(&priv->lock, flags);
3386                         dev_dbg(dev, "Failed to create pipe\n");
3387                         return -ENOMEM;
3388                 }
3389                 ep->hcpriv = pipe;
3390         } else {
3391                 pipe = ep->hcpriv;
3392         }
3393 
3394         switch (usb_pipetype(urb->pipe)) {
3395         case PIPE_ISOCHRONOUS:
3396                 dev_dbg(dev, "Submit isochronous to %d.%d\n",
3397                         usb_pipedevice(urb->pipe),
3398                         usb_pipeendpoint(urb->pipe));
3399                 /*
3400                  * Allocate a structure to use for our private list of
3401                  * isochronous packets.
3402                  */
3403                 iso_packet = kmalloc(urb->number_of_packets *
3404                                      sizeof(struct cvmx_usb_iso_packet),
3405                                      GFP_ATOMIC);
3406                 if (iso_packet) {
3407                         int i;
3408                         /* Fill the list with the data from the URB */
3409                         for (i = 0; i < urb->number_of_packets; i++) {
3410                                 iso_packet[i].offset =
3411                                         urb->iso_frame_desc[i].offset;
3412                                 iso_packet[i].length =
3413                                         urb->iso_frame_desc[i].length;
3414                                 iso_packet[i].status =
3415                                         CVMX_USB_COMPLETE_ERROR;
3416                         }
3417                         /*
3418                          * Store a pointer to the list in the URB setup_packet
3419                          * field. We know this currently isn't being used and
3420                          * this saves us a bunch of logic.
3421                          */
3422                         urb->setup_packet = (char *)iso_packet;
3423                         transaction = cvmx_usb_submit_isochronous(&priv->usb,
3424                                                                   pipe, urb);
3425                         /*
3426                          * If submit failed we need to free our private packet
3427                          * list.
3428                          */
3429                         if (!transaction) {
3430                                 urb->setup_packet = NULL;
3431                                 kfree(iso_packet);
3432                         }
3433                 }
3434                 break;
3435         case PIPE_INTERRUPT:
3436                 dev_dbg(dev, "Submit interrupt to %d.%d\n",
3437                         usb_pipedevice(urb->pipe),
3438                         usb_pipeendpoint(urb->pipe));
3439                 transaction = cvmx_usb_submit_interrupt(&priv->usb, pipe, urb);
3440                 break;
3441         case PIPE_CONTROL:
3442                 dev_dbg(dev, "Submit control to %d.%d\n",
3443                         usb_pipedevice(urb->pipe),
3444                         usb_pipeendpoint(urb->pipe));
3445                 transaction = cvmx_usb_submit_control(&priv->usb, pipe, urb);
3446                 break;
3447         case PIPE_BULK:
3448                 dev_dbg(dev, "Submit bulk to %d.%d\n",
3449                         usb_pipedevice(urb->pipe),
3450                         usb_pipeendpoint(urb->pipe));
3451                 transaction = cvmx_usb_submit_bulk(&priv->usb, pipe, urb);
3452                 break;
3453         }
3454         if (!transaction) {
3455                 spin_unlock_irqrestore(&priv->lock, flags);
3456                 dev_dbg(dev, "Failed to submit\n");
3457                 return -ENOMEM;
3458         }
3459         urb->hcpriv = transaction;
3460         spin_unlock_irqrestore(&priv->lock, flags);
3461         return 0;
3462 }
3463 
3464 static void octeon_usb_urb_dequeue_work(unsigned long arg)
3465 {
3466         struct urb *urb;
3467         struct urb *next;
3468         unsigned long flags;
3469         struct octeon_hcd *priv = (struct octeon_hcd *)arg;
3470 
3471         spin_lock_irqsave(&priv->lock, flags);
3472 
3473         list_for_each_entry_safe(urb, next, &priv->dequeue_list, urb_list) {
3474                 list_del_init(&urb->urb_list);
3475                 cvmx_usb_cancel(&priv->usb, urb->ep->hcpriv, urb->hcpriv);
3476         }
3477 
3478         spin_unlock_irqrestore(&priv->lock, flags);
3479 }
3480 
3481 static int octeon_usb_urb_dequeue(struct usb_hcd *hcd,
3482                                   struct urb *urb,
3483                                   int status)
3484 {
3485         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3486         unsigned long flags;
3487 
3488         if (!urb->dev)
3489                 return -EINVAL;
3490 
3491         spin_lock_irqsave(&priv->lock, flags);
3492 
3493         urb->status = status;
3494         list_add_tail(&urb->urb_list, &priv->dequeue_list);
3495 
3496         spin_unlock_irqrestore(&priv->lock, flags);
3497 
3498         tasklet_schedule(&priv->dequeue_tasklet);
3499 
3500         return 0;
3501 }
3502 
3503 static void octeon_usb_endpoint_disable(struct usb_hcd *hcd,
3504                                         struct usb_host_endpoint *ep)
3505 {
3506         struct device *dev = hcd->self.controller;
3507 
3508         if (ep->hcpriv) {
3509                 struct octeon_hcd *priv = hcd_to_octeon(hcd);
3510                 struct cvmx_usb_pipe *pipe = ep->hcpriv;
3511                 unsigned long flags;
3512 
3513                 spin_lock_irqsave(&priv->lock, flags);
3514                 cvmx_usb_cancel_all(&priv->usb, pipe);
3515                 if (cvmx_usb_close_pipe(&priv->usb, pipe))
3516                         dev_dbg(dev, "Closing pipe %p failed\n", pipe);
3517                 spin_unlock_irqrestore(&priv->lock, flags);
3518                 ep->hcpriv = NULL;
3519         }
3520 }
3521 
3522 static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
3523 {
3524         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3525         struct cvmx_usb_port_status port_status;
3526         unsigned long flags;
3527 
3528         spin_lock_irqsave(&priv->lock, flags);
3529         port_status = cvmx_usb_get_status(&priv->usb);
3530         spin_unlock_irqrestore(&priv->lock, flags);
3531         buf[0] = 0;
3532         buf[0] = port_status.connect_change << 1;
3533 
3534         return buf[0] != 0;
3535 }
3536 
3537 static int octeon_usb_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
3538                                 u16 wIndex, char *buf, u16 wLength)
3539 {
3540         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3541         struct device *dev = hcd->self.controller;
3542         struct cvmx_usb_port_status usb_port_status;
3543         int port_status;
3544         struct usb_hub_descriptor *desc;
3545         unsigned long flags;
3546 
3547         switch (typeReq) {
3548         case ClearHubFeature:
3549                 dev_dbg(dev, "ClearHubFeature\n");
3550                 switch (wValue) {
3551                 case C_HUB_LOCAL_POWER:
3552                 case C_HUB_OVER_CURRENT:
3553                         /* Nothing required here */
3554                         break;
3555                 default:
3556                         return -EINVAL;
3557                 }
3558                 break;
3559         case ClearPortFeature:
3560                 dev_dbg(dev, "ClearPortFeature\n");
3561                 if (wIndex != 1) {
3562                         dev_dbg(dev, " INVALID\n");
3563                         return -EINVAL;
3564                 }
3565 
3566                 switch (wValue) {
3567                 case USB_PORT_FEAT_ENABLE:
3568                         dev_dbg(dev, " ENABLE\n");
3569                         spin_lock_irqsave(&priv->lock, flags);
3570                         cvmx_usb_disable(&priv->usb);
3571                         spin_unlock_irqrestore(&priv->lock, flags);
3572                         break;
3573                 case USB_PORT_FEAT_SUSPEND:
3574                         dev_dbg(dev, " SUSPEND\n");
3575                         /* Not supported on Octeon */
3576                         break;
3577                 case USB_PORT_FEAT_POWER:
3578                         dev_dbg(dev, " POWER\n");
3579                         /* Not supported on Octeon */
3580                         break;
3581                 case USB_PORT_FEAT_INDICATOR:
3582                         dev_dbg(dev, " INDICATOR\n");
3583                         /* Port inidicator not supported */
3584                         break;
3585                 case USB_PORT_FEAT_C_CONNECTION:
3586                         dev_dbg(dev, " C_CONNECTION\n");
3587                         /* Clears drivers internal connect status change flag */
3588                         spin_lock_irqsave(&priv->lock, flags);
3589                         priv->usb.port_status =
3590                                 cvmx_usb_get_status(&priv->usb);
3591                         spin_unlock_irqrestore(&priv->lock, flags);
3592                         break;
3593                 case USB_PORT_FEAT_C_RESET:
3594                         dev_dbg(dev, " C_RESET\n");
3595                         /*
3596                          * Clears the driver's internal Port Reset Change flag.
3597                          */
3598                         spin_lock_irqsave(&priv->lock, flags);
3599                         priv->usb.port_status =
3600                                 cvmx_usb_get_status(&priv->usb);
3601                         spin_unlock_irqrestore(&priv->lock, flags);
3602                         break;
3603                 case USB_PORT_FEAT_C_ENABLE:
3604                         dev_dbg(dev, " C_ENABLE\n");
3605                         /*
3606                          * Clears the driver's internal Port Enable/Disable
3607                          * Change flag.
3608                          */
3609                         spin_lock_irqsave(&priv->lock, flags);
3610                         priv->usb.port_status =
3611                                 cvmx_usb_get_status(&priv->usb);
3612                         spin_unlock_irqrestore(&priv->lock, flags);
3613                         break;
3614                 case USB_PORT_FEAT_C_SUSPEND:
3615                         dev_dbg(dev, " C_SUSPEND\n");
3616                         /*
3617                          * Clears the driver's internal Port Suspend Change
3618                          * flag, which is set when resume signaling on the host
3619                          * port is complete.
3620                          */
3621                         break;
3622                 case USB_PORT_FEAT_C_OVER_CURRENT:
3623                         dev_dbg(dev, " C_OVER_CURRENT\n");
3624                         /* Clears the driver's overcurrent Change flag */
3625                         spin_lock_irqsave(&priv->lock, flags);
3626                         priv->usb.port_status =
3627                                 cvmx_usb_get_status(&priv->usb);
3628                         spin_unlock_irqrestore(&priv->lock, flags);
3629                         break;
3630                 default:
3631                         dev_dbg(dev, " UNKNOWN\n");
3632                         return -EINVAL;
3633                 }
3634                 break;
3635         case GetHubDescriptor:
3636                 dev_dbg(dev, "GetHubDescriptor\n");
3637                 desc = (struct usb_hub_descriptor *)buf;
3638                 desc->bDescLength = 9;
3639                 desc->bDescriptorType = 0x29;
3640                 desc->bNbrPorts = 1;
3641                 desc->wHubCharacteristics = 0x08;
3642                 desc->bPwrOn2PwrGood = 1;
3643                 desc->bHubContrCurrent = 0;
3644                 desc->u.hs.DeviceRemovable[0] = 0;
3645                 desc->u.hs.DeviceRemovable[1] = 0xff;
3646                 break;
3647         case GetHubStatus:
3648                 dev_dbg(dev, "GetHubStatus\n");
3649                 *(__le32 *) buf = 0;
3650                 break;
3651         case GetPortStatus:
3652                 dev_dbg(dev, "GetPortStatus\n");
3653                 if (wIndex != 1) {
3654                         dev_dbg(dev, " INVALID\n");
3655                         return -EINVAL;
3656                 }
3657 
3658                 spin_lock_irqsave(&priv->lock, flags);
3659                 usb_port_status = cvmx_usb_get_status(&priv->usb);
3660                 spin_unlock_irqrestore(&priv->lock, flags);
3661                 port_status = 0;
3662 
3663                 if (usb_port_status.connect_change) {
3664                         port_status |= (1 << USB_PORT_FEAT_C_CONNECTION);
3665                         dev_dbg(dev, " C_CONNECTION\n");
3666                 }
3667 
3668                 if (usb_port_status.port_enabled) {
3669                         port_status |= (1 << USB_PORT_FEAT_C_ENABLE);
3670                         dev_dbg(dev, " C_ENABLE\n");
3671                 }
3672 
3673                 if (usb_port_status.connected) {
3674                         port_status |= (1 << USB_PORT_FEAT_CONNECTION);
3675                         dev_dbg(dev, " CONNECTION\n");
3676                 }
3677 
3678                 if (usb_port_status.port_enabled) {
3679                         port_status |= (1 << USB_PORT_FEAT_ENABLE);
3680                         dev_dbg(dev, " ENABLE\n");
3681                 }
3682 
3683                 if (usb_port_status.port_over_current) {
3684                         port_status |= (1 << USB_PORT_FEAT_OVER_CURRENT);
3685                         dev_dbg(dev, " OVER_CURRENT\n");
3686                 }
3687 
3688                 if (usb_port_status.port_powered) {
3689                         port_status |= (1 << USB_PORT_FEAT_POWER);
3690                         dev_dbg(dev, " POWER\n");
3691                 }
3692 
3693                 if (usb_port_status.port_speed == CVMX_USB_SPEED_HIGH) {
3694                         port_status |= USB_PORT_STAT_HIGH_SPEED;
3695                         dev_dbg(dev, " HIGHSPEED\n");
3696                 } else if (usb_port_status.port_speed == CVMX_USB_SPEED_LOW) {
3697                         port_status |= (1 << USB_PORT_FEAT_LOWSPEED);
3698                         dev_dbg(dev, " LOWSPEED\n");
3699                 }
3700 
3701                 *((__le32 *) buf) = cpu_to_le32(port_status);
3702                 break;
3703         case SetHubFeature:
3704                 dev_dbg(dev, "SetHubFeature\n");
3705                 /* No HUB features supported */
3706                 break;
3707         case SetPortFeature:
3708                 dev_dbg(dev, "SetPortFeature\n");
3709                 if (wIndex != 1) {
3710                         dev_dbg(dev, " INVALID\n");
3711                         return -EINVAL;
3712                 }
3713 
3714                 switch (wValue) {
3715                 case USB_PORT_FEAT_SUSPEND:
3716                         dev_dbg(dev, " SUSPEND\n");
3717                         return -EINVAL;
3718                 case USB_PORT_FEAT_POWER:
3719                         dev_dbg(dev, " POWER\n");
3720                         return -EINVAL;
3721                 case USB_PORT_FEAT_RESET:
3722                         dev_dbg(dev, " RESET\n");
3723                         spin_lock_irqsave(&priv->lock, flags);
3724                         cvmx_usb_disable(&priv->usb);
3725                         if (cvmx_usb_enable(&priv->usb))
3726                                 dev_dbg(dev, "Failed to enable the port\n");
3727                         spin_unlock_irqrestore(&priv->lock, flags);
3728                         return 0;
3729                 case USB_PORT_FEAT_INDICATOR:
3730                         dev_dbg(dev, " INDICATOR\n");
3731                         /* Not supported */
3732                         break;
3733                 default:
3734                         dev_dbg(dev, " UNKNOWN\n");
3735                         return -EINVAL;
3736                 }
3737                 break;
3738         default:
3739                 dev_dbg(dev, "Unknown root hub request\n");
3740                 return -EINVAL;
3741         }
3742         return 0;
3743 }
3744 
3745 static const struct hc_driver octeon_hc_driver = {
3746         .description            = "Octeon USB",
3747         .product_desc           = "Octeon Host Controller",
3748         .hcd_priv_size          = sizeof(struct octeon_hcd),
3749         .irq                    = octeon_usb_irq,
3750         .flags                  = HCD_MEMORY | HCD_USB2,
3751         .start                  = octeon_usb_start,
3752         .stop                   = octeon_usb_stop,
3753         .urb_enqueue            = octeon_usb_urb_enqueue,
3754         .urb_dequeue            = octeon_usb_urb_dequeue,
3755         .endpoint_disable       = octeon_usb_endpoint_disable,
3756         .get_frame_number       = octeon_usb_get_frame_number,
3757         .hub_status_data        = octeon_usb_hub_status_data,
3758         .hub_control            = octeon_usb_hub_control,
3759         .map_urb_for_dma        = octeon_map_urb_for_dma,
3760         .unmap_urb_for_dma      = octeon_unmap_urb_for_dma,
3761 };
3762 
3763 static int octeon_usb_probe(struct platform_device *pdev)
3764 {
3765         int status;
3766         int initialize_flags;
3767         int usb_num;
3768         struct resource *res_mem;
3769         struct device_node *usbn_node;
3770         int irq = platform_get_irq(pdev, 0);
3771         struct device *dev = &pdev->dev;
3772         struct octeon_hcd *priv;
3773         struct usb_hcd *hcd;
3774         unsigned long flags;
3775         u32 clock_rate = 48000000;
3776         bool is_crystal_clock = false;
3777         const char *clock_type;
3778         int i;
3779 
3780         if (dev->of_node == NULL) {
3781                 dev_err(dev, "Error: empty of_node\n");
3782                 return -ENXIO;
3783         }
3784         usbn_node = dev->of_node->parent;
3785 
3786         i = of_property_read_u32(usbn_node,
3787                                  "refclk-frequency", &clock_rate);
3788         if (i) {
3789                 dev_err(dev, "No USBN \"refclk-frequency\"\n");
3790                 return -ENXIO;
3791         }
3792         switch (clock_rate) {
3793         case 12000000:
3794                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
3795                 break;
3796         case 24000000:
3797                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
3798                 break;
3799         case 48000000:
3800                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
3801                 break;
3802         default:
3803                 dev_err(dev, "Illebal USBN \"refclk-frequency\" %u\n",
3804                                 clock_rate);
3805                 return -ENXIO;
3806 
3807         }
3808 
3809         i = of_property_read_string(usbn_node,
3810                                     "refclk-type", &clock_type);
3811 
3812         if (!i && strcmp("crystal", clock_type) == 0)
3813                 is_crystal_clock = true;
3814 
3815         if (is_crystal_clock)
3816                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
3817         else
3818                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
3819 
3820         res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3821         if (res_mem == NULL) {
3822                 dev_err(dev, "found no memory resource\n");
3823                 return -ENXIO;
3824         }
3825         usb_num = (res_mem->start >> 44) & 1;
3826 
3827         if (irq < 0) {
3828                 /* Defective device tree, but we know how to fix it. */
3829                 irq_hw_number_t hwirq = usb_num ? (1 << 6) + 17 : 56;
3830 
3831                 irq = irq_create_mapping(NULL, hwirq);
3832         }
3833 
3834         /*
3835          * Set the DMA mask to 64bits so we get buffers already translated for
3836          * DMA.
3837          */
3838         dev->coherent_dma_mask = ~0;
3839         dev->dma_mask = &dev->coherent_dma_mask;
3840 
3841         /*
3842          * Only cn52XX and cn56XX have DWC_OTG USB hardware and the
3843          * IOB priority registers.  Under heavy network load USB
3844          * hardware can be starved by the IOB causing a crash.  Give
3845          * it a priority boost if it has been waiting more than 400
3846          * cycles to avoid this situation.
3847          *
3848          * Testing indicates that a cnt_val of 8192 is not sufficient,
3849          * but no failures are seen with 4096.  We choose a value of
3850          * 400 to give a safety factor of 10.
3851          */
3852         if (OCTEON_IS_MODEL(OCTEON_CN52XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
3853                 union cvmx_iob_n2c_l2c_pri_cnt pri_cnt;
3854 
3855                 pri_cnt.u64 = 0;
3856                 pri_cnt.s.cnt_enb = 1;
3857                 pri_cnt.s.cnt_val = 400;
3858                 cvmx_write_csr(CVMX_IOB_N2C_L2C_PRI_CNT, pri_cnt.u64);
3859         }
3860 
3861         hcd = usb_create_hcd(&octeon_hc_driver, dev, dev_name(dev));
3862         if (!hcd) {
3863                 dev_dbg(dev, "Failed to allocate memory for HCD\n");
3864                 return -1;
3865         }
3866         hcd->uses_new_polling = 1;
3867         priv = (struct octeon_hcd *)hcd->hcd_priv;
3868 
3869         spin_lock_init(&priv->lock);
3870 
3871         tasklet_init(&priv->dequeue_tasklet, octeon_usb_urb_dequeue_work,
3872                      (unsigned long)priv);
3873         INIT_LIST_HEAD(&priv->dequeue_list);
3874 
3875         status = cvmx_usb_initialize(&priv->usb, usb_num, initialize_flags);
3876         if (status) {
3877                 dev_dbg(dev, "USB initialization failed with %d\n", status);
3878                 kfree(hcd);
3879                 return -1;
3880         }
3881 
3882         /* This delay is needed for CN3010, but I don't know why... */
3883         mdelay(10);
3884 
3885         spin_lock_irqsave(&priv->lock, flags);
3886         cvmx_usb_poll(&priv->usb);
3887         spin_unlock_irqrestore(&priv->lock, flags);
3888 
3889         status = usb_add_hcd(hcd, irq, 0);
3890         if (status) {
3891                 dev_dbg(dev, "USB add HCD failed with %d\n", status);
3892                 kfree(hcd);
3893                 return -1;
3894         }
3895         device_wakeup_enable(hcd->self.controller);
3896 
3897         dev_info(dev, "Registered HCD for port %d on irq %d\n", usb_num, irq);
3898 
3899         return 0;
3900 }
3901 
3902 static int octeon_usb_remove(struct platform_device *pdev)
3903 {
3904         int status;
3905         struct device *dev = &pdev->dev;
3906         struct usb_hcd *hcd = dev_get_drvdata(dev);
3907         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3908         unsigned long flags;
3909 
3910         usb_remove_hcd(hcd);
3911         tasklet_kill(&priv->dequeue_tasklet);
3912         spin_lock_irqsave(&priv->lock, flags);
3913         status = cvmx_usb_shutdown(&priv->usb);
3914         spin_unlock_irqrestore(&priv->lock, flags);
3915         if (status)
3916                 dev_dbg(dev, "USB shutdown failed with %d\n", status);
3917 
3918         kfree(hcd);
3919 
3920         return 0;
3921 }
3922 
3923 static struct of_device_id octeon_usb_match[] = {
3924         {
3925                 .compatible = "cavium,octeon-5750-usbc",
3926         },
3927         {},
3928 };
3929 
3930 static struct platform_driver octeon_usb_driver = {
3931         .driver = {
3932                 .name       = "OcteonUSB",
3933                 .owner          = THIS_MODULE,
3934                 .of_match_table = octeon_usb_match,
3935         },
3936         .probe      = octeon_usb_probe,
3937         .remove     = octeon_usb_remove,
3938 };
3939 
3940 static int __init octeon_usb_driver_init(void)
3941 {
3942         if (usb_disabled())
3943                 return 0;
3944 
3945         return platform_driver_register(&octeon_usb_driver);
3946 }
3947 module_init(octeon_usb_driver_init);
3948 
3949 static void __exit octeon_usb_driver_exit(void)
3950 {
3951         if (usb_disabled())
3952                 return;
3953 
3954         platform_driver_unregister(&octeon_usb_driver);
3955 }
3956 module_exit(octeon_usb_driver_exit);
3957 
3958 MODULE_LICENSE("GPL");
3959 MODULE_AUTHOR("Cavium, Inc. <support@cavium.com>");
3960 MODULE_DESCRIPTION("Cavium Inc. OCTEON USB Host driver.");
3961 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us