Version:  2.0.40 2.2.26 2.4.37 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2

Linux/drivers/staging/octeon-usb/octeon-hcd.c

  1 /*
  2  * This file is subject to the terms and conditions of the GNU General Public
  3  * License.  See the file "COPYING" in the main directory of this archive
  4  * for more details.
  5  *
  6  * Copyright (C) 2008 Cavium Networks
  7  *
  8  * Some parts of the code were originally released under BSD license:
  9  *
 10  * Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
 11  * reserved.
 12  *
 13  * Redistribution and use in source and binary forms, with or without
 14  * modification, are permitted provided that the following conditions are
 15  * met:
 16  *
 17  *   * Redistributions of source code must retain the above copyright
 18  *     notice, this list of conditions and the following disclaimer.
 19  *
 20  *   * Redistributions in binary form must reproduce the above
 21  *     copyright notice, this list of conditions and the following
 22  *     disclaimer in the documentation and/or other materials provided
 23  *     with the distribution.
 24  *
 25  *   * Neither the name of Cavium Networks nor the names of
 26  *     its contributors may be used to endorse or promote products
 27  *     derived from this software without specific prior written
 28  *     permission.
 29  *
 30  * This Software, including technical data, may be subject to U.S. export
 31  * control laws, including the U.S. Export Administration Act and its associated
 32  * regulations, and may be subject to export or import regulations in other
 33  * countries.
 34  *
 35  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
 36  * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
 37  * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
 38  * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION
 39  * OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
 40  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
 41  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
 42  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
 43  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
 44  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
 45  */
 46 #include <linux/kernel.h>
 47 #include <linux/module.h>
 48 #include <linux/init.h>
 49 #include <linux/pci.h>
 50 #include <linux/prefetch.h>
 51 #include <linux/interrupt.h>
 52 #include <linux/platform_device.h>
 53 #include <linux/usb.h>
 54 
 55 #include <linux/time.h>
 56 #include <linux/delay.h>
 57 
 58 #include <asm/octeon/cvmx.h>
 59 #include <asm/octeon/cvmx-iob-defs.h>
 60 
 61 #include <linux/usb/hcd.h>
 62 
 63 #include <linux/err.h>
 64 
 65 #include <asm/octeon/octeon.h>
 66 #include <asm/octeon/cvmx-helper.h>
 67 #include <asm/octeon/cvmx-sysinfo.h>
 68 #include <asm/octeon/cvmx-helper-board.h>
 69 
 70 #include "octeon-hcd.h"
 71 
 72 /**
 73  * enum cvmx_usb_speed - the possible USB device speeds
 74  *
 75  * @CVMX_USB_SPEED_HIGH: Device is operation at 480Mbps
 76  * @CVMX_USB_SPEED_FULL: Device is operation at 12Mbps
 77  * @CVMX_USB_SPEED_LOW:  Device is operation at 1.5Mbps
 78  */
 79 enum cvmx_usb_speed {
 80         CVMX_USB_SPEED_HIGH = 0,
 81         CVMX_USB_SPEED_FULL = 1,
 82         CVMX_USB_SPEED_LOW = 2,
 83 };
 84 
 85 /**
 86  * enum cvmx_usb_transfer - the possible USB transfer types
 87  *
 88  * @CVMX_USB_TRANSFER_CONTROL:     USB transfer type control for hub and status
 89  *                                 transfers
 90  * @CVMX_USB_TRANSFER_ISOCHRONOUS: USB transfer type isochronous for low
 91  *                                 priority periodic transfers
 92  * @CVMX_USB_TRANSFER_BULK:        USB transfer type bulk for large low priority
 93  *                                 transfers
 94  * @CVMX_USB_TRANSFER_INTERRUPT:   USB transfer type interrupt for high priority
 95  *                                 periodic transfers
 96  */
 97 enum cvmx_usb_transfer {
 98         CVMX_USB_TRANSFER_CONTROL = 0,
 99         CVMX_USB_TRANSFER_ISOCHRONOUS = 1,
100         CVMX_USB_TRANSFER_BULK = 2,
101         CVMX_USB_TRANSFER_INTERRUPT = 3,
102 };
103 
104 /**
105  * enum cvmx_usb_direction - the transfer directions
106  *
107  * @CVMX_USB_DIRECTION_OUT: Data is transferring from Octeon to the device/host
108  * @CVMX_USB_DIRECTION_IN:  Data is transferring from the device/host to Octeon
109  */
110 enum cvmx_usb_direction {
111         CVMX_USB_DIRECTION_OUT,
112         CVMX_USB_DIRECTION_IN,
113 };
114 
115 /**
116  * enum cvmx_usb_complete - possible callback function status codes
117  *
118  * @CVMX_USB_COMPLETE_SUCCESS:    The transaction / operation finished without
119  *                                any errors
120  * @CVMX_USB_COMPLETE_SHORT:      FIXME: This is currently not implemented
121  * @CVMX_USB_COMPLETE_CANCEL:     The transaction was canceled while in flight
122  *                                by a user call to cvmx_usb_cancel
123  * @CVMX_USB_COMPLETE_ERROR:      The transaction aborted with an unexpected
124  *                                error status
125  * @CVMX_USB_COMPLETE_STALL:      The transaction received a USB STALL response
126  *                                from the device
127  * @CVMX_USB_COMPLETE_XACTERR:    The transaction failed with an error from the
128  *                                device even after a number of retries
129  * @CVMX_USB_COMPLETE_DATATGLERR: The transaction failed with a data toggle
130  *                                error even after a number of retries
131  * @CVMX_USB_COMPLETE_BABBLEERR:  The transaction failed with a babble error
132  * @CVMX_USB_COMPLETE_FRAMEERR:   The transaction failed with a frame error
133  *                                even after a number of retries
134  */
135 enum cvmx_usb_complete {
136         CVMX_USB_COMPLETE_SUCCESS,
137         CVMX_USB_COMPLETE_SHORT,
138         CVMX_USB_COMPLETE_CANCEL,
139         CVMX_USB_COMPLETE_ERROR,
140         CVMX_USB_COMPLETE_STALL,
141         CVMX_USB_COMPLETE_XACTERR,
142         CVMX_USB_COMPLETE_DATATGLERR,
143         CVMX_USB_COMPLETE_BABBLEERR,
144         CVMX_USB_COMPLETE_FRAMEERR,
145 };
146 
147 /**
148  * struct cvmx_usb_port_status - the USB port status information
149  *
150  * @port_enabled:       1 = Usb port is enabled, 0 = disabled
151  * @port_over_current:  1 = Over current detected, 0 = Over current not
152  *                      detected. Octeon doesn't support over current detection.
153  * @port_powered:       1 = Port power is being supplied to the device, 0 =
154  *                      power is off. Octeon doesn't support turning port power
155  *                      off.
156  * @port_speed:         Current port speed.
157  * @connected:          1 = A device is connected to the port, 0 = No device is
158  *                      connected.
159  * @connect_change:     1 = Device connected state changed since the last set
160  *                      status call.
161  */
162 struct cvmx_usb_port_status {
163         uint32_t reserved               : 25;
164         uint32_t port_enabled           : 1;
165         uint32_t port_over_current      : 1;
166         uint32_t port_powered           : 1;
167         enum cvmx_usb_speed port_speed  : 2;
168         uint32_t connected              : 1;
169         uint32_t connect_change         : 1;
170 };
171 
172 /**
173  * struct cvmx_usb_iso_packet - descriptor for Isochronous packets
174  *
175  * @offset:     This is the offset in bytes into the main buffer where this data
176  *              is stored.
177  * @length:     This is the length in bytes of the data.
178  * @status:     This is the status of this individual packet transfer.
179  */
180 struct cvmx_usb_iso_packet {
181         int offset;
182         int length;
183         enum cvmx_usb_complete status;
184 };
185 
186 /**
187  * enum cvmx_usb_initialize_flags - flags used by the initialization function
188  *
189  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI:    The USB port uses a 12MHz crystal
190  *                                            as clock source at USB_XO and
191  *                                            USB_XI.
192  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND:   The USB port uses 12/24/48MHz 2.5V
193  *                                            board clock source at USB_XO.
194  *                                            USB_XI should be tied to GND.
195  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK: Mask for clock speed field
196  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:    Speed of reference clock or
197  *                                            crystal
198  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:    Speed of reference clock
199  * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:    Speed of reference clock
200  * @CVMX_USB_INITIALIZE_FLAGS_NO_DMA:         Disable DMA and used polled IO for
201  *                                            data transfer use for the USB
202  */
203 enum cvmx_usb_initialize_flags {
204         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI           = 1 << 0,
205         CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND          = 1 << 1,
206         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK        = 3 << 3,
207         CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ           = 1 << 3,
208         CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ           = 2 << 3,
209         CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ           = 3 << 3,
210         /* Bits 3-4 used to encode the clock frequency */
211         CVMX_USB_INITIALIZE_FLAGS_NO_DMA                = 1 << 5,
212 };
213 
214 /**
215  * enum cvmx_usb_pipe_flags - internal flags for a pipe.
216  *
217  * @CVMX_USB_PIPE_FLAGS_SCHEDULED: Used internally to determine if a pipe is
218  *                                 actively using hardware.
219  * @CVMX_USB_PIPE_FLAGS_NEED_PING: Used internally to determine if a high speed
220  *                                 pipe is in the ping state.
221  */
222 enum cvmx_usb_pipe_flags {
223         CVMX_USB_PIPE_FLAGS_SCHEDULED   = 1 << 17,
224         CVMX_USB_PIPE_FLAGS_NEED_PING   = 1 << 18,
225 };
226 
227 /* Maximum number of times to retry failed transactions */
228 #define MAX_RETRIES             3
229 
230 /* Maximum number of hardware channels supported by the USB block */
231 #define MAX_CHANNELS            8
232 
233 /*
234  * The low level hardware can transfer a maximum of this number of bytes in each
235  * transfer. The field is 19 bits wide
236  */
237 #define MAX_TRANSFER_BYTES      ((1<<19)-1)
238 
239 /*
240  * The low level hardware can transfer a maximum of this number of packets in
241  * each transfer. The field is 10 bits wide
242  */
243 #define MAX_TRANSFER_PACKETS    ((1<<10)-1)
244 
245 /**
246  * Logical transactions may take numerous low level
247  * transactions, especially when splits are concerned. This
248  * enum represents all of the possible stages a transaction can
249  * be in. Note that split completes are always even. This is so
250  * the NAK handler can backup to the previous low level
251  * transaction with a simple clearing of bit 0.
252  */
253 enum cvmx_usb_stage {
254         CVMX_USB_STAGE_NON_CONTROL,
255         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
256         CVMX_USB_STAGE_SETUP,
257         CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
258         CVMX_USB_STAGE_DATA,
259         CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
260         CVMX_USB_STAGE_STATUS,
261         CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
262 };
263 
264 /**
265  * struct cvmx_usb_transaction - describes each pending USB transaction
266  *                               regardless of type. These are linked together
267  *                               to form a list of pending requests for a pipe.
268  *
269  * @node:               List node for transactions in the pipe.
270  * @type:               Type of transaction, duplicated of the pipe.
271  * @flags:              State flags for this transaction.
272  * @buffer:             User's physical buffer address to read/write.
273  * @buffer_length:      Size of the user's buffer in bytes.
274  * @control_header:     For control transactions, physical address of the 8
275  *                      byte standard header.
276  * @iso_start_frame:    For ISO transactions, the starting frame number.
277  * @iso_number_packets: For ISO transactions, the number of packets in the
278  *                      request.
279  * @iso_packets:        For ISO transactions, the sub packets in the request.
280  * @actual_bytes:       Actual bytes transfer for this transaction.
281  * @stage:              For control transactions, the current stage.
282  * @urb:                URB.
283  */
284 struct cvmx_usb_transaction {
285         struct list_head node;
286         enum cvmx_usb_transfer type;
287         uint64_t buffer;
288         int buffer_length;
289         uint64_t control_header;
290         int iso_start_frame;
291         int iso_number_packets;
292         struct cvmx_usb_iso_packet *iso_packets;
293         int xfersize;
294         int pktcnt;
295         int retries;
296         int actual_bytes;
297         enum cvmx_usb_stage stage;
298         struct urb *urb;
299 };
300 
301 /**
302  * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
303  *                        and some USB device. It contains a list of pending
304  *                        request to the device.
305  *
306  * @node:               List node for pipe list
307  * @next:               Pipe after this one in the list
308  * @transactions:       List of pending transactions
309  * @interval:           For periodic pipes, the interval between packets in
310  *                      frames
311  * @next_tx_frame:      The next frame this pipe is allowed to transmit on
312  * @flags:              State flags for this pipe
313  * @device_speed:       Speed of device connected to this pipe
314  * @transfer_type:      Type of transaction supported by this pipe
315  * @transfer_dir:       IN or OUT. Ignored for Control
316  * @multi_count:        Max packet in a row for the device
317  * @max_packet:         The device's maximum packet size in bytes
318  * @device_addr:        USB device address at other end of pipe
319  * @endpoint_num:       USB endpoint number at other end of pipe
320  * @hub_device_addr:    Hub address this device is connected to
321  * @hub_port:           Hub port this device is connected to
322  * @pid_toggle:         This toggles between 0/1 on every packet send to track
323  *                      the data pid needed
324  * @channel:            Hardware DMA channel for this pipe
325  * @split_sc_frame:     The low order bits of the frame number the split
326  *                      complete should be sent on
327  */
328 struct cvmx_usb_pipe {
329         struct list_head node;
330         struct list_head transactions;
331         uint64_t interval;
332         uint64_t next_tx_frame;
333         enum cvmx_usb_pipe_flags flags;
334         enum cvmx_usb_speed device_speed;
335         enum cvmx_usb_transfer transfer_type;
336         enum cvmx_usb_direction transfer_dir;
337         int multi_count;
338         uint16_t max_packet;
339         uint8_t device_addr;
340         uint8_t endpoint_num;
341         uint8_t hub_device_addr;
342         uint8_t hub_port;
343         uint8_t pid_toggle;
344         uint8_t channel;
345         int8_t split_sc_frame;
346 };
347 
348 struct cvmx_usb_tx_fifo {
349         struct {
350                 int channel;
351                 int size;
352                 uint64_t address;
353         } entry[MAX_CHANNELS+1];
354         int head;
355         int tail;
356 };
357 
358 /**
359  * struct cvmx_usb_state - the state of the USB block
360  *
361  * init_flags:             Flags passed to initialize.
362  * index:                  Which USB block this is for.
363  * idle_hardware_channels: Bit set for every idle hardware channel.
364  * usbcx_hprt:             Stored port status so we don't need to read a CSR to
365  *                         determine splits.
366  * pipe_for_channel:       Map channels to pipes.
367  * pipe:                   Storage for pipes.
368  * indent:                 Used by debug output to indent functions.
369  * port_status:            Last port status used for change notification.
370  * idle_pipes:             List of open pipes that have no transactions.
371  * active_pipes:           Active pipes indexed by transfer type.
372  * frame_number:           Increments every SOF interrupt for time keeping.
373  * active_split:           Points to the current active split, or NULL.
374  */
375 struct cvmx_usb_state {
376         int init_flags;
377         int index;
378         int idle_hardware_channels;
379         union cvmx_usbcx_hprt usbcx_hprt;
380         struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
381         int indent;
382         struct cvmx_usb_port_status port_status;
383         struct list_head idle_pipes;
384         struct list_head active_pipes[4];
385         uint64_t frame_number;
386         struct cvmx_usb_transaction *active_split;
387         struct cvmx_usb_tx_fifo periodic;
388         struct cvmx_usb_tx_fifo nonperiodic;
389 };
390 
391 struct octeon_hcd {
392         spinlock_t lock;
393         struct cvmx_usb_state usb;
394 };
395 
396 /* This macro spins on a register waiting for it to reach a condition. */
397 #define CVMX_WAIT_FOR_FIELD32(address, _union, cond, timeout_usec)          \
398         ({int result;                                                       \
399         do {                                                                \
400                 uint64_t done = cvmx_get_cycle() + (uint64_t)timeout_usec * \
401                         octeon_get_clock_rate() / 1000000;                  \
402                 union _union c;                                             \
403                                                                             \
404                 while (1) {                                                 \
405                         c.u32 = cvmx_usb_read_csr32(usb, address);          \
406                                                                             \
407                         if (cond) {                                         \
408                                 result = 0;                                 \
409                                 break;                                      \
410                         } else if (cvmx_get_cycle() > done) {               \
411                                 result = -1;                                \
412                                 break;                                      \
413                         } else                                              \
414                                 cvmx_wait(100);                             \
415                 }                                                           \
416         } while (0);                                                        \
417         result; })
418 
419 /*
420  * This macro logically sets a single field in a CSR. It does the sequence
421  * read, modify, and write
422  */
423 #define USB_SET_FIELD32(address, _union, field, value)          \
424         do {                                                    \
425                 union _union c;                                 \
426                                                                 \
427                 c.u32 = cvmx_usb_read_csr32(usb, address);      \
428                 c.s.field = value;                              \
429                 cvmx_usb_write_csr32(usb, address, c.u32);      \
430         } while (0)
431 
432 /* Returns the IO address to push/pop stuff data from the FIFOs */
433 #define USB_FIFO_ADDRESS(channel, usb_index) \
434         (CVMX_USBCX_GOTGCTL(usb_index) + ((channel)+1)*0x1000)
435 
436 /**
437  * struct octeon_temp_buffer - a bounce buffer for USB transfers
438  * @orig_buffer: the original buffer passed by the USB stack
439  * @data:        the newly allocated temporary buffer (excluding meta-data)
440  *
441  * Both the DMA engine and FIFO mode will always transfer full 32-bit words. If
442  * the buffer is too short, we need to allocate a temporary one, and this struct
443  * represents it.
444  */
445 struct octeon_temp_buffer {
446         void *orig_buffer;
447         u8 data[0];
448 };
449 
450 static inline struct octeon_hcd *cvmx_usb_to_octeon(struct cvmx_usb_state *p)
451 {
452         return container_of(p, struct octeon_hcd, usb);
453 }
454 
455 static inline struct usb_hcd *octeon_to_hcd(struct octeon_hcd *p)
456 {
457         return container_of((void *)p, struct usb_hcd, hcd_priv);
458 }
459 
460 /**
461  * octeon_alloc_temp_buffer - allocate a temporary buffer for USB transfer
462  *                            (if needed)
463  * @urb:        URB.
464  * @mem_flags:  Memory allocation flags.
465  *
466  * This function allocates a temporary bounce buffer whenever it's needed
467  * due to HW limitations.
468  */
469 static int octeon_alloc_temp_buffer(struct urb *urb, gfp_t mem_flags)
470 {
471         struct octeon_temp_buffer *temp;
472 
473         if (urb->num_sgs || urb->sg ||
474             (urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP) ||
475             !(urb->transfer_buffer_length % sizeof(u32)))
476                 return 0;
477 
478         temp = kmalloc(ALIGN(urb->transfer_buffer_length, sizeof(u32)) +
479                        sizeof(*temp), mem_flags);
480         if (!temp)
481                 return -ENOMEM;
482 
483         temp->orig_buffer = urb->transfer_buffer;
484         if (usb_urb_dir_out(urb))
485                 memcpy(temp->data, urb->transfer_buffer,
486                        urb->transfer_buffer_length);
487         urb->transfer_buffer = temp->data;
488         urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER;
489 
490         return 0;
491 }
492 
493 /**
494  * octeon_free_temp_buffer - free a temporary buffer used by USB transfers.
495  * @urb: URB.
496  *
497  * Frees a buffer allocated by octeon_alloc_temp_buffer().
498  */
499 static void octeon_free_temp_buffer(struct urb *urb)
500 {
501         struct octeon_temp_buffer *temp;
502         size_t length;
503 
504         if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER))
505                 return;
506 
507         temp = container_of(urb->transfer_buffer, struct octeon_temp_buffer,
508                             data);
509         if (usb_urb_dir_in(urb)) {
510                 if (usb_pipeisoc(urb->pipe))
511                         length = urb->transfer_buffer_length;
512                 else
513                         length = urb->actual_length;
514 
515                 memcpy(temp->orig_buffer, urb->transfer_buffer, length);
516         }
517         urb->transfer_buffer = temp->orig_buffer;
518         urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER;
519         kfree(temp);
520 }
521 
522 /**
523  * octeon_map_urb_for_dma - Octeon-specific map_urb_for_dma().
524  * @hcd:        USB HCD structure.
525  * @urb:        URB.
526  * @mem_flags:  Memory allocation flags.
527  */
528 static int octeon_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
529                                   gfp_t mem_flags)
530 {
531         int ret;
532 
533         ret = octeon_alloc_temp_buffer(urb, mem_flags);
534         if (ret)
535                 return ret;
536 
537         ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
538         if (ret)
539                 octeon_free_temp_buffer(urb);
540 
541         return ret;
542 }
543 
544 /**
545  * octeon_unmap_urb_for_dma - Octeon-specific unmap_urb_for_dma()
546  * @hcd:        USB HCD structure.
547  * @urb:        URB.
548  */
549 static void octeon_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
550 {
551         usb_hcd_unmap_urb_for_dma(hcd, urb);
552         octeon_free_temp_buffer(urb);
553 }
554 
555 /**
556  * Read a USB 32bit CSR. It performs the necessary address swizzle
557  * for 32bit CSRs and logs the value in a readable format if
558  * debugging is on.
559  *
560  * @usb:     USB block this access is for
561  * @address: 64bit address to read
562  *
563  * Returns: Result of the read
564  */
565 static inline uint32_t cvmx_usb_read_csr32(struct cvmx_usb_state *usb,
566                                            uint64_t address)
567 {
568         uint32_t result = cvmx_read64_uint32(address ^ 4);
569         return result;
570 }
571 
572 
573 /**
574  * Write a USB 32bit CSR. It performs the necessary address
575  * swizzle for 32bit CSRs and logs the value in a readable format
576  * if debugging is on.
577  *
578  * @usb:     USB block this access is for
579  * @address: 64bit address to write
580  * @value:   Value to write
581  */
582 static inline void cvmx_usb_write_csr32(struct cvmx_usb_state *usb,
583                                         uint64_t address, uint32_t value)
584 {
585         cvmx_write64_uint32(address ^ 4, value);
586         cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
587 }
588 
589 /**
590  * Return non zero if this pipe connects to a non HIGH speed
591  * device through a high speed hub.
592  *
593  * @usb:    USB block this access is for
594  * @pipe:   Pipe to check
595  *
596  * Returns: Non zero if we need to do split transactions
597  */
598 static inline int cvmx_usb_pipe_needs_split(struct cvmx_usb_state *usb,
599                                             struct cvmx_usb_pipe *pipe)
600 {
601         return pipe->device_speed != CVMX_USB_SPEED_HIGH &&
602                usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH;
603 }
604 
605 
606 /**
607  * Trivial utility function to return the correct PID for a pipe
608  *
609  * @pipe:   pipe to check
610  *
611  * Returns: PID for pipe
612  */
613 static inline int cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
614 {
615         if (pipe->pid_toggle)
616                 return 2; /* Data1 */
617         return 0; /* Data0 */
618 }
619 
620 static void cvmx_fifo_setup(struct cvmx_usb_state *usb)
621 {
622         union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
623         union cvmx_usbcx_gnptxfsiz npsiz;
624         union cvmx_usbcx_hptxfsiz psiz;
625 
626         usbcx_ghwcfg3.u32 = cvmx_usb_read_csr32(usb,
627                                                 CVMX_USBCX_GHWCFG3(usb->index));
628 
629         /*
630          * Program the USBC_GRXFSIZ register to select the size of the receive
631          * FIFO (25%).
632          */
633         USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index), cvmx_usbcx_grxfsiz,
634                         rxfdep, usbcx_ghwcfg3.s.dfifodepth / 4);
635 
636         /*
637          * Program the USBC_GNPTXFSIZ register to select the size and the start
638          * address of the non-periodic transmit FIFO for nonperiodic
639          * transactions (50%).
640          */
641         npsiz.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index));
642         npsiz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
643         npsiz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
644         cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index), npsiz.u32);
645 
646         /*
647          * Program the USBC_HPTXFSIZ register to select the size and start
648          * address of the periodic transmit FIFO for periodic transactions
649          * (25%).
650          */
651         psiz.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index));
652         psiz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
653         psiz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
654         cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index), psiz.u32);
655 
656         /* Flush all FIFOs */
657         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
658                         cvmx_usbcx_grstctl, txfnum, 0x10);
659         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
660                         cvmx_usbcx_grstctl, txfflsh, 1);
661         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
662                               cvmx_usbcx_grstctl, c.s.txfflsh == 0, 100);
663         USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
664                         cvmx_usbcx_grstctl, rxfflsh, 1);
665         CVMX_WAIT_FOR_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
666                               cvmx_usbcx_grstctl, c.s.rxfflsh == 0, 100);
667 }
668 
669 /**
670  * Shutdown a USB port after a call to cvmx_usb_initialize().
671  * The port should be disabled with all pipes closed when this
672  * function is called.
673  *
674  * @usb: USB device state populated by cvmx_usb_initialize().
675  *
676  * Returns: 0 or a negative error code.
677  */
678 static int cvmx_usb_shutdown(struct cvmx_usb_state *usb)
679 {
680         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
681 
682         /* Make sure all pipes are closed */
683         if (!list_empty(&usb->idle_pipes) ||
684             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS]) ||
685             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT]) ||
686             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_CONTROL]) ||
687             !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_BULK]))
688                 return -EBUSY;
689 
690         /* Disable the clocks and put them in power on reset */
691         usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
692         usbn_clk_ctl.s.enable = 1;
693         usbn_clk_ctl.s.por = 1;
694         usbn_clk_ctl.s.hclk_rst = 1;
695         usbn_clk_ctl.s.prst = 0;
696         usbn_clk_ctl.s.hrst = 0;
697         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
698         return 0;
699 }
700 
701 /**
702  * Initialize a USB port for use. This must be called before any
703  * other access to the Octeon USB port is made. The port starts
704  * off in the disabled state.
705  *
706  * @dev:         Pointer to struct device for logging purposes.
707  * @usb:         Pointer to struct cvmx_usb_state.
708  *
709  * Returns: 0 or a negative error code.
710  */
711 static int cvmx_usb_initialize(struct device *dev,
712                                struct cvmx_usb_state *usb)
713 {
714         int channel;
715         int divisor;
716         int retries = 0;
717         union cvmx_usbcx_hcfg usbcx_hcfg;
718         union cvmx_usbnx_clk_ctl usbn_clk_ctl;
719         union cvmx_usbcx_gintsts usbc_gintsts;
720         union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
721         union cvmx_usbcx_gintmsk usbcx_gintmsk;
722         union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
723         union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;
724 
725 retry:
726         /*
727          * Power On Reset and PHY Initialization
728          *
729          * 1. Wait for DCOK to assert (nothing to do)
730          *
731          * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
732          *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
733          */
734         usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
735         usbn_clk_ctl.s.por = 1;
736         usbn_clk_ctl.s.hrst = 0;
737         usbn_clk_ctl.s.prst = 0;
738         usbn_clk_ctl.s.hclk_rst = 0;
739         usbn_clk_ctl.s.enable = 0;
740         /*
741          * 2b. Select the USB reference clock/crystal parameters by writing
742          *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
743          */
744         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
745                 /*
746                  * The USB port uses 12/24/48MHz 2.5V board clock
747                  * source at USB_XO. USB_XI should be tied to GND.
748                  * Most Octeon evaluation boards require this setting
749                  */
750                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
751                     OCTEON_IS_MODEL(OCTEON_CN56XX) ||
752                     OCTEON_IS_MODEL(OCTEON_CN50XX))
753                         /* From CN56XX,CN50XX,CN31XX,CN30XX manuals */
754                         usbn_clk_ctl.s.p_rtype = 2; /* p_rclk=1 & p_xenbn=0 */
755                 else
756                         /* From CN52XX manual */
757                         usbn_clk_ctl.s.p_rtype = 1;
758 
759                 switch (usb->init_flags &
760                         CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
761                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
762                         usbn_clk_ctl.s.p_c_sel = 0;
763                         break;
764                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
765                         usbn_clk_ctl.s.p_c_sel = 1;
766                         break;
767                 case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
768                         usbn_clk_ctl.s.p_c_sel = 2;
769                         break;
770                 }
771         } else {
772                 /*
773                  * The USB port uses a 12MHz crystal as clock source
774                  * at USB_XO and USB_XI
775                  */
776                 if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
777                         /* From CN31XX,CN30XX manual */
778                         usbn_clk_ctl.s.p_rtype = 3; /* p_rclk=1 & p_xenbn=1 */
779                 else
780                         /* From CN56XX,CN52XX,CN50XX manuals. */
781                         usbn_clk_ctl.s.p_rtype = 0;
782 
783                 usbn_clk_ctl.s.p_c_sel = 0;
784         }
785         /*
786          * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
787          *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
788          *     such that USB is as close as possible to 125Mhz
789          */
790         divisor = DIV_ROUND_UP(octeon_get_clock_rate(), 125000000);
791         /* Lower than 4 doesn't seem to work properly */
792         if (divisor < 4)
793                 divisor = 4;
794         usbn_clk_ctl.s.divide = divisor;
795         usbn_clk_ctl.s.divide2 = 0;
796         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
797 
798         /* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
799         usbn_clk_ctl.s.hclk_rst = 1;
800         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
801         /* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
802         cvmx_wait(64);
803         /*
804          * 3. Program the power-on reset field in the USBN clock-control
805          *    register:
806          *    USBN_CLK_CTL[POR] = 0
807          */
808         usbn_clk_ctl.s.por = 0;
809         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
810         /* 4. Wait 1 ms for PHY clock to start */
811         mdelay(1);
812         /*
813          * 5. Program the Reset input from automatic test equipment field in the
814          *    USBP control and status register:
815          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
816          */
817         usbn_usbp_ctl_status.u64 =
818                 cvmx_read64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index));
819         usbn_usbp_ctl_status.s.ate_reset = 1;
820         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
821                             usbn_usbp_ctl_status.u64);
822         /* 6. Wait 10 cycles */
823         cvmx_wait(10);
824         /*
825          * 7. Clear ATE_RESET field in the USBN clock-control register:
826          *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
827          */
828         usbn_usbp_ctl_status.s.ate_reset = 0;
829         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
830                             usbn_usbp_ctl_status.u64);
831         /*
832          * 8. Program the PHY reset field in the USBN clock-control register:
833          *    USBN_CLK_CTL[PRST] = 1
834          */
835         usbn_clk_ctl.s.prst = 1;
836         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
837         /*
838          * 9. Program the USBP control and status register to select host or
839          *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
840          *    device
841          */
842         usbn_usbp_ctl_status.s.hst_mode = 0;
843         cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
844                             usbn_usbp_ctl_status.u64);
845         /* 10. Wait 1 us */
846         udelay(1);
847         /*
848          * 11. Program the hreset_n field in the USBN clock-control register:
849          *     USBN_CLK_CTL[HRST] = 1
850          */
851         usbn_clk_ctl.s.hrst = 1;
852         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
853         /* 12. Proceed to USB core initialization */
854         usbn_clk_ctl.s.enable = 1;
855         cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
856         udelay(1);
857 
858         /*
859          * USB Core Initialization
860          *
861          * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
862          *    determine USB core configuration parameters.
863          *
864          *    Nothing needed
865          *
866          * 2. Program the following fields in the global AHB configuration
867          *    register (USBC_GAHBCFG)
868          *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
869          *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
870          *    Nonperiodic TxFIFO empty level (slave mode only),
871          *    USBC_GAHBCFG[NPTXFEMPLVL]
872          *    Periodic TxFIFO empty level (slave mode only),
873          *    USBC_GAHBCFG[PTXFEMPLVL]
874          *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
875          */
876         usbcx_gahbcfg.u32 = 0;
877         usbcx_gahbcfg.s.dmaen = !(usb->init_flags &
878                                   CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
879         usbcx_gahbcfg.s.hbstlen = 0;
880         usbcx_gahbcfg.s.nptxfemplvl = 1;
881         usbcx_gahbcfg.s.ptxfemplvl = 1;
882         usbcx_gahbcfg.s.glblintrmsk = 1;
883         cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
884                              usbcx_gahbcfg.u32);
885 
886         /*
887          * 3. Program the following fields in USBC_GUSBCFG register.
888          *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
889          *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
890          *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
891          *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
892          */
893         usbcx_gusbcfg.u32 = cvmx_usb_read_csr32(usb,
894                                                 CVMX_USBCX_GUSBCFG(usb->index));
895         usbcx_gusbcfg.s.toutcal = 0;
896         usbcx_gusbcfg.s.ddrsel = 0;
897         usbcx_gusbcfg.s.usbtrdtim = 0x5;
898         usbcx_gusbcfg.s.phylpwrclksel = 0;
899         cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
900                              usbcx_gusbcfg.u32);
901 
902         /*
903          * 4. The software must unmask the following bits in the USBC_GINTMSK
904          *    register.
905          *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
906          *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
907          */
908         usbcx_gintmsk.u32 = cvmx_usb_read_csr32(usb,
909                                                 CVMX_USBCX_GINTMSK(usb->index));
910         usbcx_gintmsk.s.otgintmsk = 1;
911         usbcx_gintmsk.s.modemismsk = 1;
912         usbcx_gintmsk.s.hchintmsk = 1;
913         usbcx_gintmsk.s.sofmsk = 0;
914         /* We need RX FIFO interrupts if we don't have DMA */
915         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
916                 usbcx_gintmsk.s.rxflvlmsk = 1;
917         cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
918                              usbcx_gintmsk.u32);
919 
920         /*
921          * Disable all channel interrupts. We'll enable them per channel later.
922          */
923         for (channel = 0; channel < 8; channel++)
924                 cvmx_usb_write_csr32(usb,
925                                      CVMX_USBCX_HCINTMSKX(channel, usb->index),
926                                      0);
927 
928         /*
929          * Host Port Initialization
930          *
931          * 1. Program the host-port interrupt-mask field to unmask,
932          *    USBC_GINTMSK[PRTINT] = 1
933          */
934         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
935                         cvmx_usbcx_gintmsk, prtintmsk, 1);
936         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
937                         cvmx_usbcx_gintmsk, disconnintmsk, 1);
938 
939         /*
940          * 2. Program the USBC_HCFG register to select full-speed host
941          *    or high-speed host.
942          */
943         usbcx_hcfg.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HCFG(usb->index));
944         usbcx_hcfg.s.fslssupp = 0;
945         usbcx_hcfg.s.fslspclksel = 0;
946         cvmx_usb_write_csr32(usb, CVMX_USBCX_HCFG(usb->index), usbcx_hcfg.u32);
947 
948         cvmx_fifo_setup(usb);
949 
950         /*
951          * If the controller is getting port events right after the reset, it
952          * means the initialization failed. Try resetting the controller again
953          * in such case. This is seen to happen after cold boot on DSR-1000N.
954          */
955         usbc_gintsts.u32 = cvmx_usb_read_csr32(usb,
956                                                CVMX_USBCX_GINTSTS(usb->index));
957         cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index),
958                              usbc_gintsts.u32);
959         dev_dbg(dev, "gintsts after reset: 0x%x\n", (int)usbc_gintsts.u32);
960         if (!usbc_gintsts.s.disconnint && !usbc_gintsts.s.prtint)
961                 return 0;
962         if (retries++ >= 5)
963                 return -EAGAIN;
964         dev_info(dev, "controller reset failed (gintsts=0x%x) - retrying\n",
965                  (int)usbc_gintsts.u32);
966         msleep(50);
967         cvmx_usb_shutdown(usb);
968         msleep(50);
969         goto retry;
970 }
971 
972 /**
973  * Reset a USB port. After this call succeeds, the USB port is
974  * online and servicing requests.
975  *
976  * @usb: USB device state populated by cvmx_usb_initialize().
977  */
978 static void cvmx_usb_reset_port(struct cvmx_usb_state *usb)
979 {
980         usb->usbcx_hprt.u32 = cvmx_usb_read_csr32(usb,
981                                                   CVMX_USBCX_HPRT(usb->index));
982 
983         /* Program the port reset bit to start the reset process */
984         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
985                         prtrst, 1);
986 
987         /*
988          * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
989          * process to complete.
990          */
991         mdelay(50);
992 
993         /* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
994         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
995                         prtrst, 0);
996 
997         /*
998          * Read the port speed field to get the enumerated speed,
999          * USBC_HPRT[PRTSPD].
1000          */
1001         usb->usbcx_hprt.u32 = cvmx_usb_read_csr32(usb,
1002                                                   CVMX_USBCX_HPRT(usb->index));
1003 }
1004 
1005 
1006 /**
1007  * Disable a USB port. After this call the USB port will not
1008  * generate data transfers and will not generate events.
1009  * Transactions in process will fail and call their
1010  * associated callbacks.
1011  *
1012  * @usb: USB device state populated by cvmx_usb_initialize().
1013  *
1014  * Returns: 0 or a negative error code.
1015  */
1016 static int cvmx_usb_disable(struct cvmx_usb_state *usb)
1017 {
1018         /* Disable the port */
1019         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
1020                         prtena, 1);
1021         return 0;
1022 }
1023 
1024 
1025 /**
1026  * Get the current state of the USB port. Use this call to
1027  * determine if the usb port has anything connected, is enabled,
1028  * or has some sort of error condition. The return value of this
1029  * call has "changed" bits to signal of the value of some fields
1030  * have changed between calls.
1031  *
1032  * @usb: USB device state populated by cvmx_usb_initialize().
1033  *
1034  * Returns: Port status information
1035  */
1036 static struct cvmx_usb_port_status cvmx_usb_get_status(
1037                 struct cvmx_usb_state *usb)
1038 {
1039         union cvmx_usbcx_hprt usbc_hprt;
1040         struct cvmx_usb_port_status result;
1041 
1042         memset(&result, 0, sizeof(result));
1043 
1044         usbc_hprt.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
1045         result.port_enabled = usbc_hprt.s.prtena;
1046         result.port_over_current = usbc_hprt.s.prtovrcurract;
1047         result.port_powered = usbc_hprt.s.prtpwr;
1048         result.port_speed = usbc_hprt.s.prtspd;
1049         result.connected = usbc_hprt.s.prtconnsts;
1050         result.connect_change =
1051                 (result.connected != usb->port_status.connected);
1052 
1053         return result;
1054 }
1055 
1056 /**
1057  * Open a virtual pipe between the host and a USB device. A pipe
1058  * must be opened before data can be transferred between a device
1059  * and Octeon.
1060  *
1061  * @usb:             USB device state populated by cvmx_usb_initialize().
1062  * @device_addr:
1063  *                   USB device address to open the pipe to
1064  *                   (0-127).
1065  * @endpoint_num:
1066  *                   USB endpoint number to open the pipe to
1067  *                   (0-15).
1068  * @device_speed:
1069  *                   The speed of the device the pipe is going
1070  *                   to. This must match the device's speed,
1071  *                   which may be different than the port speed.
1072  * @max_packet:      The maximum packet length the device can
1073  *                   transmit/receive (low speed=0-8, full
1074  *                   speed=0-1023, high speed=0-1024). This value
1075  *                   comes from the standard endpoint descriptor
1076  *                   field wMaxPacketSize bits <10:0>.
1077  * @transfer_type:
1078  *                   The type of transfer this pipe is for.
1079  * @transfer_dir:
1080  *                   The direction the pipe is in. This is not
1081  *                   used for control pipes.
1082  * @interval:        For ISOCHRONOUS and INTERRUPT transfers,
1083  *                   this is how often the transfer is scheduled
1084  *                   for. All other transfers should specify
1085  *                   zero. The units are in frames (8000/sec at
1086  *                   high speed, 1000/sec for full speed).
1087  * @multi_count:
1088  *                   For high speed devices, this is the maximum
1089  *                   allowed number of packet per microframe.
1090  *                   Specify zero for non high speed devices. This
1091  *                   value comes from the standard endpoint descriptor
1092  *                   field wMaxPacketSize bits <12:11>.
1093  * @hub_device_addr:
1094  *                   Hub device address this device is connected
1095  *                   to. Devices connected directly to Octeon
1096  *                   use zero. This is only used when the device
1097  *                   is full/low speed behind a high speed hub.
1098  *                   The address will be of the high speed hub,
1099  *                   not and full speed hubs after it.
1100  * @hub_port:        Which port on the hub the device is
1101  *                   connected. Use zero for devices connected
1102  *                   directly to Octeon. Like hub_device_addr,
1103  *                   this is only used for full/low speed
1104  *                   devices behind a high speed hub.
1105  *
1106  * Returns: A non-NULL value is a pipe. NULL means an error.
1107  */
1108 static struct cvmx_usb_pipe *cvmx_usb_open_pipe(struct cvmx_usb_state *usb,
1109                                                 int device_addr,
1110                                                 int endpoint_num,
1111                                                 enum cvmx_usb_speed
1112                                                         device_speed,
1113                                                 int max_packet,
1114                                                 enum cvmx_usb_transfer
1115                                                         transfer_type,
1116                                                 enum cvmx_usb_direction
1117                                                         transfer_dir,
1118                                                 int interval, int multi_count,
1119                                                 int hub_device_addr,
1120                                                 int hub_port)
1121 {
1122         struct cvmx_usb_pipe *pipe;
1123 
1124         pipe = kzalloc(sizeof(*pipe), GFP_ATOMIC);
1125         if (!pipe)
1126                 return NULL;
1127         if ((device_speed == CVMX_USB_SPEED_HIGH) &&
1128                 (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
1129                 (transfer_type == CVMX_USB_TRANSFER_BULK))
1130                 pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
1131         pipe->device_addr = device_addr;
1132         pipe->endpoint_num = endpoint_num;
1133         pipe->device_speed = device_speed;
1134         pipe->max_packet = max_packet;
1135         pipe->transfer_type = transfer_type;
1136         pipe->transfer_dir = transfer_dir;
1137         INIT_LIST_HEAD(&pipe->transactions);
1138 
1139         /*
1140          * All pipes use interval to rate limit NAK processing. Force an
1141          * interval if one wasn't supplied
1142          */
1143         if (!interval)
1144                 interval = 1;
1145         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1146                 pipe->interval = interval*8;
1147                 /* Force start splits to be schedule on uFrame 0 */
1148                 pipe->next_tx_frame = ((usb->frame_number+7)&~7) +
1149                                         pipe->interval;
1150         } else {
1151                 pipe->interval = interval;
1152                 pipe->next_tx_frame = usb->frame_number + pipe->interval;
1153         }
1154         pipe->multi_count = multi_count;
1155         pipe->hub_device_addr = hub_device_addr;
1156         pipe->hub_port = hub_port;
1157         pipe->pid_toggle = 0;
1158         pipe->split_sc_frame = -1;
1159         list_add_tail(&pipe->node, &usb->idle_pipes);
1160 
1161         /*
1162          * We don't need to tell the hardware about this pipe yet since
1163          * it doesn't have any submitted requests
1164          */
1165 
1166         return pipe;
1167 }
1168 
1169 
1170 /**
1171  * Poll the RX FIFOs and remove data as needed. This function is only used
1172  * in non DMA mode. It is very important that this function be called quickly
1173  * enough to prevent FIFO overflow.
1174  *
1175  * @usb:        USB device state populated by cvmx_usb_initialize().
1176  */
1177 static void cvmx_usb_poll_rx_fifo(struct cvmx_usb_state *usb)
1178 {
1179         union cvmx_usbcx_grxstsph rx_status;
1180         int channel;
1181         int bytes;
1182         uint64_t address;
1183         uint32_t *ptr;
1184 
1185         rx_status.u32 = cvmx_usb_read_csr32(usb,
1186                                             CVMX_USBCX_GRXSTSPH(usb->index));
1187         /* Only read data if IN data is there */
1188         if (rx_status.s.pktsts != 2)
1189                 return;
1190         /* Check if no data is available */
1191         if (!rx_status.s.bcnt)
1192                 return;
1193 
1194         channel = rx_status.s.chnum;
1195         bytes = rx_status.s.bcnt;
1196         if (!bytes)
1197                 return;
1198 
1199         /* Get where the DMA engine would have written this data */
1200         address = cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) +
1201                                      channel * 8);
1202 
1203         ptr = cvmx_phys_to_ptr(address);
1204         cvmx_write64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel * 8,
1205                             address + bytes);
1206 
1207         /* Loop writing the FIFO data for this packet into memory */
1208         while (bytes > 0) {
1209                 *ptr++ = cvmx_usb_read_csr32(usb,
1210                                         USB_FIFO_ADDRESS(channel, usb->index));
1211                 bytes -= 4;
1212         }
1213         CVMX_SYNCW;
1214 }
1215 
1216 
1217 /**
1218  * Fill the TX hardware fifo with data out of the software
1219  * fifos
1220  *
1221  * @usb:            USB device state populated by cvmx_usb_initialize().
1222  * @fifo:           Software fifo to use
1223  * @available:      Amount of space in the hardware fifo
1224  *
1225  * Returns: Non zero if the hardware fifo was too small and needs
1226  *          to be serviced again.
1227  */
1228 static int cvmx_usb_fill_tx_hw(struct cvmx_usb_state *usb,
1229                                struct cvmx_usb_tx_fifo *fifo, int available)
1230 {
1231         /*
1232          * We're done either when there isn't anymore space or the software FIFO
1233          * is empty
1234          */
1235         while (available && (fifo->head != fifo->tail)) {
1236                 int i = fifo->tail;
1237                 const uint32_t *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
1238                 uint64_t csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel,
1239                                                         usb->index) ^ 4;
1240                 int words = available;
1241 
1242                 /* Limit the amount of data to what the SW fifo has */
1243                 if (fifo->entry[i].size <= available) {
1244                         words = fifo->entry[i].size;
1245                         fifo->tail++;
1246                         if (fifo->tail > MAX_CHANNELS)
1247                                 fifo->tail = 0;
1248                 }
1249 
1250                 /* Update the next locations and counts */
1251                 available -= words;
1252                 fifo->entry[i].address += words * 4;
1253                 fifo->entry[i].size -= words;
1254 
1255                 /*
1256                  * Write the HW fifo data. The read every three writes is due
1257                  * to an errata on CN3XXX chips
1258                  */
1259                 while (words > 3) {
1260                         cvmx_write64_uint32(csr_address, *ptr++);
1261                         cvmx_write64_uint32(csr_address, *ptr++);
1262                         cvmx_write64_uint32(csr_address, *ptr++);
1263                         cvmx_read64_uint64(
1264                                         CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1265                         words -= 3;
1266                 }
1267                 cvmx_write64_uint32(csr_address, *ptr++);
1268                 if (--words) {
1269                         cvmx_write64_uint32(csr_address, *ptr++);
1270                         if (--words)
1271                                 cvmx_write64_uint32(csr_address, *ptr++);
1272                 }
1273                 cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
1274         }
1275         return fifo->head != fifo->tail;
1276 }
1277 
1278 
1279 /**
1280  * Check the hardware FIFOs and fill them as needed
1281  *
1282  * @usb:        USB device state populated by cvmx_usb_initialize().
1283  */
1284 static void cvmx_usb_poll_tx_fifo(struct cvmx_usb_state *usb)
1285 {
1286         if (usb->periodic.head != usb->periodic.tail) {
1287                 union cvmx_usbcx_hptxsts tx_status;
1288 
1289                 tx_status.u32 = cvmx_usb_read_csr32(usb,
1290                                         CVMX_USBCX_HPTXSTS(usb->index));
1291                 if (cvmx_usb_fill_tx_hw(usb, &usb->periodic,
1292                                         tx_status.s.ptxfspcavail))
1293                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1294                                         cvmx_usbcx_gintmsk, ptxfempmsk, 1);
1295                 else
1296                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1297                                         cvmx_usbcx_gintmsk, ptxfempmsk, 0);
1298         }
1299 
1300         if (usb->nonperiodic.head != usb->nonperiodic.tail) {
1301                 union cvmx_usbcx_gnptxsts tx_status;
1302 
1303                 tx_status.u32 = cvmx_usb_read_csr32(usb,
1304                                         CVMX_USBCX_GNPTXSTS(usb->index));
1305                 if (cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic,
1306                                         tx_status.s.nptxfspcavail))
1307                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1308                                         cvmx_usbcx_gintmsk, nptxfempmsk, 1);
1309                 else
1310                         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1311                                         cvmx_usbcx_gintmsk, nptxfempmsk, 0);
1312         }
1313 }
1314 
1315 
1316 /**
1317  * Fill the TX FIFO with an outgoing packet
1318  *
1319  * @usb:          USB device state populated by cvmx_usb_initialize().
1320  * @channel:      Channel number to get packet from
1321  */
1322 static void cvmx_usb_fill_tx_fifo(struct cvmx_usb_state *usb, int channel)
1323 {
1324         union cvmx_usbcx_hccharx hcchar;
1325         union cvmx_usbcx_hcspltx usbc_hcsplt;
1326         union cvmx_usbcx_hctsizx usbc_hctsiz;
1327         struct cvmx_usb_tx_fifo *fifo;
1328 
1329         /* We only need to fill data on outbound channels */
1330         hcchar.u32 = cvmx_usb_read_csr32(usb,
1331                         CVMX_USBCX_HCCHARX(channel, usb->index));
1332         if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
1333                 return;
1334 
1335         /* OUT Splits only have data on the start and not the complete */
1336         usbc_hcsplt.u32 = cvmx_usb_read_csr32(usb,
1337                                 CVMX_USBCX_HCSPLTX(channel, usb->index));
1338         if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
1339                 return;
1340 
1341         /*
1342          * Find out how many bytes we need to fill and convert it into 32bit
1343          * words.
1344          */
1345         usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
1346                                 CVMX_USBCX_HCTSIZX(channel, usb->index));
1347         if (!usbc_hctsiz.s.xfersize)
1348                 return;
1349 
1350         if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
1351                 (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
1352                 fifo = &usb->periodic;
1353         else
1354                 fifo = &usb->nonperiodic;
1355 
1356         fifo->entry[fifo->head].channel = channel;
1357         fifo->entry[fifo->head].address =
1358                 cvmx_read64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
1359                                    channel * 8);
1360         fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize+3)>>2;
1361         fifo->head++;
1362         if (fifo->head > MAX_CHANNELS)
1363                 fifo->head = 0;
1364 
1365         cvmx_usb_poll_tx_fifo(usb);
1366 }
1367 
1368 /**
1369  * Perform channel specific setup for Control transactions. All
1370  * the generic stuff will already have been done in cvmx_usb_start_channel().
1371  *
1372  * @usb:          USB device state populated by cvmx_usb_initialize().
1373  * @channel:      Channel to setup
1374  * @pipe:         Pipe for control transaction
1375  */
1376 static void cvmx_usb_start_channel_control(struct cvmx_usb_state *usb,
1377                                            int channel,
1378                                            struct cvmx_usb_pipe *pipe)
1379 {
1380         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
1381         struct usb_hcd *hcd = octeon_to_hcd(priv);
1382         struct device *dev = hcd->self.controller;
1383         struct cvmx_usb_transaction *transaction =
1384                 list_first_entry(&pipe->transactions, typeof(*transaction),
1385                                  node);
1386         struct usb_ctrlrequest *header =
1387                 cvmx_phys_to_ptr(transaction->control_header);
1388         int bytes_to_transfer = transaction->buffer_length -
1389                 transaction->actual_bytes;
1390         int packets_to_transfer;
1391         union cvmx_usbcx_hctsizx usbc_hctsiz;
1392 
1393         usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
1394                                 CVMX_USBCX_HCTSIZX(channel, usb->index));
1395 
1396         switch (transaction->stage) {
1397         case CVMX_USB_STAGE_NON_CONTROL:
1398         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
1399                 dev_err(dev, "%s: ERROR - Non control stage\n", __func__);
1400                 break;
1401         case CVMX_USB_STAGE_SETUP:
1402                 usbc_hctsiz.s.pid = 3; /* Setup */
1403                 bytes_to_transfer = sizeof(*header);
1404                 /* All Control operations start with a setup going OUT */
1405                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1406                                 cvmx_usbcx_hccharx, epdir,
1407                                 CVMX_USB_DIRECTION_OUT);
1408                 /*
1409                  * Setup send the control header instead of the buffer data. The
1410                  * buffer data will be used in the next stage
1411                  */
1412                 cvmx_write64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
1413                                         channel * 8,
1414                                     transaction->control_header);
1415                 break;
1416         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
1417                 usbc_hctsiz.s.pid = 3; /* Setup */
1418                 bytes_to_transfer = 0;
1419                 /* All Control operations start with a setup going OUT */
1420                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1421                                 cvmx_usbcx_hccharx, epdir,
1422                                 CVMX_USB_DIRECTION_OUT);
1423 
1424                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1425                                 cvmx_usbcx_hcspltx, compsplt, 1);
1426                 break;
1427         case CVMX_USB_STAGE_DATA:
1428                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1429                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1430                         if (header->bRequestType & USB_DIR_IN)
1431                                 bytes_to_transfer = 0;
1432                         else if (bytes_to_transfer > pipe->max_packet)
1433                                 bytes_to_transfer = pipe->max_packet;
1434                 }
1435                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1436                                 cvmx_usbcx_hccharx, epdir,
1437                                 ((header->bRequestType & USB_DIR_IN) ?
1438                                         CVMX_USB_DIRECTION_IN :
1439                                         CVMX_USB_DIRECTION_OUT));
1440                 break;
1441         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
1442                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1443                 if (!(header->bRequestType & USB_DIR_IN))
1444                         bytes_to_transfer = 0;
1445                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1446                                 cvmx_usbcx_hccharx, epdir,
1447                                 ((header->bRequestType & USB_DIR_IN) ?
1448                                         CVMX_USB_DIRECTION_IN :
1449                                         CVMX_USB_DIRECTION_OUT));
1450                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1451                                 cvmx_usbcx_hcspltx, compsplt, 1);
1452                 break;
1453         case CVMX_USB_STAGE_STATUS:
1454                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1455                 bytes_to_transfer = 0;
1456                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1457                                 cvmx_usbcx_hccharx, epdir,
1458                                 ((header->bRequestType & USB_DIR_IN) ?
1459                                         CVMX_USB_DIRECTION_OUT :
1460                                         CVMX_USB_DIRECTION_IN));
1461                 break;
1462         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
1463                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1464                 bytes_to_transfer = 0;
1465                 USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1466                                 cvmx_usbcx_hccharx, epdir,
1467                                 ((header->bRequestType & USB_DIR_IN) ?
1468                                         CVMX_USB_DIRECTION_OUT :
1469                                         CVMX_USB_DIRECTION_IN));
1470                 USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
1471                                 cvmx_usbcx_hcspltx, compsplt, 1);
1472                 break;
1473         }
1474 
1475         /*
1476          * Make sure the transfer never exceeds the byte limit of the hardware.
1477          * Further bytes will be sent as continued transactions
1478          */
1479         if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1480                 /* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
1481                 bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
1482                 bytes_to_transfer *= pipe->max_packet;
1483         }
1484 
1485         /*
1486          * Calculate the number of packets to transfer. If the length is zero
1487          * we still need to transfer one packet
1488          */
1489         packets_to_transfer = DIV_ROUND_UP(bytes_to_transfer,
1490                                            pipe->max_packet);
1491         if (packets_to_transfer == 0)
1492                 packets_to_transfer = 1;
1493         else if ((packets_to_transfer > 1) &&
1494                         (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1495                 /*
1496                  * Limit to one packet when not using DMA. Channels must be
1497                  * restarted between every packet for IN transactions, so there
1498                  * is no reason to do multiple packets in a row
1499                  */
1500                 packets_to_transfer = 1;
1501                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1502         } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1503                 /*
1504                  * Limit the number of packet and data transferred to what the
1505                  * hardware can handle
1506                  */
1507                 packets_to_transfer = MAX_TRANSFER_PACKETS;
1508                 bytes_to_transfer = packets_to_transfer * pipe->max_packet;
1509         }
1510 
1511         usbc_hctsiz.s.xfersize = bytes_to_transfer;
1512         usbc_hctsiz.s.pktcnt = packets_to_transfer;
1513 
1514         cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index),
1515                              usbc_hctsiz.u32);
1516 }
1517 
1518 
1519 /**
1520  * Start a channel to perform the pipe's head transaction
1521  *
1522  * @usb:          USB device state populated by cvmx_usb_initialize().
1523  * @channel:      Channel to setup
1524  * @pipe:         Pipe to start
1525  */
1526 static void cvmx_usb_start_channel(struct cvmx_usb_state *usb, int channel,
1527                                    struct cvmx_usb_pipe *pipe)
1528 {
1529         struct cvmx_usb_transaction *transaction =
1530                 list_first_entry(&pipe->transactions, typeof(*transaction),
1531                                  node);
1532 
1533         /* Make sure all writes to the DMA region get flushed */
1534         CVMX_SYNCW;
1535 
1536         /* Attach the channel to the pipe */
1537         usb->pipe_for_channel[channel] = pipe;
1538         pipe->channel = channel;
1539         pipe->flags |= CVMX_USB_PIPE_FLAGS_SCHEDULED;
1540 
1541         /* Mark this channel as in use */
1542         usb->idle_hardware_channels &= ~(1<<channel);
1543 
1544         /* Enable the channel interrupt bits */
1545         {
1546                 union cvmx_usbcx_hcintx usbc_hcint;
1547                 union cvmx_usbcx_hcintmskx usbc_hcintmsk;
1548                 union cvmx_usbcx_haintmsk usbc_haintmsk;
1549 
1550                 /* Clear all channel status bits */
1551                 usbc_hcint.u32 = cvmx_usb_read_csr32(usb,
1552                                         CVMX_USBCX_HCINTX(channel, usb->index));
1553 
1554                 cvmx_usb_write_csr32(usb,
1555                                      CVMX_USBCX_HCINTX(channel, usb->index),
1556                                      usbc_hcint.u32);
1557 
1558                 usbc_hcintmsk.u32 = 0;
1559                 usbc_hcintmsk.s.chhltdmsk = 1;
1560                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1561                         /*
1562                          * Channels need these extra interrupts when we aren't
1563                          * in DMA mode.
1564                          */
1565                         usbc_hcintmsk.s.datatglerrmsk = 1;
1566                         usbc_hcintmsk.s.frmovrunmsk = 1;
1567                         usbc_hcintmsk.s.bblerrmsk = 1;
1568                         usbc_hcintmsk.s.xacterrmsk = 1;
1569                         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1570                                 /*
1571                                  * Splits don't generate xfercompl, so we need
1572                                  * ACK and NYET.
1573                                  */
1574                                 usbc_hcintmsk.s.nyetmsk = 1;
1575                                 usbc_hcintmsk.s.ackmsk = 1;
1576                         }
1577                         usbc_hcintmsk.s.nakmsk = 1;
1578                         usbc_hcintmsk.s.stallmsk = 1;
1579                         usbc_hcintmsk.s.xfercomplmsk = 1;
1580                 }
1581                 cvmx_usb_write_csr32(usb,
1582                                 CVMX_USBCX_HCINTMSKX(channel, usb->index),
1583                                 usbc_hcintmsk.u32);
1584 
1585                 /* Enable the channel interrupt to propagate */
1586                 usbc_haintmsk.u32 = cvmx_usb_read_csr32(usb,
1587                                         CVMX_USBCX_HAINTMSK(usb->index));
1588                 usbc_haintmsk.s.haintmsk |= 1<<channel;
1589                 cvmx_usb_write_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index),
1590                                      usbc_haintmsk.u32);
1591         }
1592 
1593         /* Setup the location the DMA engine uses. */
1594         {
1595                 uint64_t reg;
1596                 uint64_t dma_address = transaction->buffer +
1597                                         transaction->actual_bytes;
1598 
1599                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1600                         dma_address = transaction->buffer +
1601                                         transaction->iso_packets[0].offset +
1602                                         transaction->actual_bytes;
1603 
1604                 if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT)
1605                         reg = CVMX_USBNX_DMA0_OUTB_CHN0(usb->index);
1606                 else
1607                         reg = CVMX_USBNX_DMA0_INB_CHN0(usb->index);
1608                 cvmx_write64_uint64(reg + channel * 8, dma_address);
1609         }
1610 
1611         /* Setup both the size of the transfer and the SPLIT characteristics */
1612         {
1613                 union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
1614                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
1615                 int packets_to_transfer;
1616                 int bytes_to_transfer = transaction->buffer_length -
1617                         transaction->actual_bytes;
1618 
1619                 /*
1620                  * ISOCHRONOUS transactions store each individual transfer size
1621                  * in the packet structure, not the global buffer_length
1622                  */
1623                 if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
1624                         bytes_to_transfer =
1625                                 transaction->iso_packets[0].length -
1626                                 transaction->actual_bytes;
1627 
1628                 /*
1629                  * We need to do split transactions when we are talking to non
1630                  * high speed devices that are behind a high speed hub
1631                  */
1632                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
1633                         /*
1634                          * On the start split phase (stage is even) record the
1635                          * frame number we will need to send the split complete.
1636                          * We only store the lower two bits since the time ahead
1637                          * can only be two frames
1638                          */
1639                         if ((transaction->stage&1) == 0) {
1640                                 if (transaction->type == CVMX_USB_TRANSFER_BULK)
1641                                         pipe->split_sc_frame =
1642                                                 (usb->frame_number + 1) & 0x7f;
1643                                 else
1644                                         pipe->split_sc_frame =
1645                                                 (usb->frame_number + 2) & 0x7f;
1646                         } else
1647                                 pipe->split_sc_frame = -1;
1648 
1649                         usbc_hcsplt.s.spltena = 1;
1650                         usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
1651                         usbc_hcsplt.s.prtaddr = pipe->hub_port;
1652                         usbc_hcsplt.s.compsplt = (transaction->stage ==
1653                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);
1654 
1655                         /*
1656                          * SPLIT transactions can only ever transmit one data
1657                          * packet so limit the transfer size to the max packet
1658                          * size
1659                          */
1660                         if (bytes_to_transfer > pipe->max_packet)
1661                                 bytes_to_transfer = pipe->max_packet;
1662 
1663                         /*
1664                          * ISOCHRONOUS OUT splits are unique in that they limit
1665                          * data transfers to 188 byte chunks representing the
1666                          * begin/middle/end of the data or all
1667                          */
1668                         if (!usbc_hcsplt.s.compsplt &&
1669                                 (pipe->transfer_dir ==
1670                                  CVMX_USB_DIRECTION_OUT) &&
1671                                 (pipe->transfer_type ==
1672                                  CVMX_USB_TRANSFER_ISOCHRONOUS)) {
1673                                 /*
1674                                  * Clear the split complete frame number as
1675                                  * there isn't going to be a split complete
1676                                  */
1677                                 pipe->split_sc_frame = -1;
1678                                 /*
1679                                  * See if we've started this transfer and sent
1680                                  * data
1681                                  */
1682                                 if (transaction->actual_bytes == 0) {
1683                                         /*
1684                                          * Nothing sent yet, this is either a
1685                                          * begin or the entire payload
1686                                          */
1687                                         if (bytes_to_transfer <= 188)
1688                                                 /* Entire payload in one go */
1689                                                 usbc_hcsplt.s.xactpos = 3;
1690                                         else
1691                                                 /* First part of payload */
1692                                                 usbc_hcsplt.s.xactpos = 2;
1693                                 } else {
1694                                         /*
1695                                          * Continuing the previous data, we must
1696                                          * either be in the middle or at the end
1697                                          */
1698                                         if (bytes_to_transfer <= 188)
1699                                                 /* End of payload */
1700                                                 usbc_hcsplt.s.xactpos = 1;
1701                                         else
1702                                                 /* Middle of payload */
1703                                                 usbc_hcsplt.s.xactpos = 0;
1704                                 }
1705                                 /*
1706                                  * Again, the transfer size is limited to 188
1707                                  * bytes
1708                                  */
1709                                 if (bytes_to_transfer > 188)
1710                                         bytes_to_transfer = 188;
1711                         }
1712                 }
1713 
1714                 /*
1715                  * Make sure the transfer never exceeds the byte limit of the
1716                  * hardware. Further bytes will be sent as continued
1717                  * transactions
1718                  */
1719                 if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
1720                         /*
1721                          * Round MAX_TRANSFER_BYTES to a multiple of out packet
1722                          * size
1723                          */
1724                         bytes_to_transfer = MAX_TRANSFER_BYTES /
1725                                 pipe->max_packet;
1726                         bytes_to_transfer *= pipe->max_packet;
1727                 }
1728 
1729                 /*
1730                  * Calculate the number of packets to transfer. If the length is
1731                  * zero we still need to transfer one packet
1732                  */
1733                 packets_to_transfer =
1734                         DIV_ROUND_UP(bytes_to_transfer, pipe->max_packet);
1735                 if (packets_to_transfer == 0)
1736                         packets_to_transfer = 1;
1737                 else if ((packets_to_transfer > 1) &&
1738                                 (usb->init_flags &
1739                                  CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
1740                         /*
1741                          * Limit to one packet when not using DMA. Channels must
1742                          * be restarted between every packet for IN
1743                          * transactions, so there is no reason to do multiple
1744                          * packets in a row
1745                          */
1746                         packets_to_transfer = 1;
1747                         bytes_to_transfer = packets_to_transfer *
1748                                 pipe->max_packet;
1749                 } else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
1750                         /*
1751                          * Limit the number of packet and data transferred to
1752                          * what the hardware can handle
1753                          */
1754                         packets_to_transfer = MAX_TRANSFER_PACKETS;
1755                         bytes_to_transfer = packets_to_transfer *
1756                                 pipe->max_packet;
1757                 }
1758 
1759                 usbc_hctsiz.s.xfersize = bytes_to_transfer;
1760                 usbc_hctsiz.s.pktcnt = packets_to_transfer;
1761 
1762                 /* Update the DATA0/DATA1 toggle */
1763                 usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
1764                 /*
1765                  * High speed pipes may need a hardware ping before they start
1766                  */
1767                 if (pipe->flags & CVMX_USB_PIPE_FLAGS_NEED_PING)
1768                         usbc_hctsiz.s.dopng = 1;
1769 
1770                 cvmx_usb_write_csr32(usb,
1771                                      CVMX_USBCX_HCSPLTX(channel, usb->index),
1772                                      usbc_hcsplt.u32);
1773                 cvmx_usb_write_csr32(usb,
1774                                      CVMX_USBCX_HCTSIZX(channel, usb->index),
1775                                      usbc_hctsiz.u32);
1776         }
1777 
1778         /* Setup the Host Channel Characteristics Register */
1779         {
1780                 union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};
1781 
1782                 /*
1783                  * Set the startframe odd/even properly. This is only used for
1784                  * periodic
1785                  */
1786                 usbc_hcchar.s.oddfrm = usb->frame_number&1;
1787 
1788                 /*
1789                  * Set the number of back to back packets allowed by this
1790                  * endpoint. Split transactions interpret "ec" as the number of
1791                  * immediate retries of failure. These retries happen too
1792                  * quickly, so we disable these entirely for splits
1793                  */
1794                 if (cvmx_usb_pipe_needs_split(usb, pipe))
1795                         usbc_hcchar.s.ec = 1;
1796                 else if (pipe->multi_count < 1)
1797                         usbc_hcchar.s.ec = 1;
1798                 else if (pipe->multi_count > 3)
1799                         usbc_hcchar.s.ec = 3;
1800                 else
1801                         usbc_hcchar.s.ec = pipe->multi_count;
1802 
1803                 /* Set the rest of the endpoint specific settings */
1804                 usbc_hcchar.s.devaddr = pipe->device_addr;
1805                 usbc_hcchar.s.eptype = transaction->type;
1806                 usbc_hcchar.s.lspddev =
1807                         (pipe->device_speed == CVMX_USB_SPEED_LOW);
1808                 usbc_hcchar.s.epdir = pipe->transfer_dir;
1809                 usbc_hcchar.s.epnum = pipe->endpoint_num;
1810                 usbc_hcchar.s.mps = pipe->max_packet;
1811                 cvmx_usb_write_csr32(usb,
1812                                      CVMX_USBCX_HCCHARX(channel, usb->index),
1813                                      usbc_hcchar.u32);
1814         }
1815 
1816         /* Do transaction type specific fixups as needed */
1817         switch (transaction->type) {
1818         case CVMX_USB_TRANSFER_CONTROL:
1819                 cvmx_usb_start_channel_control(usb, channel, pipe);
1820                 break;
1821         case CVMX_USB_TRANSFER_BULK:
1822         case CVMX_USB_TRANSFER_INTERRUPT:
1823                 break;
1824         case CVMX_USB_TRANSFER_ISOCHRONOUS:
1825                 if (!cvmx_usb_pipe_needs_split(usb, pipe)) {
1826                         /*
1827                          * ISO transactions require different PIDs depending on
1828                          * direction and how many packets are needed
1829                          */
1830                         if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
1831                                 if (pipe->multi_count < 2) /* Need DATA0 */
1832                                         USB_SET_FIELD32(
1833                                                 CVMX_USBCX_HCTSIZX(channel,
1834                                                                    usb->index),
1835                                                 cvmx_usbcx_hctsizx, pid, 0);
1836                                 else /* Need MDATA */
1837                                         USB_SET_FIELD32(
1838                                                 CVMX_USBCX_HCTSIZX(channel,
1839                                                                    usb->index),
1840                                                 cvmx_usbcx_hctsizx, pid, 3);
1841                         }
1842                 }
1843                 break;
1844         }
1845         {
1846                 union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 =
1847                         cvmx_usb_read_csr32(usb,
1848                                 CVMX_USBCX_HCTSIZX(channel, usb->index))};
1849                 transaction->xfersize = usbc_hctsiz.s.xfersize;
1850                 transaction->pktcnt = usbc_hctsiz.s.pktcnt;
1851         }
1852         /* Remember when we start a split transaction */
1853         if (cvmx_usb_pipe_needs_split(usb, pipe))
1854                 usb->active_split = transaction;
1855         USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
1856                         cvmx_usbcx_hccharx, chena, 1);
1857         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
1858                 cvmx_usb_fill_tx_fifo(usb, channel);
1859 }
1860 
1861 
1862 /**
1863  * Find a pipe that is ready to be scheduled to hardware.
1864  * @usb:         USB device state populated by cvmx_usb_initialize().
1865  * @list:        Pipe list to search
1866  * @current_frame:
1867  *               Frame counter to use as a time reference.
1868  *
1869  * Returns: Pipe or NULL if none are ready
1870  */
1871 static struct cvmx_usb_pipe *cvmx_usb_find_ready_pipe(
1872                 struct cvmx_usb_state *usb,
1873                 struct list_head *list,
1874                 uint64_t current_frame)
1875 {
1876         struct cvmx_usb_pipe *pipe;
1877 
1878         list_for_each_entry(pipe, list, node) {
1879                 struct cvmx_usb_transaction *t =
1880                         list_first_entry(&pipe->transactions, typeof(*t),
1881                                          node);
1882                 if (!(pipe->flags & CVMX_USB_PIPE_FLAGS_SCHEDULED) && t &&
1883                         (pipe->next_tx_frame <= current_frame) &&
1884                         ((pipe->split_sc_frame == -1) ||
1885                          ((((int)current_frame - (int)pipe->split_sc_frame)
1886                            & 0x7f) < 0x40)) &&
1887                         (!usb->active_split || (usb->active_split == t))) {
1888                         prefetch(t);
1889                         return pipe;
1890                 }
1891         }
1892         return NULL;
1893 }
1894 
1895 
1896 /**
1897  * Called whenever a pipe might need to be scheduled to the
1898  * hardware.
1899  *
1900  * @usb:         USB device state populated by cvmx_usb_initialize().
1901  * @is_sof:      True if this schedule was called on a SOF interrupt.
1902  */
1903 static void cvmx_usb_schedule(struct cvmx_usb_state *usb, int is_sof)
1904 {
1905         int channel;
1906         struct cvmx_usb_pipe *pipe;
1907         int need_sof;
1908         enum cvmx_usb_transfer ttype;
1909 
1910         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
1911                 /*
1912                  * Without DMA we need to be careful to not schedule something
1913                  * at the end of a frame and cause an overrun.
1914                  */
1915                 union cvmx_usbcx_hfnum hfnum = {
1916                         .u32 = cvmx_usb_read_csr32(usb,
1917                                                 CVMX_USBCX_HFNUM(usb->index))
1918                 };
1919 
1920                 union cvmx_usbcx_hfir hfir = {
1921                         .u32 = cvmx_usb_read_csr32(usb,
1922                                                 CVMX_USBCX_HFIR(usb->index))
1923                 };
1924 
1925                 if (hfnum.s.frrem < hfir.s.frint/4)
1926                         goto done;
1927         }
1928 
1929         while (usb->idle_hardware_channels) {
1930                 /* Find an idle channel */
1931                 channel = __fls(usb->idle_hardware_channels);
1932                 if (unlikely(channel > 7))
1933                         break;
1934 
1935                 /* Find a pipe needing service */
1936                 pipe = NULL;
1937                 if (is_sof) {
1938                         /*
1939                          * Only process periodic pipes on SOF interrupts. This
1940                          * way we are sure that the periodic data is sent in the
1941                          * beginning of the frame
1942                          */
1943                         pipe = cvmx_usb_find_ready_pipe(usb,
1944                                         usb->active_pipes +
1945                                         CVMX_USB_TRANSFER_ISOCHRONOUS,
1946                                         usb->frame_number);
1947                         if (likely(!pipe))
1948                                 pipe = cvmx_usb_find_ready_pipe(usb,
1949                                                 usb->active_pipes +
1950                                                 CVMX_USB_TRANSFER_INTERRUPT,
1951                                                 usb->frame_number);
1952                 }
1953                 if (likely(!pipe)) {
1954                         pipe = cvmx_usb_find_ready_pipe(usb,
1955                                         usb->active_pipes +
1956                                         CVMX_USB_TRANSFER_CONTROL,
1957                                         usb->frame_number);
1958                         if (likely(!pipe))
1959                                 pipe = cvmx_usb_find_ready_pipe(usb,
1960                                                 usb->active_pipes +
1961                                                 CVMX_USB_TRANSFER_BULK,
1962                                                 usb->frame_number);
1963                 }
1964                 if (!pipe)
1965                         break;
1966 
1967                 cvmx_usb_start_channel(usb, channel, pipe);
1968         }
1969 
1970 done:
1971         /*
1972          * Only enable SOF interrupts when we have transactions pending in the
1973          * future that might need to be scheduled
1974          */
1975         need_sof = 0;
1976         for (ttype = CVMX_USB_TRANSFER_CONTROL;
1977                         ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
1978                 list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
1979                         if (pipe->next_tx_frame > usb->frame_number) {
1980                                 need_sof = 1;
1981                                 break;
1982                         }
1983                 }
1984         }
1985         USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
1986                         cvmx_usbcx_gintmsk, sofmsk, need_sof);
1987 }
1988 
1989 static void octeon_usb_urb_complete_callback(struct cvmx_usb_state *usb,
1990                                              enum cvmx_usb_complete status,
1991                                              struct cvmx_usb_pipe *pipe,
1992                                              struct cvmx_usb_transaction
1993                                                 *transaction,
1994                                              int bytes_transferred,
1995                                              struct urb *urb)
1996 {
1997         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
1998         struct usb_hcd *hcd = octeon_to_hcd(priv);
1999         struct device *dev = hcd->self.controller;
2000 
2001         if (likely(status == CVMX_USB_COMPLETE_SUCCESS))
2002                 urb->actual_length = bytes_transferred;
2003         else
2004                 urb->actual_length = 0;
2005 
2006         urb->hcpriv = NULL;
2007 
2008         /* For Isochronous transactions we need to update the URB packet status
2009            list from data in our private copy */
2010         if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
2011                 int i;
2012                 /*
2013                  * The pointer to the private list is stored in the setup_packet
2014                  * field.
2015                  */
2016                 struct cvmx_usb_iso_packet *iso_packet =
2017                         (struct cvmx_usb_iso_packet *) urb->setup_packet;
2018                 /* Recalculate the transfer size by adding up each packet */
2019                 urb->actual_length = 0;
2020                 for (i = 0; i < urb->number_of_packets; i++) {
2021                         if (iso_packet[i].status ==
2022                                         CVMX_USB_COMPLETE_SUCCESS) {
2023                                 urb->iso_frame_desc[i].status = 0;
2024                                 urb->iso_frame_desc[i].actual_length =
2025                                         iso_packet[i].length;
2026                                 urb->actual_length +=
2027                                         urb->iso_frame_desc[i].actual_length;
2028                         } else {
2029                                 dev_dbg(dev, "ISOCHRONOUS packet=%d of %d status=%d pipe=%p transaction=%p size=%d\n",
2030                                         i, urb->number_of_packets,
2031                                         iso_packet[i].status, pipe,
2032                                         transaction, iso_packet[i].length);
2033                                 urb->iso_frame_desc[i].status = -EREMOTEIO;
2034                         }
2035                 }
2036                 /* Free the private list now that we don't need it anymore */
2037                 kfree(iso_packet);
2038                 urb->setup_packet = NULL;
2039         }
2040 
2041         switch (status) {
2042         case CVMX_USB_COMPLETE_SUCCESS:
2043                 urb->status = 0;
2044                 break;
2045         case CVMX_USB_COMPLETE_CANCEL:
2046                 if (urb->status == 0)
2047                         urb->status = -ENOENT;
2048                 break;
2049         case CVMX_USB_COMPLETE_STALL:
2050                 dev_dbg(dev, "status=stall pipe=%p transaction=%p size=%d\n",
2051                         pipe, transaction, bytes_transferred);
2052                 urb->status = -EPIPE;
2053                 break;
2054         case CVMX_USB_COMPLETE_BABBLEERR:
2055                 dev_dbg(dev, "status=babble pipe=%p transaction=%p size=%d\n",
2056                         pipe, transaction, bytes_transferred);
2057                 urb->status = -EPIPE;
2058                 break;
2059         case CVMX_USB_COMPLETE_SHORT:
2060                 dev_dbg(dev, "status=short pipe=%p transaction=%p size=%d\n",
2061                         pipe, transaction, bytes_transferred);
2062                 urb->status = -EREMOTEIO;
2063                 break;
2064         case CVMX_USB_COMPLETE_ERROR:
2065         case CVMX_USB_COMPLETE_XACTERR:
2066         case CVMX_USB_COMPLETE_DATATGLERR:
2067         case CVMX_USB_COMPLETE_FRAMEERR:
2068                 dev_dbg(dev, "status=%d pipe=%p transaction=%p size=%d\n",
2069                         status, pipe, transaction, bytes_transferred);
2070                 urb->status = -EPROTO;
2071                 break;
2072         }
2073         usb_hcd_unlink_urb_from_ep(octeon_to_hcd(priv), urb);
2074         spin_unlock(&priv->lock);
2075         usb_hcd_giveback_urb(octeon_to_hcd(priv), urb, urb->status);
2076         spin_lock(&priv->lock);
2077 }
2078 
2079 /**
2080  * Signal the completion of a transaction and free it. The
2081  * transaction will be removed from the pipe transaction list.
2082  *
2083  * @usb:         USB device state populated by cvmx_usb_initialize().
2084  * @pipe:        Pipe the transaction is on
2085  * @transaction:
2086  *               Transaction that completed
2087  * @complete_code:
2088  *               Completion code
2089  */
2090 static void cvmx_usb_perform_complete(struct cvmx_usb_state *usb,
2091                                       struct cvmx_usb_pipe *pipe,
2092                                       struct cvmx_usb_transaction *transaction,
2093                                       enum cvmx_usb_complete complete_code)
2094 {
2095         /* If this was a split then clear our split in progress marker */
2096         if (usb->active_split == transaction)
2097                 usb->active_split = NULL;
2098 
2099         /*
2100          * Isochronous transactions need extra processing as they might not be
2101          * done after a single data transfer
2102          */
2103         if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
2104                 /* Update the number of bytes transferred in this ISO packet */
2105                 transaction->iso_packets[0].length = transaction->actual_bytes;
2106                 transaction->iso_packets[0].status = complete_code;
2107 
2108                 /*
2109                  * If there are more ISOs pending and we succeeded, schedule the
2110                  * next one
2111                  */
2112                 if ((transaction->iso_number_packets > 1) &&
2113                         (complete_code == CVMX_USB_COMPLETE_SUCCESS)) {
2114                         /* No bytes transferred for this packet as of yet */
2115                         transaction->actual_bytes = 0;
2116                         /* One less ISO waiting to transfer */
2117                         transaction->iso_number_packets--;
2118                         /* Increment to the next location in our packet array */
2119                         transaction->iso_packets++;
2120                         transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2121                         return;
2122                 }
2123         }
2124 
2125         /* Remove the transaction from the pipe list */
2126         list_del(&transaction->node);
2127         if (list_empty(&pipe->transactions))
2128                 list_move_tail(&pipe->node, &usb->idle_pipes);
2129         octeon_usb_urb_complete_callback(usb, complete_code, pipe,
2130                                          transaction,
2131                                          transaction->actual_bytes,
2132                                          transaction->urb);
2133         kfree(transaction);
2134 }
2135 
2136 
2137 /**
2138  * Submit a usb transaction to a pipe. Called for all types
2139  * of transactions.
2140  *
2141  * @usb:
2142  * @pipe:           Which pipe to submit to.
2143  * @type:           Transaction type
2144  * @buffer:         User buffer for the transaction
2145  * @buffer_length:
2146  *                  User buffer's length in bytes
2147  * @control_header:
2148  *                  For control transactions, the 8 byte standard header
2149  * @iso_start_frame:
2150  *                  For ISO transactions, the start frame
2151  * @iso_number_packets:
2152  *                  For ISO, the number of packet in the transaction.
2153  * @iso_packets:
2154  *                  A description of each ISO packet
2155  * @urb:            URB for the callback
2156  *
2157  * Returns: Transaction or NULL on failure.
2158  */
2159 static struct cvmx_usb_transaction *cvmx_usb_submit_transaction(
2160                                 struct cvmx_usb_state *usb,
2161                                 struct cvmx_usb_pipe *pipe,
2162                                 enum cvmx_usb_transfer type,
2163                                 uint64_t buffer,
2164                                 int buffer_length,
2165                                 uint64_t control_header,
2166                                 int iso_start_frame,
2167                                 int iso_number_packets,
2168                                 struct cvmx_usb_iso_packet *iso_packets,
2169                                 struct urb *urb)
2170 {
2171         struct cvmx_usb_transaction *transaction;
2172 
2173         if (unlikely(pipe->transfer_type != type))
2174                 return NULL;
2175 
2176         transaction = kzalloc(sizeof(*transaction), GFP_ATOMIC);
2177         if (unlikely(!transaction))
2178                 return NULL;
2179 
2180         transaction->type = type;
2181         transaction->buffer = buffer;
2182         transaction->buffer_length = buffer_length;
2183         transaction->control_header = control_header;
2184         /* FIXME: This is not used, implement it. */
2185         transaction->iso_start_frame = iso_start_frame;
2186         transaction->iso_number_packets = iso_number_packets;
2187         transaction->iso_packets = iso_packets;
2188         transaction->urb = urb;
2189         if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
2190                 transaction->stage = CVMX_USB_STAGE_SETUP;
2191         else
2192                 transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
2193 
2194         if (!list_empty(&pipe->transactions)) {
2195                 list_add_tail(&transaction->node, &pipe->transactions);
2196         } else {
2197                 list_add_tail(&transaction->node, &pipe->transactions);
2198                 list_move_tail(&pipe->node,
2199                                &usb->active_pipes[pipe->transfer_type]);
2200 
2201                 /*
2202                  * We may need to schedule the pipe if this was the head of the
2203                  * pipe.
2204                  */
2205                 cvmx_usb_schedule(usb, 0);
2206         }
2207 
2208         return transaction;
2209 }
2210 
2211 
2212 /**
2213  * Call to submit a USB Bulk transfer to a pipe.
2214  *
2215  * @usb:            USB device state populated by cvmx_usb_initialize().
2216  * @pipe:           Handle to the pipe for the transfer.
2217  * @urb:            URB.
2218  *
2219  * Returns: A submitted transaction or NULL on failure.
2220  */
2221 static struct cvmx_usb_transaction *cvmx_usb_submit_bulk(
2222                                                 struct cvmx_usb_state *usb,
2223                                                 struct cvmx_usb_pipe *pipe,
2224                                                 struct urb *urb)
2225 {
2226         return cvmx_usb_submit_transaction(usb, pipe, CVMX_USB_TRANSFER_BULK,
2227                                            urb->transfer_dma,
2228                                            urb->transfer_buffer_length,
2229                                            0, /* control_header */
2230                                            0, /* iso_start_frame */
2231                                            0, /* iso_number_packets */
2232                                            NULL, /* iso_packets */
2233                                            urb);
2234 }
2235 
2236 
2237 /**
2238  * Call to submit a USB Interrupt transfer to a pipe.
2239  *
2240  * @usb:            USB device state populated by cvmx_usb_initialize().
2241  * @pipe:           Handle to the pipe for the transfer.
2242  * @urb:            URB returned when the callback is called.
2243  *
2244  * Returns: A submitted transaction or NULL on failure.
2245  */
2246 static struct cvmx_usb_transaction *cvmx_usb_submit_interrupt(
2247                                                 struct cvmx_usb_state *usb,
2248                                                 struct cvmx_usb_pipe *pipe,
2249                                                 struct urb *urb)
2250 {
2251         return cvmx_usb_submit_transaction(usb, pipe,
2252                                            CVMX_USB_TRANSFER_INTERRUPT,
2253                                            urb->transfer_dma,
2254                                            urb->transfer_buffer_length,
2255                                            0, /* control_header */
2256                                            0, /* iso_start_frame */
2257                                            0, /* iso_number_packets */
2258                                            NULL, /* iso_packets */
2259                                            urb);
2260 }
2261 
2262 
2263 /**
2264  * Call to submit a USB Control transfer to a pipe.
2265  *
2266  * @usb:            USB device state populated by cvmx_usb_initialize().
2267  * @pipe:           Handle to the pipe for the transfer.
2268  * @urb:            URB.
2269  *
2270  * Returns: A submitted transaction or NULL on failure.
2271  */
2272 static struct cvmx_usb_transaction *cvmx_usb_submit_control(
2273                                                 struct cvmx_usb_state *usb,
2274                                                 struct cvmx_usb_pipe *pipe,
2275                                                 struct urb *urb)
2276 {
2277         int buffer_length = urb->transfer_buffer_length;
2278         uint64_t control_header = urb->setup_dma;
2279         struct usb_ctrlrequest *header = cvmx_phys_to_ptr(control_header);
2280 
2281         if ((header->bRequestType & USB_DIR_IN) == 0)
2282                 buffer_length = le16_to_cpu(header->wLength);
2283 
2284         return cvmx_usb_submit_transaction(usb, pipe,
2285                                            CVMX_USB_TRANSFER_CONTROL,
2286                                            urb->transfer_dma, buffer_length,
2287                                            control_header,
2288                                            0, /* iso_start_frame */
2289                                            0, /* iso_number_packets */
2290                                            NULL, /* iso_packets */
2291                                            urb);
2292 }
2293 
2294 
2295 /**
2296  * Call to submit a USB Isochronous transfer to a pipe.
2297  *
2298  * @usb:            USB device state populated by cvmx_usb_initialize().
2299  * @pipe:           Handle to the pipe for the transfer.
2300  * @urb:            URB returned when the callback is called.
2301  *
2302  * Returns: A submitted transaction or NULL on failure.
2303  */
2304 static struct cvmx_usb_transaction *cvmx_usb_submit_isochronous(
2305                                                 struct cvmx_usb_state *usb,
2306                                                 struct cvmx_usb_pipe *pipe,
2307                                                 struct urb *urb)
2308 {
2309         struct cvmx_usb_iso_packet *packets;
2310 
2311         packets = (struct cvmx_usb_iso_packet *) urb->setup_packet;
2312         return cvmx_usb_submit_transaction(usb, pipe,
2313                                            CVMX_USB_TRANSFER_ISOCHRONOUS,
2314                                            urb->transfer_dma,
2315                                            urb->transfer_buffer_length,
2316                                            0, /* control_header */
2317                                            urb->start_frame,
2318                                            urb->number_of_packets,
2319                                            packets, urb);
2320 }
2321 
2322 
2323 /**
2324  * Cancel one outstanding request in a pipe. Canceling a request
2325  * can fail if the transaction has already completed before cancel
2326  * is called. Even after a successful cancel call, it may take
2327  * a frame or two for the cvmx_usb_poll() function to call the
2328  * associated callback.
2329  *
2330  * @usb:         USB device state populated by cvmx_usb_initialize().
2331  * @pipe:        Pipe to cancel requests in.
2332  * @transaction: Transaction to cancel, returned by the submit function.
2333  *
2334  * Returns: 0 or a negative error code.
2335  */
2336 static int cvmx_usb_cancel(struct cvmx_usb_state *usb,
2337                            struct cvmx_usb_pipe *pipe,
2338                            struct cvmx_usb_transaction *transaction)
2339 {
2340         /*
2341          * If the transaction is the HEAD of the queue and scheduled. We need to
2342          * treat it special
2343          */
2344         if (list_first_entry(&pipe->transactions, typeof(*transaction), node) ==
2345             transaction && (pipe->flags & CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
2346                 union cvmx_usbcx_hccharx usbc_hcchar;
2347 
2348                 usb->pipe_for_channel[pipe->channel] = NULL;
2349                 pipe->flags &= ~CVMX_USB_PIPE_FLAGS_SCHEDULED;
2350 
2351                 CVMX_SYNCW;
2352 
2353                 usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
2354                                 CVMX_USBCX_HCCHARX(pipe->channel, usb->index));
2355                 /*
2356                  * If the channel isn't enabled then the transaction already
2357                  * completed.
2358                  */
2359                 if (usbc_hcchar.s.chena) {
2360                         usbc_hcchar.s.chdis = 1;
2361                         cvmx_usb_write_csr32(usb,
2362                                         CVMX_USBCX_HCCHARX(pipe->channel,
2363                                                 usb->index),
2364                                         usbc_hcchar.u32);
2365                 }
2366         }
2367         cvmx_usb_perform_complete(usb, pipe, transaction,
2368                                   CVMX_USB_COMPLETE_CANCEL);
2369         return 0;
2370 }
2371 
2372 
2373 /**
2374  * Cancel all outstanding requests in a pipe. Logically all this
2375  * does is call cvmx_usb_cancel() in a loop.
2376  *
2377  * @usb:         USB device state populated by cvmx_usb_initialize().
2378  * @pipe:        Pipe to cancel requests in.
2379  *
2380  * Returns: 0 or a negative error code.
2381  */
2382 static int cvmx_usb_cancel_all(struct cvmx_usb_state *usb,
2383                                struct cvmx_usb_pipe *pipe)
2384 {
2385         struct cvmx_usb_transaction *transaction, *next;
2386 
2387         /* Simply loop through and attempt to cancel each transaction */
2388         list_for_each_entry_safe(transaction, next, &pipe->transactions, node) {
2389                 int result = cvmx_usb_cancel(usb, pipe, transaction);
2390 
2391                 if (unlikely(result != 0))
2392                         return result;
2393         }
2394         return 0;
2395 }
2396 
2397 
2398 /**
2399  * Close a pipe created with cvmx_usb_open_pipe().
2400  *
2401  * @usb:         USB device state populated by cvmx_usb_initialize().
2402  * @pipe:        Pipe to close.
2403  *
2404  * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
2405  *          outstanding transfers.
2406  */
2407 static int cvmx_usb_close_pipe(struct cvmx_usb_state *usb,
2408                                struct cvmx_usb_pipe *pipe)
2409 {
2410         /* Fail if the pipe has pending transactions */
2411         if (!list_empty(&pipe->transactions))
2412                 return -EBUSY;
2413 
2414         list_del(&pipe->node);
2415         kfree(pipe);
2416 
2417         return 0;
2418 }
2419 
2420 /**
2421  * Get the current USB protocol level frame number. The frame
2422  * number is always in the range of 0-0x7ff.
2423  *
2424  * @usb: USB device state populated by cvmx_usb_initialize().
2425  *
2426  * Returns: USB frame number
2427  */
2428 static int cvmx_usb_get_frame_number(struct cvmx_usb_state *usb)
2429 {
2430         int frame_number;
2431         union cvmx_usbcx_hfnum usbc_hfnum;
2432 
2433         usbc_hfnum.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2434         frame_number = usbc_hfnum.s.frnum;
2435 
2436         return frame_number;
2437 }
2438 
2439 
2440 /**
2441  * Poll a channel for status
2442  *
2443  * @usb:     USB device
2444  * @channel: Channel to poll
2445  *
2446  * Returns: Zero on success
2447  */
2448 static int cvmx_usb_poll_channel(struct cvmx_usb_state *usb, int channel)
2449 {
2450         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
2451         struct usb_hcd *hcd = octeon_to_hcd(priv);
2452         struct device *dev = hcd->self.controller;
2453         union cvmx_usbcx_hcintx usbc_hcint;
2454         union cvmx_usbcx_hctsizx usbc_hctsiz;
2455         union cvmx_usbcx_hccharx usbc_hcchar;
2456         struct cvmx_usb_pipe *pipe;
2457         struct cvmx_usb_transaction *transaction;
2458         int bytes_this_transfer;
2459         int bytes_in_last_packet;
2460         int packets_processed;
2461         int buffer_space_left;
2462 
2463         /* Read the interrupt status bits for the channel */
2464         usbc_hcint.u32 = cvmx_usb_read_csr32(usb,
2465                                 CVMX_USBCX_HCINTX(channel, usb->index));
2466 
2467         if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
2468                 usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
2469                                 CVMX_USBCX_HCCHARX(channel, usb->index));
2470 
2471                 if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
2472                         /*
2473                          * There seems to be a bug in CN31XX which can cause
2474                          * interrupt IN transfers to get stuck until we do a
2475                          * write of HCCHARX without changing things
2476                          */
2477                         cvmx_usb_write_csr32(usb,
2478                                         CVMX_USBCX_HCCHARX(channel,
2479                                                            usb->index),
2480                                         usbc_hcchar.u32);
2481                         return 0;
2482                 }
2483 
2484                 /*
2485                  * In non DMA mode the channels don't halt themselves. We need
2486                  * to manually disable channels that are left running
2487                  */
2488                 if (!usbc_hcint.s.chhltd) {
2489                         if (usbc_hcchar.s.chena) {
2490                                 union cvmx_usbcx_hcintmskx hcintmsk;
2491                                 /* Disable all interrupts except CHHLTD */
2492                                 hcintmsk.u32 = 0;
2493                                 hcintmsk.s.chhltdmsk = 1;
2494                                 cvmx_usb_write_csr32(usb,
2495                                                 CVMX_USBCX_HCINTMSKX(channel,
2496                                                         usb->index),
2497                                                 hcintmsk.u32);
2498                                 usbc_hcchar.s.chdis = 1;
2499                                 cvmx_usb_write_csr32(usb,
2500                                                 CVMX_USBCX_HCCHARX(channel,
2501                                                         usb->index),
2502                                                 usbc_hcchar.u32);
2503                                 return 0;
2504                         } else if (usbc_hcint.s.xfercompl) {
2505                                 /*
2506                                  * Successful IN/OUT with transfer complete.
2507                                  * Channel halt isn't needed.
2508                                  */
2509                         } else {
2510                                 dev_err(dev, "USB%d: Channel %d interrupt without halt\n",
2511                                         usb->index, channel);
2512                                 return 0;
2513                         }
2514                 }
2515         } else {
2516                 /*
2517                  * There is are no interrupts that we need to process when the
2518                  * channel is still running
2519                  */
2520                 if (!usbc_hcint.s.chhltd)
2521                         return 0;
2522         }
2523 
2524         /* Disable the channel interrupts now that it is done */
2525         cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
2526         usb->idle_hardware_channels |= (1<<channel);
2527 
2528         /* Make sure this channel is tied to a valid pipe */
2529         pipe = usb->pipe_for_channel[channel];
2530         prefetch(pipe);
2531         if (!pipe)
2532                 return 0;
2533         transaction = list_first_entry(&pipe->transactions,
2534                                        typeof(*transaction),
2535                                        node);
2536         prefetch(transaction);
2537 
2538         /*
2539          * Disconnect this pipe from the HW channel. Later the schedule
2540          * function will figure out which pipe needs to go
2541          */
2542         usb->pipe_for_channel[channel] = NULL;
2543         pipe->flags &= ~CVMX_USB_PIPE_FLAGS_SCHEDULED;
2544 
2545         /*
2546          * Read the channel config info so we can figure out how much data
2547          * transferred
2548          */
2549         usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
2550                         CVMX_USBCX_HCCHARX(channel, usb->index));
2551         usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
2552                         CVMX_USBCX_HCTSIZX(channel, usb->index));
2553 
2554         /*
2555          * Calculating the number of bytes successfully transferred is dependent
2556          * on the transfer direction
2557          */
2558         packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
2559         if (usbc_hcchar.s.epdir) {
2560                 /*
2561                  * IN transactions are easy. For every byte received the
2562                  * hardware decrements xfersize. All we need to do is subtract
2563                  * the current value of xfersize from its starting value and we
2564                  * know how many bytes were written to the buffer
2565                  */
2566                 bytes_this_transfer = transaction->xfersize -
2567                         usbc_hctsiz.s.xfersize;
2568         } else {
2569                 /*
2570                  * OUT transaction don't decrement xfersize. Instead pktcnt is
2571                  * decremented on every successful packet send. The hardware
2572                  * does this when it receives an ACK, or NYET. If it doesn't
2573                  * receive one of these responses pktcnt doesn't change
2574                  */
2575                 bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
2576                 /*
2577                  * The last packet may not be a full transfer if we didn't have
2578                  * enough data
2579                  */
2580                 if (bytes_this_transfer > transaction->xfersize)
2581                         bytes_this_transfer = transaction->xfersize;
2582         }
2583         /* Figure out how many bytes were in the last packet of the transfer */
2584         if (packets_processed)
2585                 bytes_in_last_packet = bytes_this_transfer -
2586                         (packets_processed - 1) * usbc_hcchar.s.mps;
2587         else
2588                 bytes_in_last_packet = bytes_this_transfer;
2589 
2590         /*
2591          * As a special case, setup transactions output the setup header, not
2592          * the user's data. For this reason we don't count setup data as bytes
2593          * transferred
2594          */
2595         if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
2596                 (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
2597                 bytes_this_transfer = 0;
2598 
2599         /*
2600          * Add the bytes transferred to the running total. It is important that
2601          * bytes_this_transfer doesn't count any data that needs to be
2602          * retransmitted
2603          */
2604         transaction->actual_bytes += bytes_this_transfer;
2605         if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
2606                 buffer_space_left = transaction->iso_packets[0].length -
2607                         transaction->actual_bytes;
2608         else
2609                 buffer_space_left = transaction->buffer_length -
2610                         transaction->actual_bytes;
2611 
2612         /*
2613          * We need to remember the PID toggle state for the next transaction.
2614          * The hardware already updated it for the next transaction
2615          */
2616         pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);
2617 
2618         /*
2619          * For high speed bulk out, assume the next transaction will need to do
2620          * a ping before proceeding. If this isn't true the ACK processing below
2621          * will clear this flag
2622          */
2623         if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
2624                 (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
2625                 (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
2626                 pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
2627 
2628         if (unlikely(WARN_ON_ONCE(bytes_this_transfer < 0))) {
2629                 /*
2630                  * In some rare cases the DMA engine seems to get stuck and
2631                  * keeps substracting same byte count over and over again. In
2632                  * such case we just need to fail every transaction.
2633                  */
2634                 cvmx_usb_perform_complete(usb, pipe, transaction,
2635                                           CVMX_USB_COMPLETE_ERROR);
2636                 return 0;
2637         }
2638 
2639         if (usbc_hcint.s.stall) {
2640                 /*
2641                  * STALL as a response means this transaction cannot be
2642                  * completed because the device can't process transactions. Tell
2643                  * the user. Any data that was transferred will be counted on
2644                  * the actual bytes transferred
2645                  */
2646                 pipe->pid_toggle = 0;
2647                 cvmx_usb_perform_complete(usb, pipe, transaction,
2648                                           CVMX_USB_COMPLETE_STALL);
2649         } else if (usbc_hcint.s.xacterr) {
2650                 /*
2651                  * XactErr as a response means the device signaled
2652                  * something wrong with the transfer. For example, PID
2653                  * toggle errors cause these.
2654                  */
2655                 cvmx_usb_perform_complete(usb, pipe, transaction,
2656                                           CVMX_USB_COMPLETE_XACTERR);
2657         } else if (usbc_hcint.s.bblerr) {
2658                 /* Babble Error (BblErr) */
2659                 cvmx_usb_perform_complete(usb, pipe, transaction,
2660                                           CVMX_USB_COMPLETE_BABBLEERR);
2661         } else if (usbc_hcint.s.datatglerr) {
2662                 /* Data toggle error */
2663                 cvmx_usb_perform_complete(usb, pipe, transaction,
2664                                           CVMX_USB_COMPLETE_DATATGLERR);
2665         } else if (usbc_hcint.s.nyet) {
2666                 /*
2667                  * NYET as a response is only allowed in three cases: as a
2668                  * response to a ping, as a response to a split transaction, and
2669                  * as a response to a bulk out. The ping case is handled by
2670                  * hardware, so we only have splits and bulk out
2671                  */
2672                 if (!cvmx_usb_pipe_needs_split(usb, pipe)) {
2673                         transaction->retries = 0;
2674                         /*
2675                          * If there is more data to go then we need to try
2676                          * again. Otherwise this transaction is complete
2677                          */
2678                         if ((buffer_space_left == 0) ||
2679                                 (bytes_in_last_packet < pipe->max_packet))
2680                                 cvmx_usb_perform_complete(usb, pipe,
2681                                                 transaction,
2682                                                 CVMX_USB_COMPLETE_SUCCESS);
2683                 } else {
2684                         /*
2685                          * Split transactions retry the split complete 4 times
2686                          * then rewind to the start split and do the entire
2687                          * transactions again
2688                          */
2689                         transaction->retries++;
2690                         if ((transaction->retries & 0x3) == 0) {
2691                                 /*
2692                                  * Rewind to the beginning of the transaction by
2693                                  * anding off the split complete bit
2694                                  */
2695                                 transaction->stage &= ~1;
2696                                 pipe->split_sc_frame = -1;
2697                         }
2698                 }
2699         } else if (usbc_hcint.s.ack) {
2700                 transaction->retries = 0;
2701                 /*
2702                  * The ACK bit can only be checked after the other error bits.
2703                  * This is because a multi packet transfer may succeed in a
2704                  * number of packets and then get a different response on the
2705                  * last packet. In this case both ACK and the last response bit
2706                  * will be set. If none of the other response bits is set, then
2707                  * the last packet must have been an ACK
2708                  *
2709                  * Since we got an ACK, we know we don't need to do a ping on
2710                  * this pipe
2711                  */
2712                 pipe->flags &= ~CVMX_USB_PIPE_FLAGS_NEED_PING;
2713 
2714                 switch (transaction->type) {
2715                 case CVMX_USB_TRANSFER_CONTROL:
2716                         switch (transaction->stage) {
2717                         case CVMX_USB_STAGE_NON_CONTROL:
2718                         case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
2719                                 /* This should be impossible */
2720                                 cvmx_usb_perform_complete(usb, pipe,
2721                                         transaction, CVMX_USB_COMPLETE_ERROR);
2722                                 break;
2723                         case CVMX_USB_STAGE_SETUP:
2724                                 pipe->pid_toggle = 1;
2725                                 if (cvmx_usb_pipe_needs_split(usb, pipe))
2726                                         transaction->stage =
2727                                                 CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
2728                                 else {
2729                                         struct usb_ctrlrequest *header =
2730                                                 cvmx_phys_to_ptr(transaction->control_header);
2731                                         if (header->wLength)
2732                                                 transaction->stage =
2733                                                         CVMX_USB_STAGE_DATA;
2734                                         else
2735                                                 transaction->stage =
2736                                                         CVMX_USB_STAGE_STATUS;
2737                                 }
2738                                 break;
2739                         case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
2740                                 {
2741                                         struct usb_ctrlrequest *header =
2742                                                 cvmx_phys_to_ptr(transaction->control_header);
2743                                         if (header->wLength)
2744                                                 transaction->stage =
2745                                                         CVMX_USB_STAGE_DATA;
2746                                         else
2747                                                 transaction->stage =
2748                                                         CVMX_USB_STAGE_STATUS;
2749                                 }
2750                                 break;
2751                         case CVMX_USB_STAGE_DATA:
2752                                 if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2753                                         transaction->stage =
2754                                                 CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
2755                                         /*
2756                                          * For setup OUT data that are splits,
2757                                          * the hardware doesn't appear to count
2758                                          * transferred data. Here we manually
2759                                          * update the data transferred
2760                                          */
2761                                         if (!usbc_hcchar.s.epdir) {
2762                                                 if (buffer_space_left < pipe->max_packet)
2763                                                         transaction->actual_bytes +=
2764                                                                 buffer_space_left;
2765                                                 else
2766                                                         transaction->actual_bytes +=
2767                                                                 pipe->max_packet;
2768                                         }
2769                                 } else if ((buffer_space_left == 0) ||
2770                                                 (bytes_in_last_packet <
2771                                                  pipe->max_packet)) {
2772                                         pipe->pid_toggle = 1;
2773                                         transaction->stage =
2774                                                 CVMX_USB_STAGE_STATUS;
2775                                 }
2776                                 break;
2777                         case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
2778                                 if ((buffer_space_left == 0) ||
2779                                                 (bytes_in_last_packet <
2780                                                  pipe->max_packet)) {
2781                                         pipe->pid_toggle = 1;
2782                                         transaction->stage =
2783                                                 CVMX_USB_STAGE_STATUS;
2784                                 } else {
2785                                         transaction->stage =
2786                                                 CVMX_USB_STAGE_DATA;
2787                                 }
2788                                 break;
2789                         case CVMX_USB_STAGE_STATUS:
2790                                 if (cvmx_usb_pipe_needs_split(usb, pipe))
2791                                         transaction->stage =
2792                                                 CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
2793                                 else
2794                                         cvmx_usb_perform_complete(usb, pipe,
2795                                                 transaction,
2796                                                 CVMX_USB_COMPLETE_SUCCESS);
2797                                 break;
2798                         case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
2799                                 cvmx_usb_perform_complete(usb, pipe,
2800                                                 transaction,
2801                                                 CVMX_USB_COMPLETE_SUCCESS);
2802                                 break;
2803                         }
2804                         break;
2805                 case CVMX_USB_TRANSFER_BULK:
2806                 case CVMX_USB_TRANSFER_INTERRUPT:
2807                         /*
2808                          * The only time a bulk transfer isn't complete when it
2809                          * finishes with an ACK is during a split transaction.
2810                          * For splits we need to continue the transfer if more
2811                          * data is needed
2812                          */
2813                         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2814                                 if (transaction->stage ==
2815                                                 CVMX_USB_STAGE_NON_CONTROL)
2816                                         transaction->stage =
2817                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2818                                 else {
2819                                         if (buffer_space_left &&
2820                                                 (bytes_in_last_packet ==
2821                                                  pipe->max_packet))
2822                                                 transaction->stage =
2823                                                         CVMX_USB_STAGE_NON_CONTROL;
2824                                         else {
2825                                                 if (transaction->type ==
2826                                                         CVMX_USB_TRANSFER_INTERRUPT)
2827                                                         pipe->next_tx_frame +=
2828                                                                 pipe->interval;
2829                                                         cvmx_usb_perform_complete(
2830                                                                 usb,
2831                                                                 pipe,
2832                                                                 transaction,
2833                                                                 CVMX_USB_COMPLETE_SUCCESS);
2834                                         }
2835                                 }
2836                         } else {
2837                                 if ((pipe->device_speed ==
2838                                         CVMX_USB_SPEED_HIGH) &&
2839                                     (pipe->transfer_type ==
2840                                      CVMX_USB_TRANSFER_BULK) &&
2841                                     (pipe->transfer_dir ==
2842                                      CVMX_USB_DIRECTION_OUT) &&
2843                                     (usbc_hcint.s.nak))
2844                                         pipe->flags |=
2845                                                 CVMX_USB_PIPE_FLAGS_NEED_PING;
2846                                 if (!buffer_space_left ||
2847                                         (bytes_in_last_packet <
2848                                          pipe->max_packet)) {
2849                                         if (transaction->type ==
2850                                                 CVMX_USB_TRANSFER_INTERRUPT)
2851                                                 pipe->next_tx_frame +=
2852                                                         pipe->interval;
2853                                         cvmx_usb_perform_complete(usb, pipe,
2854                                                 transaction,
2855                                                 CVMX_USB_COMPLETE_SUCCESS);
2856                                 }
2857                         }
2858                         break;
2859                 case CVMX_USB_TRANSFER_ISOCHRONOUS:
2860                         if (cvmx_usb_pipe_needs_split(usb, pipe)) {
2861                                 /*
2862                                  * ISOCHRONOUS OUT splits don't require a
2863                                  * complete split stage. Instead they use a
2864                                  * sequence of begin OUT splits to transfer the
2865                                  * data 188 bytes at a time. Once the transfer
2866                                  * is complete, the pipe sleeps until the next
2867                                  * schedule interval
2868                                  */
2869                                 if (pipe->transfer_dir ==
2870                                         CVMX_USB_DIRECTION_OUT) {
2871                                         /*
2872                                          * If no space left or this wasn't a max
2873                                          * size packet then this transfer is
2874                                          * complete. Otherwise start it again to
2875                                          * send the next 188 bytes
2876                                          */
2877                                         if (!buffer_space_left ||
2878                                                 (bytes_this_transfer < 188)) {
2879                                                 pipe->next_tx_frame +=
2880                                                         pipe->interval;
2881                                                 cvmx_usb_perform_complete(usb,
2882                                                         pipe, transaction,
2883                                                         CVMX_USB_COMPLETE_SUCCESS);
2884                                         }
2885                                 } else {
2886                                         if (transaction->stage ==
2887                                                 CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
2888                                                 /*
2889                                                  * We are in the incoming data
2890                                                  * phase. Keep getting data
2891                                                  * until we run out of space or
2892                                                  * get a small packet
2893                                                  */
2894                                                 if ((buffer_space_left == 0) ||
2895                                                         (bytes_in_last_packet <
2896                                                          pipe->max_packet)) {
2897                                                         pipe->next_tx_frame +=
2898                                                                 pipe->interval;
2899                                                         cvmx_usb_perform_complete(
2900                                                                 usb,
2901                                                                 pipe,
2902                                                                 transaction,
2903                                                                 CVMX_USB_COMPLETE_SUCCESS);
2904                                                 }
2905                                         } else
2906                                                 transaction->stage =
2907                                                         CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
2908                                 }
2909                         } else {
2910                                 pipe->next_tx_frame += pipe->interval;
2911                                 cvmx_usb_perform_complete(usb, pipe,
2912                                                 transaction,
2913                                                 CVMX_USB_COMPLETE_SUCCESS);
2914                         }
2915                         break;
2916                 }
2917         } else if (usbc_hcint.s.nak) {
2918                 /*
2919                  * If this was a split then clear our split in progress marker.
2920                  */
2921                 if (usb->active_split == transaction)
2922                         usb->active_split = NULL;
2923                 /*
2924                  * NAK as a response means the device couldn't accept the
2925                  * transaction, but it should be retried in the future. Rewind
2926                  * to the beginning of the transaction by anding off the split
2927                  * complete bit. Retry in the next interval
2928                  */
2929                 transaction->retries = 0;
2930                 transaction->stage &= ~1;
2931                 pipe->next_tx_frame += pipe->interval;
2932                 if (pipe->next_tx_frame < usb->frame_number)
2933                         pipe->next_tx_frame = usb->frame_number +
2934                                 pipe->interval -
2935                                 (usb->frame_number - pipe->next_tx_frame) %
2936                                 pipe->interval;
2937         } else {
2938                 struct cvmx_usb_port_status port;
2939 
2940                 port = cvmx_usb_get_status(usb);
2941                 if (port.port_enabled) {
2942                         /* We'll retry the exact same transaction again */
2943                         transaction->retries++;
2944                 } else {
2945                         /*
2946                          * We get channel halted interrupts with no result bits
2947                          * sets when the cable is unplugged
2948                          */
2949                         cvmx_usb_perform_complete(usb, pipe, transaction,
2950                                         CVMX_USB_COMPLETE_ERROR);
2951                 }
2952         }
2953         return 0;
2954 }
2955 
2956 static void octeon_usb_port_callback(struct cvmx_usb_state *usb)
2957 {
2958         struct octeon_hcd *priv = cvmx_usb_to_octeon(usb);
2959 
2960         spin_unlock(&priv->lock);
2961         usb_hcd_poll_rh_status(octeon_to_hcd(priv));
2962         spin_lock(&priv->lock);
2963 }
2964 
2965 /**
2966  * Poll the USB block for status and call all needed callback
2967  * handlers. This function is meant to be called in the interrupt
2968  * handler for the USB controller. It can also be called
2969  * periodically in a loop for non-interrupt based operation.
2970  *
2971  * @usb: USB device state populated by cvmx_usb_initialize().
2972  *
2973  * Returns: 0 or a negative error code.
2974  */
2975 static int cvmx_usb_poll(struct cvmx_usb_state *usb)
2976 {
2977         union cvmx_usbcx_hfnum usbc_hfnum;
2978         union cvmx_usbcx_gintsts usbc_gintsts;
2979 
2980         prefetch_range(usb, sizeof(*usb));
2981 
2982         /* Update the frame counter */
2983         usbc_hfnum.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
2984         if ((usb->frame_number&0x3fff) > usbc_hfnum.s.frnum)
2985                 usb->frame_number += 0x4000;
2986         usb->frame_number &= ~0x3fffull;
2987         usb->frame_number |= usbc_hfnum.s.frnum;
2988 
2989         /* Read the pending interrupts */
2990         usbc_gintsts.u32 = cvmx_usb_read_csr32(usb,
2991                                                CVMX_USBCX_GINTSTS(usb->index));
2992 
2993         /* Clear the interrupts now that we know about them */
2994         cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index),
2995                              usbc_gintsts.u32);
2996 
2997         if (usbc_gintsts.s.rxflvl) {
2998                 /*
2999                  * RxFIFO Non-Empty (RxFLvl)
3000                  * Indicates that there is at least one packet pending to be
3001                  * read from the RxFIFO.
3002                  *
3003                  * In DMA mode this is handled by hardware
3004                  */
3005                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3006                         cvmx_usb_poll_rx_fifo(usb);
3007         }
3008         if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
3009                 /* Fill the Tx FIFOs when not in DMA mode */
3010                 if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
3011                         cvmx_usb_poll_tx_fifo(usb);
3012         }
3013         if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
3014                 union cvmx_usbcx_hprt usbc_hprt;
3015                 /*
3016                  * Disconnect Detected Interrupt (DisconnInt)
3017                  * Asserted when a device disconnect is detected.
3018                  *
3019                  * Host Port Interrupt (PrtInt)
3020                  * The core sets this bit to indicate a change in port status of
3021                  * one of the O2P USB core ports in Host mode. The application
3022                  * must read the Host Port Control and Status (HPRT) register to
3023                  * determine the exact event that caused this interrupt. The
3024                  * application must clear the appropriate status bit in the Host
3025                  * Port Control and Status register to clear this bit.
3026                  *
3027                  * Call the user's port callback
3028                  */
3029                 octeon_usb_port_callback(usb);
3030                 /* Clear the port change bits */
3031                 usbc_hprt.u32 = cvmx_usb_read_csr32(usb,
3032                                         CVMX_USBCX_HPRT(usb->index));
3033                 usbc_hprt.s.prtena = 0;
3034                 cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index),
3035                                      usbc_hprt.u32);
3036         }
3037         if (usbc_gintsts.s.hchint) {
3038                 /*
3039                  * Host Channels Interrupt (HChInt)
3040                  * The core sets this bit to indicate that an interrupt is
3041                  * pending on one of the channels of the core (in Host mode).
3042                  * The application must read the Host All Channels Interrupt
3043                  * (HAINT) register to determine the exact number of the channel
3044                  * on which the interrupt occurred, and then read the
3045                  * corresponding Host Channel-n Interrupt (HCINTn) register to
3046                  * determine the exact cause of the interrupt. The application
3047                  * must clear the appropriate status bit in the HCINTn register
3048                  * to clear this bit.
3049                  */
3050                 union cvmx_usbcx_haint usbc_haint;
3051 
3052                 usbc_haint.u32 = cvmx_usb_read_csr32(usb,
3053                                         CVMX_USBCX_HAINT(usb->index));
3054                 while (usbc_haint.u32) {
3055                         int channel;
3056 
3057                         channel = __fls(usbc_haint.u32);
3058                         cvmx_usb_poll_channel(usb, channel);
3059                         usbc_haint.u32 ^= 1<<channel;
3060                 }
3061         }
3062 
3063         cvmx_usb_schedule(usb, usbc_gintsts.s.sof);
3064 
3065         return 0;
3066 }
3067 
3068 /* convert between an HCD pointer and the corresponding struct octeon_hcd */
3069 static inline struct octeon_hcd *hcd_to_octeon(struct usb_hcd *hcd)
3070 {
3071         return (struct octeon_hcd *)(hcd->hcd_priv);
3072 }
3073 
3074 static irqreturn_t octeon_usb_irq(struct usb_hcd *hcd)
3075 {
3076         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3077         unsigned long flags;
3078 
3079         spin_lock_irqsave(&priv->lock, flags);
3080         cvmx_usb_poll(&priv->usb);
3081         spin_unlock_irqrestore(&priv->lock, flags);
3082         return IRQ_HANDLED;
3083 }
3084 
3085 static int octeon_usb_start(struct usb_hcd *hcd)
3086 {
3087         hcd->state = HC_STATE_RUNNING;
3088         return 0;
3089 }
3090 
3091 static void octeon_usb_stop(struct usb_hcd *hcd)
3092 {
3093         hcd->state = HC_STATE_HALT;
3094 }
3095 
3096 static int octeon_usb_get_frame_number(struct usb_hcd *hcd)
3097 {
3098         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3099 
3100         return cvmx_usb_get_frame_number(&priv->usb);
3101 }
3102 
3103 static int octeon_usb_urb_enqueue(struct usb_hcd *hcd,
3104                                   struct urb *urb,
3105                                   gfp_t mem_flags)
3106 {
3107         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3108         struct device *dev = hcd->self.controller;
3109         struct cvmx_usb_transaction *transaction = NULL;
3110         struct cvmx_usb_pipe *pipe;
3111         unsigned long flags;
3112         struct cvmx_usb_iso_packet *iso_packet;
3113         struct usb_host_endpoint *ep = urb->ep;
3114         int rc;
3115 
3116         urb->status = 0;
3117         spin_lock_irqsave(&priv->lock, flags);
3118 
3119         rc = usb_hcd_link_urb_to_ep(hcd, urb);
3120         if (rc) {
3121                 spin_unlock_irqrestore(&priv->lock, flags);
3122                 return rc;
3123         }
3124 
3125         if (!ep->hcpriv) {
3126                 enum cvmx_usb_transfer transfer_type;
3127                 enum cvmx_usb_speed speed;
3128                 int split_device = 0;
3129                 int split_port = 0;
3130 
3131                 switch (usb_pipetype(urb->pipe)) {
3132                 case PIPE_ISOCHRONOUS:
3133                         transfer_type = CVMX_USB_TRANSFER_ISOCHRONOUS;
3134                         break;
3135                 case PIPE_INTERRUPT:
3136                         transfer_type = CVMX_USB_TRANSFER_INTERRUPT;
3137                         break;
3138                 case PIPE_CONTROL:
3139                         transfer_type = CVMX_USB_TRANSFER_CONTROL;
3140                         break;
3141                 default:
3142                         transfer_type = CVMX_USB_TRANSFER_BULK;
3143                         break;
3144                 }
3145                 switch (urb->dev->speed) {
3146                 case USB_SPEED_LOW:
3147                         speed = CVMX_USB_SPEED_LOW;
3148                         break;
3149                 case USB_SPEED_FULL:
3150                         speed = CVMX_USB_SPEED_FULL;
3151                         break;
3152                 default:
3153                         speed = CVMX_USB_SPEED_HIGH;
3154                         break;
3155                 }
3156                 /*
3157                  * For slow devices on high speed ports we need to find the hub
3158                  * that does the speed translation so we know where to send the
3159                  * split transactions.
3160                  */
3161                 if (speed != CVMX_USB_SPEED_HIGH) {
3162                         /*
3163                          * Start at this device and work our way up the usb
3164                          * tree.
3165                          */
3166                         struct usb_device *dev = urb->dev;
3167 
3168                         while (dev->parent) {
3169                                 /*
3170                                  * If our parent is high speed then he'll
3171                                  * receive the splits.
3172                                  */
3173                                 if (dev->parent->speed == USB_SPEED_HIGH) {
3174                                         split_device = dev->parent->devnum;
3175                                         split_port = dev->portnum;
3176                                         break;
3177                                 }
3178                                 /*
3179                                  * Move up the tree one level. If we make it all
3180                                  * the way up the tree, then the port must not
3181                                  * be in high speed mode and we don't need a
3182                                  * split.
3183                                  */
3184                                 dev = dev->parent;
3185                         }
3186                 }
3187                 pipe = cvmx_usb_open_pipe(&priv->usb, usb_pipedevice(urb->pipe),
3188                                           usb_pipeendpoint(urb->pipe), speed,
3189                                           le16_to_cpu(ep->desc.wMaxPacketSize)
3190                                           & 0x7ff,
3191                                           transfer_type,
3192                                           usb_pipein(urb->pipe) ?
3193                                                 CVMX_USB_DIRECTION_IN :
3194                                                 CVMX_USB_DIRECTION_OUT,
3195                                           urb->interval,
3196                                           (le16_to_cpu(ep->desc.wMaxPacketSize)
3197                                            >> 11) & 0x3,
3198                                           split_device, split_port);
3199                 if (!pipe) {
3200                         usb_hcd_unlink_urb_from_ep(hcd, urb);
3201                         spin_unlock_irqrestore(&priv->lock, flags);
3202                         dev_dbg(dev, "Failed to create pipe\n");
3203                         return -ENOMEM;
3204                 }
3205                 ep->hcpriv = pipe;
3206         } else {
3207                 pipe = ep->hcpriv;
3208         }
3209 
3210         switch (usb_pipetype(urb->pipe)) {
3211         case PIPE_ISOCHRONOUS:
3212                 dev_dbg(dev, "Submit isochronous to %d.%d\n",
3213                         usb_pipedevice(urb->pipe),
3214                         usb_pipeendpoint(urb->pipe));
3215                 /*
3216                  * Allocate a structure to use for our private list of
3217                  * isochronous packets.
3218                  */
3219                 iso_packet = kmalloc(urb->number_of_packets *
3220                                      sizeof(struct cvmx_usb_iso_packet),
3221                                      GFP_ATOMIC);
3222                 if (iso_packet) {
3223                         int i;
3224                         /* Fill the list with the data from the URB */
3225                         for (i = 0; i < urb->number_of_packets; i++) {
3226                                 iso_packet[i].offset =
3227                                         urb->iso_frame_desc[i].offset;
3228                                 iso_packet[i].length =
3229                                         urb->iso_frame_desc[i].length;
3230                                 iso_packet[i].status =
3231                                         CVMX_USB_COMPLETE_ERROR;
3232                         }
3233                         /*
3234                          * Store a pointer to the list in the URB setup_packet
3235                          * field. We know this currently isn't being used and
3236                          * this saves us a bunch of logic.
3237                          */
3238                         urb->setup_packet = (char *)iso_packet;
3239                         transaction = cvmx_usb_submit_isochronous(&priv->usb,
3240                                                                   pipe, urb);
3241                         /*
3242                          * If submit failed we need to free our private packet
3243                          * list.
3244                          */
3245                         if (!transaction) {
3246                                 urb->setup_packet = NULL;
3247                                 kfree(iso_packet);
3248                         }
3249                 }
3250                 break;
3251         case PIPE_INTERRUPT:
3252                 dev_dbg(dev, "Submit interrupt to %d.%d\n",
3253                         usb_pipedevice(urb->pipe),
3254                         usb_pipeendpoint(urb->pipe));
3255                 transaction = cvmx_usb_submit_interrupt(&priv->usb, pipe, urb);
3256                 break;
3257         case PIPE_CONTROL:
3258                 dev_dbg(dev, "Submit control to %d.%d\n",
3259                         usb_pipedevice(urb->pipe),
3260                         usb_pipeendpoint(urb->pipe));
3261                 transaction = cvmx_usb_submit_control(&priv->usb, pipe, urb);
3262                 break;
3263         case PIPE_BULK:
3264                 dev_dbg(dev, "Submit bulk to %d.%d\n",
3265                         usb_pipedevice(urb->pipe),
3266                         usb_pipeendpoint(urb->pipe));
3267                 transaction = cvmx_usb_submit_bulk(&priv->usb, pipe, urb);
3268                 break;
3269         }
3270         if (!transaction) {
3271                 usb_hcd_unlink_urb_from_ep(hcd, urb);
3272                 spin_unlock_irqrestore(&priv->lock, flags);
3273                 dev_dbg(dev, "Failed to submit\n");
3274                 return -ENOMEM;
3275         }
3276         urb->hcpriv = transaction;
3277         spin_unlock_irqrestore(&priv->lock, flags);
3278         return 0;
3279 }
3280 
3281 static int octeon_usb_urb_dequeue(struct usb_hcd *hcd,
3282                                   struct urb *urb,
3283                                   int status)
3284 {
3285         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3286         unsigned long flags;
3287         int rc;
3288 
3289         if (!urb->dev)
3290                 return -EINVAL;
3291 
3292         spin_lock_irqsave(&priv->lock, flags);
3293 
3294         rc = usb_hcd_check_unlink_urb(hcd, urb, status);
3295         if (rc)
3296                 goto out;
3297 
3298         urb->status = status;
3299         cvmx_usb_cancel(&priv->usb, urb->ep->hcpriv, urb->hcpriv);
3300 
3301 out:
3302         spin_unlock_irqrestore(&priv->lock, flags);
3303 
3304         return rc;
3305 }
3306 
3307 static void octeon_usb_endpoint_disable(struct usb_hcd *hcd,
3308                                         struct usb_host_endpoint *ep)
3309 {
3310         struct device *dev = hcd->self.controller;
3311 
3312         if (ep->hcpriv) {
3313                 struct octeon_hcd *priv = hcd_to_octeon(hcd);
3314                 struct cvmx_usb_pipe *pipe = ep->hcpriv;
3315                 unsigned long flags;
3316 
3317                 spin_lock_irqsave(&priv->lock, flags);
3318                 cvmx_usb_cancel_all(&priv->usb, pipe);
3319                 if (cvmx_usb_close_pipe(&priv->usb, pipe))
3320                         dev_dbg(dev, "Closing pipe %p failed\n", pipe);
3321                 spin_unlock_irqrestore(&priv->lock, flags);
3322                 ep->hcpriv = NULL;
3323         }
3324 }
3325 
3326 static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
3327 {
3328         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3329         struct cvmx_usb_port_status port_status;
3330         unsigned long flags;
3331 
3332         spin_lock_irqsave(&priv->lock, flags);
3333         port_status = cvmx_usb_get_status(&priv->usb);
3334         spin_unlock_irqrestore(&priv->lock, flags);
3335         buf[0] = 0;
3336         buf[0] = port_status.connect_change << 1;
3337 
3338         return buf[0] != 0;
3339 }
3340 
3341 static int octeon_usb_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
3342                                 u16 wIndex, char *buf, u16 wLength)
3343 {
3344         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3345         struct device *dev = hcd->self.controller;
3346         struct cvmx_usb_port_status usb_port_status;
3347         struct cvmx_usb_state *usb = &priv->usb;
3348         int port_status;
3349         struct usb_hub_descriptor *desc;
3350         unsigned long flags;
3351 
3352         switch (typeReq) {
3353         case ClearHubFeature:
3354                 dev_dbg(dev, "ClearHubFeature\n");
3355                 switch (wValue) {
3356                 case C_HUB_LOCAL_POWER:
3357                 case C_HUB_OVER_CURRENT:
3358                         /* Nothing required here */
3359                         break;
3360                 default:
3361                         return -EINVAL;
3362                 }
3363                 break;
3364         case ClearPortFeature:
3365                 dev_dbg(dev, "ClearPortFeature\n");
3366                 if (wIndex != 1) {
3367                         dev_dbg(dev, " INVALID\n");
3368                         return -EINVAL;
3369                 }
3370 
3371                 switch (wValue) {
3372                 case USB_PORT_FEAT_ENABLE:
3373                         dev_dbg(dev, " ENABLE\n");
3374                         spin_lock_irqsave(&priv->lock, flags);
3375                         cvmx_usb_disable(&priv->usb);
3376                         spin_unlock_irqrestore(&priv->lock, flags);
3377                         break;
3378                 case USB_PORT_FEAT_SUSPEND:
3379                         dev_dbg(dev, " SUSPEND\n");
3380                         /* Not supported on Octeon */
3381                         break;
3382                 case USB_PORT_FEAT_POWER:
3383                         dev_dbg(dev, " POWER\n");
3384                         /* Not supported on Octeon */
3385                         break;
3386                 case USB_PORT_FEAT_INDICATOR:
3387                         dev_dbg(dev, " INDICATOR\n");
3388                         /* Port inidicator not supported */
3389                         break;
3390                 case USB_PORT_FEAT_C_CONNECTION:
3391                         dev_dbg(dev, " C_CONNECTION\n");
3392                         /* Clears drivers internal connect status change flag */
3393                         spin_lock_irqsave(&priv->lock, flags);
3394                         priv->usb.port_status =
3395                                 cvmx_usb_get_status(&priv->usb);
3396                         spin_unlock_irqrestore(&priv->lock, flags);
3397                         break;
3398                 case USB_PORT_FEAT_C_RESET:
3399                         dev_dbg(dev, " C_RESET\n");
3400                         /*
3401                          * Clears the driver's internal Port Reset Change flag.
3402                          */
3403                         spin_lock_irqsave(&priv->lock, flags);
3404                         priv->usb.port_status =
3405                                 cvmx_usb_get_status(&priv->usb);
3406                         spin_unlock_irqrestore(&priv->lock, flags);
3407                         break;
3408                 case USB_PORT_FEAT_C_ENABLE:
3409                         dev_dbg(dev, " C_ENABLE\n");
3410                         /*
3411                          * Clears the driver's internal Port Enable/Disable
3412                          * Change flag.
3413                          */
3414                         spin_lock_irqsave(&priv->lock, flags);
3415                         priv->usb.port_status =
3416                                 cvmx_usb_get_status(&priv->usb);
3417                         spin_unlock_irqrestore(&priv->lock, flags);
3418                         break;
3419                 case USB_PORT_FEAT_C_SUSPEND:
3420                         dev_dbg(dev, " C_SUSPEND\n");
3421                         /*
3422                          * Clears the driver's internal Port Suspend Change
3423                          * flag, which is set when resume signaling on the host
3424                          * port is complete.
3425                          */
3426                         break;
3427                 case USB_PORT_FEAT_C_OVER_CURRENT:
3428                         dev_dbg(dev, " C_OVER_CURRENT\n");
3429                         /* Clears the driver's overcurrent Change flag */
3430                         spin_lock_irqsave(&priv->lock, flags);
3431                         priv->usb.port_status =
3432                                 cvmx_usb_get_status(&priv->usb);
3433                         spin_unlock_irqrestore(&priv->lock, flags);
3434                         break;
3435                 default:
3436                         dev_dbg(dev, " UNKNOWN\n");
3437                         return -EINVAL;
3438                 }
3439                 break;
3440         case GetHubDescriptor:
3441                 dev_dbg(dev, "GetHubDescriptor\n");
3442                 desc = (struct usb_hub_descriptor *)buf;
3443                 desc->bDescLength = 9;
3444                 desc->bDescriptorType = 0x29;
3445                 desc->bNbrPorts = 1;
3446                 desc->wHubCharacteristics = cpu_to_le16(0x08);
3447                 desc->bPwrOn2PwrGood = 1;
3448                 desc->bHubContrCurrent = 0;
3449                 desc->u.hs.DeviceRemovable[0] = 0;
3450                 desc->u.hs.DeviceRemovable[1] = 0xff;
3451                 break;
3452         case GetHubStatus:
3453                 dev_dbg(dev, "GetHubStatus\n");
3454                 *(__le32 *) buf = 0;
3455                 break;
3456         case GetPortStatus:
3457                 dev_dbg(dev, "GetPortStatus\n");
3458                 if (wIndex != 1) {
3459                         dev_dbg(dev, " INVALID\n");
3460                         return -EINVAL;
3461                 }
3462 
3463                 spin_lock_irqsave(&priv->lock, flags);
3464                 usb_port_status = cvmx_usb_get_status(&priv->usb);
3465                 spin_unlock_irqrestore(&priv->lock, flags);
3466                 port_status = 0;
3467 
3468                 if (usb_port_status.connect_change) {
3469                         port_status |= (1 << USB_PORT_FEAT_C_CONNECTION);
3470                         dev_dbg(dev, " C_CONNECTION\n");
3471                 }
3472 
3473                 if (usb_port_status.port_enabled) {
3474                         port_status |= (1 << USB_PORT_FEAT_C_ENABLE);
3475                         dev_dbg(dev, " C_ENABLE\n");
3476                 }
3477 
3478                 if (usb_port_status.connected) {
3479                         port_status |= (1 << USB_PORT_FEAT_CONNECTION);
3480                         dev_dbg(dev, " CONNECTION\n");
3481                 }
3482 
3483                 if (usb_port_status.port_enabled) {
3484                         port_status |= (1 << USB_PORT_FEAT_ENABLE);
3485                         dev_dbg(dev, " ENABLE\n");
3486                 }
3487 
3488                 if (usb_port_status.port_over_current) {
3489                         port_status |= (1 << USB_PORT_FEAT_OVER_CURRENT);
3490                         dev_dbg(dev, " OVER_CURRENT\n");
3491                 }
3492 
3493                 if (usb_port_status.port_powered) {
3494                         port_status |= (1 << USB_PORT_FEAT_POWER);
3495                         dev_dbg(dev, " POWER\n");
3496                 }
3497 
3498                 if (usb_port_status.port_speed == CVMX_USB_SPEED_HIGH) {
3499                         port_status |= USB_PORT_STAT_HIGH_SPEED;
3500                         dev_dbg(dev, " HIGHSPEED\n");
3501                 } else if (usb_port_status.port_speed == CVMX_USB_SPEED_LOW) {
3502                         port_status |= (1 << USB_PORT_FEAT_LOWSPEED);
3503                         dev_dbg(dev, " LOWSPEED\n");
3504                 }
3505 
3506                 *((__le32 *) buf) = cpu_to_le32(port_status);
3507                 break;
3508         case SetHubFeature:
3509                 dev_dbg(dev, "SetHubFeature\n");
3510                 /* No HUB features supported */
3511                 break;
3512         case SetPortFeature:
3513                 dev_dbg(dev, "SetPortFeature\n");
3514                 if (wIndex != 1) {
3515                         dev_dbg(dev, " INVALID\n");
3516                         return -EINVAL;
3517                 }
3518 
3519                 switch (wValue) {
3520                 case USB_PORT_FEAT_SUSPEND:
3521                         dev_dbg(dev, " SUSPEND\n");
3522                         return -EINVAL;
3523                 case USB_PORT_FEAT_POWER:
3524                         dev_dbg(dev, " POWER\n");
3525                         /*
3526                          * Program the port power bit to drive VBUS on the USB.
3527                          */
3528                         spin_lock_irqsave(&priv->lock, flags);
3529                         USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index),
3530                                         cvmx_usbcx_hprt, prtpwr, 1);
3531                         spin_unlock_irqrestore(&priv->lock, flags);
3532                         return 0;
3533                 case USB_PORT_FEAT_RESET:
3534                         dev_dbg(dev, " RESET\n");
3535                         spin_lock_irqsave(&priv->lock, flags);
3536                         cvmx_usb_reset_port(&priv->usb);
3537                         spin_unlock_irqrestore(&priv->lock, flags);
3538                         return 0;
3539                 case USB_PORT_FEAT_INDICATOR:
3540                         dev_dbg(dev, " INDICATOR\n");
3541                         /* Not supported */
3542                         break;
3543                 default:
3544                         dev_dbg(dev, " UNKNOWN\n");
3545                         return -EINVAL;
3546                 }
3547                 break;
3548         default:
3549                 dev_dbg(dev, "Unknown root hub request\n");
3550                 return -EINVAL;
3551         }
3552         return 0;
3553 }
3554 
3555 static const struct hc_driver octeon_hc_driver = {
3556         .description            = "Octeon USB",
3557         .product_desc           = "Octeon Host Controller",
3558         .hcd_priv_size          = sizeof(struct octeon_hcd),
3559         .irq                    = octeon_usb_irq,
3560         .flags                  = HCD_MEMORY | HCD_USB2,
3561         .start                  = octeon_usb_start,
3562         .stop                   = octeon_usb_stop,
3563         .urb_enqueue            = octeon_usb_urb_enqueue,
3564         .urb_dequeue            = octeon_usb_urb_dequeue,
3565         .endpoint_disable       = octeon_usb_endpoint_disable,
3566         .get_frame_number       = octeon_usb_get_frame_number,
3567         .hub_status_data        = octeon_usb_hub_status_data,
3568         .hub_control            = octeon_usb_hub_control,
3569         .map_urb_for_dma        = octeon_map_urb_for_dma,
3570         .unmap_urb_for_dma      = octeon_unmap_urb_for_dma,
3571 };
3572 
3573 static int octeon_usb_probe(struct platform_device *pdev)
3574 {
3575         int status;
3576         int initialize_flags;
3577         int usb_num;
3578         struct resource *res_mem;
3579         struct device_node *usbn_node;
3580         int irq = platform_get_irq(pdev, 0);
3581         struct device *dev = &pdev->dev;
3582         struct octeon_hcd *priv;
3583         struct usb_hcd *hcd;
3584         u32 clock_rate = 48000000;
3585         bool is_crystal_clock = false;
3586         const char *clock_type;
3587         int i;
3588 
3589         if (dev->of_node == NULL) {
3590                 dev_err(dev, "Error: empty of_node\n");
3591                 return -ENXIO;
3592         }
3593         usbn_node = dev->of_node->parent;
3594 
3595         i = of_property_read_u32(usbn_node,
3596                                  "refclk-frequency", &clock_rate);
3597         if (i) {
3598                 dev_err(dev, "No USBN \"refclk-frequency\"\n");
3599                 return -ENXIO;
3600         }
3601         switch (clock_rate) {
3602         case 12000000:
3603                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
3604                 break;
3605         case 24000000:
3606                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
3607                 break;
3608         case 48000000:
3609                 initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
3610                 break;
3611         default:
3612                 dev_err(dev, "Illebal USBN \"refclk-frequency\" %u\n",
3613                                 clock_rate);
3614                 return -ENXIO;
3615 
3616         }
3617 
3618         i = of_property_read_string(usbn_node,
3619                                     "refclk-type", &clock_type);
3620 
3621         if (!i && strcmp("crystal", clock_type) == 0)
3622                 is_crystal_clock = true;
3623 
3624         if (is_crystal_clock)
3625                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
3626         else
3627                 initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;
3628 
3629         res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3630         if (res_mem == NULL) {
3631                 dev_err(dev, "found no memory resource\n");
3632                 return -ENXIO;
3633         }
3634         usb_num = (res_mem->start >> 44) & 1;
3635 
3636         if (irq < 0) {
3637                 /* Defective device tree, but we know how to fix it. */
3638                 irq_hw_number_t hwirq = usb_num ? (1 << 6) + 17 : 56;
3639 
3640                 irq = irq_create_mapping(NULL, hwirq);
3641         }
3642 
3643         /*
3644          * Set the DMA mask to 64bits so we get buffers already translated for
3645          * DMA.
3646          */
3647         dev->coherent_dma_mask = ~0;
3648         dev->dma_mask = &dev->coherent_dma_mask;
3649 
3650         /*
3651          * Only cn52XX and cn56XX have DWC_OTG USB hardware and the
3652          * IOB priority registers.  Under heavy network load USB
3653          * hardware can be starved by the IOB causing a crash.  Give
3654          * it a priority boost if it has been waiting more than 400
3655          * cycles to avoid this situation.
3656          *
3657          * Testing indicates that a cnt_val of 8192 is not sufficient,
3658          * but no failures are seen with 4096.  We choose a value of
3659          * 400 to give a safety factor of 10.
3660          */
3661         if (OCTEON_IS_MODEL(OCTEON_CN52XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
3662                 union cvmx_iob_n2c_l2c_pri_cnt pri_cnt;
3663 
3664                 pri_cnt.u64 = 0;
3665                 pri_cnt.s.cnt_enb = 1;
3666                 pri_cnt.s.cnt_val = 400;
3667                 cvmx_write_csr(CVMX_IOB_N2C_L2C_PRI_CNT, pri_cnt.u64);
3668         }
3669 
3670         hcd = usb_create_hcd(&octeon_hc_driver, dev, dev_name(dev));
3671         if (!hcd) {
3672                 dev_dbg(dev, "Failed to allocate memory for HCD\n");
3673                 return -1;
3674         }
3675         hcd->uses_new_polling = 1;
3676         priv = (struct octeon_hcd *)hcd->hcd_priv;
3677 
3678         spin_lock_init(&priv->lock);
3679 
3680         priv->usb.init_flags = initialize_flags;
3681 
3682         /* Initialize the USB state structure */
3683         priv->usb.index = usb_num;
3684         INIT_LIST_HEAD(&priv->usb.idle_pipes);
3685         for (i = 0; i < ARRAY_SIZE(priv->usb.active_pipes); i++)
3686                 INIT_LIST_HEAD(&priv->usb.active_pipes[i]);
3687 
3688         /* Due to an errata, CN31XX doesn't support DMA */
3689         if (OCTEON_IS_MODEL(OCTEON_CN31XX)) {
3690                 priv->usb.init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
3691                 /* Only use one channel with non DMA */
3692                 priv->usb.idle_hardware_channels = 0x1;
3693         } else if (OCTEON_IS_MODEL(OCTEON_CN5XXX)) {
3694                 /* CN5XXX have an errata with channel 3 */
3695                 priv->usb.idle_hardware_channels = 0xf7;
3696         } else {
3697                 priv->usb.idle_hardware_channels = 0xff;
3698         }
3699 
3700         status = cvmx_usb_initialize(dev, &priv->usb);
3701         if (status) {
3702                 dev_dbg(dev, "USB initialization failed with %d\n", status);
3703                 kfree(hcd);
3704                 return -1;
3705         }
3706 
3707         status = usb_add_hcd(hcd, irq, 0);
3708         if (status) {
3709                 dev_dbg(dev, "USB add HCD failed with %d\n", status);
3710                 kfree(hcd);
3711                 return -1;
3712         }
3713         device_wakeup_enable(hcd->self.controller);
3714 
3715         dev_info(dev, "Registered HCD for port %d on irq %d\n", usb_num, irq);
3716 
3717         return 0;
3718 }
3719 
3720 static int octeon_usb_remove(struct platform_device *pdev)
3721 {
3722         int status;
3723         struct device *dev = &pdev->dev;
3724         struct usb_hcd *hcd = dev_get_drvdata(dev);
3725         struct octeon_hcd *priv = hcd_to_octeon(hcd);
3726         unsigned long flags;
3727 
3728         usb_remove_hcd(hcd);
3729         spin_lock_irqsave(&priv->lock, flags);
3730         status = cvmx_usb_shutdown(&priv->usb);
3731         spin_unlock_irqrestore(&priv->lock, flags);
3732         if (status)
3733                 dev_dbg(dev, "USB shutdown failed with %d\n", status);
3734 
3735         kfree(hcd);
3736 
3737         return 0;
3738 }
3739 
3740 static const struct of_device_id octeon_usb_match[] = {
3741         {
3742                 .compatible = "cavium,octeon-5750-usbc",
3743         },
3744         {},
3745 };
3746 
3747 static struct platform_driver octeon_usb_driver = {
3748         .driver = {
3749                 .name       = "OcteonUSB",
3750                 .of_match_table = octeon_usb_match,
3751         },
3752         .probe      = octeon_usb_probe,
3753         .remove     = octeon_usb_remove,
3754 };
3755 
3756 static int __init octeon_usb_driver_init(void)
3757 {
3758         if (usb_disabled())
3759                 return 0;
3760 
3761         return platform_driver_register(&octeon_usb_driver);
3762 }
3763 module_init(octeon_usb_driver_init);
3764 
3765 static void __exit octeon_usb_driver_exit(void)
3766 {
3767         if (usb_disabled())
3768                 return;
3769 
3770         platform_driver_unregister(&octeon_usb_driver);
3771 }
3772 module_exit(octeon_usb_driver_exit);
3773 
3774 MODULE_LICENSE("GPL");
3775 MODULE_AUTHOR("Cavium, Inc. <support@cavium.com>");
3776 MODULE_DESCRIPTION("Cavium Inc. OCTEON USB Host driver.");
3777 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us