Version:  2.0.40 2.2.26 2.4.37 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7

Linux/drivers/spi/spi-atmel.c

  1 /*
  2  * Driver for Atmel AT32 and AT91 SPI Controllers
  3  *
  4  * Copyright (C) 2006 Atmel Corporation
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License version 2 as
  8  * published by the Free Software Foundation.
  9  */
 10 
 11 #include <linux/kernel.h>
 12 #include <linux/clk.h>
 13 #include <linux/module.h>
 14 #include <linux/platform_device.h>
 15 #include <linux/delay.h>
 16 #include <linux/dma-mapping.h>
 17 #include <linux/dmaengine.h>
 18 #include <linux/err.h>
 19 #include <linux/interrupt.h>
 20 #include <linux/spi/spi.h>
 21 #include <linux/slab.h>
 22 #include <linux/platform_data/dma-atmel.h>
 23 #include <linux/of.h>
 24 
 25 #include <linux/io.h>
 26 #include <linux/gpio.h>
 27 #include <linux/pinctrl/consumer.h>
 28 #include <linux/pm_runtime.h>
 29 
 30 /* SPI register offsets */
 31 #define SPI_CR                                  0x0000
 32 #define SPI_MR                                  0x0004
 33 #define SPI_RDR                                 0x0008
 34 #define SPI_TDR                                 0x000c
 35 #define SPI_SR                                  0x0010
 36 #define SPI_IER                                 0x0014
 37 #define SPI_IDR                                 0x0018
 38 #define SPI_IMR                                 0x001c
 39 #define SPI_CSR0                                0x0030
 40 #define SPI_CSR1                                0x0034
 41 #define SPI_CSR2                                0x0038
 42 #define SPI_CSR3                                0x003c
 43 #define SPI_FMR                                 0x0040
 44 #define SPI_FLR                                 0x0044
 45 #define SPI_VERSION                             0x00fc
 46 #define SPI_RPR                                 0x0100
 47 #define SPI_RCR                                 0x0104
 48 #define SPI_TPR                                 0x0108
 49 #define SPI_TCR                                 0x010c
 50 #define SPI_RNPR                                0x0110
 51 #define SPI_RNCR                                0x0114
 52 #define SPI_TNPR                                0x0118
 53 #define SPI_TNCR                                0x011c
 54 #define SPI_PTCR                                0x0120
 55 #define SPI_PTSR                                0x0124
 56 
 57 /* Bitfields in CR */
 58 #define SPI_SPIEN_OFFSET                        0
 59 #define SPI_SPIEN_SIZE                          1
 60 #define SPI_SPIDIS_OFFSET                       1
 61 #define SPI_SPIDIS_SIZE                         1
 62 #define SPI_SWRST_OFFSET                        7
 63 #define SPI_SWRST_SIZE                          1
 64 #define SPI_LASTXFER_OFFSET                     24
 65 #define SPI_LASTXFER_SIZE                       1
 66 #define SPI_TXFCLR_OFFSET                       16
 67 #define SPI_TXFCLR_SIZE                         1
 68 #define SPI_RXFCLR_OFFSET                       17
 69 #define SPI_RXFCLR_SIZE                         1
 70 #define SPI_FIFOEN_OFFSET                       30
 71 #define SPI_FIFOEN_SIZE                         1
 72 #define SPI_FIFODIS_OFFSET                      31
 73 #define SPI_FIFODIS_SIZE                        1
 74 
 75 /* Bitfields in MR */
 76 #define SPI_MSTR_OFFSET                         0
 77 #define SPI_MSTR_SIZE                           1
 78 #define SPI_PS_OFFSET                           1
 79 #define SPI_PS_SIZE                             1
 80 #define SPI_PCSDEC_OFFSET                       2
 81 #define SPI_PCSDEC_SIZE                         1
 82 #define SPI_FDIV_OFFSET                         3
 83 #define SPI_FDIV_SIZE                           1
 84 #define SPI_MODFDIS_OFFSET                      4
 85 #define SPI_MODFDIS_SIZE                        1
 86 #define SPI_WDRBT_OFFSET                        5
 87 #define SPI_WDRBT_SIZE                          1
 88 #define SPI_LLB_OFFSET                          7
 89 #define SPI_LLB_SIZE                            1
 90 #define SPI_PCS_OFFSET                          16
 91 #define SPI_PCS_SIZE                            4
 92 #define SPI_DLYBCS_OFFSET                       24
 93 #define SPI_DLYBCS_SIZE                         8
 94 
 95 /* Bitfields in RDR */
 96 #define SPI_RD_OFFSET                           0
 97 #define SPI_RD_SIZE                             16
 98 
 99 /* Bitfields in TDR */
100 #define SPI_TD_OFFSET                           0
101 #define SPI_TD_SIZE                             16
102 
103 /* Bitfields in SR */
104 #define SPI_RDRF_OFFSET                         0
105 #define SPI_RDRF_SIZE                           1
106 #define SPI_TDRE_OFFSET                         1
107 #define SPI_TDRE_SIZE                           1
108 #define SPI_MODF_OFFSET                         2
109 #define SPI_MODF_SIZE                           1
110 #define SPI_OVRES_OFFSET                        3
111 #define SPI_OVRES_SIZE                          1
112 #define SPI_ENDRX_OFFSET                        4
113 #define SPI_ENDRX_SIZE                          1
114 #define SPI_ENDTX_OFFSET                        5
115 #define SPI_ENDTX_SIZE                          1
116 #define SPI_RXBUFF_OFFSET                       6
117 #define SPI_RXBUFF_SIZE                         1
118 #define SPI_TXBUFE_OFFSET                       7
119 #define SPI_TXBUFE_SIZE                         1
120 #define SPI_NSSR_OFFSET                         8
121 #define SPI_NSSR_SIZE                           1
122 #define SPI_TXEMPTY_OFFSET                      9
123 #define SPI_TXEMPTY_SIZE                        1
124 #define SPI_SPIENS_OFFSET                       16
125 #define SPI_SPIENS_SIZE                         1
126 #define SPI_TXFEF_OFFSET                        24
127 #define SPI_TXFEF_SIZE                          1
128 #define SPI_TXFFF_OFFSET                        25
129 #define SPI_TXFFF_SIZE                          1
130 #define SPI_TXFTHF_OFFSET                       26
131 #define SPI_TXFTHF_SIZE                         1
132 #define SPI_RXFEF_OFFSET                        27
133 #define SPI_RXFEF_SIZE                          1
134 #define SPI_RXFFF_OFFSET                        28
135 #define SPI_RXFFF_SIZE                          1
136 #define SPI_RXFTHF_OFFSET                       29
137 #define SPI_RXFTHF_SIZE                         1
138 #define SPI_TXFPTEF_OFFSET                      30
139 #define SPI_TXFPTEF_SIZE                        1
140 #define SPI_RXFPTEF_OFFSET                      31
141 #define SPI_RXFPTEF_SIZE                        1
142 
143 /* Bitfields in CSR0 */
144 #define SPI_CPOL_OFFSET                         0
145 #define SPI_CPOL_SIZE                           1
146 #define SPI_NCPHA_OFFSET                        1
147 #define SPI_NCPHA_SIZE                          1
148 #define SPI_CSAAT_OFFSET                        3
149 #define SPI_CSAAT_SIZE                          1
150 #define SPI_BITS_OFFSET                         4
151 #define SPI_BITS_SIZE                           4
152 #define SPI_SCBR_OFFSET                         8
153 #define SPI_SCBR_SIZE                           8
154 #define SPI_DLYBS_OFFSET                        16
155 #define SPI_DLYBS_SIZE                          8
156 #define SPI_DLYBCT_OFFSET                       24
157 #define SPI_DLYBCT_SIZE                         8
158 
159 /* Bitfields in RCR */
160 #define SPI_RXCTR_OFFSET                        0
161 #define SPI_RXCTR_SIZE                          16
162 
163 /* Bitfields in TCR */
164 #define SPI_TXCTR_OFFSET                        0
165 #define SPI_TXCTR_SIZE                          16
166 
167 /* Bitfields in RNCR */
168 #define SPI_RXNCR_OFFSET                        0
169 #define SPI_RXNCR_SIZE                          16
170 
171 /* Bitfields in TNCR */
172 #define SPI_TXNCR_OFFSET                        0
173 #define SPI_TXNCR_SIZE                          16
174 
175 /* Bitfields in PTCR */
176 #define SPI_RXTEN_OFFSET                        0
177 #define SPI_RXTEN_SIZE                          1
178 #define SPI_RXTDIS_OFFSET                       1
179 #define SPI_RXTDIS_SIZE                         1
180 #define SPI_TXTEN_OFFSET                        8
181 #define SPI_TXTEN_SIZE                          1
182 #define SPI_TXTDIS_OFFSET                       9
183 #define SPI_TXTDIS_SIZE                         1
184 
185 /* Bitfields in FMR */
186 #define SPI_TXRDYM_OFFSET                       0
187 #define SPI_TXRDYM_SIZE                         2
188 #define SPI_RXRDYM_OFFSET                       4
189 #define SPI_RXRDYM_SIZE                         2
190 #define SPI_TXFTHRES_OFFSET                     16
191 #define SPI_TXFTHRES_SIZE                       6
192 #define SPI_RXFTHRES_OFFSET                     24
193 #define SPI_RXFTHRES_SIZE                       6
194 
195 /* Bitfields in FLR */
196 #define SPI_TXFL_OFFSET                         0
197 #define SPI_TXFL_SIZE                           6
198 #define SPI_RXFL_OFFSET                         16
199 #define SPI_RXFL_SIZE                           6
200 
201 /* Constants for BITS */
202 #define SPI_BITS_8_BPT                          0
203 #define SPI_BITS_9_BPT                          1
204 #define SPI_BITS_10_BPT                         2
205 #define SPI_BITS_11_BPT                         3
206 #define SPI_BITS_12_BPT                         4
207 #define SPI_BITS_13_BPT                         5
208 #define SPI_BITS_14_BPT                         6
209 #define SPI_BITS_15_BPT                         7
210 #define SPI_BITS_16_BPT                         8
211 #define SPI_ONE_DATA                            0
212 #define SPI_TWO_DATA                            1
213 #define SPI_FOUR_DATA                           2
214 
215 /* Bit manipulation macros */
216 #define SPI_BIT(name) \
217         (1 << SPI_##name##_OFFSET)
218 #define SPI_BF(name, value) \
219         (((value) & ((1 << SPI_##name##_SIZE) - 1)) << SPI_##name##_OFFSET)
220 #define SPI_BFEXT(name, value) \
221         (((value) >> SPI_##name##_OFFSET) & ((1 << SPI_##name##_SIZE) - 1))
222 #define SPI_BFINS(name, value, old) \
223         (((old) & ~(((1 << SPI_##name##_SIZE) - 1) << SPI_##name##_OFFSET)) \
224           | SPI_BF(name, value))
225 
226 /* Register access macros */
227 #ifdef CONFIG_AVR32
228 #define spi_readl(port, reg) \
229         __raw_readl((port)->regs + SPI_##reg)
230 #define spi_writel(port, reg, value) \
231         __raw_writel((value), (port)->regs + SPI_##reg)
232 
233 #define spi_readw(port, reg) \
234         __raw_readw((port)->regs + SPI_##reg)
235 #define spi_writew(port, reg, value) \
236         __raw_writew((value), (port)->regs + SPI_##reg)
237 
238 #define spi_readb(port, reg) \
239         __raw_readb((port)->regs + SPI_##reg)
240 #define spi_writeb(port, reg, value) \
241         __raw_writeb((value), (port)->regs + SPI_##reg)
242 #else
243 #define spi_readl(port, reg) \
244         readl_relaxed((port)->regs + SPI_##reg)
245 #define spi_writel(port, reg, value) \
246         writel_relaxed((value), (port)->regs + SPI_##reg)
247 
248 #define spi_readw(port, reg) \
249         readw_relaxed((port)->regs + SPI_##reg)
250 #define spi_writew(port, reg, value) \
251         writew_relaxed((value), (port)->regs + SPI_##reg)
252 
253 #define spi_readb(port, reg) \
254         readb_relaxed((port)->regs + SPI_##reg)
255 #define spi_writeb(port, reg, value) \
256         writeb_relaxed((value), (port)->regs + SPI_##reg)
257 #endif
258 /* use PIO for small transfers, avoiding DMA setup/teardown overhead and
259  * cache operations; better heuristics consider wordsize and bitrate.
260  */
261 #define DMA_MIN_BYTES   16
262 
263 #define SPI_DMA_TIMEOUT         (msecs_to_jiffies(1000))
264 
265 #define AUTOSUSPEND_TIMEOUT     2000
266 
267 struct atmel_spi_dma {
268         struct dma_chan                 *chan_rx;
269         struct dma_chan                 *chan_tx;
270         struct scatterlist              sgrx;
271         struct scatterlist              sgtx;
272         struct dma_async_tx_descriptor  *data_desc_rx;
273         struct dma_async_tx_descriptor  *data_desc_tx;
274 
275         struct at_dma_slave     dma_slave;
276 };
277 
278 struct atmel_spi_caps {
279         bool    is_spi2;
280         bool    has_wdrbt;
281         bool    has_dma_support;
282 };
283 
284 /*
285  * The core SPI transfer engine just talks to a register bank to set up
286  * DMA transfers; transfer queue progress is driven by IRQs.  The clock
287  * framework provides the base clock, subdivided for each spi_device.
288  */
289 struct atmel_spi {
290         spinlock_t              lock;
291         unsigned long           flags;
292 
293         phys_addr_t             phybase;
294         void __iomem            *regs;
295         int                     irq;
296         struct clk              *clk;
297         struct platform_device  *pdev;
298 
299         struct spi_transfer     *current_transfer;
300         int                     current_remaining_bytes;
301         int                     done_status;
302 
303         struct completion       xfer_completion;
304 
305         /* scratch buffer */
306         void                    *buffer;
307         dma_addr_t              buffer_dma;
308 
309         struct atmel_spi_caps   caps;
310 
311         bool                    use_dma;
312         bool                    use_pdc;
313         bool                    use_cs_gpios;
314         /* dmaengine data */
315         struct atmel_spi_dma    dma;
316 
317         bool                    keep_cs;
318         bool                    cs_active;
319 
320         u32                     fifo_size;
321 };
322 
323 /* Controller-specific per-slave state */
324 struct atmel_spi_device {
325         unsigned int            npcs_pin;
326         u32                     csr;
327 };
328 
329 #define BUFFER_SIZE             PAGE_SIZE
330 #define INVALID_DMA_ADDRESS     0xffffffff
331 
332 /*
333  * Version 2 of the SPI controller has
334  *  - CR.LASTXFER
335  *  - SPI_MR.DIV32 may become FDIV or must-be-zero (here: always zero)
336  *  - SPI_SR.TXEMPTY, SPI_SR.NSSR (and corresponding irqs)
337  *  - SPI_CSRx.CSAAT
338  *  - SPI_CSRx.SBCR allows faster clocking
339  */
340 static bool atmel_spi_is_v2(struct atmel_spi *as)
341 {
342         return as->caps.is_spi2;
343 }
344 
345 /*
346  * Earlier SPI controllers (e.g. on at91rm9200) have a design bug whereby
347  * they assume that spi slave device state will not change on deselect, so
348  * that automagic deselection is OK.  ("NPCSx rises if no data is to be
349  * transmitted")  Not so!  Workaround uses nCSx pins as GPIOs; or newer
350  * controllers have CSAAT and friends.
351  *
352  * Since the CSAAT functionality is a bit weird on newer controllers as
353  * well, we use GPIO to control nCSx pins on all controllers, updating
354  * MR.PCS to avoid confusing the controller.  Using GPIOs also lets us
355  * support active-high chipselects despite the controller's belief that
356  * only active-low devices/systems exists.
357  *
358  * However, at91rm9200 has a second erratum whereby nCS0 doesn't work
359  * right when driven with GPIO.  ("Mode Fault does not allow more than one
360  * Master on Chip Select 0.")  No workaround exists for that ... so for
361  * nCS0 on that chip, we (a) don't use the GPIO, (b) can't support CS_HIGH,
362  * and (c) will trigger that first erratum in some cases.
363  */
364 
365 static void cs_activate(struct atmel_spi *as, struct spi_device *spi)
366 {
367         struct atmel_spi_device *asd = spi->controller_state;
368         unsigned active = spi->mode & SPI_CS_HIGH;
369         u32 mr;
370 
371         if (atmel_spi_is_v2(as)) {
372                 spi_writel(as, CSR0 + 4 * spi->chip_select, asd->csr);
373                 /* For the low SPI version, there is a issue that PDC transfer
374                  * on CS1,2,3 needs SPI_CSR0.BITS config as SPI_CSR1,2,3.BITS
375                  */
376                 spi_writel(as, CSR0, asd->csr);
377                 if (as->caps.has_wdrbt) {
378                         spi_writel(as, MR,
379                                         SPI_BF(PCS, ~(0x01 << spi->chip_select))
380                                         | SPI_BIT(WDRBT)
381                                         | SPI_BIT(MODFDIS)
382                                         | SPI_BIT(MSTR));
383                 } else {
384                         spi_writel(as, MR,
385                                         SPI_BF(PCS, ~(0x01 << spi->chip_select))
386                                         | SPI_BIT(MODFDIS)
387                                         | SPI_BIT(MSTR));
388                 }
389 
390                 mr = spi_readl(as, MR);
391                 if (as->use_cs_gpios)
392                         gpio_set_value(asd->npcs_pin, active);
393         } else {
394                 u32 cpol = (spi->mode & SPI_CPOL) ? SPI_BIT(CPOL) : 0;
395                 int i;
396                 u32 csr;
397 
398                 /* Make sure clock polarity is correct */
399                 for (i = 0; i < spi->master->num_chipselect; i++) {
400                         csr = spi_readl(as, CSR0 + 4 * i);
401                         if ((csr ^ cpol) & SPI_BIT(CPOL))
402                                 spi_writel(as, CSR0 + 4 * i,
403                                                 csr ^ SPI_BIT(CPOL));
404                 }
405 
406                 mr = spi_readl(as, MR);
407                 mr = SPI_BFINS(PCS, ~(1 << spi->chip_select), mr);
408                 if (as->use_cs_gpios && spi->chip_select != 0)
409                         gpio_set_value(asd->npcs_pin, active);
410                 spi_writel(as, MR, mr);
411         }
412 
413         dev_dbg(&spi->dev, "activate %u%s, mr %08x\n",
414                         asd->npcs_pin, active ? " (high)" : "",
415                         mr);
416 }
417 
418 static void cs_deactivate(struct atmel_spi *as, struct spi_device *spi)
419 {
420         struct atmel_spi_device *asd = spi->controller_state;
421         unsigned active = spi->mode & SPI_CS_HIGH;
422         u32 mr;
423 
424         /* only deactivate *this* device; sometimes transfers to
425          * another device may be active when this routine is called.
426          */
427         mr = spi_readl(as, MR);
428         if (~SPI_BFEXT(PCS, mr) & (1 << spi->chip_select)) {
429                 mr = SPI_BFINS(PCS, 0xf, mr);
430                 spi_writel(as, MR, mr);
431         }
432 
433         dev_dbg(&spi->dev, "DEactivate %u%s, mr %08x\n",
434                         asd->npcs_pin, active ? " (low)" : "",
435                         mr);
436 
437         if (!as->use_cs_gpios)
438                 spi_writel(as, CR, SPI_BIT(LASTXFER));
439         else if (atmel_spi_is_v2(as) || spi->chip_select != 0)
440                 gpio_set_value(asd->npcs_pin, !active);
441 }
442 
443 static void atmel_spi_lock(struct atmel_spi *as) __acquires(&as->lock)
444 {
445         spin_lock_irqsave(&as->lock, as->flags);
446 }
447 
448 static void atmel_spi_unlock(struct atmel_spi *as) __releases(&as->lock)
449 {
450         spin_unlock_irqrestore(&as->lock, as->flags);
451 }
452 
453 static inline bool atmel_spi_use_dma(struct atmel_spi *as,
454                                 struct spi_transfer *xfer)
455 {
456         return as->use_dma && xfer->len >= DMA_MIN_BYTES;
457 }
458 
459 static int atmel_spi_dma_slave_config(struct atmel_spi *as,
460                                 struct dma_slave_config *slave_config,
461                                 u8 bits_per_word)
462 {
463         int err = 0;
464 
465         if (bits_per_word > 8) {
466                 slave_config->dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
467                 slave_config->src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
468         } else {
469                 slave_config->dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
470                 slave_config->src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
471         }
472 
473         slave_config->dst_addr = (dma_addr_t)as->phybase + SPI_TDR;
474         slave_config->src_addr = (dma_addr_t)as->phybase + SPI_RDR;
475         slave_config->src_maxburst = 1;
476         slave_config->dst_maxburst = 1;
477         slave_config->device_fc = false;
478 
479         /*
480          * This driver uses fixed peripheral select mode (PS bit set to '' in
481          * the Mode Register).
482          * So according to the datasheet, when FIFOs are available (and
483          * enabled), the Transmit FIFO operates in Multiple Data Mode.
484          * In this mode, up to 2 data, not 4, can be written into the Transmit
485          * Data Register in a single access.
486          * However, the first data has to be written into the lowest 16 bits and
487          * the second data into the highest 16 bits of the Transmit
488          * Data Register. For 8bit data (the most frequent case), it would
489          * require to rework tx_buf so each data would actualy fit 16 bits.
490          * So we'd rather write only one data at the time. Hence the transmit
491          * path works the same whether FIFOs are available (and enabled) or not.
492          */
493         slave_config->direction = DMA_MEM_TO_DEV;
494         if (dmaengine_slave_config(as->dma.chan_tx, slave_config)) {
495                 dev_err(&as->pdev->dev,
496                         "failed to configure tx dma channel\n");
497                 err = -EINVAL;
498         }
499 
500         /*
501          * This driver configures the spi controller for master mode (MSTR bit
502          * set to '1' in the Mode Register).
503          * So according to the datasheet, when FIFOs are available (and
504          * enabled), the Receive FIFO operates in Single Data Mode.
505          * So the receive path works the same whether FIFOs are available (and
506          * enabled) or not.
507          */
508         slave_config->direction = DMA_DEV_TO_MEM;
509         if (dmaengine_slave_config(as->dma.chan_rx, slave_config)) {
510                 dev_err(&as->pdev->dev,
511                         "failed to configure rx dma channel\n");
512                 err = -EINVAL;
513         }
514 
515         return err;
516 }
517 
518 static int atmel_spi_configure_dma(struct atmel_spi *as)
519 {
520         struct dma_slave_config slave_config;
521         struct device *dev = &as->pdev->dev;
522         int err;
523 
524         dma_cap_mask_t mask;
525         dma_cap_zero(mask);
526         dma_cap_set(DMA_SLAVE, mask);
527 
528         as->dma.chan_tx = dma_request_slave_channel_reason(dev, "tx");
529         if (IS_ERR(as->dma.chan_tx)) {
530                 err = PTR_ERR(as->dma.chan_tx);
531                 if (err == -EPROBE_DEFER) {
532                         dev_warn(dev, "no DMA channel available at the moment\n");
533                         return err;
534                 }
535                 dev_err(dev,
536                         "DMA TX channel not available, SPI unable to use DMA\n");
537                 err = -EBUSY;
538                 goto error;
539         }
540 
541         /*
542          * No reason to check EPROBE_DEFER here since we have already requested
543          * tx channel. If it fails here, it's for another reason.
544          */
545         as->dma.chan_rx = dma_request_slave_channel(dev, "rx");
546 
547         if (!as->dma.chan_rx) {
548                 dev_err(dev,
549                         "DMA RX channel not available, SPI unable to use DMA\n");
550                 err = -EBUSY;
551                 goto error;
552         }
553 
554         err = atmel_spi_dma_slave_config(as, &slave_config, 8);
555         if (err)
556                 goto error;
557 
558         dev_info(&as->pdev->dev,
559                         "Using %s (tx) and %s (rx) for DMA transfers\n",
560                         dma_chan_name(as->dma.chan_tx),
561                         dma_chan_name(as->dma.chan_rx));
562         return 0;
563 error:
564         if (as->dma.chan_rx)
565                 dma_release_channel(as->dma.chan_rx);
566         if (!IS_ERR(as->dma.chan_tx))
567                 dma_release_channel(as->dma.chan_tx);
568         return err;
569 }
570 
571 static void atmel_spi_stop_dma(struct atmel_spi *as)
572 {
573         if (as->dma.chan_rx)
574                 dmaengine_terminate_all(as->dma.chan_rx);
575         if (as->dma.chan_tx)
576                 dmaengine_terminate_all(as->dma.chan_tx);
577 }
578 
579 static void atmel_spi_release_dma(struct atmel_spi *as)
580 {
581         if (as->dma.chan_rx)
582                 dma_release_channel(as->dma.chan_rx);
583         if (as->dma.chan_tx)
584                 dma_release_channel(as->dma.chan_tx);
585 }
586 
587 /* This function is called by the DMA driver from tasklet context */
588 static void dma_callback(void *data)
589 {
590         struct spi_master       *master = data;
591         struct atmel_spi        *as = spi_master_get_devdata(master);
592 
593         complete(&as->xfer_completion);
594 }
595 
596 /*
597  * Next transfer using PIO without FIFO.
598  */
599 static void atmel_spi_next_xfer_single(struct spi_master *master,
600                                        struct spi_transfer *xfer)
601 {
602         struct atmel_spi        *as = spi_master_get_devdata(master);
603         unsigned long xfer_pos = xfer->len - as->current_remaining_bytes;
604 
605         dev_vdbg(master->dev.parent, "atmel_spi_next_xfer_pio\n");
606 
607         /* Make sure data is not remaining in RDR */
608         spi_readl(as, RDR);
609         while (spi_readl(as, SR) & SPI_BIT(RDRF)) {
610                 spi_readl(as, RDR);
611                 cpu_relax();
612         }
613 
614         if (xfer->tx_buf) {
615                 if (xfer->bits_per_word > 8)
616                         spi_writel(as, TDR, *(u16 *)(xfer->tx_buf + xfer_pos));
617                 else
618                         spi_writel(as, TDR, *(u8 *)(xfer->tx_buf</