Version:  2.0.40 2.2.26 2.4.37 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9

Linux/drivers/dma/at_hdmac.c

  1 /*
  2  * Driver for the Atmel AHB DMA Controller (aka HDMA or DMAC on AT91 systems)
  3  *
  4  * Copyright (C) 2008 Atmel Corporation
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License as published by
  8  * the Free Software Foundation; either version 2 of the License, or
  9  * (at your option) any later version.
 10  *
 11  *
 12  * This supports the Atmel AHB DMA Controller found in several Atmel SoCs.
 13  * The only Atmel DMA Controller that is not covered by this driver is the one
 14  * found on AT91SAM9263.
 15  */
 16 
 17 #include <dt-bindings/dma/at91.h>
 18 #include <linux/clk.h>
 19 #include <linux/dmaengine.h>
 20 #include <linux/dma-mapping.h>
 21 #include <linux/dmapool.h>
 22 #include <linux/interrupt.h>
 23 #include <linux/module.h>
 24 #include <linux/platform_device.h>
 25 #include <linux/slab.h>
 26 #include <linux/of.h>
 27 #include <linux/of_device.h>
 28 #include <linux/of_dma.h>
 29 
 30 #include "at_hdmac_regs.h"
 31 #include "dmaengine.h"
 32 
 33 /*
 34  * Glossary
 35  * --------
 36  *
 37  * at_hdmac             : Name of the ATmel AHB DMA Controller
 38  * at_dma_ / atdma      : ATmel DMA controller entity related
 39  * atc_ / atchan        : ATmel DMA Channel entity related
 40  */
 41 
 42 #define ATC_DEFAULT_CFG         (ATC_FIFOCFG_HALFFIFO)
 43 #define ATC_DEFAULT_CTRLB       (ATC_SIF(AT_DMA_MEM_IF) \
 44                                 |ATC_DIF(AT_DMA_MEM_IF))
 45 #define ATC_DMA_BUSWIDTHS\
 46         (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) |\
 47         BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |\
 48         BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |\
 49         BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
 50 
 51 #define ATC_MAX_DSCR_TRIALS     10
 52 
 53 /*
 54  * Initial number of descriptors to allocate for each channel. This could
 55  * be increased during dma usage.
 56  */
 57 static unsigned int init_nr_desc_per_channel = 64;
 58 module_param(init_nr_desc_per_channel, uint, 0644);
 59 MODULE_PARM_DESC(init_nr_desc_per_channel,
 60                  "initial descriptors per channel (default: 64)");
 61 
 62 
 63 /* prototypes */
 64 static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx);
 65 static void atc_issue_pending(struct dma_chan *chan);
 66 
 67 
 68 /*----------------------------------------------------------------------*/
 69 
 70 static inline unsigned int atc_get_xfer_width(dma_addr_t src, dma_addr_t dst,
 71                                                 size_t len)
 72 {
 73         unsigned int width;
 74 
 75         if (!((src | dst  | len) & 3))
 76                 width = 2;
 77         else if (!((src | dst | len) & 1))
 78                 width = 1;
 79         else
 80                 width = 0;
 81 
 82         return width;
 83 }
 84 
 85 static struct at_desc *atc_first_active(struct at_dma_chan *atchan)
 86 {
 87         return list_first_entry(&atchan->active_list,
 88                                 struct at_desc, desc_node);
 89 }
 90 
 91 static struct at_desc *atc_first_queued(struct at_dma_chan *atchan)
 92 {
 93         return list_first_entry(&atchan->queue,
 94                                 struct at_desc, desc_node);
 95 }
 96 
 97 /**
 98  * atc_alloc_descriptor - allocate and return an initialized descriptor
 99  * @chan: the channel to allocate descriptors for
100  * @gfp_flags: GFP allocation flags
101  *
102  * Note: The ack-bit is positioned in the descriptor flag at creation time
103  *       to make initial allocation more convenient. This bit will be cleared
104  *       and control will be given to client at usage time (during
105  *       preparation functions).
106  */
107 static struct at_desc *atc_alloc_descriptor(struct dma_chan *chan,
108                                             gfp_t gfp_flags)
109 {
110         struct at_desc  *desc = NULL;
111         struct at_dma   *atdma = to_at_dma(chan->device);
112         dma_addr_t phys;
113 
114         desc = dma_pool_alloc(atdma->dma_desc_pool, gfp_flags, &phys);
115         if (desc) {
116                 memset(desc, 0, sizeof(struct at_desc));
117                 INIT_LIST_HEAD(&desc->tx_list);
118                 dma_async_tx_descriptor_init(&desc->txd, chan);
119                 /* txd.flags will be overwritten in prep functions */
120                 desc->txd.flags = DMA_CTRL_ACK;
121                 desc->txd.tx_submit = atc_tx_submit;
122                 desc->txd.phys = phys;
123         }
124 
125         return desc;
126 }
127 
128 /**
129  * atc_desc_get - get an unused descriptor from free_list
130  * @atchan: channel we want a new descriptor for
131  */
132 static struct at_desc *atc_desc_get(struct at_dma_chan *atchan)
133 {
134         struct at_desc *desc, *_desc;
135         struct at_desc *ret = NULL;
136         unsigned long flags;
137         unsigned int i = 0;
138         LIST_HEAD(tmp_list);
139 
140         spin_lock_irqsave(&atchan->lock, flags);
141         list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) {
142                 i++;
143                 if (async_tx_test_ack(&desc->txd)) {
144                         list_del(&desc->desc_node);
145                         ret = desc;
146                         break;
147                 }
148                 dev_dbg(chan2dev(&atchan->chan_common),
149                                 "desc %p not ACKed\n", desc);
150         }
151         spin_unlock_irqrestore(&atchan->lock, flags);
152         dev_vdbg(chan2dev(&atchan->chan_common),
153                 "scanned %u descriptors on freelist\n", i);
154 
155         /* no more descriptor available in initial pool: create one more */
156         if (!ret) {
157                 ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC);
158                 if (ret) {
159                         spin_lock_irqsave(&atchan->lock, flags);
160                         atchan->descs_allocated++;
161                         spin_unlock_irqrestore(&atchan->lock, flags);
162                 } else {
163                         dev_err(chan2dev(&atchan->chan_common),
164                                         "not enough descriptors available\n");
165                 }
166         }
167 
168         return ret;
169 }
170 
171 /**
172  * atc_desc_put - move a descriptor, including any children, to the free list
173  * @atchan: channel we work on
174  * @desc: descriptor, at the head of a chain, to move to free list
175  */
176 static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc)
177 {
178         if (desc) {
179                 struct at_desc *child;
180                 unsigned long flags;
181 
182                 spin_lock_irqsave(&atchan->lock, flags);
183                 list_for_each_entry(child, &desc->tx_list, desc_node)
184                         dev_vdbg(chan2dev(&atchan->chan_common),
185                                         "moving child desc %p to freelist\n",
186                                         child);
187                 list_splice_init(&desc->tx_list, &atchan->free_list);
188                 dev_vdbg(chan2dev(&atchan->chan_common),
189                          "moving desc %p to freelist\n", desc);
190                 list_add(&desc->desc_node, &atchan->free_list);
191                 spin_unlock_irqrestore(&atchan->lock, flags);
192         }
193 }
194 
195 /**
196  * atc_desc_chain - build chain adding a descriptor
197  * @first: address of first descriptor of the chain
198  * @prev: address of previous descriptor of the chain
199  * @desc: descriptor to queue
200  *
201  * Called from prep_* functions
202  */
203 static void atc_desc_chain(struct at_desc **first, struct at_desc **prev,
204                            struct at_desc *desc)
205 {
206         if (!(*first)) {
207                 *first = desc;
208         } else {
209                 /* inform the HW lli about chaining */
210                 (*prev)->lli.dscr = desc->txd.phys;
211                 /* insert the link descriptor to the LD ring */
212                 list_add_tail(&desc->desc_node,
213                                 &(*first)->tx_list);
214         }
215         *prev = desc;
216 }
217 
218 /**
219  * atc_dostart - starts the DMA engine for real
220  * @atchan: the channel we want to start
221  * @first: first descriptor in the list we want to begin with
222  *
223  * Called with atchan->lock held and bh disabled
224  */
225 static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first)
226 {
227         struct at_dma   *atdma = to_at_dma(atchan->chan_common.device);
228 
229         /* ASSERT:  channel is idle */
230         if (atc_chan_is_enabled(atchan)) {
231                 dev_err(chan2dev(&atchan->chan_common),
232                         "BUG: Attempted to start non-idle channel\n");
233                 dev_err(chan2dev(&atchan->chan_common),
234                         "  channel: s0x%x d0x%x ctrl0x%x:0x%x l0x%x\n",
235                         channel_readl(atchan, SADDR),
236                         channel_readl(atchan, DADDR),
237                         channel_readl(atchan, CTRLA),
238                         channel_readl(atchan, CTRLB),
239                         channel_readl(atchan, DSCR));
240 
241                 /* The tasklet will hopefully advance the queue... */
242                 return;
243         }
244 
245         vdbg_dump_regs(atchan);
246 
247         channel_writel(atchan, SADDR, 0);
248         channel_writel(atchan, DADDR, 0);
249         channel_writel(atchan, CTRLA, 0);
250         channel_writel(atchan, CTRLB, 0);
251         channel_writel(atchan, DSCR, first->txd.phys);
252         channel_writel(atchan, SPIP, ATC_SPIP_HOLE(first->src_hole) |
253                        ATC_SPIP_BOUNDARY(first->boundary));
254         channel_writel(atchan, DPIP, ATC_DPIP_HOLE(first->dst_hole) |
255                        ATC_DPIP_BOUNDARY(first->boundary));
256         dma_writel(atdma, CHER, atchan->mask);
257 
258         vdbg_dump_regs(atchan);
259 }
260 
261 /*
262  * atc_get_desc_by_cookie - get the descriptor of a cookie
263  * @atchan: the DMA channel
264  * @cookie: the cookie to get the descriptor for
265  */
266 static struct at_desc *atc_get_desc_by_cookie(struct at_dma_chan *atchan,
267                                                 dma_cookie_t cookie)
268 {
269         struct at_desc *desc, *_desc;
270 
271         list_for_each_entry_safe(desc, _desc, &atchan->queue, desc_node) {
272                 if (desc->txd.cookie == cookie)
273                         return desc;
274         }
275 
276         list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) {
277                 if (desc->txd.cookie == cookie)
278                         return desc;
279         }
280 
281         return NULL;
282 }
283 
284 /**
285  * atc_calc_bytes_left - calculates the number of bytes left according to the
286  * value read from CTRLA.
287  *
288  * @current_len: the number of bytes left before reading CTRLA
289  * @ctrla: the value of CTRLA
290  */
291 static inline int atc_calc_bytes_left(int current_len, u32 ctrla)
292 {
293         u32 btsize = (ctrla & ATC_BTSIZE_MAX);
294         u32 src_width = ATC_REG_TO_SRC_WIDTH(ctrla);
295 
296         /*
297          * According to the datasheet, when reading the Control A Register
298          * (ctrla), the Buffer Transfer Size (btsize) bitfield refers to the
299          * number of transfers completed on the Source Interface.
300          * So btsize is always a number of source width transfers.
301          */
302         return current_len - (btsize << src_width);
303 }
304 
305 /**
306  * atc_get_bytes_left - get the number of bytes residue for a cookie
307  * @chan: DMA channel
308  * @cookie: transaction identifier to check status of
309  */
310 static int atc_get_bytes_left(struct dma_chan *chan, dma_cookie_t cookie)
311 {
312         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
313         struct at_desc *desc_first = atc_first_active(atchan);
314         struct at_desc *desc;
315         int ret;
316         u32 ctrla, dscr, trials;
317 
318         /*
319          * If the cookie doesn't match to the currently running transfer then
320          * we can return the total length of the associated DMA transfer,
321          * because it is still queued.
322          */
323         desc = atc_get_desc_by_cookie(atchan, cookie);
324         if (desc == NULL)
325                 return -EINVAL;
326         else if (desc != desc_first)
327                 return desc->total_len;
328 
329         /* cookie matches to the currently running transfer */
330         ret = desc_first->total_len;
331 
332         if (desc_first->lli.dscr) {
333                 /* hardware linked list transfer */
334 
335                 /*
336                  * Calculate the residue by removing the length of the child
337                  * descriptors already transferred from the total length.
338                  * To get the current child descriptor we can use the value of
339                  * the channel's DSCR register and compare it against the value
340                  * of the hardware linked list structure of each child
341                  * descriptor.
342                  *
343                  * The CTRLA register provides us with the amount of data
344                  * already read from the source for the current child
345                  * descriptor. So we can compute a more accurate residue by also
346                  * removing the number of bytes corresponding to this amount of
347                  * data.
348                  *
349                  * However, the DSCR and CTRLA registers cannot be read both
350                  * atomically. Hence a race condition may occur: the first read
351                  * register may refer to one child descriptor whereas the second
352                  * read may refer to a later child descriptor in the list
353                  * because of the DMA transfer progression inbetween the two
354                  * reads.
355                  *
356                  * One solution could have been to pause the DMA transfer, read
357                  * the DSCR and CTRLA then resume the DMA transfer. Nonetheless,
358                  * this approach presents some drawbacks:
359                  * - If the DMA transfer is paused, RX overruns or TX underruns
360                  *   are more likey to occur depending on the system latency.
361                  *   Taking the USART driver as an example, it uses a cyclic DMA
362                  *   transfer to read data from the Receive Holding Register
363                  *   (RHR) to avoid RX overruns since the RHR is not protected
364                  *   by any FIFO on most Atmel SoCs. So pausing the DMA transfer
365                  *   to compute the residue would break the USART driver design.
366                  * - The atc_pause() function masks interrupts but we'd rather
367                  *   avoid to do so for system latency purpose.
368                  *
369                  * Then we'd rather use another solution: the DSCR is read a
370                  * first time, the CTRLA is read in turn, next the DSCR is read
371                  * a second time. If the two consecutive read values of the DSCR
372                  * are the same then we assume both refers to the very same
373                  * child descriptor as well as the CTRLA value read inbetween
374                  * does. For cyclic tranfers, the assumption is that a full loop
375                  * is "not so fast".
376                  * If the two DSCR values are different, we read again the CTRLA
377                  * then the DSCR till two consecutive read values from DSCR are
378                  * equal or till the maxium trials is reach.
379                  * This algorithm is very unlikely not to find a stable value for
380                  * DSCR.
381                  */
382 
383                 dscr = channel_readl(atchan, DSCR);
384                 rmb(); /* ensure DSCR is read before CTRLA */
385                 ctrla = channel_readl(atchan, CTRLA);
386                 for (trials = 0; trials < ATC_MAX_DSCR_TRIALS; ++trials) {
387                         u32 new_dscr;
388 
389                         rmb(); /* ensure DSCR is read after CTRLA */
390                         new_dscr = channel_readl(atchan, DSCR);
391 
392                         /*
393                          * If the DSCR register value has not changed inside the
394                          * DMA controller since the previous read, we assume
395                          * that both the dscr and ctrla values refers to the
396                          * very same descriptor.
397                          */
398                         if (likely(new_dscr == dscr))
399                                 break;
400 
401                         /*
402                          * DSCR has changed inside the DMA controller, so the
403                          * previouly read value of CTRLA may refer to an already
404                          * processed descriptor hence could be outdated.
405                          * We need to update ctrla to match the current
406                          * descriptor.
407                          */
408                         dscr = new_dscr;
409                         rmb(); /* ensure DSCR is read before CTRLA */
410                         ctrla = channel_readl(atchan, CTRLA);
411                 }
412                 if (unlikely(trials >= ATC_MAX_DSCR_TRIALS))
413                         return -ETIMEDOUT;
414 
415                 /* for the first descriptor we can be more accurate */
416                 if (desc_first->lli.dscr == dscr)
417                         return atc_calc_bytes_left(ret, ctrla);
418 
419                 ret -= desc_first->len;
420                 list_for_each_entry(desc, &desc_first->tx_list, desc_node) {
421                         if (desc->lli.dscr == dscr)
422                                 break;
423 
424                         ret -= desc->len;
425                 }
426 
427                 /*
428                  * For the current descriptor in the chain we can calculate
429                  * the remaining bytes using the channel's register.
430                  */
431                 ret = atc_calc_bytes_left(ret, ctrla);
432         } else {
433                 /* single transfer */
434                 ctrla = channel_readl(atchan, CTRLA);
435                 ret = atc_calc_bytes_left(ret, ctrla);
436         }
437 
438         return ret;
439 }
440 
441 /**
442  * atc_chain_complete - finish work for one transaction chain
443  * @atchan: channel we work on
444  * @desc: descriptor at the head of the chain we want do complete
445  *
446  * Called with atchan->lock held and bh disabled */
447 static void
448 atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)
449 {
450         struct dma_async_tx_descriptor  *txd = &desc->txd;
451         struct at_dma                   *atdma = to_at_dma(atchan->chan_common.device);
452 
453         dev_vdbg(chan2dev(&atchan->chan_common),
454                 "descriptor %u complete\n", txd->cookie);
455 
456         /* mark the descriptor as complete for non cyclic cases only */
457         if (!atc_chan_is_cyclic(atchan))
458                 dma_cookie_complete(txd);
459 
460         /* If the transfer was a memset, free our temporary buffer */
461         if (desc->memset_buffer) {
462                 dma_pool_free(atdma->memset_pool, desc->memset_vaddr,
463                               desc->memset_paddr);
464                 desc->memset_buffer = false;
465         }
466 
467         /* move children to free_list */
468         list_splice_init(&desc->tx_list, &atchan->free_list);
469         /* move myself to free_list */
470         list_move(&desc->desc_node, &atchan->free_list);
471 
472         dma_descriptor_unmap(txd);
473         /* for cyclic transfers,
474          * no need to replay callback function while stopping */
475         if (!atc_chan_is_cyclic(atchan)) {
476                 /*
477                  * The API requires that no submissions are done from a
478                  * callback, so we don't need to drop the lock here
479                  */
480                 dmaengine_desc_get_callback_invoke(txd, NULL);
481         }
482 
483         dma_run_dependencies(txd);
484 }
485 
486 /**
487  * atc_complete_all - finish work for all transactions
488  * @atchan: channel to complete transactions for
489  *
490  * Eventually submit queued descriptors if any
491  *
492  * Assume channel is idle while calling this function
493  * Called with atchan->lock held and bh disabled
494  */
495 static void atc_complete_all(struct at_dma_chan *atchan)
496 {
497         struct at_desc *desc, *_desc;
498         LIST_HEAD(list);
499 
500         dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n");
501 
502         /*
503          * Submit queued descriptors ASAP, i.e. before we go through
504          * the completed ones.
505          */
506         if (!list_empty(&atchan->queue))
507                 atc_dostart(atchan, atc_first_queued(atchan));
508         /* empty active_list now it is completed */
509         list_splice_init(&atchan->active_list, &list);
510         /* empty queue list by moving descriptors (if any) to active_list */
511         list_splice_init(&atchan->queue, &atchan->active_list);
512 
513         list_for_each_entry_safe(desc, _desc, &list, desc_node)
514                 atc_chain_complete(atchan, desc);
515 }
516 
517 /**
518  * atc_advance_work - at the end of a transaction, move forward
519  * @atchan: channel where the transaction ended
520  *
521  * Called with atchan->lock held and bh disabled
522  */
523 static void atc_advance_work(struct at_dma_chan *atchan)
524 {
525         dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n");
526 
527         if (atc_chan_is_enabled(atchan))
528                 return;
529 
530         if (list_empty(&atchan->active_list) ||
531             list_is_singular(&atchan->active_list)) {
532                 atc_complete_all(atchan);
533         } else {
534                 atc_chain_complete(atchan, atc_first_active(atchan));
535                 /* advance work */
536                 atc_dostart(atchan, atc_first_active(atchan));
537         }
538 }
539 
540 
541 /**
542  * atc_handle_error - handle errors reported by DMA controller
543  * @atchan: channel where error occurs
544  *
545  * Called with atchan->lock held and bh disabled
546  */
547 static void atc_handle_error(struct at_dma_chan *atchan)
548 {
549         struct at_desc *bad_desc;
550         struct at_desc *child;
551 
552         /*
553          * The descriptor currently at the head of the active list is
554          * broked. Since we don't have any way to report errors, we'll
555          * just have to scream loudly and try to carry on.
556          */
557         bad_desc = atc_first_active(atchan);
558         list_del_init(&bad_desc->desc_node);
559 
560         /* As we are stopped, take advantage to push queued descriptors
561          * in active_list */
562         list_splice_init(&atchan->queue, atchan->active_list.prev);
563 
564         /* Try to restart the controller */
565         if (!list_empty(&atchan->active_list))
566                 atc_dostart(atchan, atc_first_active(atchan));
567 
568         /*
569          * KERN_CRITICAL may seem harsh, but since this only happens
570          * when someone submits a bad physical address in a
571          * descriptor, we should consider ourselves lucky that the
572          * controller flagged an error instead of scribbling over
573          * random memory locations.
574          */
575         dev_crit(chan2dev(&atchan->chan_common),
576                         "Bad descriptor submitted for DMA!\n");
577         dev_crit(chan2dev(&atchan->chan_common),
578                         "  cookie: %d\n", bad_desc->txd.cookie);
579         atc_dump_lli(atchan, &bad_desc->lli);
580         list_for_each_entry(child, &bad_desc->tx_list, desc_node)
581                 atc_dump_lli(atchan, &child->lli);
582 
583         /* Pretend the descriptor completed successfully */
584         atc_chain_complete(atchan, bad_desc);
585 }
586 
587 /**
588  * atc_handle_cyclic - at the end of a period, run callback function
589  * @atchan: channel used for cyclic operations
590  *
591  * Called with atchan->lock held and bh disabled
592  */
593 static void atc_handle_cyclic(struct at_dma_chan *atchan)
594 {
595         struct at_desc                  *first = atc_first_active(atchan);
596         struct dma_async_tx_descriptor  *txd = &first->txd;
597 
598         dev_vdbg(chan2dev(&atchan->chan_common),
599                         "new cyclic period llp 0x%08x\n",
600                         channel_readl(atchan, DSCR));
601 
602         dmaengine_desc_get_callback_invoke(txd, NULL);
603 }
604 
605 /*--  IRQ & Tasklet  ---------------------------------------------------*/
606 
607 static void atc_tasklet(unsigned long data)
608 {
609         struct at_dma_chan *atchan = (struct at_dma_chan *)data;
610         unsigned long flags;
611 
612         spin_lock_irqsave(&atchan->lock, flags);
613         if (test_and_clear_bit(ATC_IS_ERROR, &atchan->status))
614                 atc_handle_error(atchan);
615         else if (atc_chan_is_cyclic(atchan))
616                 atc_handle_cyclic(atchan);
617         else
618                 atc_advance_work(atchan);
619 
620         spin_unlock_irqrestore(&atchan->lock, flags);
621 }
622 
623 static irqreturn_t at_dma_interrupt(int irq, void *dev_id)
624 {
625         struct at_dma           *atdma = (struct at_dma *)dev_id;
626         struct at_dma_chan      *atchan;
627         int                     i;
628         u32                     status, pending, imr;
629         int                     ret = IRQ_NONE;
630 
631         do {
632                 imr = dma_readl(atdma, EBCIMR);
633                 status = dma_readl(atdma, EBCISR);
634                 pending = status & imr;
635 
636                 if (!pending)
637                         break;
638 
639                 dev_vdbg(atdma->dma_common.dev,
640                         "interrupt: status = 0x%08x, 0x%08x, 0x%08x\n",
641                          status, imr, pending);
642 
643                 for (i = 0; i < atdma->dma_common.chancnt; i++) {
644                         atchan = &atdma->chan[i];
645                         if (pending & (AT_DMA_BTC(i) | AT_DMA_ERR(i))) {
646                                 if (pending & AT_DMA_ERR(i)) {
647                                         /* Disable channel on AHB error */
648                                         dma_writel(atdma, CHDR,
649                                                 AT_DMA_RES(i) | atchan->mask);
650                                         /* Give information to tasklet */
651                                         set_bit(ATC_IS_ERROR, &atchan->status);
652                                 }
653                                 tasklet_schedule(&atchan->tasklet);
654                                 ret = IRQ_HANDLED;
655                         }
656                 }
657 
658         } while (pending);
659 
660         return ret;
661 }
662 
663 
664 /*--  DMA Engine API  --------------------------------------------------*/
665 
666 /**
667  * atc_tx_submit - set the prepared descriptor(s) to be executed by the engine
668  * @desc: descriptor at the head of the transaction chain
669  *
670  * Queue chain if DMA engine is working already
671  *
672  * Cookie increment and adding to active_list or queue must be atomic
673  */
674 static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx)
675 {
676         struct at_desc          *desc = txd_to_at_desc(tx);
677         struct at_dma_chan      *atchan = to_at_dma_chan(tx->chan);
678         dma_cookie_t            cookie;
679         unsigned long           flags;
680 
681         spin_lock_irqsave(&atchan->lock, flags);
682         cookie = dma_cookie_assign(tx);
683 
684         if (list_empty(&atchan->active_list)) {
685                 dev_vdbg(chan2dev(tx->chan), "tx_submit: started %u\n",
686                                 desc->txd.cookie);
687                 atc_dostart(atchan, desc);
688                 list_add_tail(&desc->desc_node, &atchan->active_list);
689         } else {
690                 dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n",
691                                 desc->txd.cookie);
692                 list_add_tail(&desc->desc_node, &atchan->queue);
693         }
694 
695         spin_unlock_irqrestore(&atchan->lock, flags);
696 
697         return cookie;
698 }
699 
700 /**
701  * atc_prep_dma_interleaved - prepare memory to memory interleaved operation
702  * @chan: the channel to prepare operation on
703  * @xt: Interleaved transfer template
704  * @flags: tx descriptor status flags
705  */
706 static struct dma_async_tx_descriptor *
707 atc_prep_dma_interleaved(struct dma_chan *chan,
708                          struct dma_interleaved_template *xt,
709                          unsigned long flags)
710 {
711         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
712         struct data_chunk       *first = xt->sgl;
713         struct at_desc          *desc = NULL;
714         size_t                  xfer_count;
715         unsigned int            dwidth;
716         u32                     ctrla;
717         u32                     ctrlb;
718         size_t                  len = 0;
719         int                     i;
720 
721         if (unlikely(!xt || xt->numf != 1 || !xt->frame_size))
722                 return NULL;
723 
724         dev_info(chan2dev(chan),
725                  "%s: src=%pad, dest=%pad, numf=%d, frame_size=%d, flags=0x%lx\n",
726                 __func__, &xt->src_start, &xt->dst_start, xt->numf,
727                 xt->frame_size, flags);
728 
729         /*
730          * The controller can only "skip" X bytes every Y bytes, so we
731          * need to make sure we are given a template that fit that
732          * description, ie a template with chunks that always have the
733          * same size, with the same ICGs.
734          */
735         for (i = 0; i < xt->frame_size; i++) {
736                 struct data_chunk *chunk = xt->sgl + i;
737 
738                 if ((chunk->size != xt->sgl->size) ||
739                     (dmaengine_get_dst_icg(xt, chunk) != dmaengine_get_dst_icg(xt, first)) ||
740                     (dmaengine_get_src_icg(xt, chunk) != dmaengine_get_src_icg(xt, first))) {
741                         dev_err(chan2dev(chan),
742                                 "%s: the controller can transfer only identical chunks\n",
743                                 __func__);
744                         return NULL;
745                 }
746 
747                 len += chunk->size;
748         }
749 
750         dwidth = atc_get_xfer_width(xt->src_start,
751                                     xt->dst_start, len);
752 
753         xfer_count = len >> dwidth;
754         if (xfer_count > ATC_BTSIZE_MAX) {
755                 dev_err(chan2dev(chan), "%s: buffer is too big\n", __func__);
756                 return NULL;
757         }
758 
759         ctrla = ATC_SRC_WIDTH(dwidth) |
760                 ATC_DST_WIDTH(dwidth);
761 
762         ctrlb =   ATC_DEFAULT_CTRLB | ATC_IEN
763                 | ATC_SRC_ADDR_MODE_INCR
764                 | ATC_DST_ADDR_MODE_INCR
765                 | ATC_SRC_PIP
766                 | ATC_DST_PIP
767                 | ATC_FC_MEM2MEM;
768 
769         /* create the transfer */
770         desc = atc_desc_get(atchan);
771         if (!desc) {
772                 dev_err(chan2dev(chan),
773                         "%s: couldn't allocate our descriptor\n", __func__);
774                 return NULL;
775         }
776 
777         desc->lli.saddr = xt->src_start;
778         desc->lli.daddr = xt->dst_start;
779         desc->lli.ctrla = ctrla | xfer_count;
780         desc->lli.ctrlb = ctrlb;
781 
782         desc->boundary = first->size >> dwidth;
783         desc->dst_hole = (dmaengine_get_dst_icg(xt, first) >> dwidth) + 1;
784         desc->src_hole = (dmaengine_get_src_icg(xt, first) >> dwidth) + 1;
785 
786         desc->txd.cookie = -EBUSY;
787         desc->total_len = desc->len = len;
788 
789         /* set end-of-link to the last link descriptor of list*/
790         set_desc_eol(desc);
791 
792         desc->txd.flags = flags; /* client is in control of this ack */
793 
794         return &desc->txd;
795 }
796 
797 /**
798  * atc_prep_dma_memcpy - prepare a memcpy operation
799  * @chan: the channel to prepare operation on
800  * @dest: operation virtual destination address
801  * @src: operation virtual source address
802  * @len: operation length
803  * @flags: tx descriptor status flags
804  */
805 static struct dma_async_tx_descriptor *
806 atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
807                 size_t len, unsigned long flags)
808 {
809         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
810         struct at_desc          *desc = NULL;
811         struct at_desc          *first = NULL;
812         struct at_desc          *prev = NULL;
813         size_t                  xfer_count;
814         size_t                  offset;
815         unsigned int            src_width;
816         unsigned int            dst_width;
817         u32                     ctrla;
818         u32                     ctrlb;
819 
820         dev_vdbg(chan2dev(chan), "prep_dma_memcpy: d%pad s%pad l0x%zx f0x%lx\n",
821                         &dest, &src, len, flags);
822 
823         if (unlikely(!len)) {
824                 dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
825                 return NULL;
826         }
827 
828         ctrlb =   ATC_DEFAULT_CTRLB | ATC_IEN
829                 | ATC_SRC_ADDR_MODE_INCR
830                 | ATC_DST_ADDR_MODE_INCR
831                 | ATC_FC_MEM2MEM;
832 
833         /*
834          * We can be a lot more clever here, but this should take care
835          * of the most common optimization.
836          */
837         src_width = dst_width = atc_get_xfer_width(src, dest, len);
838 
839         ctrla = ATC_SRC_WIDTH(src_width) |
840                 ATC_DST_WIDTH(dst_width);
841 
842         for (offset = 0; offset < len; offset += xfer_count << src_width) {
843                 xfer_count = min_t(size_t, (len - offset) >> src_width,
844                                 ATC_BTSIZE_MAX);
845 
846                 desc = atc_desc_get(atchan);
847                 if (!desc)
848                         goto err_desc_get;
849 
850                 desc->lli.saddr = src + offset;
851                 desc->lli.daddr = dest + offset;
852                 desc->lli.ctrla = ctrla | xfer_count;
853                 desc->lli.ctrlb = ctrlb;
854 
855                 desc->txd.cookie = 0;
856                 desc->len = xfer_count << src_width;
857 
858                 atc_desc_chain(&first, &prev, desc);
859         }
860 
861         /* First descriptor of the chain embedds additional information */
862         first->txd.cookie = -EBUSY;
863         first->total_len = len;
864 
865         /* set end-of-link to the last link descriptor of list*/
866         set_desc_eol(desc);
867 
868         first->txd.flags = flags; /* client is in control of this ack */
869 
870         return &first->txd;
871 
872 err_desc_get:
873         atc_desc_put(atchan, first);
874         return NULL;
875 }
876 
877 static struct at_desc *atc_create_memset_desc(struct dma_chan *chan,
878                                               dma_addr_t psrc,
879                                               dma_addr_t pdst,
880                                               size_t len)
881 {
882         struct at_dma_chan *atchan = to_at_dma_chan(chan);
883         struct at_desc *desc;
884         size_t xfer_count;
885 
886         u32 ctrla = ATC_SRC_WIDTH(2) | ATC_DST_WIDTH(2);
887         u32 ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN |
888                 ATC_SRC_ADDR_MODE_FIXED |
889                 ATC_DST_ADDR_MODE_INCR |
890                 ATC_FC_MEM2MEM;
891 
892         xfer_count = len >> 2;
893         if (xfer_count > ATC_BTSIZE_MAX) {
894                 dev_err(chan2dev(chan), "%s: buffer is too big\n",
895                         __func__);
896                 return NULL;
897         }
898 
899         desc = atc_desc_get(atchan);
900         if (!desc) {
901                 dev_err(chan2dev(chan), "%s: can't get a descriptor\n",
902                         __func__);
903                 return NULL;
904         }
905 
906         desc->lli.saddr = psrc;
907         desc->lli.daddr = pdst;
908         desc->lli.ctrla = ctrla | xfer_count;
909         desc->lli.ctrlb = ctrlb;
910 
911         desc->txd.cookie = 0;
912         desc->len = len;
913 
914         return desc;
915 }
916 
917 /**
918  * atc_prep_dma_memset - prepare a memcpy operation
919  * @chan: the channel to prepare operation on
920  * @dest: operation virtual destination address
921  * @value: value to set memory buffer to
922  * @len: operation length
923  * @flags: tx descriptor status flags
924  */
925 static struct dma_async_tx_descriptor *
926 atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value,
927                     size_t len, unsigned long flags)
928 {
929         struct at_dma           *atdma = to_at_dma(chan->device);
930         struct at_desc          *desc;
931         void __iomem            *vaddr;
932         dma_addr_t              paddr;
933 
934         dev_vdbg(chan2dev(chan), "%s: d%pad v0x%x l0x%zx f0x%lx\n", __func__,
935                 &dest, value, len, flags);
936 
937         if (unlikely(!len)) {
938                 dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__);
939                 return NULL;
940         }
941 
942         if (!is_dma_fill_aligned(chan->device, dest, 0, len)) {
943                 dev_dbg(chan2dev(chan), "%s: buffer is not aligned\n",
944                         __func__);
945                 return NULL;
946         }
947 
948         vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr);
949         if (!vaddr) {
950                 dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n",
951                         __func__);
952                 return NULL;
953         }
954         *(u32*)vaddr = value;
955 
956         desc = atc_create_memset_desc(chan, paddr, dest, len);
957         if (!desc) {
958                 dev_err(chan2dev(chan), "%s: couldn't get a descriptor\n",
959                         __func__);
960                 goto err_free_buffer;
961         }
962 
963         desc->memset_paddr = paddr;
964         desc->memset_vaddr = vaddr;
965         desc->memset_buffer = true;
966 
967         desc->txd.cookie = -EBUSY;
968         desc->total_len = len;
969 
970         /* set end-of-link on the descriptor */
971         set_desc_eol(desc);
972 
973         desc->txd.flags = flags;
974 
975         return &desc->txd;
976 
977 err_free_buffer:
978         dma_pool_free(atdma->memset_pool, vaddr, paddr);
979         return NULL;
980 }
981 
982 static struct dma_async_tx_descriptor *
983 atc_prep_dma_memset_sg(struct dma_chan *chan,
984                        struct scatterlist *sgl,
985                        unsigned int sg_len, int value,
986                        unsigned long flags)
987 {
988         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
989         struct at_dma           *atdma = to_at_dma(chan->device);
990         struct at_desc          *desc = NULL, *first = NULL, *prev = NULL;
991         struct scatterlist      *sg;
992         void __iomem            *vaddr;
993         dma_addr_t              paddr;
994         size_t                  total_len = 0;
995         int                     i;
996 
997         dev_vdbg(chan2dev(chan), "%s: v0x%x l0x%zx f0x%lx\n", __func__,
998                  value, sg_len, flags);
999 
1000         if (unlikely(!sgl || !sg_len)) {
1001                 dev_dbg(chan2dev(chan), "%s: scatterlist is empty!\n",
1002                         __func__);
1003                 return NULL;
1004         }
1005 
1006         vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr);
1007         if (!vaddr) {
1008                 dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n",
1009                         __func__);
1010                 return NULL;
1011         }
1012         *(u32*)vaddr = value;
1013 
1014         for_each_sg(sgl, sg, sg_len, i) {
1015                 dma_addr_t dest = sg_dma_address(sg);
1016                 size_t len = sg_dma_len(sg);
1017 
1018                 dev_vdbg(chan2dev(chan), "%s: d%pad, l0x%zx\n",
1019                          __func__, &dest, len);
1020 
1021                 if (!is_dma_fill_aligned(chan->device, dest, 0, len)) {
1022                         dev_err(chan2dev(chan), "%s: buffer is not aligned\n",
1023                                 __func__);
1024                         goto err_put_desc;
1025                 }
1026 
1027                 desc = atc_create_memset_desc(chan, paddr, dest, len);
1028                 if (!desc)
1029                         goto err_put_desc;
1030 
1031                 atc_desc_chain(&first, &prev, desc);
1032 
1033                 total_len += len;
1034         }
1035 
1036         /*
1037          * Only set the buffer pointers on the last descriptor to
1038          * avoid free'ing while we have our transfer still going
1039          */
1040         desc->memset_paddr = paddr;
1041         desc->memset_vaddr = vaddr;
1042         desc->memset_buffer = true;
1043 
1044         first->txd.cookie = -EBUSY;
1045         first->total_len = total_len;
1046 
1047         /* set end-of-link on the descriptor */
1048         set_desc_eol(desc);
1049 
1050         first->txd.flags = flags;
1051 
1052         return &first->txd;
1053 
1054 err_put_desc:
1055         atc_desc_put(atchan, first);
1056         return NULL;
1057 }
1058 
1059 /**
1060  * atc_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
1061  * @chan: DMA channel
1062  * @sgl: scatterlist to transfer to/from
1063  * @sg_len: number of entries in @scatterlist
1064  * @direction: DMA direction
1065  * @flags: tx descriptor status flags
1066  * @context: transaction context (ignored)
1067  */
1068 static struct dma_async_tx_descriptor *
1069 atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
1070                 unsigned int sg_len, enum dma_transfer_direction direction,
1071                 unsigned long flags, void *context)
1072 {
1073         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
1074         struct at_dma_slave     *atslave = chan->private;
1075         struct dma_slave_config *sconfig = &atchan->dma_sconfig;
1076         struct at_desc          *first = NULL;
1077         struct at_desc          *prev = NULL;
1078         u32                     ctrla;
1079         u32                     ctrlb;
1080         dma_addr_t              reg;
1081         unsigned int            reg_width;
1082         unsigned int            mem_width;
1083         unsigned int            i;
1084         struct scatterlist      *sg;
1085         size_t                  total_len = 0;
1086 
1087         dev_vdbg(chan2dev(chan), "prep_slave_sg (%d): %s f0x%lx\n",
1088                         sg_len,
1089                         direction == DMA_MEM_TO_DEV ? "TO DEVICE" : "FROM DEVICE",
1090                         flags);
1091 
1092         if (unlikely(!atslave || !sg_len)) {
1093                 dev_dbg(chan2dev(chan), "prep_slave_sg: sg length is zero!\n");
1094                 return NULL;
1095         }
1096 
1097         ctrla =   ATC_SCSIZE(sconfig->src_maxburst)
1098                 | ATC_DCSIZE(sconfig->dst_maxburst);
1099         ctrlb = ATC_IEN;
1100 
1101         switch (direction) {
1102         case DMA_MEM_TO_DEV:
1103                 reg_width = convert_buswidth(sconfig->dst_addr_width);
1104                 ctrla |=  ATC_DST_WIDTH(reg_width);
1105                 ctrlb |=  ATC_DST_ADDR_MODE_FIXED
1106                         | ATC_SRC_ADDR_MODE_INCR
1107                         | ATC_FC_MEM2PER
1108                         | ATC_SIF(atchan->mem_if) | ATC_DIF(atchan->per_if);
1109                 reg = sconfig->dst_addr;
1110                 for_each_sg(sgl, sg, sg_len, i) {
1111                         struct at_desc  *desc;
1112                         u32             len;
1113                         u32             mem;
1114 
1115                         desc = atc_desc_get(atchan);
1116                         if (!desc)
1117                                 goto err_desc_get;
1118 
1119                         mem = sg_dma_address(sg);
1120                         len = sg_dma_len(sg);
1121                         if (unlikely(!len)) {
1122                                 dev_dbg(chan2dev(chan),
1123                                         "prep_slave_sg: sg(%d) data length is zero\n", i);
1124                                 goto err;
1125                         }
1126                         mem_width = 2;
1127                         if (unlikely(mem & 3 || len & 3))
1128                                 mem_width = 0;
1129 
1130                         desc->lli.saddr = mem;
1131                         desc->lli.daddr = reg;
1132                         desc->lli.ctrla = ctrla
1133                                         | ATC_SRC_WIDTH(mem_width)
1134                                         | len >> mem_width;
1135                         desc->lli.ctrlb = ctrlb;
1136                         desc->len = len;
1137 
1138                         atc_desc_chain(&first, &prev, desc);
1139                         total_len += len;
1140                 }
1141                 break;
1142         case DMA_DEV_TO_MEM:
1143                 reg_width = convert_buswidth(sconfig->src_addr_width);
1144                 ctrla |=  ATC_SRC_WIDTH(reg_width);
1145                 ctrlb |=  ATC_DST_ADDR_MODE_INCR
1146                         | ATC_SRC_ADDR_MODE_FIXED
1147                         | ATC_FC_PER2MEM
1148                         | ATC_SIF(atchan->per_if) | ATC_DIF(atchan->mem_if);
1149 
1150                 reg = sconfig->src_addr;
1151                 for_each_sg(sgl, sg, sg_len, i) {
1152                         struct at_desc  *desc;
1153                         u32             len;
1154                         u32             mem;
1155 
1156                         desc = atc_desc_get(atchan);
1157                         if (!desc)
1158                                 goto err_desc_get;
1159 
1160                         mem = sg_dma_address(sg);
1161                         len = sg_dma_len(sg);
1162                         if (unlikely(!len)) {
1163                                 dev_dbg(chan2dev(chan),
1164                                         "prep_slave_sg: sg(%d) data length is zero\n", i);
1165                                 goto err;
1166                         }
1167                         mem_width = 2;
1168                         if (unlikely(mem & 3 || len & 3))
1169                                 mem_width = 0;
1170 
1171                         desc->lli.saddr = reg;
1172                         desc->lli.daddr = mem;
1173                         desc->lli.ctrla = ctrla
1174                                         | ATC_DST_WIDTH(mem_width)
1175                                         | len >> reg_width;
1176                         desc->lli.ctrlb = ctrlb;
1177                         desc->len = len;
1178 
1179                         atc_desc_chain(&first, &prev, desc);
1180                         total_len += len;
1181                 }
1182                 break;
1183         default:
1184                 return NULL;
1185         }
1186 
1187         /* set end-of-link to the last link descriptor of list*/
1188         set_desc_eol(prev);
1189 
1190         /* First descriptor of the chain embedds additional information */
1191         first->txd.cookie = -EBUSY;
1192         first->total_len = total_len;
1193 
1194         /* first link descriptor of list is responsible of flags */
1195         first->txd.flags = flags; /* client is in control of this ack */
1196 
1197         return &first->txd;
1198 
1199 err_desc_get:
1200         dev_err(chan2dev(chan), "not enough descriptors available\n");
1201 err:
1202         atc_desc_put(atchan, first);
1203         return NULL;
1204 }
1205 
1206 /**
1207  * atc_prep_dma_sg - prepare memory to memory scather-gather operation
1208  * @chan: the channel to prepare operation on
1209  * @dst_sg: destination scatterlist
1210  * @dst_nents: number of destination scatterlist entries
1211  * @src_sg: source scatterlist
1212  * @src_nents: number of source scatterlist entries
1213  * @flags: tx descriptor status flags
1214  */
1215 static struct dma_async_tx_descriptor *
1216 atc_prep_dma_sg(struct dma_chan *chan,
1217                 struct scatterlist *dst_sg, unsigned int dst_nents,
1218                 struct scatterlist *src_sg, unsigned int src_nents,
1219                 unsigned long flags)
1220 {
1221         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
1222         struct at_desc          *desc = NULL;
1223         struct at_desc          *first = NULL;
1224         struct at_desc          *prev = NULL;
1225         unsigned int            src_width;
1226         unsigned int            dst_width;
1227         size_t                  xfer_count;
1228         u32                     ctrla;
1229         u32                     ctrlb;
1230         size_t                  dst_len = 0, src_len = 0;
1231         dma_addr_t              dst = 0, src = 0;
1232         size_t                  len = 0, total_len = 0;
1233 
1234         if (unlikely(dst_nents == 0 || src_nents == 0))
1235                 return NULL;
1236 
1237         if (unlikely(dst_sg == NULL || src_sg == NULL))
1238                 return NULL;
1239 
1240         ctrlb =   ATC_DEFAULT_CTRLB | ATC_IEN
1241                 | ATC_SRC_ADDR_MODE_INCR
1242                 | ATC_DST_ADDR_MODE_INCR
1243                 | ATC_FC_MEM2MEM;
1244 
1245         /*
1246          * loop until there is either no more source or no more destination
1247          * scatterlist entry
1248          */
1249         while (true) {
1250 
1251                 /* prepare the next transfer */
1252                 if (dst_len == 0) {
1253 
1254                         /* no more destination scatterlist entries */
1255                         if (!dst_sg || !dst_nents)
1256                                 break;
1257 
1258                         dst = sg_dma_address(dst_sg);
1259                         dst_len = sg_dma_len(dst_sg);
1260 
1261                         dst_sg = sg_next(dst_sg);
1262                         dst_nents--;
1263                 }
1264 
1265                 if (src_len == 0) {
1266 
1267                         /* no more source scatterlist entries */
1268                         if (!src_sg || !src_nents)
1269                                 break;
1270 
1271                         src = sg_dma_address(src_sg);
1272                         src_len = sg_dma_len(src_sg);
1273 
1274                         src_sg = sg_next(src_sg);
1275                         src_nents--;
1276                 }
1277 
1278                 len = min_t(size_t, src_len, dst_len);
1279                 if (len == 0)
1280                         continue;
1281 
1282                 /* take care for the alignment */
1283                 src_width = dst_width = atc_get_xfer_width(src, dst, len);
1284 
1285                 ctrla = ATC_SRC_WIDTH(src_width) |
1286                         ATC_DST_WIDTH(dst_width);
1287 
1288                 /*
1289                  * The number of transfers to set up refer to the source width
1290                  * that depends on the alignment.
1291                  */
1292                 xfer_count = len >> src_width;
1293                 if (xfer_count > ATC_BTSIZE_MAX) {
1294                         xfer_count = ATC_BTSIZE_MAX;
1295                         len = ATC_BTSIZE_MAX << src_width;
1296                 }
1297 
1298                 /* create the transfer */
1299                 desc = atc_desc_get(atchan);
1300                 if (!desc)
1301                         goto err_desc_get;
1302 
1303                 desc->lli.saddr = src;
1304                 desc->lli.daddr = dst;
1305                 desc->lli.ctrla = ctrla | xfer_count;
1306                 desc->lli.ctrlb = ctrlb;
1307 
1308                 desc->txd.cookie = 0;
1309                 desc->len = len;
1310 
1311                 atc_desc_chain(&first, &prev, desc);
1312 
1313                 /* update the lengths and addresses for the next loop cycle */
1314                 dst_len -= len;
1315                 src_len -= len;
1316                 dst += len;
1317                 src += len;
1318 
1319                 total_len += len;
1320         }
1321 
1322         /* First descriptor of the chain embedds additional information */
1323         first->txd.cookie = -EBUSY;
1324         first->total_len = total_len;
1325 
1326         /* set end-of-link to the last link descriptor of list*/
1327         set_desc_eol(desc);
1328 
1329         first->txd.flags = flags; /* client is in control of this ack */
1330 
1331         return &first->txd;
1332 
1333 err_desc_get:
1334         atc_desc_put(atchan, first);
1335         return NULL;
1336 }
1337 
1338 /**
1339  * atc_dma_cyclic_check_values
1340  * Check for too big/unaligned periods and unaligned DMA buffer
1341  */
1342 static int
1343 atc_dma_cyclic_check_values(unsigned int reg_width, dma_addr_t buf_addr,
1344                 size_t period_len)
1345 {
1346         if (period_len > (ATC_BTSIZE_MAX << reg_width))
1347                 goto err_out;
1348         if (unlikely(period_len & ((1 << reg_width) - 1)))
1349                 goto err_out;
1350         if (unlikely(buf_addr & ((1 << reg_width) - 1)))
1351                 goto err_out;
1352 
1353         return 0;
1354 
1355 err_out:
1356         return -EINVAL;
1357 }
1358 
1359 /**
1360  * atc_dma_cyclic_fill_desc - Fill one period descriptor
1361  */
1362 static int
1363 atc_dma_cyclic_fill_desc(struct dma_chan *chan, struct at_desc *desc,
1364                 unsigned int period_index, dma_addr_t buf_addr,
1365                 unsigned int reg_width, size_t period_len,
1366                 enum dma_transfer_direction direction)
1367 {
1368         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
1369         struct dma_slave_config *sconfig = &atchan->dma_sconfig;
1370         u32                     ctrla;
1371 
1372         /* prepare common CRTLA value */
1373         ctrla =   ATC_SCSIZE(sconfig->src_maxburst)
1374                 | ATC_DCSIZE(sconfig->dst_maxburst)
1375                 | ATC_DST_WIDTH(reg_width)
1376                 | ATC_SRC_WIDTH(reg_width)
1377                 | period_len >> reg_width;
1378 
1379         switch (direction) {
1380         case DMA_MEM_TO_DEV:
1381                 desc->lli.saddr = buf_addr + (period_len * period_index);
1382                 desc->lli.daddr = sconfig->dst_addr;
1383                 desc->lli.ctrla = ctrla;
1384                 desc->lli.ctrlb = ATC_DST_ADDR_MODE_FIXED
1385                                 | ATC_SRC_ADDR_MODE_INCR
1386                                 | ATC_FC_MEM2PER
1387                                 | ATC_SIF(atchan->mem_if)
1388                                 | ATC_DIF(atchan->per_if);
1389                 desc->len = period_len;
1390                 break;
1391 
1392         case DMA_DEV_TO_MEM:
1393                 desc->lli.saddr = sconfig->src_addr;
1394                 desc->lli.daddr = buf_addr + (period_len * period_index);
1395                 desc->lli.ctrla = ctrla;
1396                 desc->lli.ctrlb = ATC_DST_ADDR_MODE_INCR
1397                                 | ATC_SRC_ADDR_MODE_FIXED
1398                                 | ATC_FC_PER2MEM
1399                                 | ATC_SIF(atchan->per_if)
1400                                 | ATC_DIF(atchan->mem_if);
1401                 desc->len = period_len;
1402                 break;
1403 
1404         default:
1405                 return -EINVAL;
1406         }
1407 
1408         return 0;
1409 }
1410 
1411 /**
1412  * atc_prep_dma_cyclic - prepare the cyclic DMA transfer
1413  * @chan: the DMA channel to prepare
1414  * @buf_addr: physical DMA address where the buffer starts
1415  * @buf_len: total number of bytes for the entire buffer
1416  * @period_len: number of bytes for each period
1417  * @direction: transfer direction, to or from device
1418  * @flags: tx descriptor status flags
1419  */
1420 static struct dma_async_tx_descriptor *
1421 atc_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1422                 size_t period_len, enum dma_transfer_direction direction,
1423                 unsigned long flags)
1424 {
1425         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
1426         struct at_dma_slave     *atslave = chan->private;
1427         struct dma_slave_config *sconfig = &atchan->dma_sconfig;
1428         struct at_desc          *first = NULL;
1429         struct at_desc          *prev = NULL;
1430         unsigned long           was_cyclic;
1431         unsigned int            reg_width;
1432         unsigned int            periods = buf_len / period_len;
1433         unsigned int            i;
1434 
1435         dev_vdbg(chan2dev(chan), "prep_dma_cyclic: %s buf@%pad - %d (%d/%d)\n",
1436                         direction == DMA_MEM_TO_DEV ? "TO DEVICE" : "FROM DEVICE",
1437                         &buf_addr,
1438                         periods, buf_len, period_len);
1439 
1440         if (unlikely(!atslave || !buf_len || !period_len)) {
1441                 dev_dbg(chan2dev(chan), "prep_dma_cyclic: length is zero!\n");
1442                 return NULL;
1443         }
1444 
1445         was_cyclic = test_and_set_bit(ATC_IS_CYCLIC, &atchan->status);
1446         if (was_cyclic) {
1447                 dev_dbg(chan2dev(chan), "prep_dma_cyclic: channel in use!\n");
1448                 return NULL;
1449         }
1450 
1451         if (unlikely(!is_slave_direction(direction)))
1452                 goto err_out;
1453 
1454         if (sconfig->direction == DMA_MEM_TO_DEV)
1455                 reg_width = convert_buswidth(sconfig->dst_addr_width);
1456         else
1457                 reg_width = convert_buswidth(sconfig->src_addr_width);
1458 
1459         /* Check for too big/unaligned periods and unaligned DMA buffer */
1460         if (atc_dma_cyclic_check_values(reg_width, buf_addr, period_len))
1461                 goto err_out;
1462 
1463         /* build cyclic linked list */
1464         for (i = 0; i < periods; i++) {
1465                 struct at_desc  *desc;
1466 
1467                 desc = atc_desc_get(atchan);
1468                 if (!desc)
1469                         goto err_desc_get;
1470 
1471                 if (atc_dma_cyclic_fill_desc(chan, desc, i, buf_addr,
1472                                              reg_width, period_len, direction))
1473                         goto err_desc_get;
1474 
1475                 atc_desc_chain(&first, &prev, desc);
1476         }
1477 
1478         /* lets make a cyclic list */
1479         prev->lli.dscr = first->txd.phys;
1480 
1481         /* First descriptor of the chain embedds additional information */
1482         first->txd.cookie = -EBUSY;
1483         first->total_len = buf_len;
1484 
1485         return &first->txd;
1486 
1487 err_desc_get:
1488         dev_err(chan2dev(chan), "not enough descriptors available\n");
1489         atc_desc_put(atchan, first);
1490 err_out:
1491         clear_bit(ATC_IS_CYCLIC, &atchan->status);
1492         return NULL;
1493 }
1494 
1495 static int atc_config(struct dma_chan *chan,
1496                       struct dma_slave_config *sconfig)
1497 {
1498         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
1499 
1500         dev_vdbg(chan2dev(chan), "%s\n", __func__);
1501 
1502         /* Check if it is chan is configured for slave transfers */
1503         if (!chan->private)
1504                 return -EINVAL;
1505 
1506         memcpy(&atchan->dma_sconfig, sconfig, sizeof(*sconfig));
1507 
1508         convert_burst(&atchan->dma_sconfig.src_maxburst);
1509         convert_burst(&atchan->dma_sconfig.dst_maxburst);
1510 
1511         return 0;
1512 }
1513 
1514 static int atc_pause(struct dma_chan *chan)
1515 {
1516         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
1517         struct at_dma           *atdma = to_at_dma(chan->device);
1518         int                     chan_id = atchan->chan_common.chan_id;
1519         unsigned long           flags;
1520 
1521         LIST_HEAD(list);
1522 
1523         dev_vdbg(chan2dev(chan), "%s\n", __func__);
1524 
1525         spin_lock_irqsave(&atchan->lock, flags);
1526 
1527         dma_writel(atdma, CHER, AT_DMA_SUSP(chan_id));
1528         set_bit(ATC_IS_PAUSED, &atchan->status);
1529 
1530         spin_unlock_irqrestore(&atchan->lock, flags);
1531 
1532         return 0;
1533 }
1534 
1535 static int atc_resume(struct dma_chan *chan)
1536 {
1537         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
1538         struct at_dma           *atdma = to_at_dma(chan->device);
1539         int                     chan_id = atchan->chan_common.chan_id;
1540         unsigned long           flags;
1541 
1542         LIST_HEAD(list);
1543 
1544         dev_vdbg(chan2dev(chan), "%s\n", __func__);
1545 
1546         if (!atc_chan_is_paused(atchan))
1547                 return 0;
1548 
1549         spin_lock_irqsave(&atchan->lock, flags);
1550 
1551         dma_writel(atdma, CHDR, AT_DMA_RES(chan_id));
1552         clear_bit(ATC_IS_PAUSED, &atchan->status);
1553 
1554         spin_unlock_irqrestore(&atchan->lock, flags);
1555 
1556         return 0;
1557 }
1558 
1559 static int atc_terminate_all(struct dma_chan *chan)
1560 {
1561         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
1562         struct at_dma           *atdma = to_at_dma(chan->device);
1563         int                     chan_id = atchan->chan_common.chan_id;
1564         struct at_desc          *desc, *_desc;
1565         unsigned long           flags;
1566 
1567         LIST_HEAD(list);
1568 
1569         dev_vdbg(chan2dev(chan), "%s\n", __func__);
1570 
1571         /*
1572          * This is only called when something went wrong elsewhere, so
1573          * we don't really care about the data. Just disable the
1574          * channel. We still have to poll the channel enable bit due
1575          * to AHB/HSB limitations.
1576          */
1577         spin_lock_irqsave(&atchan->lock, flags);
1578 
1579         /* disabling channel: must also remove suspend state */
1580         dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask);
1581 
1582         /* confirm that this channel is disabled */
1583         while (dma_readl(atdma, CHSR) & atchan->mask)
1584                 cpu_relax();
1585 
1586         /* active_list entries will end up before queued entries */
1587         list_splice_init(&atchan->queue, &list);
1588         list_splice_init(&atchan->active_list, &list);
1589 
1590         /* Flush all pending and queued descriptors */
1591         list_for_each_entry_safe(desc, _desc, &list, desc_node)
1592                 atc_chain_complete(atchan, desc);
1593 
1594         clear_bit(ATC_IS_PAUSED, &atchan->status);
1595         /* if channel dedicated to cyclic operations, free it */
1596         clear_bit(ATC_IS_CYCLIC, &atchan->status);
1597 
1598         spin_unlock_irqrestore(&atchan->lock, flags);
1599 
1600         return 0;
1601 }
1602 
1603 /**
1604  * atc_tx_status - poll for transaction completion
1605  * @chan: DMA channel
1606  * @cookie: transaction identifier to check status of
1607  * @txstate: if not %NULL updated with transaction state
1608  *
1609  * If @txstate is passed in, upon return it reflect the driver
1610  * internal state and can be used with dma_async_is_complete() to check
1611  * the status of multiple cookies without re-checking hardware state.
1612  */
1613 static enum dma_status
1614 atc_tx_status(struct dma_chan *chan,
1615                 dma_cookie_t cookie,
1616                 struct dma_tx_state *txstate)
1617 {
1618         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
1619         unsigned long           flags;
1620         enum dma_status         ret;
1621         int bytes = 0;
1622 
1623         ret = dma_cookie_status(chan, cookie, txstate);
1624         if (ret == DMA_COMPLETE)
1625                 return ret;
1626         /*
1627          * There's no point calculating the residue if there's
1628          * no txstate to store the value.
1629          */
1630         if (!txstate)
1631                 return DMA_ERROR;
1632 
1633         spin_lock_irqsave(&atchan->lock, flags);
1634 
1635         /*  Get number of bytes left in the active transactions */
1636         bytes = atc_get_bytes_left(chan, cookie);
1637 
1638         spin_unlock_irqrestore(&atchan->lock, flags);
1639 
1640         if (unlikely(bytes < 0)) {
1641                 dev_vdbg(chan2dev(chan), "get residual bytes error\n");
1642                 return DMA_ERROR;
1643         } else {
1644                 dma_set_residue(txstate, bytes);
1645         }
1646 
1647         dev_vdbg(chan2dev(chan), "tx_status %d: cookie = %d residue = %d\n",
1648                  ret, cookie, bytes);
1649 
1650         return ret;
1651 }
1652 
1653 /**
1654  * atc_issue_pending - try to finish work
1655  * @chan: target DMA channel
1656  */
1657 static void atc_issue_pending(struct dma_chan *chan)
1658 {
1659         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
1660         unsigned long           flags;
1661 
1662         dev_vdbg(chan2dev(chan), "issue_pending\n");
1663 
1664         /* Not needed for cyclic transfers */
1665         if (atc_chan_is_cyclic(atchan))
1666                 return;
1667 
1668         spin_lock_irqsave(&atchan->lock, flags);
1669         atc_advance_work(atchan);
1670         spin_unlock_irqrestore(&atchan->lock, flags);
1671 }
1672 
1673 /**
1674  * atc_alloc_chan_resources - allocate resources for DMA channel
1675  * @chan: allocate descriptor resources for this channel
1676  * @client: current client requesting the channel be ready for requests
1677  *
1678  * return - the number of allocated descriptors
1679  */
1680 static int atc_alloc_chan_resources(struct dma_chan *chan)
1681 {
1682         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
1683         struct at_dma           *atdma = to_at_dma(chan->device);
1684         struct at_desc          *desc;
1685         struct at_dma_slave     *atslave;
1686         unsigned long           flags;
1687         int                     i;
1688         u32                     cfg;
1689         LIST_HEAD(tmp_list);
1690 
1691         dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");
1692 
1693         /* ASSERT:  channel is idle */
1694         if (atc_chan_is_enabled(atchan)) {
1695                 dev_dbg(chan2dev(chan), "DMA channel not idle ?\n");
1696                 return -EIO;
1697         }
1698 
1699         cfg = ATC_DEFAULT_CFG;
1700 
1701         atslave = chan->private;
1702         if (atslave) {
1703                 /*
1704                  * We need controller-specific data to set up slave
1705                  * transfers.
1706                  */
1707                 BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_common.dev);
1708 
1709                 /* if cfg configuration specified take it instead of default */
1710                 if (atslave->cfg)
1711                         cfg = atslave->cfg;
1712         }
1713 
1714         /* have we already been set up?
1715          * reconfigure channel but no need to reallocate descriptors */
1716         if (!list_empty(&atchan->free_list))
1717                 return atchan->descs_allocated;
1718 
1719         /* Allocate initial pool of descriptors */
1720         for (i = 0; i < init_nr_desc_per_channel; i++) {
1721                 desc = atc_alloc_descriptor(chan, GFP_KERNEL);
1722                 if (!desc) {
1723                         dev_err(atdma->dma_common.dev,
1724                                 "Only %d initial descriptors\n", i);
1725                         break;
1726                 }
1727                 list_add_tail(&desc->desc_node, &tmp_list);
1728         }
1729 
1730         spin_lock_irqsave(&atchan->lock, flags);
1731         atchan->descs_allocated = i;
1732         list_splice(&tmp_list, &atchan->free_list);
1733         dma_cookie_init(chan);
1734         spin_unlock_irqrestore(&atchan->lock, flags);
1735 
1736         /* channel parameters */
1737         channel_writel(atchan, CFG, cfg);
1738 
1739         dev_dbg(chan2dev(chan),
1740                 "alloc_chan_resources: allocated %d descriptors\n",
1741                 atchan->descs_allocated);
1742 
1743         return atchan->descs_allocated;
1744 }
1745 
1746 /**
1747  * atc_free_chan_resources - free all channel resources
1748  * @chan: DMA channel
1749  */
1750 static void atc_free_chan_resources(struct dma_chan *chan)
1751 {
1752         struct at_dma_chan      *atchan = to_at_dma_chan(chan);
1753         struct at_dma           *atdma = to_at_dma(chan->device);
1754         struct at_desc          *desc, *_desc;
1755         LIST_HEAD(list);
1756 
1757         dev_dbg(chan2dev(chan), "free_chan_resources: (descs allocated=%u)\n",
1758                 atchan->descs_allocated);
1759 
1760         /* ASSERT:  channel is idle */
1761         BUG_ON(!list_empty(&atchan->active_list));
1762         BUG_ON(!list_empty(&atchan->queue));
1763         BUG_ON(atc_chan_is_enabled(atchan));
1764 
1765         list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) {
1766                 dev_vdbg(chan2dev(chan), "  freeing descriptor %p\n", desc);
1767                 list_del(&desc->desc_node);
1768                 /* free link descriptor */
1769                 dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys);
1770         }
1771         list_splice_init(&atchan->free_list, &list);
1772         atchan->descs_allocated = 0;
1773         atchan->status = 0;
1774 
1775         dev_vdbg(chan2dev(chan), "free_chan_resources: done\n");
1776 }
1777 
1778 #ifdef CONFIG_OF
1779 static bool at_dma_filter(struct dma_chan *chan, void *slave)
1780 {
1781         struct at_dma_slave *atslave = slave;
1782 
1783         if (atslave->dma_dev == chan->device->dev) {
1784                 chan->private = atslave;
1785                 return true;
1786         } else {
1787                 return false;
1788         }
1789 }
1790 
1791 static struct dma_chan *at_dma_xlate(struct of_phandle_args *dma_spec,
1792                                      struct of_dma *of_dma)
1793 {
1794         struct dma_chan *chan;
1795         struct at_dma_chan *atchan;
1796         struct at_dma_slave *atslave;
1797         dma_cap_mask_t mask;
1798         unsigned int per_id;
1799         struct platform_device *dmac_pdev;
1800 
1801         if (dma_spec->args_count != 2)
1802                 return NULL;
1803 
1804         dmac_pdev = of_find_device_by_node(dma_spec->np);
1805 
1806         dma_cap_zero(mask);
1807         dma_cap_set(DMA_SLAVE, mask);
1808 
1809         atslave = devm_kzalloc(&dmac_pdev->dev, sizeof(*atslave), GFP_KERNEL);
1810         if (!atslave)
1811                 return NULL;
1812 
1813         atslave->cfg = ATC_DST_H2SEL_HW | ATC_SRC_H2SEL_HW;
1814         /*
1815          * We can fill both SRC_PER and DST_PER, one of these fields will be
1816          * ignored depending on DMA transfer direction.
1817          */
1818         per_id = dma_spec->args[1] & AT91_DMA_CFG_PER_ID_MASK;
1819         atslave->cfg |= ATC_DST_PER_MSB(per_id) | ATC_DST_PER(per_id)
1820                      | ATC_SRC_PER_MSB(per_id) | ATC_SRC_PER(per_id);
1821         /*
1822          * We have to translate the value we get from the device tree since
1823          * the half FIFO configuration value had to be 0 to keep backward
1824          * compatibility.
1825          */
1826         switch (dma_spec->args[1] & AT91_DMA_CFG_FIFOCFG_MASK) {
1827         case AT91_DMA_CFG_FIFOCFG_ALAP:
1828                 atslave->cfg |= ATC_FIFOCFG_LARGESTBURST;
1829                 break;
1830         case AT91_DMA_CFG_FIFOCFG_ASAP:
1831                 atslave->cfg |= ATC_FIFOCFG_ENOUGHSPACE;
1832                 break;
1833         case AT91_DMA_CFG_FIFOCFG_HALF:
1834         default:
1835                 atslave->cfg |= ATC_FIFOCFG_HALFFIFO;
1836         }
1837         atslave->dma_dev = &dmac_pdev->dev;
1838 
1839         chan = dma_request_channel(mask, at_dma_filter, atslave);
1840         if (!chan)
1841                 return NULL;
1842 
1843         atchan = to_at_dma_chan(chan);
1844         atchan->per_if = dma_spec->args[0] & 0xff;
1845         atchan->mem_if = (dma_spec->args[0] >> 16) & 0xff;
1846 
1847         return chan;
1848 }
1849 #else
1850 static struct dma_chan *at_dma_xlate(struct of_phandle_args *dma_spec,
1851                                      struct of_dma *of_dma)
1852 {
1853         return NULL;
1854 }
1855 #endif
1856 
1857 /*--  Module Management  -----------------------------------------------*/
1858 
1859 /* cap_mask is a multi-u32 bitfield, fill it with proper C code. */
1860 static struct at_dma_platform_data at91sam9rl_config = {
1861         .nr_channels = 2,
1862 };
1863 static struct at_dma_platform_data at91sam9g45_config = {
1864         .nr_channels = 8,
1865 };
1866 
1867 #if defined(CONFIG_OF)
1868 static const struct of_device_id atmel_dma_dt_ids[] = {
1869         {
1870                 .compatible = "atmel,at91sam9rl-dma",
1871                 .data = &at91sam9rl_config,
1872         }, {
1873                 .compatible = "atmel,at91sam9g45-dma",
1874                 .data = &at91sam9g45_config,
1875         }, {
1876                 /* sentinel */
1877         }
1878 };
1879 
1880 MODULE_DEVICE_TABLE(of, atmel_dma_dt_ids);
1881 #endif
1882 
1883 static const struct platform_device_id atdma_devtypes[] = {
1884         {
1885                 .name = "at91sam9rl_dma",
1886                 .driver_data = (unsigned long) &at91sam9rl_config,
1887         }, {
1888                 .name = "at91sam9g45_dma",
1889                 .driver_data = (unsigned long) &at91sam9g45_config,
1890         }, {
1891                 /* sentinel */
1892         }
1893 };
1894 
1895 static inline const struct at_dma_platform_data * __init at_dma_get_driver_data(
1896                                                 struct platform_device *pdev)
1897 {
1898         if (pdev->dev.of_node) {
1899                 const struct of_device_id *match;
1900                 match = of_match_node(atmel_dma_dt_ids, pdev->dev.of_node);
1901                 if (match == NULL)
1902                         return NULL;
1903                 return match->data;
1904         }
1905         return (struct at_dma_platform_data *)
1906                         platform_get_device_id(pdev)->driver_data;
1907 }
1908 
1909 /**
1910  * at_dma_off - disable DMA controller
1911  * @atdma: the Atmel HDAMC device
1912  */
1913 static void at_dma_off(struct at_dma *atdma)
1914 {
1915         dma_writel(atdma, EN, 0);
1916 
1917         /* disable all interrupts */
1918         dma_writel(atdma, EBCIDR, -1L);
1919 
1920         /* confirm that all channels are disabled */
1921         while (dma_readl(atdma, CHSR) & atdma->all_chan_mask)
1922                 cpu_relax();
1923 }
1924 
1925 static int __init at_dma_probe(struct platform_device *pdev)
1926 {
1927         struct resource         *io;
1928         struct at_dma           *atdma;
1929         size_t                  size;
1930         int                     irq;
1931         int                     err;
1932         int                     i;
1933         const struct at_dma_platform_data *plat_dat;
1934 
1935         /* setup platform data for each SoC */
1936         dma_cap_set(DMA_MEMCPY, at91sam9rl_config.cap_mask);
1937         dma_cap_set(DMA_SG, at91sam9rl_config.cap_mask);
1938         dma_cap_set(DMA_INTERLEAVE, at91sam9g45_config.cap_mask);
1939         dma_cap_set(DMA_MEMCPY, at91sam9g45_config.cap_mask);
1940         dma_cap_set(DMA_MEMSET, at91sam9g45_config.cap_mask);
1941         dma_cap_set(DMA_MEMSET_SG, at91sam9g45_config.cap_mask);
1942         dma_cap_set(DMA_PRIVATE, at91sam9g45_config.cap_mask);
1943         dma_cap_set(DMA_SLAVE, at91sam9g45_config.cap_mask);
1944         dma_cap_set(DMA_SG, at91sam9g45_config.cap_mask);
1945 
1946         /* get DMA parameters from controller type */
1947         plat_dat = at_dma_get_driver_data(pdev);
1948         if (!plat_dat)
1949                 return -ENODEV;
1950 
1951         io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1952         if (!io)
1953                 return -EINVAL;
1954 
1955         irq = platform_get_irq(pdev, 0);
1956         if (irq < 0)
1957                 return irq;
1958 
1959         size = sizeof(struct at_dma);
1960         size += plat_dat->nr_channels * sizeof(struct at_dma_chan);
1961         atdma = kzalloc(size, GFP_KERNEL);
1962         if (!atdma)
1963                 return -ENOMEM;
1964 
1965         /* discover transaction capabilities */
1966         atdma->dma_common.cap_mask = plat_dat->cap_mask;
1967         atdma->all_chan_mask = (1 << plat_dat->nr_channels) - 1;
1968 
1969         size = resource_size(io);
1970         if (!request_mem_region(io->start, size, pdev->dev.driver->name)) {
1971                 err = -EBUSY;
1972                 goto err_kfree;
1973         }
1974 
1975         atdma->regs = ioremap(io->start, size);
1976         if (!atdma->regs) {
1977                 err = -ENOMEM;
1978                 goto err_release_r;
1979         }
1980 
1981         atdma->clk = clk_get(&pdev->dev, "dma_clk");
1982         if (IS_ERR(atdma->clk)) {
1983                 err = PTR_ERR(atdma->clk);
1984                 goto err_clk;
1985         }
1986         err = clk_prepare_enable(atdma->clk);
1987         if (err)
1988                 goto err_clk_prepare;
1989 
1990         /* force dma off, just in case */
1991         at_dma_off(atdma);
1992 
1993         err = request_irq(irq, at_dma_interrupt, 0, "at_hdmac", atdma);
1994         if (err)
1995                 goto err_irq;
1996 
1997         platform_set_drvdata(pdev, atdma);
1998 
1999         /* create a pool of consistent memory blocks for hardware descriptors */
2000         atdma->dma_desc_pool = dma_pool_create("at_hdmac_desc_pool",
2001                         &pdev->dev, sizeof(struct at_desc),
2002                         4 /* word alignment */, 0);
2003         if (!atdma->dma_desc_pool) {
2004                 dev_err(&pdev->dev, "No memory for descriptors dma pool\n");
2005                 err = -ENOMEM;
2006                 goto err_desc_pool_create;
2007         }
2008 
2009         /* create a pool of consistent memory blocks for memset blocks */
2010         atdma->memset_pool = dma_pool_create("at_hdmac_memset_pool",
2011                                              &pdev->dev, sizeof(int), 4, 0);
2012         if (!atdma->memset_pool) {
2013                 dev_err(&pdev->dev, "No memory for memset dma pool\n");
2014                 err = -ENOMEM;
2015                 goto err_memset_pool_create;
2016         }
2017 
2018         /* clear any pending interrupt */
2019         while (dma_readl(atdma, EBCISR))
2020                 cpu_relax();
2021 
2022         /* initialize channels related values */
2023         INIT_LIST_HEAD(&atdma->dma_common.channels);
2024         for (i = 0; i < plat_dat->nr_channels; i++) {
2025                 struct at_dma_chan      *atchan = &atdma->chan[i];
2026 
2027                 atchan->mem_if = AT_DMA_MEM_IF;
2028                 atchan->per_if = AT_DMA_PER_IF;
2029                 atchan->chan_common.device = &atdma->dma_common;
2030                 dma_cookie_init(&atchan->chan_common);
2031                 list_add_tail(&atchan->chan_common.device_node,
2032                                 &atdma->dma_common.channels);
2033 
2034                 atchan->ch_regs = atdma->regs + ch_regs(i);
2035                 spin_lock_init(&atchan->lock);
2036                 atchan->mask = 1 << i;
2037 
2038                 INIT_LIST_HEAD(&atchan->active_list);
2039                 INIT_LIST_HEAD(&atchan->queue);
2040                 INIT_LIST_HEAD(&atchan->free_list);
2041 
2042                 tasklet_init(&atchan->tasklet, atc_tasklet,
2043                                 (unsigned long)atchan);
2044                 atc_enable_chan_irq(atdma, i);
2045         }
2046 
2047         /* set base routines */
2048         atdma->dma_common.device_alloc_chan_resources = atc_alloc_chan_resources;
2049         atdma->dma_common.device_free_chan_resources = atc_free_chan_resources;
2050         atdma->dma_common.device_tx_status = atc_tx_status;
2051         atdma->dma_common.device_issue_pending = atc_issue_pending;
2052         atdma->dma_common.dev = &pdev->dev;
2053 
2054         /* set prep routines based on capability */
2055         if (dma_has_cap(DMA_INTERLEAVE, atdma->dma_common.cap_mask))
2056                 atdma->dma_common.device_prep_interleaved_dma = atc_prep_dma_interleaved;
2057 
2058         if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask))
2059                 atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy;
2060 
2061         if (dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask)) {
2062                 atdma->dma_common.device_prep_dma_memset = atc_prep_dma_memset;
2063                 atdma->dma_common.device_prep_dma_memset_sg = atc_prep_dma_memset_sg;
2064                 atdma->dma_common.fill_align = DMAENGINE_ALIGN_4_BYTES;
2065         }
2066 
2067         if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) {
2068                 atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg;
2069                 /* controller can do slave DMA: can trigger cyclic transfers */
2070                 dma_cap_set(DMA_CYCLIC, atdma->dma_common.cap_mask);
2071                 atdma->dma_common.device_prep_dma_cyclic = atc_prep_dma_cyclic;
2072                 atdma->dma_common.device_config = atc_config;
2073                 atdma->dma_common.device_pause = atc_pause;
2074                 atdma->dma_common.device_resume = atc_resume;
2075                 atdma->dma_common.device_terminate_all = atc_terminate_all;
2076                 atdma->dma_common.src_addr_widths = ATC_DMA_BUSWIDTHS;
2077                 atdma->dma_common.dst_addr_widths = ATC_DMA_BUSWIDTHS;
2078                 atdma->dma_common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
2079                 atdma->dma_common.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
2080         }
2081 
2082         if (dma_has_cap(DMA_SG, atdma->dma_common.cap_mask))
2083                 atdma->dma_common.device_prep_dma_sg = atc_prep_dma_sg;
2084 
2085         dma_writel(atdma, EN, AT_DMA_ENABLE);
2086 
2087         dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s%s%s), %d channels\n",
2088           dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "",
2089           dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask) ? "set " : "",
2090           dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)  ? "slave " : "",
2091           dma_has_cap(DMA_SG, atdma->dma_common.cap_mask)  ? "sg-cpy " : "",
2092           plat_dat->nr_channels);
2093 
2094         dma_async_device_register(&atdma->dma_common);
2095 
2096         /*
2097          * Do not return an error if the dmac node is not present in order to
2098          * not break the existing way of requesting channel with
2099          * dma_request_channel().
2100          */
2101         if (pdev->dev.of_node) {
2102                 err = of_dma_controller_register(pdev->dev.of_node,
2103                                                  at_dma_xlate, atdma);
2104                 if (err) {
2105                         dev_err(&pdev->dev, "could not register of_dma_controller\n");
2106                         goto err_of_dma_controller_register;
2107                 }
2108         }
2109 
2110         return 0;
2111 
2112 err_of_dma_controller_register:
2113         dma_async_device_unregister(&atdma->dma_common);
2114         dma_pool_destroy(atdma->memset_pool);
2115 err_memset_pool_create:
2116         dma_pool_destroy(atdma->dma_desc_pool);
2117 err_desc_pool_create:
2118         free_irq(platform_get_irq(pdev, 0), atdma);
2119 err_irq:
2120         clk_disable_unprepare(atdma->clk);
2121 err_clk_prepare:
2122         clk_put(atdma->clk);
2123 err_clk:
2124         iounmap(atdma->regs);
2125         atdma->regs = NULL;
2126 err_release_r:
2127         release_mem_region(io->start, size);
2128 err_kfree:
2129         kfree(atdma);
2130         return err;
2131 }
2132 
2133 static int at_dma_remove(struct platform_device *pdev)
2134 {
2135         struct at_dma           *atdma = platform_get_drvdata(pdev);
2136         struct dma_chan         *chan, *_chan;
2137         struct resource         *io;
2138 
2139         at_dma_off(atdma);
2140         dma_async_device_unregister(&atdma->dma_common);
2141 
2142         dma_pool_destroy(atdma->memset_pool);
2143         dma_pool_destroy(atdma->dma_desc_pool);
2144         free_irq(platform_get_irq(pdev, 0), atdma);
2145 
2146         list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,
2147                         device_node) {
2148                 struct at_dma_chan      *atchan = to_at_dma_chan(chan);
2149 
2150                 /* Disable interrupts */
2151                 atc_disable_chan_irq(atdma, chan->chan_id);
2152 
2153                 tasklet_kill(&atchan->tasklet);
2154                 list_del(&chan->device_node);
2155         }
2156 
2157         clk_disable_unprepare(atdma->clk);
2158         clk_put(atdma->clk);
2159 
2160         iounmap(atdma->regs);
2161         atdma->regs = NULL;
2162 
2163         io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2164         release_mem_region(io->start, resource_size(io));
2165 
2166         kfree(atdma);
2167 
2168         return 0;
2169 }
2170 
2171 static void at_dma_shutdown(struct platform_device *pdev)
2172 {
2173         struct at_dma   *atdma = platform_get_drvdata(pdev);
2174 
2175         at_dma_off(platform_get_drvdata(pdev));
2176         clk_disable_unprepare(atdma->clk);
2177 }
2178 
2179 static int at_dma_prepare(struct device *dev)
2180 {
2181         struct platform_device *pdev = to_platform_device(dev);
2182         struct at_dma *atdma = platform_get_drvdata(pdev);
2183         struct dma_chan *chan, *_chan;
2184 
2185         list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,
2186                         device_node) {
2187                 struct at_dma_chan *atchan = to_at_dma_chan(chan);
2188                 /* wait for transaction completion (except in cyclic case) */
2189                 if (atc_chan_is_enabled(atchan) && !atc_chan_is_cyclic(atchan))
2190                         return -EAGAIN;
2191         }
2192         return 0;
2193 }
2194 
2195 static void atc_suspend_cyclic(struct at_dma_chan *atchan)
2196 {
2197         struct dma_chan *chan = &atchan->chan_common;
2198 
2199         /* Channel should be paused by user
2200          * do it anyway even if it is not done already */
2201         if (!atc_chan_is_paused(atchan)) {
2202                 dev_warn(chan2dev(chan),
2203                 "cyclic channel not paused, should be done by channel user\n");
2204                 atc_pause(chan);
2205         }
2206 
2207         /* now preserve additional data for cyclic operations */
2208         /* next descriptor address in the cyclic list */
2209         atchan->save_dscr = channel_readl(atchan, DSCR);
2210 
2211         vdbg_dump_regs(atchan);
2212 }
2213 
2214 static int at_dma_suspend_noirq(struct device *dev)
2215 {
2216         struct platform_device *pdev = to_platform_device(dev);
2217         struct at_dma *atdma = platform_get_drvdata(pdev);
2218         struct dma_chan *chan, *_chan;
2219 
2220         /* preserve data */
2221         list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,
2222                         device_node) {
2223                 struct at_dma_chan *atchan = to_at_dma_chan(chan);
2224 
2225                 if (atc_chan_is_cyclic(atchan))
2226                         atc_suspend_cyclic(atchan);
2227                 atchan->save_cfg = channel_readl(atchan, CFG);
2228         }
2229         atdma->save_imr = dma_readl(atdma, EBCIMR);
2230 
2231         /* disable DMA controller */
2232         at_dma_off(atdma);
2233         clk_disable_unprepare(atdma->clk);
2234         return 0;
2235 }
2236 
2237 static void atc_resume_cyclic(struct at_dma_chan *atchan)
2238 {
2239         struct at_dma   *atdma = to_at_dma(atchan->chan_common.device);
2240 
2241         /* restore channel status for cyclic descriptors list:
2242          * next descriptor in the cyclic list at the time of suspend */
2243         channel_writel(atchan, SADDR, 0);
2244         channel_writel(atchan, DADDR, 0);
2245         channel_writel(atchan, CTRLA, 0);
2246         channel_writel(atchan, CTRLB, 0);
2247         channel_writel(atchan, DSCR, atchan->save_dscr);
2248         dma_writel(atdma, CHER, atchan->mask);
2249 
2250         /* channel pause status should be removed by channel user
2251          * We cannot take the initiative to do it here */
2252 
2253         vdbg_dump_regs(atchan);
2254 }
2255 
2256 static int at_dma_resume_noirq(struct device *dev)
2257 {
2258         struct platform_device *pdev = to_platform_device(dev);
2259         struct at_dma *atdma = platform_get_drvdata(pdev);
2260         struct dma_chan *chan, *_chan;
2261 
2262         /* bring back DMA controller */
2263         clk_prepare_enable(atdma->clk);
2264         dma_writel(atdma, EN, AT_DMA_ENABLE);
2265 
2266         /* clear any pending interrupt */
2267         while (dma_readl(atdma, EBCISR))
2268                 cpu_relax();
2269 
2270         /* restore saved data */
2271         dma_writel(atdma, EBCIER, atdma->save_imr);
2272         list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,
2273                         device_node) {
2274                 struct at_dma_chan *atchan = to_at_dma_chan(chan);
2275 
2276                 channel_writel(atchan, CFG, atchan->save_cfg);
2277                 if (atc_chan_is_cyclic(atchan))
2278                         atc_resume_cyclic(atchan);
2279         }
2280         return 0;
2281 }
2282 
2283 static const struct dev_pm_ops at_dma_dev_pm_ops = {
2284         .prepare = at_dma_prepare,
2285         .suspend_noirq = at_dma_suspend_noirq,
2286         .resume_noirq = at_dma_resume_noirq,
2287 };
2288 
2289 static struct platform_driver at_dma_driver = {
2290         .remove         = at_dma_remove,
2291         .shutdown       = at_dma_shutdown,
2292         .id_table       = atdma_devtypes,
2293         .driver = {
2294                 .name   = "at_hdmac",
2295                 .pm     = &at_dma_dev_pm_ops,
2296                 .of_match_table = of_match_ptr(atmel_dma_dt_ids),
2297         },
2298 };
2299 
2300 static int __init at_dma_init(void)
2301 {
2302         return platform_driver_probe(&at_dma_driver, at_dma_probe);
2303 }
2304 subsys_initcall(at_dma_init);
2305 
2306 static void __exit at_dma_exit(void)
2307 {
2308         platform_driver_unregister(&at_dma_driver);
2309 }
2310 module_exit(at_dma_exit);
2311 
2312 MODULE_DESCRIPTION("Atmel AHB DMA Controller driver");
2313 MODULE_AUTHOR("Nicolas Ferre <nicolas.ferre@atmel.com>");
2314 MODULE_LICENSE("GPL");
2315 MODULE_ALIAS("platform:at_hdmac");
2316 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us