Version:  2.0.40 2.2.26 2.4.37 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17

Linux/drivers/dma/fsldma.c

  1 /*
  2  * Freescale MPC85xx, MPC83xx DMA Engine support
  3  *
  4  * Copyright (C) 2007-2010 Freescale Semiconductor, Inc. All rights reserved.
  5  *
  6  * Author:
  7  *   Zhang Wei <wei.zhang@freescale.com>, Jul 2007
  8  *   Ebony Zhu <ebony.zhu@freescale.com>, May 2007
  9  *
 10  * Description:
 11  *   DMA engine driver for Freescale MPC8540 DMA controller, which is
 12  *   also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc.
 13  *   The support for MPC8349 DMA controller is also added.
 14  *
 15  * This driver instructs the DMA controller to issue the PCI Read Multiple
 16  * command for PCI read operations, instead of using the default PCI Read Line
 17  * command. Please be aware that this setting may result in read pre-fetching
 18  * on some platforms.
 19  *
 20  * This is free software; you can redistribute it and/or modify
 21  * it under the terms of the GNU General Public License as published by
 22  * the Free Software Foundation; either version 2 of the License, or
 23  * (at your option) any later version.
 24  *
 25  */
 26 
 27 #include <linux/init.h>
 28 #include <linux/module.h>
 29 #include <linux/pci.h>
 30 #include <linux/slab.h>
 31 #include <linux/interrupt.h>
 32 #include <linux/dmaengine.h>
 33 #include <linux/delay.h>
 34 #include <linux/dma-mapping.h>
 35 #include <linux/dmapool.h>
 36 #include <linux/of_address.h>
 37 #include <linux/of_irq.h>
 38 #include <linux/of_platform.h>
 39 
 40 #include "dmaengine.h"
 41 #include "fsldma.h"
 42 
 43 #define chan_dbg(chan, fmt, arg...)                                     \
 44         dev_dbg(chan->dev, "%s: " fmt, chan->name, ##arg)
 45 #define chan_err(chan, fmt, arg...)                                     \
 46         dev_err(chan->dev, "%s: " fmt, chan->name, ##arg)
 47 
 48 static const char msg_ld_oom[] = "No free memory for link descriptor";
 49 
 50 /*
 51  * Register Helpers
 52  */
 53 
 54 static void set_sr(struct fsldma_chan *chan, u32 val)
 55 {
 56         DMA_OUT(chan, &chan->regs->sr, val, 32);
 57 }
 58 
 59 static u32 get_sr(struct fsldma_chan *chan)
 60 {
 61         return DMA_IN(chan, &chan->regs->sr, 32);
 62 }
 63 
 64 static void set_mr(struct fsldma_chan *chan, u32 val)
 65 {
 66         DMA_OUT(chan, &chan->regs->mr, val, 32);
 67 }
 68 
 69 static u32 get_mr(struct fsldma_chan *chan)
 70 {
 71         return DMA_IN(chan, &chan->regs->mr, 32);
 72 }
 73 
 74 static void set_cdar(struct fsldma_chan *chan, dma_addr_t addr)
 75 {
 76         DMA_OUT(chan, &chan->regs->cdar, addr | FSL_DMA_SNEN, 64);
 77 }
 78 
 79 static dma_addr_t get_cdar(struct fsldma_chan *chan)
 80 {
 81         return DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN;
 82 }
 83 
 84 static void set_bcr(struct fsldma_chan *chan, u32 val)
 85 {
 86         DMA_OUT(chan, &chan->regs->bcr, val, 32);
 87 }
 88 
 89 static u32 get_bcr(struct fsldma_chan *chan)
 90 {
 91         return DMA_IN(chan, &chan->regs->bcr, 32);
 92 }
 93 
 94 /*
 95  * Descriptor Helpers
 96  */
 97 
 98 static void set_desc_cnt(struct fsldma_chan *chan,
 99                                 struct fsl_dma_ld_hw *hw, u32 count)
100 {
101         hw->count = CPU_TO_DMA(chan, count, 32);
102 }
103 
104 static void set_desc_src(struct fsldma_chan *chan,
105                          struct fsl_dma_ld_hw *hw, dma_addr_t src)
106 {
107         u64 snoop_bits;
108 
109         snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
110                 ? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0;
111         hw->src_addr = CPU_TO_DMA(chan, snoop_bits | src, 64);
112 }
113 
114 static void set_desc_dst(struct fsldma_chan *chan,
115                          struct fsl_dma_ld_hw *hw, dma_addr_t dst)
116 {
117         u64 snoop_bits;
118 
119         snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
120                 ? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0;
121         hw->dst_addr = CPU_TO_DMA(chan, snoop_bits | dst, 64);
122 }
123 
124 static void set_desc_next(struct fsldma_chan *chan,
125                           struct fsl_dma_ld_hw *hw, dma_addr_t next)
126 {
127         u64 snoop_bits;
128 
129         snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)
130                 ? FSL_DMA_SNEN : 0;
131         hw->next_ln_addr = CPU_TO_DMA(chan, snoop_bits | next, 64);
132 }
133 
134 static void set_ld_eol(struct fsldma_chan *chan, struct fsl_desc_sw *desc)
135 {
136         u64 snoop_bits;
137 
138         snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)
139                 ? FSL_DMA_SNEN : 0;
140 
141         desc->hw.next_ln_addr = CPU_TO_DMA(chan,
142                 DMA_TO_CPU(chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL
143                         | snoop_bits, 64);
144 }
145 
146 /*
147  * DMA Engine Hardware Control Helpers
148  */
149 
150 static void dma_init(struct fsldma_chan *chan)
151 {
152         /* Reset the channel */
153         set_mr(chan, 0);
154 
155         switch (chan->feature & FSL_DMA_IP_MASK) {
156         case FSL_DMA_IP_85XX:
157                 /* Set the channel to below modes:
158                  * EIE - Error interrupt enable
159                  * EOLNIE - End of links interrupt enable
160                  * BWC - Bandwidth sharing among channels
161                  */
162                 set_mr(chan, FSL_DMA_MR_BWC | FSL_DMA_MR_EIE
163                         | FSL_DMA_MR_EOLNIE);
164                 break;
165         case FSL_DMA_IP_83XX:
166                 /* Set the channel to below modes:
167                  * EOTIE - End-of-transfer interrupt enable
168                  * PRC_RM - PCI read multiple
169                  */
170                 set_mr(chan, FSL_DMA_MR_EOTIE | FSL_DMA_MR_PRC_RM);
171                 break;
172         }
173 }
174 
175 static int dma_is_idle(struct fsldma_chan *chan)
176 {
177         u32 sr = get_sr(chan);
178         return (!(sr & FSL_DMA_SR_CB)) || (sr & FSL_DMA_SR_CH);
179 }
180 
181 /*
182  * Start the DMA controller
183  *
184  * Preconditions:
185  * - the CDAR register must point to the start descriptor
186  * - the MRn[CS] bit must be cleared
187  */
188 static void dma_start(struct fsldma_chan *chan)
189 {
190         u32 mode;
191 
192         mode = get_mr(chan);
193 
194         if (chan->feature & FSL_DMA_CHAN_PAUSE_EXT) {
195                 set_bcr(chan, 0);
196                 mode |= FSL_DMA_MR_EMP_EN;
197         } else {
198                 mode &= ~FSL_DMA_MR_EMP_EN;
199         }
200 
201         if (chan->feature & FSL_DMA_CHAN_START_EXT) {
202                 mode |= FSL_DMA_MR_EMS_EN;
203         } else {
204                 mode &= ~FSL_DMA_MR_EMS_EN;
205                 mode |= FSL_DMA_MR_CS;
206         }
207 
208         set_mr(chan, mode);
209 }
210 
211 static void dma_halt(struct fsldma_chan *chan)
212 {
213         u32 mode;
214         int i;
215 
216         /* read the mode register */
217         mode = get_mr(chan);
218 
219         /*
220          * The 85xx controller supports channel abort, which will stop
221          * the current transfer. On 83xx, this bit is the transfer error
222          * mask bit, which should not be changed.
223          */
224         if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {
225                 mode |= FSL_DMA_MR_CA;
226                 set_mr(chan, mode);
227 
228                 mode &= ~FSL_DMA_MR_CA;
229         }
230 
231         /* stop the DMA controller */
232         mode &= ~(FSL_DMA_MR_CS | FSL_DMA_MR_EMS_EN);
233         set_mr(chan, mode);
234 
235         /* wait for the DMA controller to become idle */
236         for (i = 0; i < 100; i++) {
237                 if (dma_is_idle(chan))
238                         return;
239 
240                 udelay(10);
241         }
242 
243         if (!dma_is_idle(chan))
244                 chan_err(chan, "DMA halt timeout!\n");
245 }
246 
247 /**
248  * fsl_chan_set_src_loop_size - Set source address hold transfer size
249  * @chan : Freescale DMA channel
250  * @size     : Address loop size, 0 for disable loop
251  *
252  * The set source address hold transfer size. The source
253  * address hold or loop transfer size is when the DMA transfer
254  * data from source address (SA), if the loop size is 4, the DMA will
255  * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA,
256  * SA + 1 ... and so on.
257  */
258 static void fsl_chan_set_src_loop_size(struct fsldma_chan *chan, int size)
259 {
260         u32 mode;
261 
262         mode = get_mr(chan);
263 
264         switch (size) {
265         case 0:
266                 mode &= ~FSL_DMA_MR_SAHE;
267                 break;
268         case 1:
269         case 2:
270         case 4:
271         case 8:
272                 mode |= FSL_DMA_MR_SAHE | (__ilog2(size) << 14);
273                 break;
274         }
275 
276         set_mr(chan, mode);
277 }
278 
279 /**
280  * fsl_chan_set_dst_loop_size - Set destination address hold transfer size
281  * @chan : Freescale DMA channel
282  * @size     : Address loop size, 0 for disable loop
283  *
284  * The set destination address hold transfer size. The destination
285  * address hold or loop transfer size is when the DMA transfer
286  * data to destination address (TA), if the loop size is 4, the DMA will
287  * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA,
288  * TA + 1 ... and so on.
289  */
290 static void fsl_chan_set_dst_loop_size(struct fsldma_chan *chan, int size)
291 {
292         u32 mode;
293 
294         mode = get_mr(chan);
295 
296         switch (size) {
297         case 0:
298                 mode &= ~FSL_DMA_MR_DAHE;
299                 break;
300         case 1:
301         case 2:
302         case 4:
303         case 8:
304                 mode |= FSL_DMA_MR_DAHE | (__ilog2(size) << 16);
305                 break;
306         }
307 
308         set_mr(chan, mode);
309 }
310 
311 /**
312  * fsl_chan_set_request_count - Set DMA Request Count for external control
313  * @chan : Freescale DMA channel
314  * @size     : Number of bytes to transfer in a single request
315  *
316  * The Freescale DMA channel can be controlled by the external signal DREQ#.
317  * The DMA request count is how many bytes are allowed to transfer before
318  * pausing the channel, after which a new assertion of DREQ# resumes channel
319  * operation.
320  *
321  * A size of 0 disables external pause control. The maximum size is 1024.
322  */
323 static void fsl_chan_set_request_count(struct fsldma_chan *chan, int size)
324 {
325         u32 mode;
326 
327         BUG_ON(size > 1024);
328 
329         mode = get_mr(chan);
330         mode |= (__ilog2(size) << 24) & 0x0f000000;
331 
332         set_mr(chan, mode);
333 }
334 
335 /**
336  * fsl_chan_toggle_ext_pause - Toggle channel external pause status
337  * @chan : Freescale DMA channel
338  * @enable   : 0 is disabled, 1 is enabled.
339  *
340  * The Freescale DMA channel can be controlled by the external signal DREQ#.
341  * The DMA Request Count feature should be used in addition to this feature
342  * to set the number of bytes to transfer before pausing the channel.
343  */
344 static void fsl_chan_toggle_ext_pause(struct fsldma_chan *chan, int enable)
345 {
346         if (enable)
347                 chan->feature |= FSL_DMA_CHAN_PAUSE_EXT;
348         else
349                 chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT;
350 }
351 
352 /**
353  * fsl_chan_toggle_ext_start - Toggle channel external start status
354  * @chan : Freescale DMA channel
355  * @enable   : 0 is disabled, 1 is enabled.
356  *
357  * If enable the external start, the channel can be started by an
358  * external DMA start pin. So the dma_start() does not start the
359  * transfer immediately. The DMA channel will wait for the
360  * control pin asserted.
361  */
362 static void fsl_chan_toggle_ext_start(struct fsldma_chan *chan, int enable)
363 {
364         if (enable)
365                 chan->feature |= FSL_DMA_CHAN_START_EXT;
366         else
367                 chan->feature &= ~FSL_DMA_CHAN_START_EXT;
368 }
369 
370 static void append_ld_queue(struct fsldma_chan *chan, struct fsl_desc_sw *desc)
371 {
372         struct fsl_desc_sw *tail = to_fsl_desc(chan->ld_pending.prev);
373 
374         if (list_empty(&chan->ld_pending))
375                 goto out_splice;
376 
377         /*
378          * Add the hardware descriptor to the chain of hardware descriptors
379          * that already exists in memory.
380          *
381          * This will un-set the EOL bit of the existing transaction, and the
382          * last link in this transaction will become the EOL descriptor.
383          */
384         set_desc_next(chan, &tail->hw, desc->async_tx.phys);
385 
386         /*
387          * Add the software descriptor and all children to the list
388          * of pending transactions
389          */
390 out_splice:
391         list_splice_tail_init(&desc->tx_list, &chan->ld_pending);
392 }
393 
394 static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
395 {
396         struct fsldma_chan *chan = to_fsl_chan(tx->chan);
397         struct fsl_desc_sw *desc = tx_to_fsl_desc(tx);
398         struct fsl_desc_sw *child;
399         dma_cookie_t cookie = -EINVAL;
400 
401         spin_lock_bh(&chan->desc_lock);
402 
403 #ifdef CONFIG_PM
404         if (unlikely(chan->pm_state != RUNNING)) {
405                 chan_dbg(chan, "cannot submit due to suspend\n");
406                 spin_unlock_bh(&chan->desc_lock);
407                 return -1;
408         }
409 #endif
410 
411         /*
412          * assign cookies to all of the software descriptors
413          * that make up this transaction
414          */
415         list_for_each_entry(child, &desc->tx_list, node) {
416                 cookie = dma_cookie_assign(&child->async_tx);
417         }
418 
419         /* put this transaction onto the tail of the pending queue */
420         append_ld_queue(chan, desc);
421 
422         spin_unlock_bh(&chan->desc_lock);
423 
424         return cookie;
425 }
426 
427 /**
428  * fsl_dma_free_descriptor - Free descriptor from channel's DMA pool.
429  * @chan : Freescale DMA channel
430  * @desc: descriptor to be freed
431  */
432 static void fsl_dma_free_descriptor(struct fsldma_chan *chan,
433                 struct fsl_desc_sw *desc)
434 {
435         list_del(&desc->node);
436         chan_dbg(chan, "LD %p free\n", desc);
437         dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
438 }
439 
440 /**
441  * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool.
442  * @chan : Freescale DMA channel
443  *
444  * Return - The descriptor allocated. NULL for failed.
445  */
446 static struct fsl_desc_sw *fsl_dma_alloc_descriptor(struct fsldma_chan *chan)
447 {
448         struct fsl_desc_sw *desc;
449         dma_addr_t pdesc;
450 
451         desc = dma_pool_alloc(chan->desc_pool, GFP_ATOMIC, &pdesc);
452         if (!desc) {
453                 chan_dbg(chan, "out of memory for link descriptor\n");
454                 return NULL;
455         }
456 
457         memset(desc, 0, sizeof(*desc));
458         INIT_LIST_HEAD(&desc->tx_list);
459         dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);
460         desc->async_tx.tx_submit = fsl_dma_tx_submit;
461         desc->async_tx.phys = pdesc;
462 
463         chan_dbg(chan, "LD %p allocated\n", desc);
464 
465         return desc;
466 }
467 
468 /**
469  * fsldma_clean_completed_descriptor - free all descriptors which
470  * has been completed and acked
471  * @chan: Freescale DMA channel
472  *
473  * This function is used on all completed and acked descriptors.
474  * All descriptors should only be freed in this function.
475  */
476 static void fsldma_clean_completed_descriptor(struct fsldma_chan *chan)
477 {
478         struct fsl_desc_sw *desc, *_desc;
479 
480         /* Run the callback for each descriptor, in order */
481         list_for_each_entry_safe(desc, _desc, &chan->ld_completed, node)
482                 if (async_tx_test_ack(&desc->async_tx))
483                         fsl_dma_free_descriptor(chan, desc);
484 }
485 
486 /**
487  * fsldma_run_tx_complete_actions - cleanup a single link descriptor
488  * @chan: Freescale DMA channel
489  * @desc: descriptor to cleanup and free
490  * @cookie: Freescale DMA transaction identifier
491  *
492  * This function is used on a descriptor which has been executed by the DMA
493  * controller. It will run any callbacks, submit any dependencies.
494  */
495 static dma_cookie_t fsldma_run_tx_complete_actions(struct fsldma_chan *chan,
496                 struct fsl_desc_sw *desc, dma_cookie_t cookie)
497 {
498         struct dma_async_tx_descriptor *txd = &desc->async_tx;
499         dma_cookie_t ret = cookie;
500 
501         BUG_ON(txd->cookie < 0);
502 
503         if (txd->cookie > 0) {
504                 ret = txd->cookie;
505 
506                 /* Run the link descriptor callback function */
507                 if (txd->callback) {
508                         chan_dbg(chan, "LD %p callback\n", desc);
509                         txd->callback(txd->callback_param);
510                 }
511         }
512 
513         /* Run any dependencies */
514         dma_run_dependencies(txd);
515 
516         return ret;
517 }
518 
519 /**
520  * fsldma_clean_running_descriptor - move the completed descriptor from
521  * ld_running to ld_completed
522  * @chan: Freescale DMA channel
523  * @desc: the descriptor which is completed
524  *
525  * Free the descriptor directly if acked by async_tx api, or move it to
526  * queue ld_completed.
527  */
528 static void fsldma_clean_running_descriptor(struct fsldma_chan *chan,
529                 struct fsl_desc_sw *desc)
530 {
531         /* Remove from the list of transactions */
532         list_del(&desc->node);
533 
534         /*
535          * the client is allowed to attach dependent operations
536          * until 'ack' is set
537          */
538         if (!async_tx_test_ack(&desc->async_tx)) {
539                 /*
540                  * Move this descriptor to the list of descriptors which is
541                  * completed, but still awaiting the 'ack' bit to be set.
542                  */
543                 list_add_tail(&desc->node, &chan->ld_completed);
544                 return;
545         }
546 
547         dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
548 }
549 
550 /**
551  * fsl_chan_xfer_ld_queue - transfer any pending transactions
552  * @chan : Freescale DMA channel
553  *
554  * HARDWARE STATE: idle
555  * LOCKING: must hold chan->desc_lock
556  */
557 static void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan)
558 {
559         struct fsl_desc_sw *desc;
560 
561         /*
562          * If the list of pending descriptors is empty, then we
563          * don't need to do any work at all
564          */
565         if (list_empty(&chan->ld_pending)) {
566                 chan_dbg(chan, "no pending LDs\n");
567                 return;
568         }
569 
570         /*
571          * The DMA controller is not idle, which means that the interrupt
572          * handler will start any queued transactions when it runs after
573          * this transaction finishes
574          */
575         if (!chan->idle) {
576                 chan_dbg(chan, "DMA controller still busy\n");
577                 return;
578         }
579 
580         /*
581          * If there are some link descriptors which have not been
582          * transferred, we need to start the controller
583          */
584 
585         /*
586          * Move all elements from the queue of pending transactions
587          * onto the list of running transactions
588          */
589         chan_dbg(chan, "idle, starting controller\n");
590         desc = list_first_entry(&chan->ld_pending, struct fsl_desc_sw, node);
591         list_splice_tail_init(&chan->ld_pending, &chan->ld_running);
592 
593         /*
594          * The 85xx DMA controller doesn't clear the channel start bit
595          * automatically at the end of a transfer. Therefore we must clear
596          * it in software before starting the transfer.
597          */
598         if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) {
599                 u32 mode;
600 
601                 mode = get_mr(chan);
602                 mode &= ~FSL_DMA_MR_CS;
603                 set_mr(chan, mode);
604         }
605 
606         /*
607          * Program the descriptor's address into the DMA controller,
608          * then start the DMA transaction
609          */
610         set_cdar(chan, desc->async_tx.phys);
611         get_cdar(chan);
612 
613         dma_start(chan);
614         chan->idle = false;
615 }
616 
617 /**
618  * fsldma_cleanup_descriptors - cleanup link descriptors which are completed
619  * and move them to ld_completed to free until flag 'ack' is set
620  * @chan: Freescale DMA channel
621  *
622  * This function is used on descriptors which have been executed by the DMA
623  * controller. It will run any callbacks, submit any dependencies, then
624  * free these descriptors if flag 'ack' is set.
625  */
626 static void fsldma_cleanup_descriptors(struct fsldma_chan *chan)
627 {
628         struct fsl_desc_sw *desc, *_desc;
629         dma_cookie_t cookie = 0;
630         dma_addr_t curr_phys = get_cdar(chan);
631         int seen_current = 0;
632 
633         fsldma_clean_completed_descriptor(chan);
634 
635         /* Run the callback for each descriptor, in order */
636         list_for_each_entry_safe(desc, _desc, &chan->ld_running, node) {
637                 /*
638                  * do not advance past the current descriptor loaded into the
639                  * hardware channel, subsequent descriptors are either in
640                  * process or have not been submitted
641                  */
642                 if (seen_current)
643                         break;
644 
645                 /*
646                  * stop the search if we reach the current descriptor and the
647                  * channel is busy
648                  */
649                 if (desc->async_tx.phys == curr_phys) {
650                         seen_current = 1;
651                         if (!dma_is_idle(chan))
652                                 break;
653                 }
654 
655                 cookie = fsldma_run_tx_complete_actions(chan, desc, cookie);
656 
657                 fsldma_clean_running_descriptor(chan, desc);
658         }
659 
660         /*
661          * Start any pending transactions automatically
662          *
663          * In the ideal case, we keep the DMA controller busy while we go
664          * ahead and free the descriptors below.
665          */
666         fsl_chan_xfer_ld_queue(chan);
667 
668         if (cookie > 0)
669                 chan->common.completed_cookie = cookie;
670 }
671 
672 /**
673  * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel.
674  * @chan : Freescale DMA channel
675  *
676  * This function will create a dma pool for descriptor allocation.
677  *
678  * Return - The number of descriptors allocated.
679  */
680 static int fsl_dma_alloc_chan_resources(struct dma_chan *dchan)
681 {
682         struct fsldma_chan *chan = to_fsl_chan(dchan);
683 
684         /* Has this channel already been allocated? */
685         if (chan->desc_pool)
686                 return 1;
687 
688         /*
689          * We need the descriptor to be aligned to 32bytes
690          * for meeting FSL DMA specification requirement.
691          */
692         chan->desc_pool = dma_pool_create(chan->name, chan->dev,
693                                           sizeof(struct fsl_desc_sw),
694                                           __alignof__(struct fsl_desc_sw), 0);
695         if (!chan->desc_pool) {
696                 chan_err(chan, "unable to allocate descriptor pool\n");
697                 return -ENOMEM;
698         }
699 
700         /* there is at least one descriptor free to be allocated */
701         return 1;
702 }
703 
704 /**
705  * fsldma_free_desc_list - Free all descriptors in a queue
706  * @chan: Freescae DMA channel
707  * @list: the list to free
708  *
709  * LOCKING: must hold chan->desc_lock
710  */
711 static void fsldma_free_desc_list(struct fsldma_chan *chan,
712                                   struct list_head *list)
713 {
714         struct fsl_desc_sw *desc, *_desc;
715 
716         list_for_each_entry_safe(desc, _desc, list, node)
717                 fsl_dma_free_descriptor(chan, desc);
718 }
719 
720 static void fsldma_free_desc_list_reverse(struct fsldma_chan *chan,
721                                           struct list_head *list)
722 {
723         struct fsl_desc_sw *desc, *_desc;
724 
725         list_for_each_entry_safe_reverse(desc, _desc, list, node)
726                 fsl_dma_free_descriptor(chan, desc);
727 }
728 
729 /**
730  * fsl_dma_free_chan_resources - Free all resources of the channel.
731  * @chan : Freescale DMA channel
732  */
733 static void fsl_dma_free_chan_resources(struct dma_chan *dchan)
734 {
735         struct fsldma_chan *chan = to_fsl_chan(dchan);
736 
737         chan_dbg(chan, "free all channel resources\n");
738         spin_lock_bh(&chan->desc_lock);
739         fsldma_cleanup_descriptors(chan);
740         fsldma_free_desc_list(chan, &chan->ld_pending);
741         fsldma_free_desc_list(chan, &chan->ld_running);
742         fsldma_free_desc_list(chan, &chan->ld_completed);
743         spin_unlock_bh(&chan->desc_lock);
744 
745         dma_pool_destroy(chan->desc_pool);
746         chan->desc_pool = NULL;
747 }
748 
749 static struct dma_async_tx_descriptor *
750 fsl_dma_prep_memcpy(struct dma_chan *dchan,
751         dma_addr_t dma_dst, dma_addr_t dma_src,
752         size_t len, unsigned long flags)
753 {
754         struct fsldma_chan *chan;
755         struct fsl_desc_sw *first = NULL, *prev = NULL, *new;
756         size_t copy;
757 
758         if (!dchan)
759                 return NULL;
760 
761         if (!len)
762                 return NULL;
763 
764         chan = to_fsl_chan(dchan);
765 
766         do {
767 
768                 /* Allocate the link descriptor from DMA pool */
769                 new = fsl_dma_alloc_descriptor(chan);
770                 if (!new) {
771                         chan_err(chan, "%s\n", msg_ld_oom);
772                         goto fail;
773                 }
774 
775                 copy = min(len, (size_t)FSL_DMA_BCR_MAX_CNT);
776 
777                 set_desc_cnt(chan, &new->hw, copy);
778                 set_desc_src(chan, &new->hw, dma_src);
779                 set_desc_dst(chan, &new->hw, dma_dst);
780 
781                 if (!first)
782                         first = new;
783                 else
784                         set_desc_next(chan, &prev->hw, new->async_tx.phys);
785 
786                 new->async_tx.cookie = 0;
787                 async_tx_ack(&new->async_tx);
788 
789                 prev = new;
790                 len -= copy;
791                 dma_src += copy;
792                 dma_dst += copy;
793 
794                 /* Insert the link descriptor to the LD ring */
795                 list_add_tail(&new->node, &first->tx_list);
796         } while (len);
797 
798         new->async_tx.flags = flags; /* client is in control of this ack */
799         new->async_tx.cookie = -EBUSY;
800 
801         /* Set End-of-link to the last link descriptor of new list */
802         set_ld_eol(chan, new);
803 
804         return &first->async_tx;
805 
806 fail:
807         if (!first)
808                 return NULL;
809 
810         fsldma_free_desc_list_reverse(chan, &first->tx_list);
811         return NULL;
812 }
813 
814 static struct dma_async_tx_descriptor *fsl_dma_prep_sg(struct dma_chan *dchan,
815         struct scatterlist *dst_sg, unsigned int dst_nents,
816         struct scatterlist *src_sg, unsigned int src_nents,
817         unsigned long flags)
818 {
819         struct fsl_desc_sw *first = NULL, *prev = NULL, *new = NULL;
820         struct fsldma_chan *chan = to_fsl_chan(dchan);
821         size_t dst_avail, src_avail;
822         dma_addr_t dst, src;
823         size_t len;
824 
825         /* basic sanity checks */
826         if (dst_nents == 0 || src_nents == 0)
827                 return NULL;
828 
829         if (dst_sg == NULL || src_sg == NULL)
830                 return NULL;
831 
832         /*
833          * TODO: should we check that both scatterlists have the same
834          * TODO: number of bytes in total? Is that really an error?
835          */
836 
837         /* get prepared for the loop */
838         dst_avail = sg_dma_len(dst_sg);
839         src_avail = sg_dma_len(src_sg);
840 
841         /* run until we are out of scatterlist entries */
842         while (true) {
843 
844                 /* create the largest transaction possible */
845                 len = min_t(size_t, src_avail, dst_avail);
846                 len = min_t(size_t, len, FSL_DMA_BCR_MAX_CNT);
847                 if (len == 0)
848                         goto fetch;
849 
850                 dst = sg_dma_address(dst_sg) + sg_dma_len(dst_sg) - dst_avail;
851                 src = sg_dma_address(src_sg) + sg_dma_len(src_sg) - src_avail;
852 
853                 /* allocate and populate the descriptor */
854                 new = fsl_dma_alloc_descriptor(chan);
855                 if (!new) {
856                         chan_err(chan, "%s\n", msg_ld_oom);
857                         goto fail;
858                 }
859 
860                 set_desc_cnt(chan, &new->hw, len);
861                 set_desc_src(chan, &new->hw, src);
862                 set_desc_dst(chan, &new->hw, dst);
863 
864                 if (!first)
865                         first = new;
866                 else
867                         set_desc_next(chan, &prev->hw, new->async_tx.phys);
868 
869                 new->async_tx.cookie = 0;
870                 async_tx_ack(&new->async_tx);
871                 prev = new;
872 
873                 /* Insert the link descriptor to the LD ring */
874                 list_add_tail(&new->node, &first->tx_list);
875 
876                 /* update metadata */
877                 dst_avail -= len;
878                 src_avail -= len;
879 
880 fetch:
881                 /* fetch the next dst scatterlist entry */
882                 if (dst_avail == 0) {
883 
884                         /* no more entries: we're done */
885                         if (dst_nents == 0)
886                                 break;
887 
888                         /* fetch the next entry: if there are no more: done */
889                         dst_sg = sg_next(dst_sg);
890                         if (dst_sg == NULL)
891                                 break;
892 
893                         dst_nents--;
894                         dst_avail = sg_dma_len(dst_sg);
895                 }
896 
897                 /* fetch the next src scatterlist entry */
898                 if (src_avail == 0) {
899 
900                         /* no more entries: we're done */
901                         if (src_nents == 0)
902                                 break;
903 
904                         /* fetch the next entry: if there are no more: done */
905                         src_sg = sg_next(src_sg);
906                         if (src_sg == NULL)
907                                 break;
908 
909                         src_nents--;
910                         src_avail = sg_dma_len(src_sg);
911                 }
912         }
913 
914         new->async_tx.flags = flags; /* client is in control of this ack */
915         new->async_tx.cookie = -EBUSY;
916 
917         /* Set End-of-link to the last link descriptor of new list */
918         set_ld_eol(chan, new);
919 
920         return &first->async_tx;
921 
922 fail:
923         if (!first)
924                 return NULL;
925 
926         fsldma_free_desc_list_reverse(chan, &first->tx_list);
927         return NULL;
928 }
929 
930 /**
931  * fsl_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
932  * @chan: DMA channel
933  * @sgl: scatterlist to transfer to/from
934  * @sg_len: number of entries in @scatterlist
935  * @direction: DMA direction
936  * @flags: DMAEngine flags
937  * @context: transaction context (ignored)
938  *
939  * Prepare a set of descriptors for a DMA_SLAVE transaction. Following the
940  * DMA_SLAVE API, this gets the device-specific information from the
941  * chan->private variable.
942  */
943 static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg(
944         struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len,
945         enum dma_transfer_direction direction, unsigned long flags,
946         void *context)
947 {
948         /*
949          * This operation is not supported on the Freescale DMA controller
950          *
951          * However, we need to provide the function pointer to allow the
952          * device_control() method to work.
953          */
954         return NULL;
955 }
956 
957 static int fsl_dma_device_control(struct dma_chan *dchan,
958                                   enum dma_ctrl_cmd cmd, unsigned long arg)
959 {
960         struct dma_slave_config *config;
961         struct fsldma_chan *chan;
962         int size;
963 
964         if (!dchan)
965                 return -EINVAL;
966 
967         chan = to_fsl_chan(dchan);
968 
969         switch (cmd) {
970         case DMA_TERMINATE_ALL:
971                 spin_lock_bh(&chan->desc_lock);
972 
973                 /* Halt the DMA engine */
974                 dma_halt(chan);
975 
976                 /* Remove and free all of the descriptors in the LD queue */
977                 fsldma_free_desc_list(chan, &chan->ld_pending);
978                 fsldma_free_desc_list(chan, &chan->ld_running);
979                 fsldma_free_desc_list(chan, &chan->ld_completed);
980                 chan->idle = true;
981 
982                 spin_unlock_bh(&chan->desc_lock);
983                 return 0;
984 
985         case DMA_SLAVE_CONFIG:
986                 config = (struct dma_slave_config *)arg;
987 
988                 /* make sure the channel supports setting burst size */
989                 if (!chan->set_request_count)
990                         return -ENXIO;
991 
992                 /* we set the controller burst size depending on direction */
993                 if (config->direction == DMA_MEM_TO_DEV)
994                         size = config->dst_addr_width * config->dst_maxburst;
995                 else
996                         size = config->src_addr_width * config->src_maxburst;
997 
998                 chan->set_request_count(chan, size);
999                 return 0;
1000 
1001         case FSLDMA_EXTERNAL_START:
1002 
1003                 /* make sure the channel supports external start */
1004                 if (!chan->toggle_ext_start)
1005                         return -ENXIO;
1006 
1007                 chan->toggle_ext_start(chan, arg);
1008                 return 0;
1009 
1010         default:
1011                 return -ENXIO;
1012         }
1013 
1014         return 0;
1015 }
1016 
1017 /**
1018  * fsl_dma_memcpy_issue_pending - Issue the DMA start command
1019  * @chan : Freescale DMA channel
1020  */
1021 static void fsl_dma_memcpy_issue_pending(struct dma_chan *dchan)
1022 {
1023         struct fsldma_chan *chan = to_fsl_chan(dchan);
1024 
1025         spin_lock_bh(&chan->desc_lock);
1026         fsl_chan_xfer_ld_queue(chan);
1027         spin_unlock_bh(&chan->desc_lock);
1028 }
1029 
1030 /**
1031  * fsl_tx_status - Determine the DMA status
1032  * @chan : Freescale DMA channel
1033  */
1034 static enum dma_status fsl_tx_status(struct dma_chan *dchan,
1035                                         dma_cookie_t cookie,
1036                                         struct dma_tx_state *txstate)
1037 {
1038         struct fsldma_chan *chan = to_fsl_chan(dchan);
1039         enum dma_status ret;
1040 
1041         ret = dma_cookie_status(dchan, cookie, txstate);
1042         if (ret == DMA_COMPLETE)
1043                 return ret;
1044 
1045         spin_lock_bh(&chan->desc_lock);
1046         fsldma_cleanup_descriptors(chan);
1047         spin_unlock_bh(&chan->desc_lock);
1048 
1049         return dma_cookie_status(dchan, cookie, txstate);
1050 }
1051 
1052 /*----------------------------------------------------------------------------*/
1053 /* Interrupt Handling                                                         */
1054 /*----------------------------------------------------------------------------*/
1055 
1056 static irqreturn_t fsldma_chan_irq(int irq, void *data)
1057 {
1058         struct fsldma_chan *chan = data;
1059         u32 stat;
1060 
1061         /* save and clear the status register */
1062         stat = get_sr(chan);
1063         set_sr(chan, stat);
1064         chan_dbg(chan, "irq: stat = 0x%x\n", stat);
1065 
1066         /* check that this was really our device */
1067         stat &= ~(FSL_DMA_SR_CB | FSL_DMA_SR_CH);
1068         if (!stat)
1069                 return IRQ_NONE;
1070 
1071         if (stat & FSL_DMA_SR_TE)
1072                 chan_err(chan, "Transfer Error!\n");
1073 
1074         /*
1075          * Programming Error
1076          * The DMA_INTERRUPT async_tx is a NULL transfer, which will
1077          * trigger a PE interrupt.
1078          */
1079         if (stat & FSL_DMA_SR_PE) {
1080                 chan_dbg(chan, "irq: Programming Error INT\n");
1081                 stat &= ~FSL_DMA_SR_PE;
1082                 if (get_bcr(chan) != 0)
1083                         chan_err(chan, "Programming Error!\n");
1084         }
1085 
1086         /*
1087          * For MPC8349, EOCDI event need to update cookie
1088          * and start the next transfer if it exist.
1089          */
1090         if (stat & FSL_DMA_SR_EOCDI) {
1091                 chan_dbg(chan, "irq: End-of-Chain link INT\n");
1092                 stat &= ~FSL_DMA_SR_EOCDI;
1093         }
1094 
1095         /*
1096          * If it current transfer is the end-of-transfer,
1097          * we should clear the Channel Start bit for
1098          * prepare next transfer.
1099          */
1100         if (stat & FSL_DMA_SR_EOLNI) {
1101                 chan_dbg(chan, "irq: End-of-link INT\n");
1102                 stat &= ~FSL_DMA_SR_EOLNI;
1103         }
1104 
1105         /* check that the DMA controller is really idle */
1106         if (!dma_is_idle(chan))
1107                 chan_err(chan, "irq: controller not idle!\n");
1108 
1109         /* check that we handled all of the bits */
1110         if (stat)
1111                 chan_err(chan, "irq: unhandled sr 0x%08x\n", stat);
1112 
1113         /*
1114          * Schedule the tasklet to handle all cleanup of the current
1115          * transaction. It will start a new transaction if there is
1116          * one pending.
1117          */
1118         tasklet_schedule(&chan->tasklet);
1119         chan_dbg(chan, "irq: Exit\n");
1120         return IRQ_HANDLED;
1121 }
1122 
1123 static void dma_do_tasklet(unsigned long data)
1124 {
1125         struct fsldma_chan *chan = (struct fsldma_chan *)data;
1126 
1127         chan_dbg(chan, "tasklet entry\n");
1128 
1129         spin_lock_bh(&chan->desc_lock);
1130 
1131         /* the hardware is now idle and ready for more */
1132         chan->idle = true;
1133 
1134         /* Run all cleanup for descriptors which have been completed */
1135         fsldma_cleanup_descriptors(chan);
1136 
1137         spin_unlock_bh(&chan->desc_lock);
1138 
1139         chan_dbg(chan, "tasklet exit\n");
1140 }
1141 
1142 static irqreturn_t fsldma_ctrl_irq(int irq, void *data)
1143 {
1144         struct fsldma_device *fdev = data;
1145         struct fsldma_chan *chan;
1146         unsigned int handled = 0;
1147         u32 gsr, mask;
1148         int i;
1149 
1150         gsr = (fdev->feature & FSL_DMA_BIG_ENDIAN) ? in_be32(fdev->regs)
1151                                                    : in_le32(fdev->regs);
1152         mask = 0xff000000;
1153         dev_dbg(fdev->dev, "IRQ: gsr 0x%.8x\n", gsr);
1154 
1155         for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
1156                 chan = fdev->chan[i];
1157                 if (!chan)
1158                         continue;
1159 
1160                 if (gsr & mask) {
1161                         dev_dbg(fdev->dev, "IRQ: chan %d\n", chan->id);
1162                         fsldma_chan_irq(irq, chan);
1163                         handled++;
1164                 }
1165 
1166                 gsr &= ~mask;
1167                 mask >>= 8;
1168         }
1169 
1170         return IRQ_RETVAL(handled);
1171 }
1172 
1173 static void fsldma_free_irqs(struct fsldma_device *fdev)
1174 {
1175         struct fsldma_chan *chan;
1176         int i;
1177 
1178         if (fdev->irq != NO_IRQ) {
1179                 dev_dbg(fdev->dev, "free per-controller IRQ\n");
1180                 free_irq(fdev->irq, fdev);
1181                 return;
1182         }
1183 
1184         for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
1185                 chan = fdev->chan[i];
1186                 if (chan && chan->irq != NO_IRQ) {
1187                         chan_dbg(chan, "free per-channel IRQ\n");
1188                         free_irq(chan->irq, chan);
1189                 }
1190         }
1191 }
1192 
1193 static int fsldma_request_irqs(struct fsldma_device *fdev)
1194 {
1195         struct fsldma_chan *chan;
1196         int ret;
1197         int i;
1198 
1199         /* if we have a per-controller IRQ, use that */
1200         if (fdev->irq != NO_IRQ) {
1201                 dev_dbg(fdev->dev, "request per-controller IRQ\n");
1202                 ret = request_irq(fdev->irq, fsldma_ctrl_irq, IRQF_SHARED,
1203                                   "fsldma-controller", fdev);
1204                 return ret;
1205         }
1206 
1207         /* no per-controller IRQ, use the per-channel IRQs */
1208         for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
1209                 chan = fdev->chan[i];
1210                 if (!chan)
1211                         continue;
1212 
1213                 if (chan->irq == NO_IRQ) {
1214                         chan_err(chan, "interrupts property missing in device tree\n");
1215                         ret = -ENODEV;
1216                         goto out_unwind;
1217                 }
1218 
1219                 chan_dbg(chan, "request per-channel IRQ\n");
1220                 ret = request_irq(chan->irq, fsldma_chan_irq, IRQF_SHARED,
1221                                   "fsldma-chan", chan);
1222                 if (ret) {
1223                         chan_err(chan, "unable to request per-channel IRQ\n");
1224                         goto out_unwind;
1225                 }
1226         }
1227 
1228         return 0;
1229 
1230 out_unwind:
1231         for (/* none */; i >= 0; i--) {
1232                 chan = fdev->chan[i];
1233                 if (!chan)
1234                         continue;
1235 
1236                 if (chan->irq == NO_IRQ)
1237                         continue;
1238 
1239                 free_irq(chan->irq, chan);
1240         }
1241 
1242         return ret;
1243 }
1244 
1245 /*----------------------------------------------------------------------------*/
1246 /* OpenFirmware Subsystem                                                     */
1247 /*----------------------------------------------------------------------------*/
1248 
1249 static int fsl_dma_chan_probe(struct fsldma_device *fdev,
1250         struct device_node *node, u32 feature, const char *compatible)
1251 {
1252         struct fsldma_chan *chan;
1253         struct resource res;
1254         int err;
1255 
1256         /* alloc channel */
1257         chan = kzalloc(sizeof(*chan), GFP_KERNEL);
1258         if (!chan) {
1259                 dev_err(fdev->dev, "no free memory for DMA channels!\n");
1260                 err = -ENOMEM;
1261                 goto out_return;
1262         }
1263 
1264         /* ioremap registers for use */
1265         chan->regs = of_iomap(node, 0);
1266         if (!chan->regs) {
1267                 dev_err(fdev->dev, "unable to ioremap registers\n");
1268                 err = -ENOMEM;
1269                 goto out_free_chan;
1270         }
1271 
1272         err = of_address_to_resource(node, 0, &res);
1273         if (err) {
1274                 dev_err(fdev->dev, "unable to find 'reg' property\n");
1275                 goto out_iounmap_regs;
1276         }
1277 
1278         chan->feature = feature;
1279         if (!fdev->feature)
1280                 fdev->feature = chan->feature;
1281 
1282         /*
1283          * If the DMA device's feature is different than the feature
1284          * of its channels, report the bug
1285          */
1286         WARN_ON(fdev->feature != chan->feature);
1287 
1288         chan->dev = fdev->dev;
1289         chan->id = (res.start & 0xfff) < 0x300 ?
1290                    ((res.start - 0x100) & 0xfff) >> 7 :
1291                    ((res.start - 0x200) & 0xfff) >> 7;
1292         if (chan->id >= FSL_DMA_MAX_CHANS_PER_DEVICE) {
1293                 dev_err(fdev->dev, "too many channels for device\n");
1294                 err = -EINVAL;
1295                 goto out_iounmap_regs;
1296         }
1297 
1298         fdev->chan[chan->id] = chan;
1299         tasklet_init(&chan->tasklet, dma_do_tasklet, (unsigned long)chan);
1300         snprintf(chan->name, sizeof(chan->name), "chan%d", chan->id);
1301 
1302         /* Initialize the channel */
1303         dma_init(chan);
1304 
1305         /* Clear cdar registers */
1306         set_cdar(chan, 0);
1307 
1308         switch (chan->feature & FSL_DMA_IP_MASK) {
1309         case FSL_DMA_IP_85XX:
1310                 chan->toggle_ext_pause = fsl_chan_toggle_ext_pause;
1311         case FSL_DMA_IP_83XX:
1312                 chan->toggle_ext_start = fsl_chan_toggle_ext_start;
1313                 chan->set_src_loop_size = fsl_chan_set_src_loop_size;
1314                 chan->set_dst_loop_size = fsl_chan_set_dst_loop_size;
1315                 chan->set_request_count = fsl_chan_set_request_count;
1316         }
1317 
1318         spin_lock_init(&chan->desc_lock);
1319         INIT_LIST_HEAD(&chan->ld_pending);
1320         INIT_LIST_HEAD(&chan->ld_running);
1321         INIT_LIST_HEAD(&chan->ld_completed);
1322         chan->idle = true;
1323 #ifdef CONFIG_PM
1324         chan->pm_state = RUNNING;
1325 #endif
1326 
1327         chan->common.device = &fdev->common;
1328         dma_cookie_init(&chan->common);
1329 
1330         /* find the IRQ line, if it exists in the device tree */
1331         chan->irq = irq_of_parse_and_map(node, 0);
1332 
1333         /* Add the channel to DMA device channel list */
1334         list_add_tail(&chan->common.device_node, &fdev->common.channels);
1335         fdev->common.chancnt++;
1336 
1337         dev_info(fdev->dev, "#%d (%s), irq %d\n", chan->id, compatible,
1338                  chan->irq != NO_IRQ ? chan->irq : fdev->irq);
1339 
1340         return 0;
1341 
1342 out_iounmap_regs:
1343         iounmap(chan->regs);
1344 out_free_chan:
1345         kfree(chan);
1346 out_return:
1347         return err;
1348 }
1349 
1350 static void fsl_dma_chan_remove(struct fsldma_chan *chan)
1351 {
1352         irq_dispose_mapping(chan->irq);
1353         list_del(&chan->common.device_node);
1354         iounmap(chan->regs);
1355         kfree(chan);
1356 }
1357 
1358 static int fsldma_of_probe(struct platform_device *op)
1359 {
1360         struct fsldma_device *fdev;
1361         struct device_node *child;
1362         int err;
1363 
1364         fdev = kzalloc(sizeof(*fdev), GFP_KERNEL);
1365         if (!fdev) {
1366                 dev_err(&op->dev, "No enough memory for 'priv'\n");
1367                 err = -ENOMEM;
1368                 goto out_return;
1369         }
1370 
1371         fdev->dev = &op->dev;
1372         INIT_LIST_HEAD(&fdev->common.channels);
1373 
1374         /* ioremap the registers for use */
1375         fdev->regs = of_iomap(op->dev.of_node, 0);
1376         if (!fdev->regs) {
1377                 dev_err(&op->dev, "unable to ioremap registers\n");
1378                 err = -ENOMEM;
1379                 goto out_free_fdev;
1380         }
1381 
1382         /* map the channel IRQ if it exists, but don't hookup the handler yet */
1383         fdev->irq = irq_of_parse_and_map(op->dev.of_node, 0);
1384 
1385         dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);
1386         dma_cap_set(DMA_SG, fdev->common.cap_mask);
1387         dma_cap_set(DMA_SLAVE, fdev->common.cap_mask);
1388         fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources;
1389         fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources;
1390         fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy;
1391         fdev->common.device_prep_dma_sg = fsl_dma_prep_sg;
1392         fdev->common.device_tx_status = fsl_tx_status;
1393         fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending;
1394         fdev->common.device_prep_slave_sg = fsl_dma_prep_slave_sg;
1395         fdev->common.device_control = fsl_dma_device_control;
1396         fdev->common.dev = &op->dev;
1397 
1398         dma_set_mask(&(op->dev), DMA_BIT_MASK(36));
1399 
1400         platform_set_drvdata(op, fdev);
1401 
1402         /*
1403          * We cannot use of_platform_bus_probe() because there is no
1404          * of_platform_bus_remove(). Instead, we manually instantiate every DMA
1405          * channel object.
1406          */
1407         for_each_child_of_node(op->dev.of_node, child) {
1408                 if (of_device_is_compatible(child, "fsl,eloplus-dma-channel")) {
1409                         fsl_dma_chan_probe(fdev, child,
1410                                 FSL_DMA_IP_85XX | FSL_DMA_BIG_ENDIAN,
1411                                 "fsl,eloplus-dma-channel");
1412                 }
1413 
1414                 if (of_device_is_compatible(child, "fsl,elo-dma-channel")) {
1415                         fsl_dma_chan_probe(fdev, child,
1416                                 FSL_DMA_IP_83XX | FSL_DMA_LITTLE_ENDIAN,
1417                                 "fsl,elo-dma-channel");
1418                 }
1419         }
1420 
1421         /*
1422          * Hookup the IRQ handler(s)
1423          *
1424          * If we have a per-controller interrupt, we prefer that to the
1425          * per-channel interrupts to reduce the number of shared interrupt
1426          * handlers on the same IRQ line
1427          */
1428         err = fsldma_request_irqs(fdev);
1429         if (err) {
1430                 dev_err(fdev->dev, "unable to request IRQs\n");
1431                 goto out_free_fdev;
1432         }
1433 
1434         dma_async_device_register(&fdev->common);
1435         return 0;
1436 
1437 out_free_fdev:
1438         irq_dispose_mapping(fdev->irq);
1439         kfree(fdev);
1440 out_return:
1441         return err;
1442 }
1443 
1444 static int fsldma_of_remove(struct platform_device *op)
1445 {
1446         struct fsldma_device *fdev;
1447         unsigned int i;
1448 
1449         fdev = platform_get_drvdata(op);
1450         dma_async_device_unregister(&fdev->common);
1451 
1452         fsldma_free_irqs(fdev);
1453 
1454         for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
1455                 if (fdev->chan[i])
1456                         fsl_dma_chan_remove(fdev->chan[i]);
1457         }
1458 
1459         iounmap(fdev->regs);
1460         kfree(fdev);
1461 
1462         return 0;
1463 }
1464 
1465 #ifdef CONFIG_PM
1466 static int fsldma_suspend_late(struct device *dev)
1467 {
1468         struct platform_device *pdev = to_platform_device(dev);
1469         struct fsldma_device *fdev = platform_get_drvdata(pdev);
1470         struct fsldma_chan *chan;
1471         int i;
1472 
1473         for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
1474                 chan = fdev->chan[i];
1475                 if (!chan)
1476                         continue;
1477 
1478                 spin_lock_bh(&chan->desc_lock);
1479                 if (unlikely(!chan->idle))
1480                         goto out;
1481                 chan->regs_save.mr = get_mr(chan);
1482                 chan->pm_state = SUSPENDED;
1483                 spin_unlock_bh(&chan->desc_lock);
1484         }
1485         return 0;
1486 
1487 out:
1488         for (; i >= 0; i--) {
1489                 chan = fdev->chan[i];
1490                 if (!chan)
1491                         continue;
1492                 chan->pm_state = RUNNING;
1493                 spin_unlock_bh(&chan->desc_lock);
1494         }
1495         return -EBUSY;
1496 }
1497 
1498 static int fsldma_resume_early(struct device *dev)
1499 {
1500         struct platform_device *pdev = to_platform_device(dev);
1501         struct fsldma_device *fdev = platform_get_drvdata(pdev);
1502         struct fsldma_chan *chan;
1503         u32 mode;
1504         int i;
1505 
1506         for (i = 0; i < FSL_DMA_MAX_CHANS_PER_DEVICE; i++) {
1507                 chan = fdev->chan[i];
1508                 if (!chan)
1509                         continue;
1510 
1511                 spin_lock_bh(&chan->desc_lock);
1512                 mode = chan->regs_save.mr
1513                         & ~FSL_DMA_MR_CS & ~FSL_DMA_MR_CC & ~FSL_DMA_MR_CA;
1514                 set_mr(chan, mode);
1515                 chan->pm_state = RUNNING;
1516                 spin_unlock_bh(&chan->desc_lock);
1517         }
1518 
1519         return 0;
1520 }
1521 
1522 static const struct dev_pm_ops fsldma_pm_ops = {
1523         .suspend_late   = fsldma_suspend_late,
1524         .resume_early   = fsldma_resume_early,
1525 };
1526 #endif
1527 
1528 static const struct of_device_id fsldma_of_ids[] = {
1529         { .compatible = "fsl,elo3-dma", },
1530         { .compatible = "fsl,eloplus-dma", },
1531         { .compatible = "fsl,elo-dma", },
1532         {}
1533 };
1534 
1535 static struct platform_driver fsldma_of_driver = {
1536         .driver = {
1537                 .name = "fsl-elo-dma",
1538                 .owner = THIS_MODULE,
1539                 .of_match_table = fsldma_of_ids,
1540 #ifdef CONFIG_PM
1541                 .pm = &fsldma_pm_ops,
1542 #endif
1543         },
1544         .probe = fsldma_of_probe,
1545         .remove = fsldma_of_remove,
1546 };
1547 
1548 /*----------------------------------------------------------------------------*/
1549 /* Module Init / Exit                                                         */
1550 /*----------------------------------------------------------------------------*/
1551 
1552 static __init int fsldma_init(void)
1553 {
1554         pr_info("Freescale Elo series DMA driver\n");
1555         return platform_driver_register(&fsldma_of_driver);
1556 }
1557 
1558 static void __exit fsldma_exit(void)
1559 {
1560         platform_driver_unregister(&fsldma_of_driver);
1561 }
1562 
1563 subsys_initcall(fsldma_init);
1564 module_exit(fsldma_exit);
1565 
1566 MODULE_DESCRIPTION("Freescale Elo series DMA driver");
1567 MODULE_LICENSE("GPL");
1568 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us