Version:  2.0.40 2.2.26 2.4.37 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16

Linux/drivers/dma/edma.c

  1 /*
  2  * TI EDMA DMA engine driver
  3  *
  4  * Copyright 2012 Texas Instruments
  5  *
  6  * This program is free software; you can redistribute it and/or
  7  * modify it under the terms of the GNU General Public License as
  8  * published by the Free Software Foundation version 2.
  9  *
 10  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
 11  * kind, whether express or implied; without even the implied warranty
 12  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13  * GNU General Public License for more details.
 14  */
 15 
 16 #include <linux/dmaengine.h>
 17 #include <linux/dma-mapping.h>
 18 #include <linux/err.h>
 19 #include <linux/init.h>
 20 #include <linux/interrupt.h>
 21 #include <linux/list.h>
 22 #include <linux/module.h>
 23 #include <linux/platform_device.h>
 24 #include <linux/slab.h>
 25 #include <linux/spinlock.h>
 26 
 27 #include <linux/platform_data/edma.h>
 28 
 29 #include "dmaengine.h"
 30 #include "virt-dma.h"
 31 
 32 /*
 33  * This will go away when the private EDMA API is folded
 34  * into this driver and the platform device(s) are
 35  * instantiated in the arch code. We can only get away
 36  * with this simplification because DA8XX may not be built
 37  * in the same kernel image with other DaVinci parts. This
 38  * avoids having to sprinkle dmaengine driver platform devices
 39  * and data throughout all the existing board files.
 40  */
 41 #ifdef CONFIG_ARCH_DAVINCI_DA8XX
 42 #define EDMA_CTLRS      2
 43 #define EDMA_CHANS      32
 44 #else
 45 #define EDMA_CTLRS      1
 46 #define EDMA_CHANS      64
 47 #endif /* CONFIG_ARCH_DAVINCI_DA8XX */
 48 
 49 /*
 50  * Max of 20 segments per channel to conserve PaRAM slots
 51  * Also note that MAX_NR_SG should be atleast the no.of periods
 52  * that are required for ASoC, otherwise DMA prep calls will
 53  * fail. Today davinci-pcm is the only user of this driver and
 54  * requires atleast 17 slots, so we setup the default to 20.
 55  */
 56 #define MAX_NR_SG               20
 57 #define EDMA_MAX_SLOTS          MAX_NR_SG
 58 #define EDMA_DESCRIPTORS        16
 59 
 60 struct edma_pset {
 61         u32                             len;
 62         dma_addr_t                      addr;
 63         struct edmacc_param             param;
 64 };
 65 
 66 struct edma_desc {
 67         struct virt_dma_desc            vdesc;
 68         struct list_head                node;
 69         enum dma_transfer_direction     direction;
 70         int                             cyclic;
 71         int                             absync;
 72         int                             pset_nr;
 73         struct edma_chan                *echan;
 74         int                             processed;
 75 
 76         /*
 77          * The following 4 elements are used for residue accounting.
 78          *
 79          * - processed_stat: the number of SG elements we have traversed
 80          * so far to cover accounting. This is updated directly to processed
 81          * during edma_callback and is always <= processed, because processed
 82          * refers to the number of pending transfer (programmed to EDMA
 83          * controller), where as processed_stat tracks number of transfers
 84          * accounted for so far.
 85          *
 86          * - residue: The amount of bytes we have left to transfer for this desc
 87          *
 88          * - residue_stat: The residue in bytes of data we have covered
 89          * so far for accounting. This is updated directly to residue
 90          * during callbacks to keep it current.
 91          *
 92          * - sg_len: Tracks the length of the current intermediate transfer,
 93          * this is required to update the residue during intermediate transfer
 94          * completion callback.
 95          */
 96         int                             processed_stat;
 97         u32                             sg_len;
 98         u32                             residue;
 99         u32                             residue_stat;
100 
101         struct edma_pset                pset[0];
102 };
103 
104 struct edma_cc;
105 
106 struct edma_chan {
107         struct virt_dma_chan            vchan;
108         struct list_head                node;
109         struct edma_desc                *edesc;
110         struct edma_cc                  *ecc;
111         int                             ch_num;
112         bool                            alloced;
113         int                             slot[EDMA_MAX_SLOTS];
114         int                             missed;
115         struct dma_slave_config         cfg;
116 };
117 
118 struct edma_cc {
119         int                             ctlr;
120         struct dma_device               dma_slave;
121         struct edma_chan                slave_chans[EDMA_CHANS];
122         int                             num_slave_chans;
123         int                             dummy_slot;
124 };
125 
126 static inline struct edma_cc *to_edma_cc(struct dma_device *d)
127 {
128         return container_of(d, struct edma_cc, dma_slave);
129 }
130 
131 static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
132 {
133         return container_of(c, struct edma_chan, vchan.chan);
134 }
135 
136 static inline struct edma_desc
137 *to_edma_desc(struct dma_async_tx_descriptor *tx)
138 {
139         return container_of(tx, struct edma_desc, vdesc.tx);
140 }
141 
142 static void edma_desc_free(struct virt_dma_desc *vdesc)
143 {
144         kfree(container_of(vdesc, struct edma_desc, vdesc));
145 }
146 
147 /* Dispatch a queued descriptor to the controller (caller holds lock) */
148 static void edma_execute(struct edma_chan *echan)
149 {
150         struct virt_dma_desc *vdesc;
151         struct edma_desc *edesc;
152         struct device *dev = echan->vchan.chan.device->dev;
153         int i, j, left, nslots;
154 
155         /* If either we processed all psets or we're still not started */
156         if (!echan->edesc ||
157             echan->edesc->pset_nr == echan->edesc->processed) {
158                 /* Get next vdesc */
159                 vdesc = vchan_next_desc(&echan->vchan);
160                 if (!vdesc) {
161                         echan->edesc = NULL;
162                         return;
163                 }
164                 list_del(&vdesc->node);
165                 echan->edesc = to_edma_desc(&vdesc->tx);
166         }
167 
168         edesc = echan->edesc;
169 
170         /* Find out how many left */
171         left = edesc->pset_nr - edesc->processed;
172         nslots = min(MAX_NR_SG, left);
173         edesc->sg_len = 0;
174 
175         /* Write descriptor PaRAM set(s) */
176         for (i = 0; i < nslots; i++) {
177                 j = i + edesc->processed;
178                 edma_write_slot(echan->slot[i], &edesc->pset[j].param);
179                 edesc->sg_len += edesc->pset[j].len;
180                 dev_vdbg(echan->vchan.chan.device->dev,
181                         "\n pset[%d]:\n"
182                         "  chnum\t%d\n"
183                         "  slot\t%d\n"
184                         "  opt\t%08x\n"
185                         "  src\t%08x\n"
186                         "  dst\t%08x\n"
187                         "  abcnt\t%08x\n"
188                         "  ccnt\t%08x\n"
189                         "  bidx\t%08x\n"
190                         "  cidx\t%08x\n"
191                         "  lkrld\t%08x\n",
192                         j, echan->ch_num, echan->slot[i],
193                         edesc->pset[j].param.opt,
194                         edesc->pset[j].param.src,
195                         edesc->pset[j].param.dst,
196                         edesc->pset[j].param.a_b_cnt,
197                         edesc->pset[j].param.ccnt,
198                         edesc->pset[j].param.src_dst_bidx,
199                         edesc->pset[j].param.src_dst_cidx,
200                         edesc->pset[j].param.link_bcntrld);
201                 /* Link to the previous slot if not the last set */
202                 if (i != (nslots - 1))
203                         edma_link(echan->slot[i], echan->slot[i+1]);
204         }
205 
206         edesc->processed += nslots;
207 
208         /*
209          * If this is either the last set in a set of SG-list transactions
210          * then setup a link to the dummy slot, this results in all future
211          * events being absorbed and that's OK because we're done
212          */
213         if (edesc->processed == edesc->pset_nr) {
214                 if (edesc->cyclic)
215                         edma_link(echan->slot[nslots-1], echan->slot[1]);
216                 else
217                         edma_link(echan->slot[nslots-1],
218                                   echan->ecc->dummy_slot);
219         }
220 
221         if (edesc->processed <= MAX_NR_SG) {
222                 dev_dbg(dev, "first transfer starting on channel %d\n",
223                         echan->ch_num);
224                 edma_start(echan->ch_num);
225         } else {
226                 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
227                         echan->ch_num, edesc->processed);
228                 edma_resume(echan->ch_num);
229         }
230 
231         /*
232          * This happens due to setup times between intermediate transfers
233          * in long SG lists which have to be broken up into transfers of
234          * MAX_NR_SG
235          */
236         if (echan->missed) {
237                 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
238                 edma_clean_channel(echan->ch_num);
239                 edma_stop(echan->ch_num);
240                 edma_start(echan->ch_num);
241                 edma_trigger_channel(echan->ch_num);
242                 echan->missed = 0;
243         }
244 }
245 
246 static int edma_terminate_all(struct edma_chan *echan)
247 {
248         unsigned long flags;
249         LIST_HEAD(head);
250 
251         spin_lock_irqsave(&echan->vchan.lock, flags);
252 
253         /*
254          * Stop DMA activity: we assume the callback will not be called
255          * after edma_dma() returns (even if it does, it will see
256          * echan->edesc is NULL and exit.)
257          */
258         if (echan->edesc) {
259                 echan->edesc = NULL;
260                 edma_stop(echan->ch_num);
261         }
262 
263         vchan_get_all_descriptors(&echan->vchan, &head);
264         spin_unlock_irqrestore(&echan->vchan.lock, flags);
265         vchan_dma_desc_free_list(&echan->vchan, &head);
266 
267         return 0;
268 }
269 
270 static int edma_slave_config(struct edma_chan *echan,
271         struct dma_slave_config *cfg)
272 {
273         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
274             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
275                 return -EINVAL;
276 
277         memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
278 
279         return 0;
280 }
281 
282 static int edma_dma_pause(struct edma_chan *echan)
283 {
284         /* Pause/Resume only allowed with cyclic mode */
285         if (!echan->edesc->cyclic)
286                 return -EINVAL;
287 
288         edma_pause(echan->ch_num);
289         return 0;
290 }
291 
292 static int edma_dma_resume(struct edma_chan *echan)
293 {
294         /* Pause/Resume only allowed with cyclic mode */
295         if (!echan->edesc->cyclic)
296                 return -EINVAL;
297 
298         edma_resume(echan->ch_num);
299         return 0;
300 }
301 
302 static int edma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
303                         unsigned long arg)
304 {
305         int ret = 0;
306         struct dma_slave_config *config;
307         struct edma_chan *echan = to_edma_chan(chan);
308 
309         switch (cmd) {
310         case DMA_TERMINATE_ALL:
311                 edma_terminate_all(echan);
312                 break;
313         case DMA_SLAVE_CONFIG:
314                 config = (struct dma_slave_config *)arg;
315                 ret = edma_slave_config(echan, config);
316                 break;
317         case DMA_PAUSE:
318                 ret = edma_dma_pause(echan);
319                 break;
320 
321         case DMA_RESUME:
322                 ret = edma_dma_resume(echan);
323                 break;
324 
325         default:
326                 ret = -ENOSYS;
327         }
328 
329         return ret;
330 }
331 
332 /*
333  * A PaRAM set configuration abstraction used by other modes
334  * @chan: Channel who's PaRAM set we're configuring
335  * @pset: PaRAM set to initialize and setup.
336  * @src_addr: Source address of the DMA
337  * @dst_addr: Destination address of the DMA
338  * @burst: In units of dev_width, how much to send
339  * @dev_width: How much is the dev_width
340  * @dma_length: Total length of the DMA transfer
341  * @direction: Direction of the transfer
342  */
343 static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
344         dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
345         enum dma_slave_buswidth dev_width, unsigned int dma_length,
346         enum dma_transfer_direction direction)
347 {
348         struct edma_chan *echan = to_edma_chan(chan);
349         struct device *dev = chan->device->dev;
350         struct edmacc_param *param = &epset->param;
351         int acnt, bcnt, ccnt, cidx;
352         int src_bidx, dst_bidx, src_cidx, dst_cidx;
353         int absync;
354 
355         acnt = dev_width;
356 
357         /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
358         if (!burst)
359                 burst = 1;
360         /*
361          * If the maxburst is equal to the fifo width, use
362          * A-synced transfers. This allows for large contiguous
363          * buffer transfers using only one PaRAM set.
364          */
365         if (burst == 1) {
366                 /*
367                  * For the A-sync case, bcnt and ccnt are the remainder
368                  * and quotient respectively of the division of:
369                  * (dma_length / acnt) by (SZ_64K -1). This is so
370                  * that in case bcnt over flows, we have ccnt to use.
371                  * Note: In A-sync tranfer only, bcntrld is used, but it
372                  * only applies for sg_dma_len(sg) >= SZ_64K.
373                  * In this case, the best way adopted is- bccnt for the
374                  * first frame will be the remainder below. Then for
375                  * every successive frame, bcnt will be SZ_64K-1. This
376                  * is assured as bcntrld = 0xffff in end of function.
377                  */
378                 absync = false;
379                 ccnt = dma_length / acnt / (SZ_64K - 1);
380                 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
381                 /*
382                  * If bcnt is non-zero, we have a remainder and hence an
383                  * extra frame to transfer, so increment ccnt.
384                  */
385                 if (bcnt)
386                         ccnt++;
387                 else
388                         bcnt = SZ_64K - 1;
389                 cidx = acnt;
390         } else {
391                 /*
392                  * If maxburst is greater than the fifo address_width,
393                  * use AB-synced transfers where A count is the fifo
394                  * address_width and B count is the maxburst. In this
395                  * case, we are limited to transfers of C count frames
396                  * of (address_width * maxburst) where C count is limited
397                  * to SZ_64K-1. This places an upper bound on the length
398                  * of an SG segment that can be handled.
399                  */
400                 absync = true;
401                 bcnt = burst;
402                 ccnt = dma_length / (acnt * bcnt);
403                 if (ccnt > (SZ_64K - 1)) {
404                         dev_err(dev, "Exceeded max SG segment size\n");
405                         return -EINVAL;
406                 }
407                 cidx = acnt * bcnt;
408         }
409 
410         epset->len = dma_length;
411 
412         if (direction == DMA_MEM_TO_DEV) {
413                 src_bidx = acnt;
414                 src_cidx = cidx;
415                 dst_bidx = 0;
416                 dst_cidx = 0;
417                 epset->addr = src_addr;
418         } else if (direction == DMA_DEV_TO_MEM)  {
419                 src_bidx = 0;
420                 src_cidx = 0;
421                 dst_bidx = acnt;
422                 dst_cidx = cidx;
423                 epset->addr = dst_addr;
424         } else if (direction == DMA_MEM_TO_MEM)  {
425                 src_bidx = acnt;
426                 src_cidx = cidx;
427                 dst_bidx = acnt;
428                 dst_cidx = cidx;
429         } else {
430                 dev_err(dev, "%s: direction not implemented yet\n", __func__);
431                 return -EINVAL;
432         }
433 
434         param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
435         /* Configure A or AB synchronized transfers */
436         if (absync)
437                 param->opt |= SYNCDIM;
438 
439         param->src = src_addr;
440         param->dst = dst_addr;
441 
442         param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
443         param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
444 
445         param->a_b_cnt = bcnt << 16 | acnt;
446         param->ccnt = ccnt;
447         /*
448          * Only time when (bcntrld) auto reload is required is for
449          * A-sync case, and in this case, a requirement of reload value
450          * of SZ_64K-1 only is assured. 'link' is initially set to NULL
451          * and then later will be populated by edma_execute.
452          */
453         param->link_bcntrld = 0xffffffff;
454         return absync;
455 }
456 
457 static struct dma_async_tx_descriptor *edma_prep_slave_sg(
458         struct dma_chan *chan, struct scatterlist *sgl,
459         unsigned int sg_len, enum dma_transfer_direction direction,
460         unsigned long tx_flags, void *context)
461 {
462         struct edma_chan *echan = to_edma_chan(chan);
463         struct device *dev = chan->device->dev;
464         struct edma_desc *edesc;
465         dma_addr_t src_addr = 0, dst_addr = 0;
466         enum dma_slave_buswidth dev_width;
467         u32 burst;
468         struct scatterlist *sg;
469         int i, nslots, ret;
470 
471         if (unlikely(!echan || !sgl || !sg_len))
472                 return NULL;
473 
474         if (direction == DMA_DEV_TO_MEM) {
475                 src_addr = echan->cfg.src_addr;
476                 dev_width = echan->cfg.src_addr_width;
477                 burst = echan->cfg.src_maxburst;
478         } else if (direction == DMA_MEM_TO_DEV) {
479                 dst_addr = echan->cfg.dst_addr;
480                 dev_width = echan->cfg.dst_addr_width;
481                 burst = echan->cfg.dst_maxburst;
482         } else {
483                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
484                 return NULL;
485         }
486 
487         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
488                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
489                 return NULL;
490         }
491 
492         edesc = kzalloc(sizeof(*edesc) + sg_len *
493                 sizeof(edesc->pset[0]), GFP_ATOMIC);
494         if (!edesc) {
495                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
496                 return NULL;
497         }
498 
499         edesc->pset_nr = sg_len;
500         edesc->residue = 0;
501         edesc->direction = direction;
502         edesc->echan = echan;
503 
504         /* Allocate a PaRAM slot, if needed */
505         nslots = min_t(unsigned, MAX_NR_SG, sg_len);
506 
507         for (i = 0; i < nslots; i++) {
508                 if (echan->slot[i] < 0) {
509                         echan->slot[i] =
510                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
511                                                 EDMA_SLOT_ANY);
512                         if (echan->slot[i] < 0) {
513                                 kfree(edesc);
514                                 dev_err(dev, "%s: Failed to allocate slot\n",
515                                         __func__);
516                                 return NULL;
517                         }
518                 }
519         }
520 
521         /* Configure PaRAM sets for each SG */
522         for_each_sg(sgl, sg, sg_len, i) {
523                 /* Get address for each SG */
524                 if (direction == DMA_DEV_TO_MEM)
525                         dst_addr = sg_dma_address(sg);
526                 else
527                         src_addr = sg_dma_address(sg);
528 
529                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
530                                        dst_addr, burst, dev_width,
531                                        sg_dma_len(sg), direction);
532                 if (ret < 0) {
533                         kfree(edesc);
534                         return NULL;
535                 }
536 
537                 edesc->absync = ret;
538                 edesc->residue += sg_dma_len(sg);
539 
540                 /* If this is the last in a current SG set of transactions,
541                    enable interrupts so that next set is processed */
542                 if (!((i+1) % MAX_NR_SG))
543                         edesc->pset[i].param.opt |= TCINTEN;
544 
545                 /* If this is the last set, enable completion interrupt flag */
546                 if (i == sg_len - 1)
547                         edesc->pset[i].param.opt |= TCINTEN;
548         }
549         edesc->residue_stat = edesc->residue;
550 
551         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
552 }
553 
554 struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
555         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
556         size_t len, unsigned long tx_flags)
557 {
558         int ret;
559         struct edma_desc *edesc;
560         struct device *dev = chan->device->dev;
561         struct edma_chan *echan = to_edma_chan(chan);
562 
563         if (unlikely(!echan || !len))
564                 return NULL;
565 
566         edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC);
567         if (!edesc) {
568                 dev_dbg(dev, "Failed to allocate a descriptor\n");
569                 return NULL;
570         }
571 
572         edesc->pset_nr = 1;
573 
574         ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
575                                DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM);
576         if (ret < 0)
577                 return NULL;
578 
579         edesc->absync = ret;
580 
581         /*
582          * Enable intermediate transfer chaining to re-trigger channel
583          * on completion of every TR, and enable transfer-completion
584          * interrupt on completion of the whole transfer.
585          */
586         edesc->pset[0].param.opt |= ITCCHEN;
587         edesc->pset[0].param.opt |= TCINTEN;
588 
589         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
590 }
591 
592 static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
593         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
594         size_t period_len, enum dma_transfer_direction direction,
595         unsigned long tx_flags, void *context)
596 {
597         struct edma_chan *echan = to_edma_chan(chan);
598         struct device *dev = chan->device->dev;
599         struct edma_desc *edesc;
600         dma_addr_t src_addr, dst_addr;
601         enum dma_slave_buswidth dev_width;
602         u32 burst;
603         int i, ret, nslots;
604 
605         if (unlikely(!echan || !buf_len || !period_len))
606                 return NULL;
607 
608         if (direction == DMA_DEV_TO_MEM) {
609                 src_addr = echan->cfg.src_addr;
610                 dst_addr = buf_addr;
611                 dev_width = echan->cfg.src_addr_width;
612                 burst = echan->cfg.src_maxburst;
613         } else if (direction == DMA_MEM_TO_DEV) {
614                 src_addr = buf_addr;
615                 dst_addr = echan->cfg.dst_addr;
616                 dev_width = echan->cfg.dst_addr_width;
617                 burst = echan->cfg.dst_maxburst;
618         } else {
619                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
620                 return NULL;
621         }
622 
623         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
624                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
625                 return NULL;
626         }
627 
628         if (unlikely(buf_len % period_len)) {
629                 dev_err(dev, "Period should be multiple of Buffer length\n");
630                 return NULL;
631         }
632 
633         nslots = (buf_len / period_len) + 1;
634 
635         /*
636          * Cyclic DMA users such as audio cannot tolerate delays introduced
637          * by cases where the number of periods is more than the maximum
638          * number of SGs the EDMA driver can handle at a time. For DMA types
639          * such as Slave SGs, such delays are tolerable and synchronized,
640          * but the synchronization is difficult to achieve with Cyclic and
641          * cannot be guaranteed, so we error out early.
642          */
643         if (nslots > MAX_NR_SG)
644                 return NULL;
645 
646         edesc = kzalloc(sizeof(*edesc) + nslots *
647                 sizeof(edesc->pset[0]), GFP_ATOMIC);
648         if (!edesc) {
649                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
650                 return NULL;
651         }
652 
653         edesc->cyclic = 1;
654         edesc->pset_nr = nslots;
655         edesc->residue = edesc->residue_stat = buf_len;
656         edesc->direction = direction;
657         edesc->echan = echan;
658 
659         dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
660                 __func__, echan->ch_num, nslots, period_len, buf_len);
661 
662         for (i = 0; i < nslots; i++) {
663                 /* Allocate a PaRAM slot, if needed */
664                 if (echan->slot[i] < 0) {
665                         echan->slot[i] =
666                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
667                                                 EDMA_SLOT_ANY);
668                         if (echan->slot[i] < 0) {
669                                 kfree(edesc);
670                                 dev_err(dev, "%s: Failed to allocate slot\n",
671                                         __func__);
672                                 return NULL;
673                         }
674                 }
675 
676                 if (i == nslots - 1) {
677                         memcpy(&edesc->pset[i], &edesc->pset[0],
678                                sizeof(edesc->pset[0]));
679                         break;
680                 }
681 
682                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
683                                        dst_addr, burst, dev_width, period_len,
684                                        direction);
685                 if (ret < 0) {
686                         kfree(edesc);
687                         return NULL;
688                 }
689 
690                 if (direction == DMA_DEV_TO_MEM)
691                         dst_addr += period_len;
692                 else
693                         src_addr += period_len;
694 
695                 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
696                 dev_vdbg(dev,
697                         "\n pset[%d]:\n"
698                         "  chnum\t%d\n"
699                         "  slot\t%d\n"
700                         "  opt\t%08x\n"
701                         "  src\t%08x\n"
702                         "  dst\t%08x\n"
703                         "  abcnt\t%08x\n"
704                         "  ccnt\t%08x\n"
705                         "  bidx\t%08x\n"
706                         "  cidx\t%08x\n"
707                         "  lkrld\t%08x\n",
708                         i, echan->ch_num, echan->slot[i],
709                         edesc->pset[i].param.opt,
710                         edesc->pset[i].param.src,
711                         edesc->pset[i].param.dst,
712                         edesc->pset[i].param.a_b_cnt,
713                         edesc->pset[i].param.ccnt,
714                         edesc->pset[i].param.src_dst_bidx,
715                         edesc->pset[i].param.src_dst_cidx,
716                         edesc->pset[i].param.link_bcntrld);
717 
718                 edesc->absync = ret;
719 
720                 /*
721                  * Enable interrupts for every period because callback
722                  * has to be called for every period.
723                  */
724                 edesc->pset[i].param.opt |= TCINTEN;
725         }
726 
727         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
728 }
729 
730 static void edma_callback(unsigned ch_num, u16 ch_status, void *data)
731 {
732         struct edma_chan *echan = data;
733         struct device *dev = echan->vchan.chan.device->dev;
734         struct edma_desc *edesc;
735         struct edmacc_param p;
736 
737         edesc = echan->edesc;
738 
739         /* Pause the channel for non-cyclic */
740         if (!edesc || (edesc && !edesc->cyclic))
741                 edma_pause(echan->ch_num);
742 
743         switch (ch_status) {
744         case EDMA_DMA_COMPLETE:
745                 spin_lock(&echan->vchan.lock);
746 
747                 if (edesc) {
748                         if (edesc->cyclic) {
749                                 vchan_cyclic_callback(&edesc->vdesc);
750                         } else if (edesc->processed == edesc->pset_nr) {
751                                 dev_dbg(dev, "Transfer complete, stopping channel %d\n", ch_num);
752                                 edesc->residue = 0;
753                                 edma_stop(echan->ch_num);
754                                 vchan_cookie_complete(&edesc->vdesc);
755                                 edma_execute(echan);
756                         } else {
757                                 dev_dbg(dev, "Intermediate transfer complete on channel %d\n", ch_num);
758 
759                                 /* Update statistics for tx_status */
760                                 edesc->residue -= edesc->sg_len;
761                                 edesc->residue_stat = edesc->residue;
762                                 edesc->processed_stat = edesc->processed;
763 
764                                 edma_execute(echan);
765                         }
766                 }
767 
768                 spin_unlock(&echan->vchan.lock);
769 
770                 break;
771         case EDMA_DMA_CC_ERROR:
772                 spin_lock(&echan->vchan.lock);
773 
774                 edma_read_slot(EDMA_CHAN_SLOT(echan->slot[0]), &p);
775 
776                 /*
777                  * Issue later based on missed flag which will be sure
778                  * to happen as:
779                  * (1) we finished transmitting an intermediate slot and
780                  *     edma_execute is coming up.
781                  * (2) or we finished current transfer and issue will
782                  *     call edma_execute.
783                  *
784                  * Important note: issuing can be dangerous here and
785                  * lead to some nasty recursion when we are in a NULL
786                  * slot. So we avoid doing so and set the missed flag.
787                  */
788                 if (p.a_b_cnt == 0 && p.ccnt == 0) {
789                         dev_dbg(dev, "Error occurred, looks like slot is null, just setting miss\n");
790                         echan->missed = 1;
791                 } else {
792                         /*
793                          * The slot is already programmed but the event got
794                          * missed, so its safe to issue it here.
795                          */
796                         dev_dbg(dev, "Error occurred but slot is non-null, TRIGGERING\n");
797                         edma_clean_channel(echan->ch_num);
798                         edma_stop(echan->ch_num);
799                         edma_start(echan->ch_num);
800                         edma_trigger_channel(echan->ch_num);
801                 }
802 
803                 spin_unlock(&echan->vchan.lock);
804 
805                 break;
806         default:
807                 break;
808         }
809 }
810 
811 /* Alloc channel resources */
812 static int edma_alloc_chan_resources(struct dma_chan *chan)
813 {
814         struct edma_chan *echan = to_edma_chan(chan);
815         struct device *dev = chan->device->dev;
816         int ret;
817         int a_ch_num;
818         LIST_HEAD(descs);
819 
820         a_ch_num = edma_alloc_channel(echan->ch_num, edma_callback,
821                                         chan, EVENTQ_DEFAULT);
822 
823         if (a_ch_num < 0) {
824                 ret = -ENODEV;
825                 goto err_no_chan;
826         }
827 
828         if (a_ch_num != echan->ch_num) {
829                 dev_err(dev, "failed to allocate requested channel %u:%u\n",
830                         EDMA_CTLR(echan->ch_num),
831                         EDMA_CHAN_SLOT(echan->ch_num));
832                 ret = -ENODEV;
833                 goto err_wrong_chan;
834         }
835 
836         echan->alloced = true;
837         echan->slot[0] = echan->ch_num;
838 
839         dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num,
840                 EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num));
841 
842         return 0;
843 
844 err_wrong_chan:
845         edma_free_channel(a_ch_num);
846 err_no_chan:
847         return ret;
848 }
849 
850 /* Free channel resources */
851 static void edma_free_chan_resources(struct dma_chan *chan)
852 {
853         struct edma_chan *echan = to_edma_chan(chan);
854         struct device *dev = chan->device->dev;
855         int i;
856 
857         /* Terminate transfers */
858         edma_stop(echan->ch_num);
859 
860         vchan_free_chan_resources(&echan->vchan);
861 
862         /* Free EDMA PaRAM slots */
863         for (i = 1; i < EDMA_MAX_SLOTS; i++) {
864                 if (echan->slot[i] >= 0) {
865                         edma_free_slot(echan->slot[i]);
866                         echan->slot[i] = -1;
867                 }
868         }
869 
870         /* Free EDMA channel */
871         if (echan->alloced) {
872                 edma_free_channel(echan->ch_num);
873                 echan->alloced = false;
874         }
875 
876         dev_dbg(dev, "freeing channel for %u\n", echan->ch_num);
877 }
878 
879 /* Send pending descriptor to hardware */
880 static void edma_issue_pending(struct dma_chan *chan)
881 {
882         struct edma_chan *echan = to_edma_chan(chan);
883         unsigned long flags;
884 
885         spin_lock_irqsave(&echan->vchan.lock, flags);
886         if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
887                 edma_execute(echan);
888         spin_unlock_irqrestore(&echan->vchan.lock, flags);
889 }
890 
891 static u32 edma_residue(struct edma_desc *edesc)
892 {
893         bool dst = edesc->direction == DMA_DEV_TO_MEM;
894         struct edma_pset *pset = edesc->pset;
895         dma_addr_t done, pos;
896         int i;
897 
898         /*
899          * We always read the dst/src position from the first RamPar
900          * pset. That's the one which is active now.
901          */
902         pos = edma_get_position(edesc->echan->slot[0], dst);
903 
904         /*
905          * Cyclic is simple. Just subtract pset[0].addr from pos.
906          *
907          * We never update edesc->residue in the cyclic case, so we
908          * can tell the remaining room to the end of the circular
909          * buffer.
910          */
911         if (edesc->cyclic) {
912                 done = pos - pset->addr;
913                 edesc->residue_stat = edesc->residue - done;
914                 return edesc->residue_stat;
915         }
916 
917         /*
918          * For SG operation we catch up with the last processed
919          * status.
920          */
921         pset += edesc->processed_stat;
922 
923         for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
924                 /*
925                  * If we are inside this pset address range, we know
926                  * this is the active one. Get the current delta and
927                  * stop walking the psets.
928                  */
929                 if (pos >= pset->addr && pos < pset->addr + pset->len)
930                         return edesc->residue_stat - (pos - pset->addr);
931 
932                 /* Otherwise mark it done and update residue_stat. */
933                 edesc->processed_stat++;
934                 edesc->residue_stat -= pset->len;
935         }
936         return edesc->residue_stat;
937 }
938 
939 /* Check request completion status */
940 static enum dma_status edma_tx_status(struct dma_chan *chan,
941                                       dma_cookie_t cookie,
942                                       struct dma_tx_state *txstate)
943 {
944         struct edma_chan *echan = to_edma_chan(chan);
945         struct virt_dma_desc *vdesc;
946         enum dma_status ret;
947         unsigned long flags;
948 
949         ret = dma_cookie_status(chan, cookie, txstate);
950         if (ret == DMA_COMPLETE || !txstate)
951                 return ret;
952 
953         spin_lock_irqsave(&echan->vchan.lock, flags);
954         if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
955                 txstate->residue = edma_residue(echan->edesc);
956         else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
957                 txstate->residue = to_edma_desc(&vdesc->tx)->residue;
958         spin_unlock_irqrestore(&echan->vchan.lock, flags);
959 
960         return ret;
961 }
962 
963 static void __init edma_chan_init(struct edma_cc *ecc,
964                                   struct dma_device *dma,
965                                   struct edma_chan *echans)
966 {
967         int i, j;
968 
969         for (i = 0; i < EDMA_CHANS; i++) {
970                 struct edma_chan *echan = &echans[i];
971                 echan->ch_num = EDMA_CTLR_CHAN(ecc->ctlr, i);
972                 echan->ecc = ecc;
973                 echan->vchan.desc_free = edma_desc_free;
974 
975                 vchan_init(&echan->vchan, dma);
976 
977                 INIT_LIST_HEAD(&echan->node);
978                 for (j = 0; j < EDMA_MAX_SLOTS; j++)
979                         echan->slot[j] = -1;
980         }
981 }
982 
983 #define EDMA_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
984                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
985                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
986 
987 static int edma_dma_device_slave_caps(struct dma_chan *dchan,
988                                       struct dma_slave_caps *caps)
989 {
990         caps->src_addr_widths = EDMA_DMA_BUSWIDTHS;
991         caps->dstn_addr_widths = EDMA_DMA_BUSWIDTHS;
992         caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
993         caps->cmd_pause = true;
994         caps->cmd_terminate = true;
995         caps->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
996 
997         return 0;
998 }
999 
1000 static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
1001                           struct device *dev)
1002 {
1003         dma->device_prep_slave_sg = edma_prep_slave_sg;
1004         dma->device_prep_dma_cyclic = edma_prep_dma_cyclic;
1005         dma->device_prep_dma_memcpy = edma_prep_dma_memcpy;
1006         dma->device_alloc_chan_resources = edma_alloc_chan_resources;
1007         dma->device_free_chan_resources = edma_free_chan_resources;
1008         dma->device_issue_pending = edma_issue_pending;
1009         dma->device_tx_status = edma_tx_status;
1010         dma->device_control = edma_control;
1011         dma->device_slave_caps = edma_dma_device_slave_caps;
1012         dma->dev = dev;
1013 
1014         /*
1015          * code using dma memcpy must make sure alignment of
1016          * length is at dma->copy_align boundary.
1017          */
1018         dma->copy_align = DMA_SLAVE_BUSWIDTH_4_BYTES;
1019 
1020         INIT_LIST_HEAD(&dma->channels);
1021 }
1022 
1023 static int edma_probe(struct platform_device *pdev)
1024 {
1025         struct edma_cc *ecc;
1026         int ret;
1027 
1028         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1029         if (ret)
1030                 return ret;
1031 
1032         ecc = devm_kzalloc(&pdev->dev, sizeof(*ecc), GFP_KERNEL);
1033         if (!ecc) {
1034                 dev_err(&pdev->dev, "Can't allocate controller\n");
1035                 return -ENOMEM;
1036         }
1037 
1038         ecc->ctlr = pdev->id;
1039         ecc->dummy_slot = edma_alloc_slot(ecc->ctlr, EDMA_SLOT_ANY);
1040         if (ecc->dummy_slot < 0) {
1041                 dev_err(&pdev->dev, "Can't allocate PaRAM dummy slot\n");
1042                 return -EIO;
1043         }
1044 
1045         dma_cap_zero(ecc->dma_slave.cap_mask);
1046         dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask);
1047         dma_cap_set(DMA_CYCLIC, ecc->dma_slave.cap_mask);
1048         dma_cap_set(DMA_MEMCPY, ecc->dma_slave.cap_mask);
1049 
1050         edma_dma_init(ecc, &ecc->dma_slave, &pdev->dev);
1051 
1052         edma_chan_init(ecc, &ecc->dma_slave, ecc->slave_chans);
1053 
1054         ret = dma_async_device_register(&ecc->dma_slave);
1055         if (ret)
1056                 goto err_reg1;
1057 
1058         platform_set_drvdata(pdev, ecc);
1059 
1060         dev_info(&pdev->dev, "TI EDMA DMA engine driver\n");
1061 
1062         return 0;
1063 
1064 err_reg1:
1065         edma_free_slot(ecc->dummy_slot);
1066         return ret;
1067 }
1068 
1069 static int edma_remove(struct platform_device *pdev)
1070 {
1071         struct device *dev = &pdev->dev;
1072         struct edma_cc *ecc = dev_get_drvdata(dev);
1073 
1074         dma_async_device_unregister(&ecc->dma_slave);
1075         edma_free_slot(ecc->dummy_slot);
1076 
1077         return 0;
1078 }
1079 
1080 static struct platform_driver edma_driver = {
1081         .probe          = edma_probe,
1082         .remove         = edma_remove,
1083         .driver = {
1084                 .name = "edma-dma-engine",
1085                 .owner = THIS_MODULE,
1086         },
1087 };
1088 
1089 bool edma_filter_fn(struct dma_chan *chan, void *param)
1090 {
1091         if (chan->device->dev->driver == &edma_driver.driver) {
1092                 struct edma_chan *echan = to_edma_chan(chan);
1093                 unsigned ch_req = *(unsigned *)param;
1094                 return ch_req == echan->ch_num;
1095         }
1096         return false;
1097 }
1098 EXPORT_SYMBOL(edma_filter_fn);
1099 
1100 static struct platform_device *pdev0, *pdev1;
1101 
1102 static const struct platform_device_info edma_dev_info0 = {
1103         .name = "edma-dma-engine",
1104         .id = 0,
1105         .dma_mask = DMA_BIT_MASK(32),
1106 };
1107 
1108 static const struct platform_device_info edma_dev_info1 = {
1109         .name = "edma-dma-engine",
1110         .id = 1,
1111         .dma_mask = DMA_BIT_MASK(32),
1112 };
1113 
1114 static int edma_init(void)
1115 {
1116         int ret = platform_driver_register(&edma_driver);
1117 
1118         if (ret == 0) {
1119                 pdev0 = platform_device_register_full(&edma_dev_info0);
1120                 if (IS_ERR(pdev0)) {
1121                         platform_driver_unregister(&edma_driver);
1122                         ret = PTR_ERR(pdev0);
1123                         goto out;
1124                 }
1125         }
1126 
1127         if (EDMA_CTLRS == 2) {
1128                 pdev1 = platform_device_register_full(&edma_dev_info1);
1129                 if (IS_ERR(pdev1)) {
1130                         platform_driver_unregister(&edma_driver);
1131                         platform_device_unregister(pdev0);
1132                         ret = PTR_ERR(pdev1);
1133                 }
1134         }
1135 
1136 out:
1137         return ret;
1138 }
1139 subsys_initcall(edma_init);
1140 
1141 static void __exit edma_exit(void)
1142 {
1143         platform_device_unregister(pdev0);
1144         if (pdev1)
1145                 platform_device_unregister(pdev1);
1146         platform_driver_unregister(&edma_driver);
1147 }
1148 module_exit(edma_exit);
1149 
1150 MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
1151 MODULE_DESCRIPTION("TI EDMA DMA engine driver");
1152 MODULE_LICENSE("GPL v2");
1153 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us