Version:  2.0.40 2.2.26 2.4.37 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2

Linux/drivers/dma/edma.c

  1 /*
  2  * TI EDMA DMA engine driver
  3  *
  4  * Copyright 2012 Texas Instruments
  5  *
  6  * This program is free software; you can redistribute it and/or
  7  * modify it under the terms of the GNU General Public License as
  8  * published by the Free Software Foundation version 2.
  9  *
 10  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
 11  * kind, whether express or implied; without even the implied warranty
 12  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13  * GNU General Public License for more details.
 14  */
 15 
 16 #include <linux/dmaengine.h>
 17 #include <linux/dma-mapping.h>
 18 #include <linux/edma.h>
 19 #include <linux/err.h>
 20 #include <linux/init.h>
 21 #include <linux/interrupt.h>
 22 #include <linux/list.h>
 23 #include <linux/module.h>
 24 #include <linux/platform_device.h>
 25 #include <linux/slab.h>
 26 #include <linux/spinlock.h>
 27 #include <linux/of.h>
 28 
 29 #include <linux/platform_data/edma.h>
 30 
 31 #include "dmaengine.h"
 32 #include "virt-dma.h"
 33 
 34 /*
 35  * This will go away when the private EDMA API is folded
 36  * into this driver and the platform device(s) are
 37  * instantiated in the arch code. We can only get away
 38  * with this simplification because DA8XX may not be built
 39  * in the same kernel image with other DaVinci parts. This
 40  * avoids having to sprinkle dmaengine driver platform devices
 41  * and data throughout all the existing board files.
 42  */
 43 #ifdef CONFIG_ARCH_DAVINCI_DA8XX
 44 #define EDMA_CTLRS      2
 45 #define EDMA_CHANS      32
 46 #else
 47 #define EDMA_CTLRS      1
 48 #define EDMA_CHANS      64
 49 #endif /* CONFIG_ARCH_DAVINCI_DA8XX */
 50 
 51 /*
 52  * Max of 20 segments per channel to conserve PaRAM slots
 53  * Also note that MAX_NR_SG should be atleast the no.of periods
 54  * that are required for ASoC, otherwise DMA prep calls will
 55  * fail. Today davinci-pcm is the only user of this driver and
 56  * requires atleast 17 slots, so we setup the default to 20.
 57  */
 58 #define MAX_NR_SG               20
 59 #define EDMA_MAX_SLOTS          MAX_NR_SG
 60 #define EDMA_DESCRIPTORS        16
 61 
 62 struct edma_pset {
 63         u32                             len;
 64         dma_addr_t                      addr;
 65         struct edmacc_param             param;
 66 };
 67 
 68 struct edma_desc {
 69         struct virt_dma_desc            vdesc;
 70         struct list_head                node;
 71         enum dma_transfer_direction     direction;
 72         int                             cyclic;
 73         int                             absync;
 74         int                             pset_nr;
 75         struct edma_chan                *echan;
 76         int                             processed;
 77 
 78         /*
 79          * The following 4 elements are used for residue accounting.
 80          *
 81          * - processed_stat: the number of SG elements we have traversed
 82          * so far to cover accounting. This is updated directly to processed
 83          * during edma_callback and is always <= processed, because processed
 84          * refers to the number of pending transfer (programmed to EDMA
 85          * controller), where as processed_stat tracks number of transfers
 86          * accounted for so far.
 87          *
 88          * - residue: The amount of bytes we have left to transfer for this desc
 89          *
 90          * - residue_stat: The residue in bytes of data we have covered
 91          * so far for accounting. This is updated directly to residue
 92          * during callbacks to keep it current.
 93          *
 94          * - sg_len: Tracks the length of the current intermediate transfer,
 95          * this is required to update the residue during intermediate transfer
 96          * completion callback.
 97          */
 98         int                             processed_stat;
 99         u32                             sg_len;
100         u32                             residue;
101         u32                             residue_stat;
102 
103         struct edma_pset                pset[0];
104 };
105 
106 struct edma_cc;
107 
108 struct edma_chan {
109         struct virt_dma_chan            vchan;
110         struct list_head                node;
111         struct edma_desc                *edesc;
112         struct edma_cc                  *ecc;
113         int                             ch_num;
114         bool                            alloced;
115         int                             slot[EDMA_MAX_SLOTS];
116         int                             missed;
117         struct dma_slave_config         cfg;
118 };
119 
120 struct edma_cc {
121         int                             ctlr;
122         struct dma_device               dma_slave;
123         struct edma_chan                slave_chans[EDMA_CHANS];
124         int                             num_slave_chans;
125         int                             dummy_slot;
126 };
127 
128 static inline struct edma_cc *to_edma_cc(struct dma_device *d)
129 {
130         return container_of(d, struct edma_cc, dma_slave);
131 }
132 
133 static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
134 {
135         return container_of(c, struct edma_chan, vchan.chan);
136 }
137 
138 static inline struct edma_desc
139 *to_edma_desc(struct dma_async_tx_descriptor *tx)
140 {
141         return container_of(tx, struct edma_desc, vdesc.tx);
142 }
143 
144 static void edma_desc_free(struct virt_dma_desc *vdesc)
145 {
146         kfree(container_of(vdesc, struct edma_desc, vdesc));
147 }
148 
149 /* Dispatch a queued descriptor to the controller (caller holds lock) */
150 static void edma_execute(struct edma_chan *echan)
151 {
152         struct virt_dma_desc *vdesc;
153         struct edma_desc *edesc;
154         struct device *dev = echan->vchan.chan.device->dev;
155         int i, j, left, nslots;
156 
157         /* If either we processed all psets or we're still not started */
158         if (!echan->edesc ||
159             echan->edesc->pset_nr == echan->edesc->processed) {
160                 /* Get next vdesc */
161                 vdesc = vchan_next_desc(&echan->vchan);
162                 if (!vdesc) {
163                         echan->edesc = NULL;
164                         return;
165                 }
166                 list_del(&vdesc->node);
167                 echan->edesc = to_edma_desc(&vdesc->tx);
168         }
169 
170         edesc = echan->edesc;
171 
172         /* Find out how many left */
173         left = edesc->pset_nr - edesc->processed;
174         nslots = min(MAX_NR_SG, left);
175         edesc->sg_len = 0;
176 
177         /* Write descriptor PaRAM set(s) */
178         for (i = 0; i < nslots; i++) {
179                 j = i + edesc->processed;
180                 edma_write_slot(echan->slot[i], &edesc->pset[j].param);
181                 edesc->sg_len += edesc->pset[j].len;
182                 dev_vdbg(echan->vchan.chan.device->dev,
183                         "\n pset[%d]:\n"
184                         "  chnum\t%d\n"
185                         "  slot\t%d\n"
186                         "  opt\t%08x\n"
187                         "  src\t%08x\n"
188                         "  dst\t%08x\n"
189                         "  abcnt\t%08x\n"
190                         "  ccnt\t%08x\n"
191                         "  bidx\t%08x\n"
192                         "  cidx\t%08x\n"
193                         "  lkrld\t%08x\n",
194                         j, echan->ch_num, echan->slot[i],
195                         edesc->pset[j].param.opt,
196                         edesc->pset[j].param.src,
197                         edesc->pset[j].param.dst,
198                         edesc->pset[j].param.a_b_cnt,
199                         edesc->pset[j].param.ccnt,
200                         edesc->pset[j].param.src_dst_bidx,
201                         edesc->pset[j].param.src_dst_cidx,
202                         edesc->pset[j].param.link_bcntrld);
203                 /* Link to the previous slot if not the last set */
204                 if (i != (nslots - 1))
205                         edma_link(echan->slot[i], echan->slot[i+1]);
206         }
207 
208         edesc->processed += nslots;
209 
210         /*
211          * If this is either the last set in a set of SG-list transactions
212          * then setup a link to the dummy slot, this results in all future
213          * events being absorbed and that's OK because we're done
214          */
215         if (edesc->processed == edesc->pset_nr) {
216                 if (edesc->cyclic)
217                         edma_link(echan->slot[nslots-1], echan->slot[1]);
218                 else
219                         edma_link(echan->slot[nslots-1],
220                                   echan->ecc->dummy_slot);
221         }
222 
223         if (edesc->processed <= MAX_NR_SG) {
224                 dev_dbg(dev, "first transfer starting on channel %d\n",
225                         echan->ch_num);
226                 edma_start(echan->ch_num);
227         } else {
228                 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
229                         echan->ch_num, edesc->processed);
230                 edma_resume(echan->ch_num);
231         }
232 
233         /*
234          * This happens due to setup times between intermediate transfers
235          * in long SG lists which have to be broken up into transfers of
236          * MAX_NR_SG
237          */
238         if (echan->missed) {
239                 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
240                 edma_clean_channel(echan->ch_num);
241                 edma_stop(echan->ch_num);
242                 edma_start(echan->ch_num);
243                 edma_trigger_channel(echan->ch_num);
244                 echan->missed = 0;
245         }
246 }
247 
248 static int edma_terminate_all(struct dma_chan *chan)
249 {
250         struct edma_chan *echan = to_edma_chan(chan);
251         unsigned long flags;
252         LIST_HEAD(head);
253 
254         spin_lock_irqsave(&echan->vchan.lock, flags);
255 
256         /*
257          * Stop DMA activity: we assume the callback will not be called
258          * after edma_dma() returns (even if it does, it will see
259          * echan->edesc is NULL and exit.)
260          */
261         if (echan->edesc) {
262                 int cyclic = echan->edesc->cyclic;
263 
264                 /*
265                  * free the running request descriptor
266                  * since it is not in any of the vdesc lists
267                  */
268                 edma_desc_free(&echan->edesc->vdesc);
269 
270                 echan->edesc = NULL;
271                 edma_stop(echan->ch_num);
272                 /* Move the cyclic channel back to default queue */
273                 if (cyclic)
274                         edma_assign_channel_eventq(echan->ch_num,
275                                                    EVENTQ_DEFAULT);
276         }
277 
278         vchan_get_all_descriptors(&echan->vchan, &head);
279         spin_unlock_irqrestore(&echan->vchan.lock, flags);
280         vchan_dma_desc_free_list(&echan->vchan, &head);
281 
282         return 0;
283 }
284 
285 static int edma_slave_config(struct dma_chan *chan,
286         struct dma_slave_config *cfg)
287 {
288         struct edma_chan *echan = to_edma_chan(chan);
289 
290         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
291             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
292                 return -EINVAL;
293 
294         memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
295 
296         return 0;
297 }
298 
299 static int edma_dma_pause(struct dma_chan *chan)
300 {
301         struct edma_chan *echan = to_edma_chan(chan);
302 
303         if (!echan->edesc)
304                 return -EINVAL;
305 
306         edma_pause(echan->ch_num);
307         return 0;
308 }
309 
310 static int edma_dma_resume(struct dma_chan *chan)
311 {
312         struct edma_chan *echan = to_edma_chan(chan);
313 
314         edma_resume(echan->ch_num);
315         return 0;
316 }
317 
318 /*
319  * A PaRAM set configuration abstraction used by other modes
320  * @chan: Channel who's PaRAM set we're configuring
321  * @pset: PaRAM set to initialize and setup.
322  * @src_addr: Source address of the DMA
323  * @dst_addr: Destination address of the DMA
324  * @burst: In units of dev_width, how much to send
325  * @dev_width: How much is the dev_width
326  * @dma_length: Total length of the DMA transfer
327  * @direction: Direction of the transfer
328  */
329 static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
330         dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
331         enum dma_slave_buswidth dev_width, unsigned int dma_length,
332         enum dma_transfer_direction direction)
333 {
334         struct edma_chan *echan = to_edma_chan(chan);
335         struct device *dev = chan->device->dev;
336         struct edmacc_param *param = &epset->param;
337         int acnt, bcnt, ccnt, cidx;
338         int src_bidx, dst_bidx, src_cidx, dst_cidx;
339         int absync;
340 
341         acnt = dev_width;
342 
343         /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
344         if (!burst)
345                 burst = 1;
346         /*
347          * If the maxburst is equal to the fifo width, use
348          * A-synced transfers. This allows for large contiguous
349          * buffer transfers using only one PaRAM set.
350          */
351         if (burst == 1) {
352                 /*
353                  * For the A-sync case, bcnt and ccnt are the remainder
354                  * and quotient respectively of the division of:
355                  * (dma_length / acnt) by (SZ_64K -1). This is so
356                  * that in case bcnt over flows, we have ccnt to use.
357                  * Note: In A-sync tranfer only, bcntrld is used, but it
358                  * only applies for sg_dma_len(sg) >= SZ_64K.
359                  * In this case, the best way adopted is- bccnt for the
360                  * first frame will be the remainder below. Then for
361                  * every successive frame, bcnt will be SZ_64K-1. This
362                  * is assured as bcntrld = 0xffff in end of function.
363                  */
364                 absync = false;
365                 ccnt = dma_length / acnt / (SZ_64K - 1);
366                 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
367                 /*
368                  * If bcnt is non-zero, we have a remainder and hence an
369                  * extra frame to transfer, so increment ccnt.
370                  */
371                 if (bcnt)
372                         ccnt++;
373                 else
374                         bcnt = SZ_64K - 1;
375                 cidx = acnt;
376         } else {
377                 /*
378                  * If maxburst is greater than the fifo address_width,
379                  * use AB-synced transfers where A count is the fifo
380                  * address_width and B count is the maxburst. In this
381                  * case, we are limited to transfers of C count frames
382                  * of (address_width * maxburst) where C count is limited
383                  * to SZ_64K-1. This places an upper bound on the length
384                  * of an SG segment that can be handled.
385                  */
386                 absync = true;
387                 bcnt = burst;
388                 ccnt = dma_length / (acnt * bcnt);
389                 if (ccnt > (SZ_64K - 1)) {
390                         dev_err(dev, "Exceeded max SG segment size\n");
391                         return -EINVAL;
392                 }
393                 cidx = acnt * bcnt;
394         }
395 
396         epset->len = dma_length;
397 
398         if (direction == DMA_MEM_TO_DEV) {
399                 src_bidx = acnt;
400                 src_cidx = cidx;
401                 dst_bidx = 0;
402                 dst_cidx = 0;
403                 epset->addr = src_addr;
404         } else if (direction == DMA_DEV_TO_MEM)  {
405                 src_bidx = 0;
406                 src_cidx = 0;
407                 dst_bidx = acnt;
408                 dst_cidx = cidx;
409                 epset->addr = dst_addr;
410         } else if (direction == DMA_MEM_TO_MEM)  {
411                 src_bidx = acnt;
412                 src_cidx = cidx;
413                 dst_bidx = acnt;
414                 dst_cidx = cidx;
415         } else {
416                 dev_err(dev, "%s: direction not implemented yet\n", __func__);
417                 return -EINVAL;
418         }
419 
420         param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
421         /* Configure A or AB synchronized transfers */
422         if (absync)
423                 param->opt |= SYNCDIM;
424 
425         param->src = src_addr;
426         param->dst = dst_addr;
427 
428         param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
429         param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
430 
431         param->a_b_cnt = bcnt << 16 | acnt;
432         param->ccnt = ccnt;
433         /*
434          * Only time when (bcntrld) auto reload is required is for
435          * A-sync case, and in this case, a requirement of reload value
436          * of SZ_64K-1 only is assured. 'link' is initially set to NULL
437          * and then later will be populated by edma_execute.
438          */
439         param->link_bcntrld = 0xffffffff;
440         return absync;
441 }
442 
443 static struct dma_async_tx_descriptor *edma_prep_slave_sg(
444         struct dma_chan *chan, struct scatterlist *sgl,
445         unsigned int sg_len, enum dma_transfer_direction direction,
446         unsigned long tx_flags, void *context)
447 {
448         struct edma_chan *echan = to_edma_chan(chan);
449         struct device *dev = chan->device->dev;
450         struct edma_desc *edesc;
451         dma_addr_t src_addr = 0, dst_addr = 0;
452         enum dma_slave_buswidth dev_width;
453         u32 burst;
454         struct scatterlist *sg;
455         int i, nslots, ret;
456 
457         if (unlikely(!echan || !sgl || !sg_len))
458                 return NULL;
459 
460         if (direction == DMA_DEV_TO_MEM) {
461                 src_addr = echan->cfg.src_addr;
462                 dev_width = echan->cfg.src_addr_width;
463                 burst = echan->cfg.src_maxburst;
464         } else if (direction == DMA_MEM_TO_DEV) {
465                 dst_addr = echan->cfg.dst_addr;
466                 dev_width = echan->cfg.dst_addr_width;
467                 burst = echan->cfg.dst_maxburst;
468         } else {
469                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
470                 return NULL;
471         }
472 
473         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
474                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
475                 return NULL;
476         }
477 
478         edesc = kzalloc(sizeof(*edesc) + sg_len *
479                 sizeof(edesc->pset[0]), GFP_ATOMIC);
480         if (!edesc) {
481                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
482                 return NULL;
483         }
484 
485         edesc->pset_nr = sg_len;
486         edesc->residue = 0;
487         edesc->direction = direction;
488         edesc->echan = echan;
489 
490         /* Allocate a PaRAM slot, if needed */
491         nslots = min_t(unsigned, MAX_NR_SG, sg_len);
492 
493         for (i = 0; i < nslots; i++) {
494                 if (echan->slot[i] < 0) {
495                         echan->slot[i] =
496                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
497                                                 EDMA_SLOT_ANY);
498                         if (echan->slot[i] < 0) {
499                                 kfree(edesc);
500                                 dev_err(dev, "%s: Failed to allocate slot\n",
501                                         __func__);
502                                 return NULL;
503                         }
504                 }
505         }
506 
507         /* Configure PaRAM sets for each SG */
508         for_each_sg(sgl, sg, sg_len, i) {
509                 /* Get address for each SG */
510                 if (direction == DMA_DEV_TO_MEM)
511                         dst_addr = sg_dma_address(sg);
512                 else
513                         src_addr = sg_dma_address(sg);
514 
515                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
516                                        dst_addr, burst, dev_width,
517                                        sg_dma_len(sg), direction);
518                 if (ret < 0) {
519                         kfree(edesc);
520                         return NULL;
521                 }
522 
523                 edesc->absync = ret;
524                 edesc->residue += sg_dma_len(sg);
525 
526                 /* If this is the last in a current SG set of transactions,
527                    enable interrupts so that next set is processed */
528                 if (!((i+1) % MAX_NR_SG))
529                         edesc->pset[i].param.opt |= TCINTEN;
530 
531                 /* If this is the last set, enable completion interrupt flag */
532                 if (i == sg_len - 1)
533                         edesc->pset[i].param.opt |= TCINTEN;
534         }
535         edesc->residue_stat = edesc->residue;
536 
537         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
538 }
539 
540 static struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
541         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
542         size_t len, unsigned long tx_flags)
543 {
544         int ret;
545         struct edma_desc *edesc;
546         struct device *dev = chan->device->dev;
547         struct edma_chan *echan = to_edma_chan(chan);
548 
549         if (unlikely(!echan || !len))
550                 return NULL;
551 
552         edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC);
553         if (!edesc) {
554                 dev_dbg(dev, "Failed to allocate a descriptor\n");
555                 return NULL;
556         }
557 
558         edesc->pset_nr = 1;
559 
560         ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
561                                DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM);
562         if (ret < 0)
563                 return NULL;
564 
565         edesc->absync = ret;
566 
567         /*
568          * Enable intermediate transfer chaining to re-trigger channel
569          * on completion of every TR, and enable transfer-completion
570          * interrupt on completion of the whole transfer.
571          */
572         edesc->pset[0].param.opt |= ITCCHEN;
573         edesc->pset[0].param.opt |= TCINTEN;
574 
575         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
576 }
577 
578 static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
579         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
580         size_t period_len, enum dma_transfer_direction direction,
581         unsigned long tx_flags)
582 {
583         struct edma_chan *echan = to_edma_chan(chan);
584         struct device *dev = chan->device->dev;
585         struct edma_desc *edesc;
586         dma_addr_t src_addr, dst_addr;
587         enum dma_slave_buswidth dev_width;
588         u32 burst;
589         int i, ret, nslots;
590 
591         if (unlikely(!echan || !buf_len || !period_len))
592                 return NULL;
593 
594         if (direction == DMA_DEV_TO_MEM) {
595                 src_addr = echan->cfg.src_addr;
596                 dst_addr = buf_addr;
597                 dev_width = echan->cfg.src_addr_width;
598                 burst = echan->cfg.src_maxburst;
599         } else if (direction == DMA_MEM_TO_DEV) {
600                 src_addr = buf_addr;
601                 dst_addr = echan->cfg.dst_addr;
602                 dev_width = echan->cfg.dst_addr_width;
603                 burst = echan->cfg.dst_maxburst;
604         } else {
605                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
606                 return NULL;
607         }
608 
609         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
610                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
611                 return NULL;
612         }
613 
614         if (unlikely(buf_len % period_len)) {
615                 dev_err(dev, "Period should be multiple of Buffer length\n");
616                 return NULL;
617         }
618 
619         nslots = (buf_len / period_len) + 1;
620 
621         /*
622          * Cyclic DMA users such as audio cannot tolerate delays introduced
623          * by cases where the number of periods is more than the maximum
624          * number of SGs the EDMA driver can handle at a time. For DMA types
625          * such as Slave SGs, such delays are tolerable and synchronized,
626          * but the synchronization is difficult to achieve with Cyclic and
627          * cannot be guaranteed, so we error out early.
628          */
629         if (nslots > MAX_NR_SG)
630                 return NULL;
631 
632         edesc = kzalloc(sizeof(*edesc) + nslots *
633                 sizeof(edesc->pset[0]), GFP_ATOMIC);
634         if (!edesc) {
635                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
636                 return NULL;
637         }
638 
639         edesc->cyclic = 1;
640         edesc->pset_nr = nslots;
641         edesc->residue = edesc->residue_stat = buf_len;
642         edesc->direction = direction;
643         edesc->echan = echan;
644 
645         dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
646                 __func__, echan->ch_num, nslots, period_len, buf_len);
647 
648         for (i = 0; i < nslots; i++) {
649                 /* Allocate a PaRAM slot, if needed */
650                 if (echan->slot[i] < 0) {
651                         echan->slot[i] =
652                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
653                                                 EDMA_SLOT_ANY);
654                         if (echan->slot[i] < 0) {
655                                 kfree(edesc);
656                                 dev_err(dev, "%s: Failed to allocate slot\n",
657                                         __func__);
658                                 return NULL;
659                         }
660                 }
661 
662                 if (i == nslots - 1) {
663                         memcpy(&edesc->pset[i], &edesc->pset[0],
664                                sizeof(edesc->pset[0]));
665                         break;
666                 }
667 
668                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
669                                        dst_addr, burst, dev_width, period_len,
670                                        direction);
671                 if (ret < 0) {
672                         kfree(edesc);
673                         return NULL;
674                 }
675 
676                 if (direction == DMA_DEV_TO_MEM)
677                         dst_addr += period_len;
678                 else
679                         src_addr += period_len;
680 
681                 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
682                 dev_vdbg(dev,
683                         "\n pset[%d]:\n"
684                         "  chnum\t%d\n"
685                         "  slot\t%d\n"
686                         "  opt\t%08x\n"
687                         "  src\t%08x\n"
688                         "  dst\t%08x\n"
689                         "  abcnt\t%08x\n"
690                         "  ccnt\t%08x\n"
691                         "  bidx\t%08x\n"
692                         "  cidx\t%08x\n"
693                         "  lkrld\t%08x\n",
694                         i, echan->ch_num, echan->slot[i],
695                         edesc->pset[i].param.opt,
696                         edesc->pset[i].param.src,
697                         edesc->pset[i].param.dst,
698                         edesc->pset[i].param.a_b_cnt,
699                         edesc->pset[i].param.ccnt,
700                         edesc->pset[i].param.src_dst_bidx,
701                         edesc->pset[i].param.src_dst_cidx,
702                         edesc->pset[i].param.link_bcntrld);
703 
704                 edesc->absync = ret;
705 
706                 /*
707                  * Enable period interrupt only if it is requested
708                  */
709                 if (tx_flags & DMA_PREP_INTERRUPT)
710                         edesc->pset[i].param.opt |= TCINTEN;
711         }
712 
713         /* Place the cyclic channel to highest priority queue */
714         edma_assign_channel_eventq(echan->ch_num, EVENTQ_0);
715 
716         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
717 }
718 
719 static void edma_callback(unsigned ch_num, u16 ch_status, void *data)
720 {
721         struct edma_chan *echan = data;
722         struct device *dev = echan->vchan.chan.device->dev;
723         struct edma_desc *edesc;
724         struct edmacc_param p;
725 
726         edesc = echan->edesc;
727 
728         /* Pause the channel for non-cyclic */
729         if (!edesc || (edesc && !edesc->cyclic))
730                 edma_pause(echan->ch_num);
731 
732         switch (ch_status) {
733         case EDMA_DMA_COMPLETE:
734                 spin_lock(&echan->vchan.lock);
735 
736                 if (edesc) {
737                         if (edesc->cyclic) {
738                                 vchan_cyclic_callback(&edesc->vdesc);
739                         } else if (edesc->processed == edesc->pset_nr) {
740                                 dev_dbg(dev, "Transfer complete, stopping channel %d\n", ch_num);
741                                 edesc->residue = 0;
742                                 edma_stop(echan->ch_num);
743                                 vchan_cookie_complete(&edesc->vdesc);
744                                 edma_execute(echan);
745                         } else {
746                                 dev_dbg(dev, "Intermediate transfer complete on channel %d\n", ch_num);
747 
748                                 /* Update statistics for tx_status */
749                                 edesc->residue -= edesc->sg_len;
750                                 edesc->residue_stat = edesc->residue;
751                                 edesc->processed_stat = edesc->processed;
752 
753                                 edma_execute(echan);
754                         }
755                 }
756 
757                 spin_unlock(&echan->vchan.lock);
758 
759                 break;
760         case EDMA_DMA_CC_ERROR:
761                 spin_lock(&echan->vchan.lock);
762 
763                 edma_read_slot(EDMA_CHAN_SLOT(echan->slot[0]), &p);
764 
765                 /*
766                  * Issue later based on missed flag which will be sure
767                  * to happen as:
768                  * (1) we finished transmitting an intermediate slot and
769                  *     edma_execute is coming up.
770                  * (2) or we finished current transfer and issue will
771                  *     call edma_execute.
772                  *
773                  * Important note: issuing can be dangerous here and
774                  * lead to some nasty recursion when we are in a NULL
775                  * slot. So we avoid doing so and set the missed flag.
776                  */
777                 if (p.a_b_cnt == 0 && p.ccnt == 0) {
778                         dev_dbg(dev, "Error occurred, looks like slot is null, just setting miss\n");
779                         echan->missed = 1;
780                 } else {
781                         /*
782                          * The slot is already programmed but the event got
783                          * missed, so its safe to issue it here.
784                          */
785                         dev_dbg(dev, "Error occurred but slot is non-null, TRIGGERING\n");
786                         edma_clean_channel(echan->ch_num);
787                         edma_stop(echan->ch_num);
788                         edma_start(echan->ch_num);
789                         edma_trigger_channel(echan->ch_num);
790                 }
791 
792                 spin_unlock(&echan->vchan.lock);
793 
794                 break;
795         default:
796                 break;
797         }
798 }
799 
800 /* Alloc channel resources */
801 static int edma_alloc_chan_resources(struct dma_chan *chan)
802 {
803         struct edma_chan *echan = to_edma_chan(chan);
804         struct device *dev = chan->device->dev;
805         int ret;
806         int a_ch_num;
807         LIST_HEAD(descs);
808 
809         a_ch_num = edma_alloc_channel(echan->ch_num, edma_callback,
810                                         echan, EVENTQ_DEFAULT);
811 
812         if (a_ch_num < 0) {
813                 ret = -ENODEV;
814                 goto err_no_chan;
815         }
816 
817         if (a_ch_num != echan->ch_num) {
818                 dev_err(dev, "failed to allocate requested channel %u:%u\n",
819                         EDMA_CTLR(echan->ch_num),
820                         EDMA_CHAN_SLOT(echan->ch_num));
821                 ret = -ENODEV;
822                 goto err_wrong_chan;
823         }
824 
825         echan->alloced = true;
826         echan->slot[0] = echan->ch_num;
827 
828         dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num,
829                 EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num));
830 
831         return 0;
832 
833 err_wrong_chan:
834         edma_free_channel(a_ch_num);
835 err_no_chan:
836         return ret;
837 }
838 
839 /* Free channel resources */
840 static void edma_free_chan_resources(struct dma_chan *chan)
841 {
842         struct edma_chan *echan = to_edma_chan(chan);
843         struct device *dev = chan->device->dev;
844         int i;
845 
846         /* Terminate transfers */
847         edma_stop(echan->ch_num);
848 
849         vchan_free_chan_resources(&echan->vchan);
850 
851         /* Free EDMA PaRAM slots */
852         for (i = 1; i < EDMA_MAX_SLOTS; i++) {
853                 if (echan->slot[i] >= 0) {
854                         edma_free_slot(echan->slot[i]);
855                         echan->slot[i] = -1;
856                 }
857         }
858 
859         /* Free EDMA channel */
860         if (echan->alloced) {
861                 edma_free_channel(echan->ch_num);
862                 echan->alloced = false;
863         }
864 
865         dev_dbg(dev, "freeing channel for %u\n", echan->ch_num);
866 }
867 
868 /* Send pending descriptor to hardware */
869 static void edma_issue_pending(struct dma_chan *chan)
870 {
871         struct edma_chan *echan = to_edma_chan(chan);
872         unsigned long flags;
873 
874         spin_lock_irqsave(&echan->vchan.lock, flags);
875         if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
876                 edma_execute(echan);
877         spin_unlock_irqrestore(&echan->vchan.lock, flags);
878 }
879 
880 static u32 edma_residue(struct edma_desc *edesc)
881 {
882         bool dst = edesc->direction == DMA_DEV_TO_MEM;
883         struct edma_pset *pset = edesc->pset;
884         dma_addr_t done, pos;
885         int i;
886 
887         /*
888          * We always read the dst/src position from the first RamPar
889          * pset. That's the one which is active now.
890          */
891         pos = edma_get_position(edesc->echan->slot[0], dst);
892 
893         /*
894          * Cyclic is simple. Just subtract pset[0].addr from pos.
895          *
896          * We never update edesc->residue in the cyclic case, so we
897          * can tell the remaining room to the end of the circular
898          * buffer.
899          */
900         if (edesc->cyclic) {
901                 done = pos - pset->addr;
902                 edesc->residue_stat = edesc->residue - done;
903                 return edesc->residue_stat;
904         }
905 
906         /*
907          * For SG operation we catch up with the last processed
908          * status.
909          */
910         pset += edesc->processed_stat;
911 
912         for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
913                 /*
914                  * If we are inside this pset address range, we know
915                  * this is the active one. Get the current delta and
916                  * stop walking the psets.
917                  */
918                 if (pos >= pset->addr && pos < pset->addr + pset->len)
919                         return edesc->residue_stat - (pos - pset->addr);
920 
921                 /* Otherwise mark it done and update residue_stat. */
922                 edesc->processed_stat++;
923                 edesc->residue_stat -= pset->len;
924         }
925         return edesc->residue_stat;
926 }
927 
928 /* Check request completion status */
929 static enum dma_status edma_tx_status(struct dma_chan *chan,
930                                       dma_cookie_t cookie,
931                                       struct dma_tx_state *txstate)
932 {
933         struct edma_chan *echan = to_edma_chan(chan);
934         struct virt_dma_desc *vdesc;
935         enum dma_status ret;
936         unsigned long flags;
937 
938         ret = dma_cookie_status(chan, cookie, txstate);
939         if (ret == DMA_COMPLETE || !txstate)
940                 return ret;
941 
942         spin_lock_irqsave(&echan->vchan.lock, flags);
943         if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
944                 txstate->residue = edma_residue(echan->edesc);
945         else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
946                 txstate->residue = to_edma_desc(&vdesc->tx)->residue;
947         spin_unlock_irqrestore(&echan->vchan.lock, flags);
948 
949         return ret;
950 }
951 
952 static void __init edma_chan_init(struct edma_cc *ecc,
953                                   struct dma_device *dma,
954                                   struct edma_chan *echans)
955 {
956         int i, j;
957 
958         for (i = 0; i < EDMA_CHANS; i++) {
959                 struct edma_chan *echan = &echans[i];
960                 echan->ch_num = EDMA_CTLR_CHAN(ecc->ctlr, i);
961                 echan->ecc = ecc;
962                 echan->vchan.desc_free = edma_desc_free;
963 
964                 vchan_init(&echan->vchan, dma);
965 
966                 INIT_LIST_HEAD(&echan->node);
967                 for (j = 0; j < EDMA_MAX_SLOTS; j++)
968                         echan->slot[j] = -1;
969         }
970 }
971 
972 #define EDMA_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
973                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
974                                  BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
975                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
976 
977 static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
978                           struct device *dev)
979 {
980         dma->device_prep_slave_sg = edma_prep_slave_sg;
981         dma->device_prep_dma_cyclic = edma_prep_dma_cyclic;
982         dma->device_prep_dma_memcpy = edma_prep_dma_memcpy;
983         dma->device_alloc_chan_resources = edma_alloc_chan_resources;
984         dma->device_free_chan_resources = edma_free_chan_resources;
985         dma->device_issue_pending = edma_issue_pending;
986         dma->device_tx_status = edma_tx_status;
987         dma->device_config = edma_slave_config;
988         dma->device_pause = edma_dma_pause;
989         dma->device_resume = edma_dma_resume;
990         dma->device_terminate_all = edma_terminate_all;
991 
992         dma->src_addr_widths = EDMA_DMA_BUSWIDTHS;
993         dma->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
994         dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
995         dma->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
996 
997         dma->dev = dev;
998 
999         /*
1000          * code using dma memcpy must make sure alignment of
1001          * length is at dma->copy_align boundary.
1002          */
1003         dma->copy_align = DMA_SLAVE_BUSWIDTH_4_BYTES;
1004 
1005         INIT_LIST_HEAD(&dma->channels);
1006 }
1007 
1008 static int edma_probe(struct platform_device *pdev)
1009 {
1010         struct edma_cc *ecc;
1011         int ret;
1012 
1013         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1014         if (ret)
1015                 return ret;
1016 
1017         ecc = devm_kzalloc(&pdev->dev, sizeof(*ecc), GFP_KERNEL);
1018         if (!ecc) {
1019                 dev_err(&pdev->dev, "Can't allocate controller\n");
1020                 return -ENOMEM;
1021         }
1022 
1023         ecc->ctlr = pdev->id;
1024         ecc->dummy_slot = edma_alloc_slot(ecc->ctlr, EDMA_SLOT_ANY);
1025         if (ecc->dummy_slot < 0) {
1026                 dev_err(&pdev->dev, "Can't allocate PaRAM dummy slot\n");
1027                 return ecc->dummy_slot;
1028         }
1029 
1030         dma_cap_zero(ecc->dma_slave.cap_mask);
1031         dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask);
1032         dma_cap_set(DMA_CYCLIC, ecc->dma_slave.cap_mask);
1033         dma_cap_set(DMA_MEMCPY, ecc->dma_slave.cap_mask);
1034 
1035         edma_dma_init(ecc, &ecc->dma_slave, &pdev->dev);
1036 
1037         edma_chan_init(ecc, &ecc->dma_slave, ecc->slave_chans);
1038 
1039         ret = dma_async_device_register(&ecc->dma_slave);
1040         if (ret)
1041                 goto err_reg1;
1042 
1043         platform_set_drvdata(pdev, ecc);
1044 
1045         dev_info(&pdev->dev, "TI EDMA DMA engine driver\n");
1046 
1047         return 0;
1048 
1049 err_reg1:
1050         edma_free_slot(ecc->dummy_slot);
1051         return ret;
1052 }
1053 
1054 static int edma_remove(struct platform_device *pdev)
1055 {
1056         struct device *dev = &pdev->dev;
1057         struct edma_cc *ecc = dev_get_drvdata(dev);
1058 
1059         dma_async_device_unregister(&ecc->dma_slave);
1060         edma_free_slot(ecc->dummy_slot);
1061 
1062         return 0;
1063 }
1064 
1065 static struct platform_driver edma_driver = {
1066         .probe          = edma_probe,
1067         .remove         = edma_remove,
1068         .driver = {
1069                 .name = "edma-dma-engine",
1070         },
1071 };
1072 
1073 bool edma_filter_fn(struct dma_chan *chan, void *param)
1074 {
1075         if (chan->device->dev->driver == &edma_driver.driver) {
1076                 struct edma_chan *echan = to_edma_chan(chan);
1077                 unsigned ch_req = *(unsigned *)param;
1078                 return ch_req == echan->ch_num;
1079         }
1080         return false;
1081 }
1082 EXPORT_SYMBOL(edma_filter_fn);
1083 
1084 static int edma_init(void)
1085 {
1086         return platform_driver_register(&edma_driver);
1087 }
1088 subsys_initcall(edma_init);
1089 
1090 static void __exit edma_exit(void)
1091 {
1092         platform_driver_unregister(&edma_driver);
1093 }
1094 module_exit(edma_exit);
1095 
1096 MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
1097 MODULE_DESCRIPTION("TI EDMA DMA engine driver");
1098 MODULE_LICENSE("GPL v2");
1099 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us