Version:  2.0.40 2.2.26 2.4.37 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1

Linux/drivers/dma/edma.c

  1 /*
  2  * TI EDMA DMA engine driver
  3  *
  4  * Copyright 2012 Texas Instruments
  5  *
  6  * This program is free software; you can redistribute it and/or
  7  * modify it under the terms of the GNU General Public License as
  8  * published by the Free Software Foundation version 2.
  9  *
 10  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
 11  * kind, whether express or implied; without even the implied warranty
 12  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13  * GNU General Public License for more details.
 14  */
 15 
 16 #include <linux/dmaengine.h>
 17 #include <linux/dma-mapping.h>
 18 #include <linux/edma.h>
 19 #include <linux/err.h>
 20 #include <linux/init.h>
 21 #include <linux/interrupt.h>
 22 #include <linux/list.h>
 23 #include <linux/module.h>
 24 #include <linux/platform_device.h>
 25 #include <linux/slab.h>
 26 #include <linux/spinlock.h>
 27 #include <linux/of.h>
 28 
 29 #include <linux/platform_data/edma.h>
 30 
 31 #include "dmaengine.h"
 32 #include "virt-dma.h"
 33 
 34 /*
 35  * This will go away when the private EDMA API is folded
 36  * into this driver and the platform device(s) are
 37  * instantiated in the arch code. We can only get away
 38  * with this simplification because DA8XX may not be built
 39  * in the same kernel image with other DaVinci parts. This
 40  * avoids having to sprinkle dmaengine driver platform devices
 41  * and data throughout all the existing board files.
 42  */
 43 #ifdef CONFIG_ARCH_DAVINCI_DA8XX
 44 #define EDMA_CTLRS      2
 45 #define EDMA_CHANS      32
 46 #else
 47 #define EDMA_CTLRS      1
 48 #define EDMA_CHANS      64
 49 #endif /* CONFIG_ARCH_DAVINCI_DA8XX */
 50 
 51 /*
 52  * Max of 20 segments per channel to conserve PaRAM slots
 53  * Also note that MAX_NR_SG should be atleast the no.of periods
 54  * that are required for ASoC, otherwise DMA prep calls will
 55  * fail. Today davinci-pcm is the only user of this driver and
 56  * requires atleast 17 slots, so we setup the default to 20.
 57  */
 58 #define MAX_NR_SG               20
 59 #define EDMA_MAX_SLOTS          MAX_NR_SG
 60 #define EDMA_DESCRIPTORS        16
 61 
 62 struct edma_pset {
 63         u32                             len;
 64         dma_addr_t                      addr;
 65         struct edmacc_param             param;
 66 };
 67 
 68 struct edma_desc {
 69         struct virt_dma_desc            vdesc;
 70         struct list_head                node;
 71         enum dma_transfer_direction     direction;
 72         int                             cyclic;
 73         int                             absync;
 74         int                             pset_nr;
 75         struct edma_chan                *echan;
 76         int                             processed;
 77 
 78         /*
 79          * The following 4 elements are used for residue accounting.
 80          *
 81          * - processed_stat: the number of SG elements we have traversed
 82          * so far to cover accounting. This is updated directly to processed
 83          * during edma_callback and is always <= processed, because processed
 84          * refers to the number of pending transfer (programmed to EDMA
 85          * controller), where as processed_stat tracks number of transfers
 86          * accounted for so far.
 87          *
 88          * - residue: The amount of bytes we have left to transfer for this desc
 89          *
 90          * - residue_stat: The residue in bytes of data we have covered
 91          * so far for accounting. This is updated directly to residue
 92          * during callbacks to keep it current.
 93          *
 94          * - sg_len: Tracks the length of the current intermediate transfer,
 95          * this is required to update the residue during intermediate transfer
 96          * completion callback.
 97          */
 98         int                             processed_stat;
 99         u32                             sg_len;
100         u32                             residue;
101         u32                             residue_stat;
102 
103         struct edma_pset                pset[0];
104 };
105 
106 struct edma_cc;
107 
108 struct edma_chan {
109         struct virt_dma_chan            vchan;
110         struct list_head                node;
111         struct edma_desc                *edesc;
112         struct edma_cc                  *ecc;
113         int                             ch_num;
114         bool                            alloced;
115         int                             slot[EDMA_MAX_SLOTS];
116         int                             missed;
117         struct dma_slave_config         cfg;
118 };
119 
120 struct edma_cc {
121         int                             ctlr;
122         struct dma_device               dma_slave;
123         struct edma_chan                slave_chans[EDMA_CHANS];
124         int                             num_slave_chans;
125         int                             dummy_slot;
126 };
127 
128 static inline struct edma_cc *to_edma_cc(struct dma_device *d)
129 {
130         return container_of(d, struct edma_cc, dma_slave);
131 }
132 
133 static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
134 {
135         return container_of(c, struct edma_chan, vchan.chan);
136 }
137 
138 static inline struct edma_desc
139 *to_edma_desc(struct dma_async_tx_descriptor *tx)
140 {
141         return container_of(tx, struct edma_desc, vdesc.tx);
142 }
143 
144 static void edma_desc_free(struct virt_dma_desc *vdesc)
145 {
146         kfree(container_of(vdesc, struct edma_desc, vdesc));
147 }
148 
149 /* Dispatch a queued descriptor to the controller (caller holds lock) */
150 static void edma_execute(struct edma_chan *echan)
151 {
152         struct virt_dma_desc *vdesc;
153         struct edma_desc *edesc;
154         struct device *dev = echan->vchan.chan.device->dev;
155         int i, j, left, nslots;
156 
157         /* If either we processed all psets or we're still not started */
158         if (!echan->edesc ||
159             echan->edesc->pset_nr == echan->edesc->processed) {
160                 /* Get next vdesc */
161                 vdesc = vchan_next_desc(&echan->vchan);
162                 if (!vdesc) {
163                         echan->edesc = NULL;
164                         return;
165                 }
166                 list_del(&vdesc->node);
167                 echan->edesc = to_edma_desc(&vdesc->tx);
168         }
169 
170         edesc = echan->edesc;
171 
172         /* Find out how many left */
173         left = edesc->pset_nr - edesc->processed;
174         nslots = min(MAX_NR_SG, left);
175         edesc->sg_len = 0;
176 
177         /* Write descriptor PaRAM set(s) */
178         for (i = 0; i < nslots; i++) {
179                 j = i + edesc->processed;
180                 edma_write_slot(echan->slot[i], &edesc->pset[j].param);
181                 edesc->sg_len += edesc->pset[j].len;
182                 dev_vdbg(echan->vchan.chan.device->dev,
183                         "\n pset[%d]:\n"
184                         "  chnum\t%d\n"
185                         "  slot\t%d\n"
186                         "  opt\t%08x\n"
187                         "  src\t%08x\n"
188                         "  dst\t%08x\n"
189                         "  abcnt\t%08x\n"
190                         "  ccnt\t%08x\n"
191                         "  bidx\t%08x\n"
192                         "  cidx\t%08x\n"
193                         "  lkrld\t%08x\n",
194                         j, echan->ch_num, echan->slot[i],
195                         edesc->pset[j].param.opt,
196                         edesc->pset[j].param.src,
197                         edesc->pset[j].param.dst,
198                         edesc->pset[j].param.a_b_cnt,
199                         edesc->pset[j].param.ccnt,
200                         edesc->pset[j].param.src_dst_bidx,
201                         edesc->pset[j].param.src_dst_cidx,
202                         edesc->pset[j].param.link_bcntrld);
203                 /* Link to the previous slot if not the last set */
204                 if (i != (nslots - 1))
205                         edma_link(echan->slot[i], echan->slot[i+1]);
206         }
207 
208         edesc->processed += nslots;
209 
210         /*
211          * If this is either the last set in a set of SG-list transactions
212          * then setup a link to the dummy slot, this results in all future
213          * events being absorbed and that's OK because we're done
214          */
215         if (edesc->processed == edesc->pset_nr) {
216                 if (edesc->cyclic)
217                         edma_link(echan->slot[nslots-1], echan->slot[1]);
218                 else
219                         edma_link(echan->slot[nslots-1],
220                                   echan->ecc->dummy_slot);
221         }
222 
223         if (edesc->processed <= MAX_NR_SG) {
224                 dev_dbg(dev, "first transfer starting on channel %d\n",
225                         echan->ch_num);
226                 edma_start(echan->ch_num);
227         } else {
228                 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
229                         echan->ch_num, edesc->processed);
230                 edma_resume(echan->ch_num);
231         }
232 
233         /*
234          * This happens due to setup times between intermediate transfers
235          * in long SG lists which have to be broken up into transfers of
236          * MAX_NR_SG
237          */
238         if (echan->missed) {
239                 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
240                 edma_clean_channel(echan->ch_num);
241                 edma_stop(echan->ch_num);
242                 edma_start(echan->ch_num);
243                 edma_trigger_channel(echan->ch_num);
244                 echan->missed = 0;
245         }
246 }
247 
248 static int edma_terminate_all(struct dma_chan *chan)
249 {
250         struct edma_chan *echan = to_edma_chan(chan);
251         unsigned long flags;
252         LIST_HEAD(head);
253 
254         spin_lock_irqsave(&echan->vchan.lock, flags);
255 
256         /*
257          * Stop DMA activity: we assume the callback will not be called
258          * after edma_dma() returns (even if it does, it will see
259          * echan->edesc is NULL and exit.)
260          */
261         if (echan->edesc) {
262                 int cyclic = echan->edesc->cyclic;
263 
264                 /*
265                  * free the running request descriptor
266                  * since it is not in any of the vdesc lists
267                  */
268                 edma_desc_free(&echan->edesc->vdesc);
269 
270                 echan->edesc = NULL;
271                 edma_stop(echan->ch_num);
272                 /* Move the cyclic channel back to default queue */
273                 if (cyclic)
274                         edma_assign_channel_eventq(echan->ch_num,
275                                                    EVENTQ_DEFAULT);
276         }
277 
278         vchan_get_all_descriptors(&echan->vchan, &head);
279         spin_unlock_irqrestore(&echan->vchan.lock, flags);
280         vchan_dma_desc_free_list(&echan->vchan, &head);
281 
282         return 0;
283 }
284 
285 static int edma_slave_config(struct dma_chan *chan,
286         struct dma_slave_config *cfg)
287 {
288         struct edma_chan *echan = to_edma_chan(chan);
289 
290         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
291             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
292                 return -EINVAL;
293 
294         memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
295 
296         return 0;
297 }
298 
299 static int edma_dma_pause(struct dma_chan *chan)
300 {
301         struct edma_chan *echan = to_edma_chan(chan);
302 
303         /* Pause/Resume only allowed with cyclic mode */
304         if (!echan->edesc || !echan->edesc->cyclic)
305                 return -EINVAL;
306 
307         edma_pause(echan->ch_num);
308         return 0;
309 }
310 
311 static int edma_dma_resume(struct dma_chan *chan)
312 {
313         struct edma_chan *echan = to_edma_chan(chan);
314 
315         /* Pause/Resume only allowed with cyclic mode */
316         if (!echan->edesc->cyclic)
317                 return -EINVAL;
318 
319         edma_resume(echan->ch_num);
320         return 0;
321 }
322 
323 /*
324  * A PaRAM set configuration abstraction used by other modes
325  * @chan: Channel who's PaRAM set we're configuring
326  * @pset: PaRAM set to initialize and setup.
327  * @src_addr: Source address of the DMA
328  * @dst_addr: Destination address of the DMA
329  * @burst: In units of dev_width, how much to send
330  * @dev_width: How much is the dev_width
331  * @dma_length: Total length of the DMA transfer
332  * @direction: Direction of the transfer
333  */
334 static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
335         dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
336         enum dma_slave_buswidth dev_width, unsigned int dma_length,
337         enum dma_transfer_direction direction)
338 {
339         struct edma_chan *echan = to_edma_chan(chan);
340         struct device *dev = chan->device->dev;
341         struct edmacc_param *param = &epset->param;
342         int acnt, bcnt, ccnt, cidx;
343         int src_bidx, dst_bidx, src_cidx, dst_cidx;
344         int absync;
345 
346         acnt = dev_width;
347 
348         /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
349         if (!burst)
350                 burst = 1;
351         /*
352          * If the maxburst is equal to the fifo width, use
353          * A-synced transfers. This allows for large contiguous
354          * buffer transfers using only one PaRAM set.
355          */
356         if (burst == 1) {
357                 /*
358                  * For the A-sync case, bcnt and ccnt are the remainder
359                  * and quotient respectively of the division of:
360                  * (dma_length / acnt) by (SZ_64K -1). This is so
361                  * that in case bcnt over flows, we have ccnt to use.
362                  * Note: In A-sync tranfer only, bcntrld is used, but it
363                  * only applies for sg_dma_len(sg) >= SZ_64K.
364                  * In this case, the best way adopted is- bccnt for the
365                  * first frame will be the remainder below. Then for
366                  * every successive frame, bcnt will be SZ_64K-1. This
367                  * is assured as bcntrld = 0xffff in end of function.
368                  */
369                 absync = false;
370                 ccnt = dma_length / acnt / (SZ_64K - 1);
371                 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
372                 /*
373                  * If bcnt is non-zero, we have a remainder and hence an
374                  * extra frame to transfer, so increment ccnt.
375                  */
376                 if (bcnt)
377                         ccnt++;
378                 else
379                         bcnt = SZ_64K - 1;
380                 cidx = acnt;
381         } else {
382                 /*
383                  * If maxburst is greater than the fifo address_width,
384                  * use AB-synced transfers where A count is the fifo
385                  * address_width and B count is the maxburst. In this
386                  * case, we are limited to transfers of C count frames
387                  * of (address_width * maxburst) where C count is limited
388                  * to SZ_64K-1. This places an upper bound on the length
389                  * of an SG segment that can be handled.
390                  */
391                 absync = true;
392                 bcnt = burst;
393                 ccnt = dma_length / (acnt * bcnt);
394                 if (ccnt > (SZ_64K - 1)) {
395                         dev_err(dev, "Exceeded max SG segment size\n");
396                         return -EINVAL;
397                 }
398                 cidx = acnt * bcnt;
399         }
400 
401         epset->len = dma_length;
402 
403         if (direction == DMA_MEM_TO_DEV) {
404                 src_bidx = acnt;
405                 src_cidx = cidx;
406                 dst_bidx = 0;
407                 dst_cidx = 0;
408                 epset->addr = src_addr;
409         } else if (direction == DMA_DEV_TO_MEM)  {
410                 src_bidx = 0;
411                 src_cidx = 0;
412                 dst_bidx = acnt;
413                 dst_cidx = cidx;
414                 epset->addr = dst_addr;
415         } else if (direction == DMA_MEM_TO_MEM)  {
416                 src_bidx = acnt;
417                 src_cidx = cidx;
418                 dst_bidx = acnt;
419                 dst_cidx = cidx;
420         } else {
421                 dev_err(dev, "%s: direction not implemented yet\n", __func__);
422                 return -EINVAL;
423         }
424 
425         param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
426         /* Configure A or AB synchronized transfers */
427         if (absync)
428                 param->opt |= SYNCDIM;
429 
430         param->src = src_addr;
431         param->dst = dst_addr;
432 
433         param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
434         param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
435 
436         param->a_b_cnt = bcnt << 16 | acnt;
437         param->ccnt = ccnt;
438         /*
439          * Only time when (bcntrld) auto reload is required is for
440          * A-sync case, and in this case, a requirement of reload value
441          * of SZ_64K-1 only is assured. 'link' is initially set to NULL
442          * and then later will be populated by edma_execute.
443          */
444         param->link_bcntrld = 0xffffffff;
445         return absync;
446 }
447 
448 static struct dma_async_tx_descriptor *edma_prep_slave_sg(
449         struct dma_chan *chan, struct scatterlist *sgl,
450         unsigned int sg_len, enum dma_transfer_direction direction,
451         unsigned long tx_flags, void *context)
452 {
453         struct edma_chan *echan = to_edma_chan(chan);
454         struct device *dev = chan->device->dev;
455         struct edma_desc *edesc;
456         dma_addr_t src_addr = 0, dst_addr = 0;
457         enum dma_slave_buswidth dev_width;
458         u32 burst;
459         struct scatterlist *sg;
460         int i, nslots, ret;
461 
462         if (unlikely(!echan || !sgl || !sg_len))
463                 return NULL;
464 
465         if (direction == DMA_DEV_TO_MEM) {
466                 src_addr = echan->cfg.src_addr;
467                 dev_width = echan->cfg.src_addr_width;
468                 burst = echan->cfg.src_maxburst;
469         } else if (direction == DMA_MEM_TO_DEV) {
470                 dst_addr = echan->cfg.dst_addr;
471                 dev_width = echan->cfg.dst_addr_width;
472                 burst = echan->cfg.dst_maxburst;
473         } else {
474                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
475                 return NULL;
476         }
477 
478         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
479                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
480                 return NULL;
481         }
482 
483         edesc = kzalloc(sizeof(*edesc) + sg_len *
484                 sizeof(edesc->pset[0]), GFP_ATOMIC);
485         if (!edesc) {
486                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
487                 return NULL;
488         }
489 
490         edesc->pset_nr = sg_len;
491         edesc->residue = 0;
492         edesc->direction = direction;
493         edesc->echan = echan;
494 
495         /* Allocate a PaRAM slot, if needed */
496         nslots = min_t(unsigned, MAX_NR_SG, sg_len);
497 
498         for (i = 0; i < nslots; i++) {
499                 if (echan->slot[i] < 0) {
500                         echan->slot[i] =
501                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
502                                                 EDMA_SLOT_ANY);
503                         if (echan->slot[i] < 0) {
504                                 kfree(edesc);
505                                 dev_err(dev, "%s: Failed to allocate slot\n",
506                                         __func__);
507                                 return NULL;
508                         }
509                 }
510         }
511 
512         /* Configure PaRAM sets for each SG */
513         for_each_sg(sgl, sg, sg_len, i) {
514                 /* Get address for each SG */
515                 if (direction == DMA_DEV_TO_MEM)
516                         dst_addr = sg_dma_address(sg);
517                 else
518                         src_addr = sg_dma_address(sg);
519 
520                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
521                                        dst_addr, burst, dev_width,
522                                        sg_dma_len(sg), direction);
523                 if (ret < 0) {
524                         kfree(edesc);
525                         return NULL;
526                 }
527 
528                 edesc->absync = ret;
529                 edesc->residue += sg_dma_len(sg);
530 
531                 /* If this is the last in a current SG set of transactions,
532                    enable interrupts so that next set is processed */
533                 if (!((i+1) % MAX_NR_SG))
534                         edesc->pset[i].param.opt |= TCINTEN;
535 
536                 /* If this is the last set, enable completion interrupt flag */
537                 if (i == sg_len - 1)
538                         edesc->pset[i].param.opt |= TCINTEN;
539         }
540         edesc->residue_stat = edesc->residue;
541 
542         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
543 }
544 
545 static struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
546         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
547         size_t len, unsigned long tx_flags)
548 {
549         int ret;
550         struct edma_desc *edesc;
551         struct device *dev = chan->device->dev;
552         struct edma_chan *echan = to_edma_chan(chan);
553 
554         if (unlikely(!echan || !len))
555                 return NULL;
556 
557         edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC);
558         if (!edesc) {
559                 dev_dbg(dev, "Failed to allocate a descriptor\n");
560                 return NULL;
561         }
562 
563         edesc->pset_nr = 1;
564 
565         ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
566                                DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM);
567         if (ret < 0)
568                 return NULL;
569 
570         edesc->absync = ret;
571 
572         /*
573          * Enable intermediate transfer chaining to re-trigger channel
574          * on completion of every TR, and enable transfer-completion
575          * interrupt on completion of the whole transfer.
576          */
577         edesc->pset[0].param.opt |= ITCCHEN;
578         edesc->pset[0].param.opt |= TCINTEN;
579 
580         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
581 }
582 
583 static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
584         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
585         size_t period_len, enum dma_transfer_direction direction,
586         unsigned long tx_flags)
587 {
588         struct edma_chan *echan = to_edma_chan(chan);
589         struct device *dev = chan->device->dev;
590         struct edma_desc *edesc;
591         dma_addr_t src_addr, dst_addr;
592         enum dma_slave_buswidth dev_width;
593         u32 burst;
594         int i, ret, nslots;
595 
596         if (unlikely(!echan || !buf_len || !period_len))
597                 return NULL;
598 
599         if (direction == DMA_DEV_TO_MEM) {
600                 src_addr = echan->cfg.src_addr;
601                 dst_addr = buf_addr;
602                 dev_width = echan->cfg.src_addr_width;
603                 burst = echan->cfg.src_maxburst;
604         } else if (direction == DMA_MEM_TO_DEV) {
605                 src_addr = buf_addr;
606                 dst_addr = echan->cfg.dst_addr;
607                 dev_width = echan->cfg.dst_addr_width;
608                 burst = echan->cfg.dst_maxburst;
609         } else {
610                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
611                 return NULL;
612         }
613 
614         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
615                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
616                 return NULL;
617         }
618 
619         if (unlikely(buf_len % period_len)) {
620                 dev_err(dev, "Period should be multiple of Buffer length\n");
621                 return NULL;
622         }
623 
624         nslots = (buf_len / period_len) + 1;
625 
626         /*
627          * Cyclic DMA users such as audio cannot tolerate delays introduced
628          * by cases where the number of periods is more than the maximum
629          * number of SGs the EDMA driver can handle at a time. For DMA types
630          * such as Slave SGs, such delays are tolerable and synchronized,
631          * but the synchronization is difficult to achieve with Cyclic and
632          * cannot be guaranteed, so we error out early.
633          */
634         if (nslots > MAX_NR_SG)
635                 return NULL;
636 
637         edesc = kzalloc(sizeof(*edesc) + nslots *
638                 sizeof(edesc->pset[0]), GFP_ATOMIC);
639         if (!edesc) {
640                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
641                 return NULL;
642         }
643 
644         edesc->cyclic = 1;
645         edesc->pset_nr = nslots;
646         edesc->residue = edesc->residue_stat = buf_len;
647         edesc->direction = direction;
648         edesc->echan = echan;
649 
650         dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
651                 __func__, echan->ch_num, nslots, period_len, buf_len);
652 
653         for (i = 0; i < nslots; i++) {
654                 /* Allocate a PaRAM slot, if needed */
655                 if (echan->slot[i] < 0) {
656                         echan->slot[i] =
657                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
658                                                 EDMA_SLOT_ANY);
659                         if (echan->slot[i] < 0) {
660                                 kfree(edesc);
661                                 dev_err(dev, "%s: Failed to allocate slot\n",
662                                         __func__);
663                                 return NULL;
664                         }
665                 }
666 
667                 if (i == nslots - 1) {
668                         memcpy(&edesc->pset[i], &edesc->pset[0],
669                                sizeof(edesc->pset[0]));
670                         break;
671                 }
672 
673                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
674                                        dst_addr, burst, dev_width, period_len,
675                                        direction);
676                 if (ret < 0) {
677                         kfree(edesc);
678                         return NULL;
679                 }
680 
681                 if (direction == DMA_DEV_TO_MEM)
682                         dst_addr += period_len;
683                 else
684                         src_addr += period_len;
685 
686                 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
687                 dev_vdbg(dev,
688                         "\n pset[%d]:\n"
689                         "  chnum\t%d\n"
690                         "  slot\t%d\n"
691                         "  opt\t%08x\n"
692                         "  src\t%08x\n"
693                         "  dst\t%08x\n"
694                         "  abcnt\t%08x\n"
695                         "  ccnt\t%08x\n"
696                         "  bidx\t%08x\n"
697                         "  cidx\t%08x\n"
698                         "  lkrld\t%08x\n",
699                         i, echan->ch_num, echan->slot[i],
700                         edesc->pset[i].param.opt,
701                         edesc->pset[i].param.src,
702                         edesc->pset[i].param.dst,
703                         edesc->pset[i].param.a_b_cnt,
704                         edesc->pset[i].param.ccnt,
705                         edesc->pset[i].param.src_dst_bidx,
706                         edesc->pset[i].param.src_dst_cidx,
707                         edesc->pset[i].param.link_bcntrld);
708 
709                 edesc->absync = ret;
710 
711                 /*
712                  * Enable period interrupt only if it is requested
713                  */
714                 if (tx_flags & DMA_PREP_INTERRUPT)
715                         edesc->pset[i].param.opt |= TCINTEN;
716         }
717 
718         /* Place the cyclic channel to highest priority queue */
719         edma_assign_channel_eventq(echan->ch_num, EVENTQ_0);
720 
721         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
722 }
723 
724 static void edma_callback(unsigned ch_num, u16 ch_status, void *data)
725 {
726         struct edma_chan *echan = data;
727         struct device *dev = echan->vchan.chan.device->dev;
728         struct edma_desc *edesc;
729         struct edmacc_param p;
730 
731         edesc = echan->edesc;
732 
733         /* Pause the channel for non-cyclic */
734         if (!edesc || (edesc && !edesc->cyclic))
735                 edma_pause(echan->ch_num);
736 
737         switch (ch_status) {
738         case EDMA_DMA_COMPLETE:
739                 spin_lock(&echan->vchan.lock);
740 
741                 if (edesc) {
742                         if (edesc->cyclic) {
743                                 vchan_cyclic_callback(&edesc->vdesc);
744                         } else if (edesc->processed == edesc->pset_nr) {
745                                 dev_dbg(dev, "Transfer complete, stopping channel %d\n", ch_num);
746                                 edesc->residue = 0;
747                                 edma_stop(echan->ch_num);
748                                 vchan_cookie_complete(&edesc->vdesc);
749                                 edma_execute(echan);
750                         } else {
751                                 dev_dbg(dev, "Intermediate transfer complete on channel %d\n", ch_num);
752 
753                                 /* Update statistics for tx_status */
754                                 edesc->residue -= edesc->sg_len;
755                                 edesc->residue_stat = edesc->residue;
756                                 edesc->processed_stat = edesc->processed;
757 
758                                 edma_execute(echan);
759                         }
760                 }
761 
762                 spin_unlock(&echan->vchan.lock);
763 
764                 break;
765         case EDMA_DMA_CC_ERROR:
766                 spin_lock(&echan->vchan.lock);
767 
768                 edma_read_slot(EDMA_CHAN_SLOT(echan->slot[0]), &p);
769 
770                 /*
771                  * Issue later based on missed flag which will be sure
772                  * to happen as:
773                  * (1) we finished transmitting an intermediate slot and
774                  *     edma_execute is coming up.
775                  * (2) or we finished current transfer and issue will
776                  *     call edma_execute.
777                  *
778                  * Important note: issuing can be dangerous here and
779                  * lead to some nasty recursion when we are in a NULL
780                  * slot. So we avoid doing so and set the missed flag.
781                  */
782                 if (p.a_b_cnt == 0 && p.ccnt == 0) {
783                         dev_dbg(dev, "Error occurred, looks like slot is null, just setting miss\n");
784                         echan->missed = 1;
785                 } else {
786                         /*
787                          * The slot is already programmed but the event got
788                          * missed, so its safe to issue it here.
789                          */
790                         dev_dbg(dev, "Error occurred but slot is non-null, TRIGGERING\n");
791                         edma_clean_channel(echan->ch_num);
792                         edma_stop(echan->ch_num);
793                         edma_start(echan->ch_num);
794                         edma_trigger_channel(echan->ch_num);
795                 }
796 
797                 spin_unlock(&echan->vchan.lock);
798 
799                 break;
800         default:
801                 break;
802         }
803 }
804 
805 /* Alloc channel resources */
806 static int edma_alloc_chan_resources(struct dma_chan *chan)
807 {
808         struct edma_chan *echan = to_edma_chan(chan);
809         struct device *dev = chan->device->dev;
810         int ret;
811         int a_ch_num;
812         LIST_HEAD(descs);
813 
814         a_ch_num = edma_alloc_channel(echan->ch_num, edma_callback,
815                                         echan, EVENTQ_DEFAULT);
816 
817         if (a_ch_num < 0) {
818                 ret = -ENODEV;
819                 goto err_no_chan;
820         }
821 
822         if (a_ch_num != echan->ch_num) {
823                 dev_err(dev, "failed to allocate requested channel %u:%u\n",
824                         EDMA_CTLR(echan->ch_num),
825                         EDMA_CHAN_SLOT(echan->ch_num));
826                 ret = -ENODEV;
827                 goto err_wrong_chan;
828         }
829 
830         echan->alloced = true;
831         echan->slot[0] = echan->ch_num;
832 
833         dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num,
834                 EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num));
835 
836         return 0;
837 
838 err_wrong_chan:
839         edma_free_channel(a_ch_num);
840 err_no_chan:
841         return ret;
842 }
843 
844 /* Free channel resources */
845 static void edma_free_chan_resources(struct dma_chan *chan)
846 {
847         struct edma_chan *echan = to_edma_chan(chan);
848         struct device *dev = chan->device->dev;
849         int i;
850 
851         /* Terminate transfers */
852         edma_stop(echan->ch_num);
853 
854         vchan_free_chan_resources(&echan->vchan);
855 
856         /* Free EDMA PaRAM slots */
857         for (i = 1; i < EDMA_MAX_SLOTS; i++) {
858                 if (echan->slot[i] >= 0) {
859                         edma_free_slot(echan->slot[i]);
860                         echan->slot[i] = -1;
861                 }
862         }
863 
864         /* Free EDMA channel */
865         if (echan->alloced) {
866                 edma_free_channel(echan->ch_num);
867                 echan->alloced = false;
868         }
869 
870         dev_dbg(dev, "freeing channel for %u\n", echan->ch_num);
871 }
872 
873 /* Send pending descriptor to hardware */
874 static void edma_issue_pending(struct dma_chan *chan)
875 {
876         struct edma_chan *echan = to_edma_chan(chan);
877         unsigned long flags;
878 
879         spin_lock_irqsave(&echan->vchan.lock, flags);
880         if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
881                 edma_execute(echan);
882         spin_unlock_irqrestore(&echan->vchan.lock, flags);
883 }
884 
885 static u32 edma_residue(struct edma_desc *edesc)
886 {
887         bool dst = edesc->direction == DMA_DEV_TO_MEM;
888         struct edma_pset *pset = edesc->pset;
889         dma_addr_t done, pos;
890         int i;
891 
892         /*
893          * We always read the dst/src position from the first RamPar
894          * pset. That's the one which is active now.
895          */
896         pos = edma_get_position(edesc->echan->slot[0], dst);
897 
898         /*
899          * Cyclic is simple. Just subtract pset[0].addr from pos.
900          *
901          * We never update edesc->residue in the cyclic case, so we
902          * can tell the remaining room to the end of the circular
903          * buffer.
904          */
905         if (edesc->cyclic) {
906                 done = pos - pset->addr;
907                 edesc->residue_stat = edesc->residue - done;
908                 return edesc->residue_stat;
909         }
910 
911         /*
912          * For SG operation we catch up with the last processed
913          * status.
914          */
915         pset += edesc->processed_stat;
916 
917         for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
918                 /*
919                  * If we are inside this pset address range, we know
920                  * this is the active one. Get the current delta and
921                  * stop walking the psets.
922                  */
923                 if (pos >= pset->addr && pos < pset->addr + pset->len)
924                         return edesc->residue_stat - (pos - pset->addr);
925 
926                 /* Otherwise mark it done and update residue_stat. */
927                 edesc->processed_stat++;
928                 edesc->residue_stat -= pset->len;
929         }
930         return edesc->residue_stat;
931 }
932 
933 /* Check request completion status */
934 static enum dma_status edma_tx_status(struct dma_chan *chan,
935                                       dma_cookie_t cookie,
936                                       struct dma_tx_state *txstate)
937 {
938         struct edma_chan *echan = to_edma_chan(chan);
939         struct virt_dma_desc *vdesc;
940         enum dma_status ret;
941         unsigned long flags;
942 
943         ret = dma_cookie_status(chan, cookie, txstate);
944         if (ret == DMA_COMPLETE || !txstate)
945                 return ret;
946 
947         spin_lock_irqsave(&echan->vchan.lock, flags);
948         if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
949                 txstate->residue = edma_residue(echan->edesc);
950         else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
951                 txstate->residue = to_edma_desc(&vdesc->tx)->residue;
952         spin_unlock_irqrestore(&echan->vchan.lock, flags);
953 
954         return ret;
955 }
956 
957 static void __init edma_chan_init(struct edma_cc *ecc,
958                                   struct dma_device *dma,
959                                   struct edma_chan *echans)
960 {
961         int i, j;
962 
963         for (i = 0; i < EDMA_CHANS; i++) {
964                 struct edma_chan *echan = &echans[i];
965                 echan->ch_num = EDMA_CTLR_CHAN(ecc->ctlr, i);
966                 echan->ecc = ecc;
967                 echan->vchan.desc_free = edma_desc_free;
968 
969                 vchan_init(&echan->vchan, dma);
970 
971                 INIT_LIST_HEAD(&echan->node);
972                 for (j = 0; j < EDMA_MAX_SLOTS; j++)
973                         echan->slot[j] = -1;
974         }
975 }
976 
977 #define EDMA_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
978                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
979                                  BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
980                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
981 
982 static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
983                           struct device *dev)
984 {
985         dma->device_prep_slave_sg = edma_prep_slave_sg;
986         dma->device_prep_dma_cyclic = edma_prep_dma_cyclic;
987         dma->device_prep_dma_memcpy = edma_prep_dma_memcpy;
988         dma->device_alloc_chan_resources = edma_alloc_chan_resources;
989         dma->device_free_chan_resources = edma_free_chan_resources;
990         dma->device_issue_pending = edma_issue_pending;
991         dma->device_tx_status = edma_tx_status;
992         dma->device_config = edma_slave_config;
993         dma->device_pause = edma_dma_pause;
994         dma->device_resume = edma_dma_resume;
995         dma->device_terminate_all = edma_terminate_all;
996 
997         dma->src_addr_widths = EDMA_DMA_BUSWIDTHS;
998         dma->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
999         dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1000         dma->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1001 
1002         dma->dev = dev;
1003 
1004         /*
1005          * code using dma memcpy must make sure alignment of
1006          * length is at dma->copy_align boundary.
1007          */
1008         dma->copy_align = DMA_SLAVE_BUSWIDTH_4_BYTES;
1009 
1010         INIT_LIST_HEAD(&dma->channels);
1011 }
1012 
1013 static int edma_probe(struct platform_device *pdev)
1014 {
1015         struct edma_cc *ecc;
1016         int ret;
1017 
1018         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1019         if (ret)
1020                 return ret;
1021 
1022         ecc = devm_kzalloc(&pdev->dev, sizeof(*ecc), GFP_KERNEL);
1023         if (!ecc) {
1024                 dev_err(&pdev->dev, "Can't allocate controller\n");
1025                 return -ENOMEM;
1026         }
1027 
1028         ecc->ctlr = pdev->id;
1029         ecc->dummy_slot = edma_alloc_slot(ecc->ctlr, EDMA_SLOT_ANY);
1030         if (ecc->dummy_slot < 0) {
1031                 dev_err(&pdev->dev, "Can't allocate PaRAM dummy slot\n");
1032                 return ecc->dummy_slot;
1033         }
1034 
1035         dma_cap_zero(ecc->dma_slave.cap_mask);
1036         dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask);
1037         dma_cap_set(DMA_CYCLIC, ecc->dma_slave.cap_mask);
1038         dma_cap_set(DMA_MEMCPY, ecc->dma_slave.cap_mask);
1039 
1040         edma_dma_init(ecc, &ecc->dma_slave, &pdev->dev);
1041 
1042         edma_chan_init(ecc, &ecc->dma_slave, ecc->slave_chans);
1043 
1044         ret = dma_async_device_register(&ecc->dma_slave);
1045         if (ret)
1046                 goto err_reg1;
1047 
1048         platform_set_drvdata(pdev, ecc);
1049 
1050         dev_info(&pdev->dev, "TI EDMA DMA engine driver\n");
1051 
1052         return 0;
1053 
1054 err_reg1:
1055         edma_free_slot(ecc->dummy_slot);
1056         return ret;
1057 }
1058 
1059 static int edma_remove(struct platform_device *pdev)
1060 {
1061         struct device *dev = &pdev->dev;
1062         struct edma_cc *ecc = dev_get_drvdata(dev);
1063 
1064         dma_async_device_unregister(&ecc->dma_slave);
1065         edma_free_slot(ecc->dummy_slot);
1066 
1067         return 0;
1068 }
1069 
1070 static struct platform_driver edma_driver = {
1071         .probe          = edma_probe,
1072         .remove         = edma_remove,
1073         .driver = {
1074                 .name = "edma-dma-engine",
1075         },
1076 };
1077 
1078 bool edma_filter_fn(struct dma_chan *chan, void *param)
1079 {
1080         if (chan->device->dev->driver == &edma_driver.driver) {
1081                 struct edma_chan *echan = to_edma_chan(chan);
1082                 unsigned ch_req = *(unsigned *)param;
1083                 return ch_req == echan->ch_num;
1084         }
1085         return false;
1086 }
1087 EXPORT_SYMBOL(edma_filter_fn);
1088 
1089 static int edma_init(void)
1090 {
1091         return platform_driver_register(&edma_driver);
1092 }
1093 subsys_initcall(edma_init);
1094 
1095 static void __exit edma_exit(void)
1096 {
1097         platform_driver_unregister(&edma_driver);
1098 }
1099 module_exit(edma_exit);
1100 
1101 MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
1102 MODULE_DESCRIPTION("TI EDMA DMA engine driver");
1103 MODULE_LICENSE("GPL v2");
1104 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us