Version:  2.0.40 2.2.26 2.4.37 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17

Linux/drivers/dma/edma.c

  1 /*
  2  * TI EDMA DMA engine driver
  3  *
  4  * Copyright 2012 Texas Instruments
  5  *
  6  * This program is free software; you can redistribute it and/or
  7  * modify it under the terms of the GNU General Public License as
  8  * published by the Free Software Foundation version 2.
  9  *
 10  * This program is distributed "as is" WITHOUT ANY WARRANTY of any
 11  * kind, whether express or implied; without even the implied warranty
 12  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13  * GNU General Public License for more details.
 14  */
 15 
 16 #include <linux/dmaengine.h>
 17 #include <linux/dma-mapping.h>
 18 #include <linux/err.h>
 19 #include <linux/init.h>
 20 #include <linux/interrupt.h>
 21 #include <linux/list.h>
 22 #include <linux/module.h>
 23 #include <linux/platform_device.h>
 24 #include <linux/slab.h>
 25 #include <linux/spinlock.h>
 26 #include <linux/of.h>
 27 
 28 #include <linux/platform_data/edma.h>
 29 
 30 #include "dmaengine.h"
 31 #include "virt-dma.h"
 32 
 33 /*
 34  * This will go away when the private EDMA API is folded
 35  * into this driver and the platform device(s) are
 36  * instantiated in the arch code. We can only get away
 37  * with this simplification because DA8XX may not be built
 38  * in the same kernel image with other DaVinci parts. This
 39  * avoids having to sprinkle dmaengine driver platform devices
 40  * and data throughout all the existing board files.
 41  */
 42 #ifdef CONFIG_ARCH_DAVINCI_DA8XX
 43 #define EDMA_CTLRS      2
 44 #define EDMA_CHANS      32
 45 #else
 46 #define EDMA_CTLRS      1
 47 #define EDMA_CHANS      64
 48 #endif /* CONFIG_ARCH_DAVINCI_DA8XX */
 49 
 50 /*
 51  * Max of 20 segments per channel to conserve PaRAM slots
 52  * Also note that MAX_NR_SG should be atleast the no.of periods
 53  * that are required for ASoC, otherwise DMA prep calls will
 54  * fail. Today davinci-pcm is the only user of this driver and
 55  * requires atleast 17 slots, so we setup the default to 20.
 56  */
 57 #define MAX_NR_SG               20
 58 #define EDMA_MAX_SLOTS          MAX_NR_SG
 59 #define EDMA_DESCRIPTORS        16
 60 
 61 struct edma_pset {
 62         u32                             len;
 63         dma_addr_t                      addr;
 64         struct edmacc_param             param;
 65 };
 66 
 67 struct edma_desc {
 68         struct virt_dma_desc            vdesc;
 69         struct list_head                node;
 70         enum dma_transfer_direction     direction;
 71         int                             cyclic;
 72         int                             absync;
 73         int                             pset_nr;
 74         struct edma_chan                *echan;
 75         int                             processed;
 76 
 77         /*
 78          * The following 4 elements are used for residue accounting.
 79          *
 80          * - processed_stat: the number of SG elements we have traversed
 81          * so far to cover accounting. This is updated directly to processed
 82          * during edma_callback and is always <= processed, because processed
 83          * refers to the number of pending transfer (programmed to EDMA
 84          * controller), where as processed_stat tracks number of transfers
 85          * accounted for so far.
 86          *
 87          * - residue: The amount of bytes we have left to transfer for this desc
 88          *
 89          * - residue_stat: The residue in bytes of data we have covered
 90          * so far for accounting. This is updated directly to residue
 91          * during callbacks to keep it current.
 92          *
 93          * - sg_len: Tracks the length of the current intermediate transfer,
 94          * this is required to update the residue during intermediate transfer
 95          * completion callback.
 96          */
 97         int                             processed_stat;
 98         u32                             sg_len;
 99         u32                             residue;
100         u32                             residue_stat;
101 
102         struct edma_pset                pset[0];
103 };
104 
105 struct edma_cc;
106 
107 struct edma_chan {
108         struct virt_dma_chan            vchan;
109         struct list_head                node;
110         struct edma_desc                *edesc;
111         struct edma_cc                  *ecc;
112         int                             ch_num;
113         bool                            alloced;
114         int                             slot[EDMA_MAX_SLOTS];
115         int                             missed;
116         struct dma_slave_config         cfg;
117 };
118 
119 struct edma_cc {
120         int                             ctlr;
121         struct dma_device               dma_slave;
122         struct edma_chan                slave_chans[EDMA_CHANS];
123         int                             num_slave_chans;
124         int                             dummy_slot;
125 };
126 
127 static inline struct edma_cc *to_edma_cc(struct dma_device *d)
128 {
129         return container_of(d, struct edma_cc, dma_slave);
130 }
131 
132 static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
133 {
134         return container_of(c, struct edma_chan, vchan.chan);
135 }
136 
137 static inline struct edma_desc
138 *to_edma_desc(struct dma_async_tx_descriptor *tx)
139 {
140         return container_of(tx, struct edma_desc, vdesc.tx);
141 }
142 
143 static void edma_desc_free(struct virt_dma_desc *vdesc)
144 {
145         kfree(container_of(vdesc, struct edma_desc, vdesc));
146 }
147 
148 /* Dispatch a queued descriptor to the controller (caller holds lock) */
149 static void edma_execute(struct edma_chan *echan)
150 {
151         struct virt_dma_desc *vdesc;
152         struct edma_desc *edesc;
153         struct device *dev = echan->vchan.chan.device->dev;
154         int i, j, left, nslots;
155 
156         /* If either we processed all psets or we're still not started */
157         if (!echan->edesc ||
158             echan->edesc->pset_nr == echan->edesc->processed) {
159                 /* Get next vdesc */
160                 vdesc = vchan_next_desc(&echan->vchan);
161                 if (!vdesc) {
162                         echan->edesc = NULL;
163                         return;
164                 }
165                 list_del(&vdesc->node);
166                 echan->edesc = to_edma_desc(&vdesc->tx);
167         }
168 
169         edesc = echan->edesc;
170 
171         /* Find out how many left */
172         left = edesc->pset_nr - edesc->processed;
173         nslots = min(MAX_NR_SG, left);
174         edesc->sg_len = 0;
175 
176         /* Write descriptor PaRAM set(s) */
177         for (i = 0; i < nslots; i++) {
178                 j = i + edesc->processed;
179                 edma_write_slot(echan->slot[i], &edesc->pset[j].param);
180                 edesc->sg_len += edesc->pset[j].len;
181                 dev_vdbg(echan->vchan.chan.device->dev,
182                         "\n pset[%d]:\n"
183                         "  chnum\t%d\n"
184                         "  slot\t%d\n"
185                         "  opt\t%08x\n"
186                         "  src\t%08x\n"
187                         "  dst\t%08x\n"
188                         "  abcnt\t%08x\n"
189                         "  ccnt\t%08x\n"
190                         "  bidx\t%08x\n"
191                         "  cidx\t%08x\n"
192                         "  lkrld\t%08x\n",
193                         j, echan->ch_num, echan->slot[i],
194                         edesc->pset[j].param.opt,
195                         edesc->pset[j].param.src,
196                         edesc->pset[j].param.dst,
197                         edesc->pset[j].param.a_b_cnt,
198                         edesc->pset[j].param.ccnt,
199                         edesc->pset[j].param.src_dst_bidx,
200                         edesc->pset[j].param.src_dst_cidx,
201                         edesc->pset[j].param.link_bcntrld);
202                 /* Link to the previous slot if not the last set */
203                 if (i != (nslots - 1))
204                         edma_link(echan->slot[i], echan->slot[i+1]);
205         }
206 
207         edesc->processed += nslots;
208 
209         /*
210          * If this is either the last set in a set of SG-list transactions
211          * then setup a link to the dummy slot, this results in all future
212          * events being absorbed and that's OK because we're done
213          */
214         if (edesc->processed == edesc->pset_nr) {
215                 if (edesc->cyclic)
216                         edma_link(echan->slot[nslots-1], echan->slot[1]);
217                 else
218                         edma_link(echan->slot[nslots-1],
219                                   echan->ecc->dummy_slot);
220         }
221 
222         if (edesc->processed <= MAX_NR_SG) {
223                 dev_dbg(dev, "first transfer starting on channel %d\n",
224                         echan->ch_num);
225                 edma_start(echan->ch_num);
226         } else {
227                 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
228                         echan->ch_num, edesc->processed);
229                 edma_resume(echan->ch_num);
230         }
231 
232         /*
233          * This happens due to setup times between intermediate transfers
234          * in long SG lists which have to be broken up into transfers of
235          * MAX_NR_SG
236          */
237         if (echan->missed) {
238                 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
239                 edma_clean_channel(echan->ch_num);
240                 edma_stop(echan->ch_num);
241                 edma_start(echan->ch_num);
242                 edma_trigger_channel(echan->ch_num);
243                 echan->missed = 0;
244         }
245 }
246 
247 static int edma_terminate_all(struct edma_chan *echan)
248 {
249         unsigned long flags;
250         LIST_HEAD(head);
251 
252         spin_lock_irqsave(&echan->vchan.lock, flags);
253 
254         /*
255          * Stop DMA activity: we assume the callback will not be called
256          * after edma_dma() returns (even if it does, it will see
257          * echan->edesc is NULL and exit.)
258          */
259         if (echan->edesc) {
260                 int cyclic = echan->edesc->cyclic;
261                 echan->edesc = NULL;
262                 edma_stop(echan->ch_num);
263                 /* Move the cyclic channel back to default queue */
264                 if (cyclic)
265                         edma_assign_channel_eventq(echan->ch_num,
266                                                    EVENTQ_DEFAULT);
267         }
268 
269         vchan_get_all_descriptors(&echan->vchan, &head);
270         spin_unlock_irqrestore(&echan->vchan.lock, flags);
271         vchan_dma_desc_free_list(&echan->vchan, &head);
272 
273         return 0;
274 }
275 
276 static int edma_slave_config(struct edma_chan *echan,
277         struct dma_slave_config *cfg)
278 {
279         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
280             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
281                 return -EINVAL;
282 
283         memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
284 
285         return 0;
286 }
287 
288 static int edma_dma_pause(struct edma_chan *echan)
289 {
290         /* Pause/Resume only allowed with cyclic mode */
291         if (!echan->edesc->cyclic)
292                 return -EINVAL;
293 
294         edma_pause(echan->ch_num);
295         return 0;
296 }
297 
298 static int edma_dma_resume(struct edma_chan *echan)
299 {
300         /* Pause/Resume only allowed with cyclic mode */
301         if (!echan->edesc->cyclic)
302                 return -EINVAL;
303 
304         edma_resume(echan->ch_num);
305         return 0;
306 }
307 
308 static int edma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
309                         unsigned long arg)
310 {
311         int ret = 0;
312         struct dma_slave_config *config;
313         struct edma_chan *echan = to_edma_chan(chan);
314 
315         switch (cmd) {
316         case DMA_TERMINATE_ALL:
317                 edma_terminate_all(echan);
318                 break;
319         case DMA_SLAVE_CONFIG:
320                 config = (struct dma_slave_config *)arg;
321                 ret = edma_slave_config(echan, config);
322                 break;
323         case DMA_PAUSE:
324                 ret = edma_dma_pause(echan);
325                 break;
326 
327         case DMA_RESUME:
328                 ret = edma_dma_resume(echan);
329                 break;
330 
331         default:
332                 ret = -ENOSYS;
333         }
334 
335         return ret;
336 }
337 
338 /*
339  * A PaRAM set configuration abstraction used by other modes
340  * @chan: Channel who's PaRAM set we're configuring
341  * @pset: PaRAM set to initialize and setup.
342  * @src_addr: Source address of the DMA
343  * @dst_addr: Destination address of the DMA
344  * @burst: In units of dev_width, how much to send
345  * @dev_width: How much is the dev_width
346  * @dma_length: Total length of the DMA transfer
347  * @direction: Direction of the transfer
348  */
349 static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
350         dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
351         enum dma_slave_buswidth dev_width, unsigned int dma_length,
352         enum dma_transfer_direction direction)
353 {
354         struct edma_chan *echan = to_edma_chan(chan);
355         struct device *dev = chan->device->dev;
356         struct edmacc_param *param = &epset->param;
357         int acnt, bcnt, ccnt, cidx;
358         int src_bidx, dst_bidx, src_cidx, dst_cidx;
359         int absync;
360 
361         acnt = dev_width;
362 
363         /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
364         if (!burst)
365                 burst = 1;
366         /*
367          * If the maxburst is equal to the fifo width, use
368          * A-synced transfers. This allows for large contiguous
369          * buffer transfers using only one PaRAM set.
370          */
371         if (burst == 1) {
372                 /*
373                  * For the A-sync case, bcnt and ccnt are the remainder
374                  * and quotient respectively of the division of:
375                  * (dma_length / acnt) by (SZ_64K -1). This is so
376                  * that in case bcnt over flows, we have ccnt to use.
377                  * Note: In A-sync tranfer only, bcntrld is used, but it
378                  * only applies for sg_dma_len(sg) >= SZ_64K.
379                  * In this case, the best way adopted is- bccnt for the
380                  * first frame will be the remainder below. Then for
381                  * every successive frame, bcnt will be SZ_64K-1. This
382                  * is assured as bcntrld = 0xffff in end of function.
383                  */
384                 absync = false;
385                 ccnt = dma_length / acnt / (SZ_64K - 1);
386                 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
387                 /*
388                  * If bcnt is non-zero, we have a remainder and hence an
389                  * extra frame to transfer, so increment ccnt.
390                  */
391                 if (bcnt)
392                         ccnt++;
393                 else
394                         bcnt = SZ_64K - 1;
395                 cidx = acnt;
396         } else {
397                 /*
398                  * If maxburst is greater than the fifo address_width,
399                  * use AB-synced transfers where A count is the fifo
400                  * address_width and B count is the maxburst. In this
401                  * case, we are limited to transfers of C count frames
402                  * of (address_width * maxburst) where C count is limited
403                  * to SZ_64K-1. This places an upper bound on the length
404                  * of an SG segment that can be handled.
405                  */
406                 absync = true;
407                 bcnt = burst;
408                 ccnt = dma_length / (acnt * bcnt);
409                 if (ccnt > (SZ_64K - 1)) {
410                         dev_err(dev, "Exceeded max SG segment size\n");
411                         return -EINVAL;
412                 }
413                 cidx = acnt * bcnt;
414         }
415 
416         epset->len = dma_length;
417 
418         if (direction == DMA_MEM_TO_DEV) {
419                 src_bidx = acnt;
420                 src_cidx = cidx;
421                 dst_bidx = 0;
422                 dst_cidx = 0;
423                 epset->addr = src_addr;
424         } else if (direction == DMA_DEV_TO_MEM)  {
425                 src_bidx = 0;
426                 src_cidx = 0;
427                 dst_bidx = acnt;
428                 dst_cidx = cidx;
429                 epset->addr = dst_addr;
430         } else if (direction == DMA_MEM_TO_MEM)  {
431                 src_bidx = acnt;
432                 src_cidx = cidx;
433                 dst_bidx = acnt;
434                 dst_cidx = cidx;
435         } else {
436                 dev_err(dev, "%s: direction not implemented yet\n", __func__);
437                 return -EINVAL;
438         }
439 
440         param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
441         /* Configure A or AB synchronized transfers */
442         if (absync)
443                 param->opt |= SYNCDIM;
444 
445         param->src = src_addr;
446         param->dst = dst_addr;
447 
448         param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
449         param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
450 
451         param->a_b_cnt = bcnt << 16 | acnt;
452         param->ccnt = ccnt;
453         /*
454          * Only time when (bcntrld) auto reload is required is for
455          * A-sync case, and in this case, a requirement of reload value
456          * of SZ_64K-1 only is assured. 'link' is initially set to NULL
457          * and then later will be populated by edma_execute.
458          */
459         param->link_bcntrld = 0xffffffff;
460         return absync;
461 }
462 
463 static struct dma_async_tx_descriptor *edma_prep_slave_sg(
464         struct dma_chan *chan, struct scatterlist *sgl,
465         unsigned int sg_len, enum dma_transfer_direction direction,
466         unsigned long tx_flags, void *context)
467 {
468         struct edma_chan *echan = to_edma_chan(chan);
469         struct device *dev = chan->device->dev;
470         struct edma_desc *edesc;
471         dma_addr_t src_addr = 0, dst_addr = 0;
472         enum dma_slave_buswidth dev_width;
473         u32 burst;
474         struct scatterlist *sg;
475         int i, nslots, ret;
476 
477         if (unlikely(!echan || !sgl || !sg_len))
478                 return NULL;
479 
480         if (direction == DMA_DEV_TO_MEM) {
481                 src_addr = echan->cfg.src_addr;
482                 dev_width = echan->cfg.src_addr_width;
483                 burst = echan->cfg.src_maxburst;
484         } else if (direction == DMA_MEM_TO_DEV) {
485                 dst_addr = echan->cfg.dst_addr;
486                 dev_width = echan->cfg.dst_addr_width;
487                 burst = echan->cfg.dst_maxburst;
488         } else {
489                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
490                 return NULL;
491         }
492 
493         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
494                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
495                 return NULL;
496         }
497 
498         edesc = kzalloc(sizeof(*edesc) + sg_len *
499                 sizeof(edesc->pset[0]), GFP_ATOMIC);
500         if (!edesc) {
501                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
502                 return NULL;
503         }
504 
505         edesc->pset_nr = sg_len;
506         edesc->residue = 0;
507         edesc->direction = direction;
508         edesc->echan = echan;
509 
510         /* Allocate a PaRAM slot, if needed */
511         nslots = min_t(unsigned, MAX_NR_SG, sg_len);
512 
513         for (i = 0; i < nslots; i++) {
514                 if (echan->slot[i] < 0) {
515                         echan->slot[i] =
516                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
517                                                 EDMA_SLOT_ANY);
518                         if (echan->slot[i] < 0) {
519                                 kfree(edesc);
520                                 dev_err(dev, "%s: Failed to allocate slot\n",
521                                         __func__);
522                                 return NULL;
523                         }
524                 }
525         }
526 
527         /* Configure PaRAM sets for each SG */
528         for_each_sg(sgl, sg, sg_len, i) {
529                 /* Get address for each SG */
530                 if (direction == DMA_DEV_TO_MEM)
531                         dst_addr = sg_dma_address(sg);
532                 else
533                         src_addr = sg_dma_address(sg);
534 
535                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
536                                        dst_addr, burst, dev_width,
537                                        sg_dma_len(sg), direction);
538                 if (ret < 0) {
539                         kfree(edesc);
540                         return NULL;
541                 }
542 
543                 edesc->absync = ret;
544                 edesc->residue += sg_dma_len(sg);
545 
546                 /* If this is the last in a current SG set of transactions,
547                    enable interrupts so that next set is processed */
548                 if (!((i+1) % MAX_NR_SG))
549                         edesc->pset[i].param.opt |= TCINTEN;
550 
551                 /* If this is the last set, enable completion interrupt flag */
552                 if (i == sg_len - 1)
553                         edesc->pset[i].param.opt |= TCINTEN;
554         }
555         edesc->residue_stat = edesc->residue;
556 
557         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
558 }
559 
560 struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
561         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
562         size_t len, unsigned long tx_flags)
563 {
564         int ret;
565         struct edma_desc *edesc;
566         struct device *dev = chan->device->dev;
567         struct edma_chan *echan = to_edma_chan(chan);
568 
569         if (unlikely(!echan || !len))
570                 return NULL;
571 
572         edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC);
573         if (!edesc) {
574                 dev_dbg(dev, "Failed to allocate a descriptor\n");
575                 return NULL;
576         }
577 
578         edesc->pset_nr = 1;
579 
580         ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
581                                DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM);
582         if (ret < 0)
583                 return NULL;
584 
585         edesc->absync = ret;
586 
587         /*
588          * Enable intermediate transfer chaining to re-trigger channel
589          * on completion of every TR, and enable transfer-completion
590          * interrupt on completion of the whole transfer.
591          */
592         edesc->pset[0].param.opt |= ITCCHEN;
593         edesc->pset[0].param.opt |= TCINTEN;
594 
595         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
596 }
597 
598 static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
599         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
600         size_t period_len, enum dma_transfer_direction direction,
601         unsigned long tx_flags)
602 {
603         struct edma_chan *echan = to_edma_chan(chan);
604         struct device *dev = chan->device->dev;
605         struct edma_desc *edesc;
606         dma_addr_t src_addr, dst_addr;
607         enum dma_slave_buswidth dev_width;
608         u32 burst;
609         int i, ret, nslots;
610 
611         if (unlikely(!echan || !buf_len || !period_len))
612                 return NULL;
613 
614         if (direction == DMA_DEV_TO_MEM) {
615                 src_addr = echan->cfg.src_addr;
616                 dst_addr = buf_addr;
617                 dev_width = echan->cfg.src_addr_width;
618                 burst = echan->cfg.src_maxburst;
619         } else if (direction == DMA_MEM_TO_DEV) {
620                 src_addr = buf_addr;
621                 dst_addr = echan->cfg.dst_addr;
622                 dev_width = echan->cfg.dst_addr_width;
623                 burst = echan->cfg.dst_maxburst;
624         } else {
625                 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
626                 return NULL;
627         }
628 
629         if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
630                 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
631                 return NULL;
632         }
633 
634         if (unlikely(buf_len % period_len)) {
635                 dev_err(dev, "Period should be multiple of Buffer length\n");
636                 return NULL;
637         }
638 
639         nslots = (buf_len / period_len) + 1;
640 
641         /*
642          * Cyclic DMA users such as audio cannot tolerate delays introduced
643          * by cases where the number of periods is more than the maximum
644          * number of SGs the EDMA driver can handle at a time. For DMA types
645          * such as Slave SGs, such delays are tolerable and synchronized,
646          * but the synchronization is difficult to achieve with Cyclic and
647          * cannot be guaranteed, so we error out early.
648          */
649         if (nslots > MAX_NR_SG)
650                 return NULL;
651 
652         edesc = kzalloc(sizeof(*edesc) + nslots *
653                 sizeof(edesc->pset[0]), GFP_ATOMIC);
654         if (!edesc) {
655                 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
656                 return NULL;
657         }
658 
659         edesc->cyclic = 1;
660         edesc->pset_nr = nslots;
661         edesc->residue = edesc->residue_stat = buf_len;
662         edesc->direction = direction;
663         edesc->echan = echan;
664 
665         dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
666                 __func__, echan->ch_num, nslots, period_len, buf_len);
667 
668         for (i = 0; i < nslots; i++) {
669                 /* Allocate a PaRAM slot, if needed */
670                 if (echan->slot[i] < 0) {
671                         echan->slot[i] =
672                                 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
673                                                 EDMA_SLOT_ANY);
674                         if (echan->slot[i] < 0) {
675                                 kfree(edesc);
676                                 dev_err(dev, "%s: Failed to allocate slot\n",
677                                         __func__);
678                                 return NULL;
679                         }
680                 }
681 
682                 if (i == nslots - 1) {
683                         memcpy(&edesc->pset[i], &edesc->pset[0],
684                                sizeof(edesc->pset[0]));
685                         break;
686                 }
687 
688                 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
689                                        dst_addr, burst, dev_width, period_len,
690                                        direction);
691                 if (ret < 0) {
692                         kfree(edesc);
693                         return NULL;
694                 }
695 
696                 if (direction == DMA_DEV_TO_MEM)
697                         dst_addr += period_len;
698                 else
699                         src_addr += period_len;
700 
701                 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
702                 dev_vdbg(dev,
703                         "\n pset[%d]:\n"
704                         "  chnum\t%d\n"
705                         "  slot\t%d\n"
706                         "  opt\t%08x\n"
707                         "  src\t%08x\n"
708                         "  dst\t%08x\n"
709                         "  abcnt\t%08x\n"
710                         "  ccnt\t%08x\n"
711                         "  bidx\t%08x\n"
712                         "  cidx\t%08x\n"
713                         "  lkrld\t%08x\n",
714                         i, echan->ch_num, echan->slot[i],
715                         edesc->pset[i].param.opt,
716                         edesc->pset[i].param.src,
717                         edesc->pset[i].param.dst,
718                         edesc->pset[i].param.a_b_cnt,
719                         edesc->pset[i].param.ccnt,
720                         edesc->pset[i].param.src_dst_bidx,
721                         edesc->pset[i].param.src_dst_cidx,
722                         edesc->pset[i].param.link_bcntrld);
723 
724                 edesc->absync = ret;
725 
726                 /*
727                  * Enable period interrupt only if it is requested
728                  */
729                 if (tx_flags & DMA_PREP_INTERRUPT)
730                         edesc->pset[i].param.opt |= TCINTEN;
731         }
732 
733         /* Place the cyclic channel to highest priority queue */
734         edma_assign_channel_eventq(echan->ch_num, EVENTQ_0);
735 
736         return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
737 }
738 
739 static void edma_callback(unsigned ch_num, u16 ch_status, void *data)
740 {
741         struct edma_chan *echan = data;
742         struct device *dev = echan->vchan.chan.device->dev;
743         struct edma_desc *edesc;
744         struct edmacc_param p;
745 
746         edesc = echan->edesc;
747 
748         /* Pause the channel for non-cyclic */
749         if (!edesc || (edesc && !edesc->cyclic))
750                 edma_pause(echan->ch_num);
751 
752         switch (ch_status) {
753         case EDMA_DMA_COMPLETE:
754                 spin_lock(&echan->vchan.lock);
755 
756                 if (edesc) {
757                         if (edesc->cyclic) {
758                                 vchan_cyclic_callback(&edesc->vdesc);
759                         } else if (edesc->processed == edesc->pset_nr) {
760                                 dev_dbg(dev, "Transfer complete, stopping channel %d\n", ch_num);
761                                 edesc->residue = 0;
762                                 edma_stop(echan->ch_num);
763                                 vchan_cookie_complete(&edesc->vdesc);
764                                 edma_execute(echan);
765                         } else {
766                                 dev_dbg(dev, "Intermediate transfer complete on channel %d\n", ch_num);
767 
768                                 /* Update statistics for tx_status */
769                                 edesc->residue -= edesc->sg_len;
770                                 edesc->residue_stat = edesc->residue;
771                                 edesc->processed_stat = edesc->processed;
772 
773                                 edma_execute(echan);
774                         }
775                 }
776 
777                 spin_unlock(&echan->vchan.lock);
778 
779                 break;
780         case EDMA_DMA_CC_ERROR:
781                 spin_lock(&echan->vchan.lock);
782 
783                 edma_read_slot(EDMA_CHAN_SLOT(echan->slot[0]), &p);
784 
785                 /*
786                  * Issue later based on missed flag which will be sure
787                  * to happen as:
788                  * (1) we finished transmitting an intermediate slot and
789                  *     edma_execute is coming up.
790                  * (2) or we finished current transfer and issue will
791                  *     call edma_execute.
792                  *
793                  * Important note: issuing can be dangerous here and
794                  * lead to some nasty recursion when we are in a NULL
795                  * slot. So we avoid doing so and set the missed flag.
796                  */
797                 if (p.a_b_cnt == 0 && p.ccnt == 0) {
798                         dev_dbg(dev, "Error occurred, looks like slot is null, just setting miss\n");
799                         echan->missed = 1;
800                 } else {
801                         /*
802                          * The slot is already programmed but the event got
803                          * missed, so its safe to issue it here.
804                          */
805                         dev_dbg(dev, "Error occurred but slot is non-null, TRIGGERING\n");
806                         edma_clean_channel(echan->ch_num);
807                         edma_stop(echan->ch_num);
808                         edma_start(echan->ch_num);
809                         edma_trigger_channel(echan->ch_num);
810                 }
811 
812                 spin_unlock(&echan->vchan.lock);
813 
814                 break;
815         default:
816                 break;
817         }
818 }
819 
820 /* Alloc channel resources */
821 static int edma_alloc_chan_resources(struct dma_chan *chan)
822 {
823         struct edma_chan *echan = to_edma_chan(chan);
824         struct device *dev = chan->device->dev;
825         int ret;
826         int a_ch_num;
827         LIST_HEAD(descs);
828 
829         a_ch_num = edma_alloc_channel(echan->ch_num, edma_callback,
830                                         chan, EVENTQ_DEFAULT);
831 
832         if (a_ch_num < 0) {
833                 ret = -ENODEV;
834                 goto err_no_chan;
835         }
836 
837         if (a_ch_num != echan->ch_num) {
838                 dev_err(dev, "failed to allocate requested channel %u:%u\n",
839                         EDMA_CTLR(echan->ch_num),
840                         EDMA_CHAN_SLOT(echan->ch_num));
841                 ret = -ENODEV;
842                 goto err_wrong_chan;
843         }
844 
845         echan->alloced = true;
846         echan->slot[0] = echan->ch_num;
847 
848         dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num,
849                 EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num));
850 
851         return 0;
852 
853 err_wrong_chan:
854         edma_free_channel(a_ch_num);
855 err_no_chan:
856         return ret;
857 }
858 
859 /* Free channel resources */
860 static void edma_free_chan_resources(struct dma_chan *chan)
861 {
862         struct edma_chan *echan = to_edma_chan(chan);
863         struct device *dev = chan->device->dev;
864         int i;
865 
866         /* Terminate transfers */
867         edma_stop(echan->ch_num);
868 
869         vchan_free_chan_resources(&echan->vchan);
870 
871         /* Free EDMA PaRAM slots */
872         for (i = 1; i < EDMA_MAX_SLOTS; i++) {
873                 if (echan->slot[i] >= 0) {
874                         edma_free_slot(echan->slot[i]);
875                         echan->slot[i] = -1;
876                 }
877         }
878 
879         /* Free EDMA channel */
880         if (echan->alloced) {
881                 edma_free_channel(echan->ch_num);
882                 echan->alloced = false;
883         }
884 
885         dev_dbg(dev, "freeing channel for %u\n", echan->ch_num);
886 }
887 
888 /* Send pending descriptor to hardware */
889 static void edma_issue_pending(struct dma_chan *chan)
890 {
891         struct edma_chan *echan = to_edma_chan(chan);
892         unsigned long flags;
893 
894         spin_lock_irqsave(&echan->vchan.lock, flags);
895         if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
896                 edma_execute(echan);
897         spin_unlock_irqrestore(&echan->vchan.lock, flags);
898 }
899 
900 static u32 edma_residue(struct edma_desc *edesc)
901 {
902         bool dst = edesc->direction == DMA_DEV_TO_MEM;
903         struct edma_pset *pset = edesc->pset;
904         dma_addr_t done, pos;
905         int i;
906 
907         /*
908          * We always read the dst/src position from the first RamPar
909          * pset. That's the one which is active now.
910          */
911         pos = edma_get_position(edesc->echan->slot[0], dst);
912 
913         /*
914          * Cyclic is simple. Just subtract pset[0].addr from pos.
915          *
916          * We never update edesc->residue in the cyclic case, so we
917          * can tell the remaining room to the end of the circular
918          * buffer.
919          */
920         if (edesc->cyclic) {
921                 done = pos - pset->addr;
922                 edesc->residue_stat = edesc->residue - done;
923                 return edesc->residue_stat;
924         }
925 
926         /*
927          * For SG operation we catch up with the last processed
928          * status.
929          */
930         pset += edesc->processed_stat;
931 
932         for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
933                 /*
934                  * If we are inside this pset address range, we know
935                  * this is the active one. Get the current delta and
936                  * stop walking the psets.
937                  */
938                 if (pos >= pset->addr && pos < pset->addr + pset->len)
939                         return edesc->residue_stat - (pos - pset->addr);
940 
941                 /* Otherwise mark it done and update residue_stat. */
942                 edesc->processed_stat++;
943                 edesc->residue_stat -= pset->len;
944         }
945         return edesc->residue_stat;
946 }
947 
948 /* Check request completion status */
949 static enum dma_status edma_tx_status(struct dma_chan *chan,
950                                       dma_cookie_t cookie,
951                                       struct dma_tx_state *txstate)
952 {
953         struct edma_chan *echan = to_edma_chan(chan);
954         struct virt_dma_desc *vdesc;
955         enum dma_status ret;
956         unsigned long flags;
957 
958         ret = dma_cookie_status(chan, cookie, txstate);
959         if (ret == DMA_COMPLETE || !txstate)
960                 return ret;
961 
962         spin_lock_irqsave(&echan->vchan.lock, flags);
963         if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
964                 txstate->residue = edma_residue(echan->edesc);
965         else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
966                 txstate->residue = to_edma_desc(&vdesc->tx)->residue;
967         spin_unlock_irqrestore(&echan->vchan.lock, flags);
968 
969         return ret;
970 }
971 
972 static void __init edma_chan_init(struct edma_cc *ecc,
973                                   struct dma_device *dma,
974                                   struct edma_chan *echans)
975 {
976         int i, j;
977 
978         for (i = 0; i < EDMA_CHANS; i++) {
979                 struct edma_chan *echan = &echans[i];
980                 echan->ch_num = EDMA_CTLR_CHAN(ecc->ctlr, i);
981                 echan->ecc = ecc;
982                 echan->vchan.desc_free = edma_desc_free;
983 
984                 vchan_init(&echan->vchan, dma);
985 
986                 INIT_LIST_HEAD(&echan->node);
987                 for (j = 0; j < EDMA_MAX_SLOTS; j++)
988                         echan->slot[j] = -1;
989         }
990 }
991 
992 #define EDMA_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
993                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
994                                  BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
995                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
996 
997 static int edma_dma_device_slave_caps(struct dma_chan *dchan,
998                                       struct dma_slave_caps *caps)
999 {
1000         caps->src_addr_widths = EDMA_DMA_BUSWIDTHS;
1001         caps->dstn_addr_widths = EDMA_DMA_BUSWIDTHS;
1002         caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1003         caps->cmd_pause = true;
1004         caps->cmd_terminate = true;
1005         caps->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1006 
1007         return 0;
1008 }
1009 
1010 static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
1011                           struct device *dev)
1012 {
1013         dma->device_prep_slave_sg = edma_prep_slave_sg;
1014         dma->device_prep_dma_cyclic = edma_prep_dma_cyclic;
1015         dma->device_prep_dma_memcpy = edma_prep_dma_memcpy;
1016         dma->device_alloc_chan_resources = edma_alloc_chan_resources;
1017         dma->device_free_chan_resources = edma_free_chan_resources;
1018         dma->device_issue_pending = edma_issue_pending;
1019         dma->device_tx_status = edma_tx_status;
1020         dma->device_control = edma_control;
1021         dma->device_slave_caps = edma_dma_device_slave_caps;
1022         dma->dev = dev;
1023 
1024         /*
1025          * code using dma memcpy must make sure alignment of
1026          * length is at dma->copy_align boundary.
1027          */
1028         dma->copy_align = DMA_SLAVE_BUSWIDTH_4_BYTES;
1029 
1030         INIT_LIST_HEAD(&dma->channels);
1031 }
1032 
1033 static int edma_probe(struct platform_device *pdev)
1034 {
1035         struct edma_cc *ecc;
1036         int ret;
1037 
1038         ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1039         if (ret)
1040                 return ret;
1041 
1042         ecc = devm_kzalloc(&pdev->dev, sizeof(*ecc), GFP_KERNEL);
1043         if (!ecc) {
1044                 dev_err(&pdev->dev, "Can't allocate controller\n");
1045                 return -ENOMEM;
1046         }
1047 
1048         ecc->ctlr = pdev->id;
1049         ecc->dummy_slot = edma_alloc_slot(ecc->ctlr, EDMA_SLOT_ANY);
1050         if (ecc->dummy_slot < 0) {
1051                 dev_err(&pdev->dev, "Can't allocate PaRAM dummy slot\n");
1052                 return ecc->dummy_slot;
1053         }
1054 
1055         dma_cap_zero(ecc->dma_slave.cap_mask);
1056         dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask);
1057         dma_cap_set(DMA_CYCLIC, ecc->dma_slave.cap_mask);
1058         dma_cap_set(DMA_MEMCPY, ecc->dma_slave.cap_mask);
1059 
1060         edma_dma_init(ecc, &ecc->dma_slave, &pdev->dev);
1061 
1062         edma_chan_init(ecc, &ecc->dma_slave, ecc->slave_chans);
1063 
1064         ret = dma_async_device_register(&ecc->dma_slave);
1065         if (ret)
1066                 goto err_reg1;
1067 
1068         platform_set_drvdata(pdev, ecc);
1069 
1070         dev_info(&pdev->dev, "TI EDMA DMA engine driver\n");
1071 
1072         return 0;
1073 
1074 err_reg1:
1075         edma_free_slot(ecc->dummy_slot);
1076         return ret;
1077 }
1078 
1079 static int edma_remove(struct platform_device *pdev)
1080 {
1081         struct device *dev = &pdev->dev;
1082         struct edma_cc *ecc = dev_get_drvdata(dev);
1083 
1084         dma_async_device_unregister(&ecc->dma_slave);
1085         edma_free_slot(ecc->dummy_slot);
1086 
1087         return 0;
1088 }
1089 
1090 static struct platform_driver edma_driver = {
1091         .probe          = edma_probe,
1092         .remove         = edma_remove,
1093         .driver = {
1094                 .name = "edma-dma-engine",
1095                 .owner = THIS_MODULE,
1096         },
1097 };
1098 
1099 bool edma_filter_fn(struct dma_chan *chan, void *param)
1100 {
1101         if (chan->device->dev->driver == &edma_driver.driver) {
1102                 struct edma_chan *echan = to_edma_chan(chan);
1103                 unsigned ch_req = *(unsigned *)param;
1104                 return ch_req == echan->ch_num;
1105         }
1106         return false;
1107 }
1108 EXPORT_SYMBOL(edma_filter_fn);
1109 
1110 static struct platform_device *pdev0, *pdev1;
1111 
1112 static const struct platform_device_info edma_dev_info0 = {
1113         .name = "edma-dma-engine",
1114         .id = 0,
1115         .dma_mask = DMA_BIT_MASK(32),
1116 };
1117 
1118 static const struct platform_device_info edma_dev_info1 = {
1119         .name = "edma-dma-engine",
1120         .id = 1,
1121         .dma_mask = DMA_BIT_MASK(32),
1122 };
1123 
1124 static int edma_init(void)
1125 {
1126         int ret = platform_driver_register(&edma_driver);
1127 
1128         if (ret == 0) {
1129                 pdev0 = platform_device_register_full(&edma_dev_info0);
1130                 if (IS_ERR(pdev0)) {
1131                         platform_driver_unregister(&edma_driver);
1132                         ret = PTR_ERR(pdev0);
1133                         goto out;
1134                 }
1135         }
1136 
1137         if (!of_have_populated_dt() && EDMA_CTLRS == 2) {
1138                 pdev1 = platform_device_register_full(&edma_dev_info1);
1139                 if (IS_ERR(pdev1)) {
1140                         platform_driver_unregister(&edma_driver);
1141                         platform_device_unregister(pdev0);
1142                         ret = PTR_ERR(pdev1);
1143                 }
1144         }
1145 
1146 out:
1147         return ret;
1148 }
1149 subsys_initcall(edma_init);
1150 
1151 static void __exit edma_exit(void)
1152 {
1153         platform_device_unregister(pdev0);
1154         if (pdev1)
1155                 platform_device_unregister(pdev1);
1156         platform_driver_unregister(&edma_driver);
1157 }
1158 module_exit(edma_exit);
1159 
1160 MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
1161 MODULE_DESCRIPTION("TI EDMA DMA engine driver");
1162 MODULE_LICENSE("GPL v2");
1163 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us