Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/drivers/dma/omap-dma.c

  1 /*
  2  * OMAP DMAengine support
  3  *
  4  * This program is free software; you can redistribute it and/or modify
  5  * it under the terms of the GNU General Public License version 2 as
  6  * published by the Free Software Foundation.
  7  */
  8 #include <linux/delay.h>
  9 #include <linux/dmaengine.h>
 10 #include <linux/dma-mapping.h>
 11 #include <linux/dmapool.h>
 12 #include <linux/err.h>
 13 #include <linux/init.h>
 14 #include <linux/interrupt.h>
 15 #include <linux/list.h>
 16 #include <linux/module.h>
 17 #include <linux/omap-dma.h>
 18 #include <linux/platform_device.h>
 19 #include <linux/slab.h>
 20 #include <linux/spinlock.h>
 21 #include <linux/of_dma.h>
 22 #include <linux/of_device.h>
 23 
 24 #include "virt-dma.h"
 25 
 26 #define OMAP_SDMA_REQUESTS      127
 27 #define OMAP_SDMA_CHANNELS      32
 28 
 29 struct omap_dmadev {
 30         struct dma_device ddev;
 31         spinlock_t lock;
 32         void __iomem *base;
 33         const struct omap_dma_reg *reg_map;
 34         struct omap_system_dma_plat_info *plat;
 35         bool legacy;
 36         bool ll123_supported;
 37         struct dma_pool *desc_pool;
 38         unsigned dma_requests;
 39         spinlock_t irq_lock;
 40         uint32_t irq_enable_mask;
 41         struct omap_chan **lch_map;
 42 };
 43 
 44 struct omap_chan {
 45         struct virt_dma_chan vc;
 46         void __iomem *channel_base;
 47         const struct omap_dma_reg *reg_map;
 48         uint32_t ccr;
 49 
 50         struct dma_slave_config cfg;
 51         unsigned dma_sig;
 52         bool cyclic;
 53         bool paused;
 54         bool running;
 55 
 56         int dma_ch;
 57         struct omap_desc *desc;
 58         unsigned sgidx;
 59 };
 60 
 61 #define DESC_NXT_SV_REFRESH     (0x1 << 24)
 62 #define DESC_NXT_SV_REUSE       (0x2 << 24)
 63 #define DESC_NXT_DV_REFRESH     (0x1 << 26)
 64 #define DESC_NXT_DV_REUSE       (0x2 << 26)
 65 #define DESC_NTYPE_TYPE2        (0x2 << 29)
 66 
 67 /* Type 2 descriptor with Source or Destination address update */
 68 struct omap_type2_desc {
 69         uint32_t next_desc;
 70         uint32_t en;
 71         uint32_t addr; /* src or dst */
 72         uint16_t fn;
 73         uint16_t cicr;
 74         int16_t cdei;
 75         int16_t csei;
 76         int32_t cdfi;
 77         int32_t csfi;
 78 } __packed;
 79 
 80 struct omap_sg {
 81         dma_addr_t addr;
 82         uint32_t en;            /* number of elements (24-bit) */
 83         uint32_t fn;            /* number of frames (16-bit) */
 84         int32_t fi;             /* for double indexing */
 85         int16_t ei;             /* for double indexing */
 86 
 87         /* Linked list */
 88         struct omap_type2_desc *t2_desc;
 89         dma_addr_t t2_desc_paddr;
 90 };
 91 
 92 struct omap_desc {
 93         struct virt_dma_desc vd;
 94         bool using_ll;
 95         enum dma_transfer_direction dir;
 96         dma_addr_t dev_addr;
 97 
 98         int32_t fi;             /* for OMAP_DMA_SYNC_PACKET / double indexing */
 99         int16_t ei;             /* for double indexing */
100         uint8_t es;             /* CSDP_DATA_TYPE_xxx */
101         uint32_t ccr;           /* CCR value */
102         uint16_t clnk_ctrl;     /* CLNK_CTRL value */
103         uint16_t cicr;          /* CICR value */
104         uint32_t csdp;          /* CSDP value */
105 
106         unsigned sglen;
107         struct omap_sg sg[0];
108 };
109 
110 enum {
111         CAPS_0_SUPPORT_LL123    = BIT(20),      /* Linked List type1/2/3 */
112         CAPS_0_SUPPORT_LL4      = BIT(21),      /* Linked List type4 */
113 
114         CCR_FS                  = BIT(5),
115         CCR_READ_PRIORITY       = BIT(6),
116         CCR_ENABLE              = BIT(7),
117         CCR_AUTO_INIT           = BIT(8),       /* OMAP1 only */
118         CCR_REPEAT              = BIT(9),       /* OMAP1 only */
119         CCR_OMAP31_DISABLE      = BIT(10),      /* OMAP1 only */
120         CCR_SUSPEND_SENSITIVE   = BIT(8),       /* OMAP2+ only */
121         CCR_RD_ACTIVE           = BIT(9),       /* OMAP2+ only */
122         CCR_WR_ACTIVE           = BIT(10),      /* OMAP2+ only */
123         CCR_SRC_AMODE_CONSTANT  = 0 << 12,
124         CCR_SRC_AMODE_POSTINC   = 1 << 12,
125         CCR_SRC_AMODE_SGLIDX    = 2 << 12,
126         CCR_SRC_AMODE_DBLIDX    = 3 << 12,
127         CCR_DST_AMODE_CONSTANT  = 0 << 14,
128         CCR_DST_AMODE_POSTINC   = 1 << 14,
129         CCR_DST_AMODE_SGLIDX    = 2 << 14,
130         CCR_DST_AMODE_DBLIDX    = 3 << 14,
131         CCR_CONSTANT_FILL       = BIT(16),
132         CCR_TRANSPARENT_COPY    = BIT(17),
133         CCR_BS                  = BIT(18),
134         CCR_SUPERVISOR          = BIT(22),
135         CCR_PREFETCH            = BIT(23),
136         CCR_TRIGGER_SRC         = BIT(24),
137         CCR_BUFFERING_DISABLE   = BIT(25),
138         CCR_WRITE_PRIORITY      = BIT(26),
139         CCR_SYNC_ELEMENT        = 0,
140         CCR_SYNC_FRAME          = CCR_FS,
141         CCR_SYNC_BLOCK          = CCR_BS,
142         CCR_SYNC_PACKET         = CCR_BS | CCR_FS,
143 
144         CSDP_DATA_TYPE_8        = 0,
145         CSDP_DATA_TYPE_16       = 1,
146         CSDP_DATA_TYPE_32       = 2,
147         CSDP_SRC_PORT_EMIFF     = 0 << 2, /* OMAP1 only */
148         CSDP_SRC_PORT_EMIFS     = 1 << 2, /* OMAP1 only */
149         CSDP_SRC_PORT_OCP_T1    = 2 << 2, /* OMAP1 only */
150         CSDP_SRC_PORT_TIPB      = 3 << 2, /* OMAP1 only */
151         CSDP_SRC_PORT_OCP_T2    = 4 << 2, /* OMAP1 only */
152         CSDP_SRC_PORT_MPUI      = 5 << 2, /* OMAP1 only */
153         CSDP_SRC_PACKED         = BIT(6),
154         CSDP_SRC_BURST_1        = 0 << 7,
155         CSDP_SRC_BURST_16       = 1 << 7,
156         CSDP_SRC_BURST_32       = 2 << 7,
157         CSDP_SRC_BURST_64       = 3 << 7,
158         CSDP_DST_PORT_EMIFF     = 0 << 9, /* OMAP1 only */
159         CSDP_DST_PORT_EMIFS     = 1 << 9, /* OMAP1 only */
160         CSDP_DST_PORT_OCP_T1    = 2 << 9, /* OMAP1 only */
161         CSDP_DST_PORT_TIPB      = 3 << 9, /* OMAP1 only */
162         CSDP_DST_PORT_OCP_T2    = 4 << 9, /* OMAP1 only */
163         CSDP_DST_PORT_MPUI      = 5 << 9, /* OMAP1 only */
164         CSDP_DST_PACKED         = BIT(13),
165         CSDP_DST_BURST_1        = 0 << 14,
166         CSDP_DST_BURST_16       = 1 << 14,
167         CSDP_DST_BURST_32       = 2 << 14,
168         CSDP_DST_BURST_64       = 3 << 14,
169         CSDP_WRITE_NON_POSTED   = 0 << 16,
170         CSDP_WRITE_POSTED       = 1 << 16,
171         CSDP_WRITE_LAST_NON_POSTED = 2 << 16,
172 
173         CICR_TOUT_IE            = BIT(0),       /* OMAP1 only */
174         CICR_DROP_IE            = BIT(1),
175         CICR_HALF_IE            = BIT(2),
176         CICR_FRAME_IE           = BIT(3),
177         CICR_LAST_IE            = BIT(4),
178         CICR_BLOCK_IE           = BIT(5),
179         CICR_PKT_IE             = BIT(7),       /* OMAP2+ only */
180         CICR_TRANS_ERR_IE       = BIT(8),       /* OMAP2+ only */
181         CICR_SUPERVISOR_ERR_IE  = BIT(10),      /* OMAP2+ only */
182         CICR_MISALIGNED_ERR_IE  = BIT(11),      /* OMAP2+ only */
183         CICR_DRAIN_IE           = BIT(12),      /* OMAP2+ only */
184         CICR_SUPER_BLOCK_IE     = BIT(14),      /* OMAP2+ only */
185 
186         CLNK_CTRL_ENABLE_LNK    = BIT(15),
187 
188         CDP_DST_VALID_INC       = 0 << 0,
189         CDP_DST_VALID_RELOAD    = 1 << 0,
190         CDP_DST_VALID_REUSE     = 2 << 0,
191         CDP_SRC_VALID_INC       = 0 << 2,
192         CDP_SRC_VALID_RELOAD    = 1 << 2,
193         CDP_SRC_VALID_REUSE     = 2 << 2,
194         CDP_NTYPE_TYPE1         = 1 << 4,
195         CDP_NTYPE_TYPE2         = 2 << 4,
196         CDP_NTYPE_TYPE3         = 3 << 4,
197         CDP_TMODE_NORMAL        = 0 << 8,
198         CDP_TMODE_LLIST         = 1 << 8,
199         CDP_FAST                = BIT(10),
200 };
201 
202 static const unsigned es_bytes[] = {
203         [CSDP_DATA_TYPE_8] = 1,
204         [CSDP_DATA_TYPE_16] = 2,
205         [CSDP_DATA_TYPE_32] = 4,
206 };
207 
208 static struct of_dma_filter_info omap_dma_info = {
209         .filter_fn = omap_dma_filter_fn,
210 };
211 
212 static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d)
213 {
214         return container_of(d, struct omap_dmadev, ddev);
215 }
216 
217 static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c)
218 {
219         return container_of(c, struct omap_chan, vc.chan);
220 }
221 
222 static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t)
223 {
224         return container_of(t, struct omap_desc, vd.tx);
225 }
226 
227 static void omap_dma_desc_free(struct virt_dma_desc *vd)
228 {
229         struct omap_desc *d = to_omap_dma_desc(&vd->tx);
230 
231         if (d->using_ll) {
232                 struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device);
233                 int i;
234 
235                 for (i = 0; i < d->sglen; i++) {
236                         if (d->sg[i].t2_desc)
237                                 dma_pool_free(od->desc_pool, d->sg[i].t2_desc,
238                                               d->sg[i].t2_desc_paddr);
239                 }
240         }
241 
242         kfree(d);
243 }
244 
245 static void omap_dma_fill_type2_desc(struct omap_desc *d, int idx,
246                                      enum dma_transfer_direction dir, bool last)
247 {
248         struct omap_sg *sg = &d->sg[idx];
249         struct omap_type2_desc *t2_desc = sg->t2_desc;
250 
251         if (idx)
252                 d->sg[idx - 1].t2_desc->next_desc = sg->t2_desc_paddr;
253         if (last)
254                 t2_desc->next_desc = 0xfffffffc;
255 
256         t2_desc->en = sg->en;
257         t2_desc->addr = sg->addr;
258         t2_desc->fn = sg->fn & 0xffff;
259         t2_desc->cicr = d->cicr;
260         if (!last)
261                 t2_desc->cicr &= ~CICR_BLOCK_IE;
262 
263         switch (dir) {
264         case DMA_DEV_TO_MEM:
265                 t2_desc->cdei = sg->ei;
266                 t2_desc->csei = d->ei;
267                 t2_desc->cdfi = sg->fi;
268                 t2_desc->csfi = d->fi;
269 
270                 t2_desc->en |= DESC_NXT_DV_REFRESH;
271                 t2_desc->en |= DESC_NXT_SV_REUSE;
272                 break;
273         case DMA_MEM_TO_DEV:
274                 t2_desc->cdei = d->ei;
275                 t2_desc->csei = sg->ei;
276                 t2_desc->cdfi = d->fi;
277                 t2_desc->csfi = sg->fi;
278 
279                 t2_desc->en |= DESC_NXT_SV_REFRESH;
280                 t2_desc->en |= DESC_NXT_DV_REUSE;
281                 break;
282         default:
283                 return;
284         }
285 
286         t2_desc->en |= DESC_NTYPE_TYPE2;
287 }
288 
289 static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr)
290 {
291         switch (type) {
292         case OMAP_DMA_REG_16BIT:
293                 writew_relaxed(val, addr);
294                 break;
295         case OMAP_DMA_REG_2X16BIT:
296                 writew_relaxed(val, addr);
297                 writew_relaxed(val >> 16, addr + 2);
298                 break;
299         case OMAP_DMA_REG_32BIT:
300                 writel_relaxed(val, addr);
301                 break;
302         default:
303                 WARN_ON(1);
304         }
305 }
306 
307 static unsigned omap_dma_read(unsigned type, void __iomem *addr)
308 {
309         unsigned val;
310 
311         switch (type) {
312         case OMAP_DMA_REG_16BIT:
313                 val = readw_relaxed(addr);
314                 break;
315         case OMAP_DMA_REG_2X16BIT:
316                 val = readw_relaxed(addr);
317                 val |= readw_relaxed(addr + 2) << 16;
318                 break;
319         case OMAP_DMA_REG_32BIT:
320                 val = readl_relaxed(addr);
321                 break;
322         default:
323                 WARN_ON(1);
324                 val = 0;
325         }
326 
327         return val;
328 }
329 
330 static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val)
331 {
332         const struct omap_dma_reg *r = od->reg_map + reg;
333 
334         WARN_ON(r->stride);
335 
336         omap_dma_write(val, r->type, od->base + r->offset);
337 }
338 
339 static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg)
340 {
341         const struct omap_dma_reg *r = od->reg_map + reg;
342 
343         WARN_ON(r->stride);
344 
345         return omap_dma_read(r->type, od->base + r->offset);
346 }
347 
348 static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val)
349 {
350         const struct omap_dma_reg *r = c->reg_map + reg;
351 
352         omap_dma_write(val, r->type, c->channel_base + r->offset);
353 }
354 
355 static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg)
356 {
357         const struct omap_dma_reg *r = c->reg_map + reg;
358 
359         return omap_dma_read(r->type, c->channel_base + r->offset);
360 }
361 
362 static void omap_dma_clear_csr(struct omap_chan *c)
363 {
364         if (dma_omap1())
365                 omap_dma_chan_read(c, CSR);
366         else
367                 omap_dma_chan_write(c, CSR, ~0);
368 }
369 
370 static unsigned omap_dma_get_csr(struct omap_chan *c)
371 {
372         unsigned val = omap_dma_chan_read(c, CSR);
373 
374         if (!dma_omap1())
375                 omap_dma_chan_write(c, CSR, val);
376 
377         return val;
378 }
379 
380 static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c,
381         unsigned lch)
382 {
383         c->channel_base = od->base + od->plat->channel_stride * lch;
384 
385         od->lch_map[lch] = c;
386 }
387 
388 static void omap_dma_start(struct omap_chan *c, struct omap_desc *d)
389 {
390         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
391         uint16_t cicr = d->cicr;
392 
393         if (__dma_omap15xx(od->plat->dma_attr))
394                 omap_dma_chan_write(c, CPC, 0);
395         else
396                 omap_dma_chan_write(c, CDAC, 0);
397 
398         omap_dma_clear_csr(c);
399 
400         if (d->using_ll) {
401                 uint32_t cdp = CDP_TMODE_LLIST | CDP_NTYPE_TYPE2 | CDP_FAST;
402 
403                 if (d->dir == DMA_DEV_TO_MEM)
404                         cdp |= (CDP_DST_VALID_RELOAD | CDP_SRC_VALID_REUSE);
405                 else
406                         cdp |= (CDP_DST_VALID_REUSE | CDP_SRC_VALID_RELOAD);
407                 omap_dma_chan_write(c, CDP, cdp);
408 
409                 omap_dma_chan_write(c, CNDP, d->sg[0].t2_desc_paddr);
410                 omap_dma_chan_write(c, CCDN, 0);
411                 omap_dma_chan_write(c, CCFN, 0xffff);
412                 omap_dma_chan_write(c, CCEN, 0xffffff);
413 
414                 cicr &= ~CICR_BLOCK_IE;
415         } else if (od->ll123_supported) {
416                 omap_dma_chan_write(c, CDP, 0);
417         }
418 
419         /* Enable interrupts */
420         omap_dma_chan_write(c, CICR, cicr);
421 
422         /* Enable channel */
423         omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE);
424 
425         c->running = true;
426 }
427 
428 static void omap_dma_drain_chan(struct omap_chan *c)
429 {
430         int i;
431         u32 val;
432 
433         /* Wait for sDMA FIFO to drain */
434         for (i = 0; ; i++) {
435                 val = omap_dma_chan_read(c, CCR);
436                 if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)))
437                         break;
438 
439                 if (i > 100)
440                         break;
441 
442                 udelay(5);
443         }
444 
445         if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))
446                 dev_err(c->vc.chan.device->dev,
447                         "DMA drain did not complete on lch %d\n",
448                         c->dma_ch);
449 }
450 
451 static int omap_dma_stop(struct omap_chan *c)
452 {
453         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
454         uint32_t val;
455 
456         /* disable irq */
457         omap_dma_chan_write(c, CICR, 0);
458 
459         omap_dma_clear_csr(c);
460 
461         val = omap_dma_chan_read(c, CCR);
462         if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) {
463                 uint32_t sysconfig;
464 
465                 sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG);
466                 val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK;
467                 val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE);
468                 omap_dma_glbl_write(od, OCP_SYSCONFIG, val);
469 
470                 val = omap_dma_chan_read(c, CCR);
471                 val &= ~CCR_ENABLE;
472                 omap_dma_chan_write(c, CCR, val);
473 
474                 if (!(c->ccr & CCR_BUFFERING_DISABLE))
475                         omap_dma_drain_chan(c);
476 
477                 omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig);
478         } else {
479                 if (!(val & CCR_ENABLE))
480                         return -EINVAL;
481 
482                 val &= ~CCR_ENABLE;
483                 omap_dma_chan_write(c, CCR, val);
484 
485                 if (!(c->ccr & CCR_BUFFERING_DISABLE))
486                         omap_dma_drain_chan(c);
487         }
488 
489         mb();
490 
491         if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) {
492                 val = omap_dma_chan_read(c, CLNK_CTRL);
493 
494                 if (dma_omap1())
495                         val |= 1 << 14; /* set the STOP_LNK bit */
496                 else
497                         val &= ~CLNK_CTRL_ENABLE_LNK;
498 
499                 omap_dma_chan_write(c, CLNK_CTRL, val);
500         }
501         c->running = false;
502         return 0;
503 }
504 
505 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d)
506 {
507         struct omap_sg *sg = d->sg + c->sgidx;
508         unsigned cxsa, cxei, cxfi;
509 
510         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
511                 cxsa = CDSA;
512                 cxei = CDEI;
513                 cxfi = CDFI;
514         } else {
515                 cxsa = CSSA;
516                 cxei = CSEI;
517                 cxfi = CSFI;
518         }
519 
520         omap_dma_chan_write(c, cxsa, sg->addr);
521         omap_dma_chan_write(c, cxei, sg->ei);
522         omap_dma_chan_write(c, cxfi, sg->fi);
523         omap_dma_chan_write(c, CEN, sg->en);
524         omap_dma_chan_write(c, CFN, sg->fn);
525 
526         omap_dma_start(c, d);
527         c->sgidx++;
528 }
529 
530 static void omap_dma_start_desc(struct omap_chan *c)
531 {
532         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
533         struct omap_desc *d;
534         unsigned cxsa, cxei, cxfi;
535 
536         if (!vd) {
537                 c->desc = NULL;
538                 return;
539         }
540 
541         list_del(&vd->node);
542 
543         c->desc = d = to_omap_dma_desc(&vd->tx);
544         c->sgidx = 0;
545 
546         /*
547          * This provides the necessary barrier to ensure data held in
548          * DMA coherent memory is visible to the DMA engine prior to
549          * the transfer starting.
550          */
551         mb();
552 
553         omap_dma_chan_write(c, CCR, d->ccr);
554         if (dma_omap1())
555                 omap_dma_chan_write(c, CCR2, d->ccr >> 16);
556 
557         if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) {
558                 cxsa = CSSA;
559                 cxei = CSEI;
560                 cxfi = CSFI;
561         } else {
562                 cxsa = CDSA;
563                 cxei = CDEI;
564                 cxfi = CDFI;
565         }
566 
567         omap_dma_chan_write(c, cxsa, d->dev_addr);
568         omap_dma_chan_write(c, cxei, d->ei);
569         omap_dma_chan_write(c, cxfi, d->fi);
570         omap_dma_chan_write(c, CSDP, d->csdp);
571         omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl);
572 
573         omap_dma_start_sg(c, d);
574 }
575 
576 static void omap_dma_callback(int ch, u16 status, void *data)
577 {
578         struct omap_chan *c = data;
579         struct omap_desc *d;
580         unsigned long flags;
581 
582         spin_lock_irqsave(&c->vc.lock, flags);
583         d = c->desc;
584         if (d) {
585                 if (c->cyclic) {
586                         vchan_cyclic_callback(&d->vd);
587                 } else if (d->using_ll || c->sgidx == d->sglen) {
588                         omap_dma_start_desc(c);
589                         vchan_cookie_complete(&d->vd);
590                 } else {
591                         omap_dma_start_sg(c, d);
592                 }
593         }
594         spin_unlock_irqrestore(&c->vc.lock, flags);
595 }
596 
597 static irqreturn_t omap_dma_irq(int irq, void *devid)
598 {
599         struct omap_dmadev *od = devid;
600         unsigned status, channel;
601 
602         spin_lock(&od->irq_lock);
603 
604         status = omap_dma_glbl_read(od, IRQSTATUS_L1);
605         status &= od->irq_enable_mask;
606         if (status == 0) {
607                 spin_unlock(&od->irq_lock);
608                 return IRQ_NONE;
609         }
610 
611         while ((channel = ffs(status)) != 0) {
612                 unsigned mask, csr;
613                 struct omap_chan *c;
614 
615                 channel -= 1;
616                 mask = BIT(channel);
617                 status &= ~mask;
618 
619                 c = od->lch_map[channel];
620                 if (c == NULL) {
621                         /* This should never happen */
622                         dev_err(od->ddev.dev, "invalid channel %u\n", channel);
623                         continue;
624                 }
625 
626                 csr = omap_dma_get_csr(c);
627                 omap_dma_glbl_write(od, IRQSTATUS_L1, mask);
628 
629                 omap_dma_callback(channel, csr, c);
630         }
631 
632         spin_unlock(&od->irq_lock);
633 
634         return IRQ_HANDLED;
635 }
636 
637 static int omap_dma_alloc_chan_resources(struct dma_chan *chan)
638 {
639         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
640         struct omap_chan *c = to_omap_dma_chan(chan);
641         struct device *dev = od->ddev.dev;
642         int ret;
643 
644         if (od->legacy) {
645                 ret = omap_request_dma(c->dma_sig, "DMA engine",
646                                        omap_dma_callback, c, &c->dma_ch);
647         } else {
648                 ret = omap_request_dma(c->dma_sig, "DMA engine", NULL, NULL,
649                                        &c->dma_ch);
650         }
651 
652         dev_dbg(dev, "allocating channel %u for %u\n", c->dma_ch, c->dma_sig);
653 
654         if (ret >= 0) {
655                 omap_dma_assign(od, c, c->dma_ch);
656 
657                 if (!od->legacy) {
658                         unsigned val;
659 
660                         spin_lock_irq(&od->irq_lock);
661                         val = BIT(c->dma_ch);
662                         omap_dma_glbl_write(od, IRQSTATUS_L1, val);
663                         od->irq_enable_mask |= val;
664                         omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
665 
666                         val = omap_dma_glbl_read(od, IRQENABLE_L0);
667                         val &= ~BIT(c->dma_ch);
668                         omap_dma_glbl_write(od, IRQENABLE_L0, val);
669                         spin_unlock_irq(&od->irq_lock);
670                 }
671         }
672 
673         if (dma_omap1()) {
674                 if (__dma_omap16xx(od->plat->dma_attr)) {
675                         c->ccr = CCR_OMAP31_DISABLE;
676                         /* Duplicate what plat-omap/dma.c does */
677                         c->ccr |= c->dma_ch + 1;
678                 } else {
679                         c->ccr = c->dma_sig & 0x1f;
680                 }
681         } else {
682                 c->ccr = c->dma_sig & 0x1f;
683                 c->ccr |= (c->dma_sig & ~0x1f) << 14;
684         }
685         if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING)
686                 c->ccr |= CCR_BUFFERING_DISABLE;
687 
688         return ret;
689 }
690 
691 static void omap_dma_free_chan_resources(struct dma_chan *chan)
692 {
693         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
694         struct omap_chan *c = to_omap_dma_chan(chan);
695 
696         if (!od->legacy) {
697                 spin_lock_irq(&od->irq_lock);
698                 od->irq_enable_mask &= ~BIT(c->dma_ch);
699                 omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask);
700                 spin_unlock_irq(&od->irq_lock);
701         }
702 
703         c->channel_base = NULL;
704         od->lch_map[c->dma_ch] = NULL;
705         vchan_free_chan_resources(&c->vc);
706         omap_free_dma(c->dma_ch);
707 
708         dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch,
709                 c->dma_sig);
710         c->dma_sig = 0;
711 }
712 
713 static size_t omap_dma_sg_size(struct omap_sg *sg)
714 {
715         return sg->en * sg->fn;
716 }
717 
718 static size_t omap_dma_desc_size(struct omap_desc *d)
719 {
720         unsigned i;
721         size_t size;
722 
723         for (size = i = 0; i < d->sglen; i++)
724                 size += omap_dma_sg_size(&d->sg[i]);
725 
726         return size * es_bytes[d->es];
727 }
728 
729 static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr)
730 {
731         unsigned i;
732         size_t size, es_size = es_bytes[d->es];
733 
734         for (size = i = 0; i < d->sglen; i++) {
735                 size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size;
736 
737                 if (size)
738                         size += this_size;
739                 else if (addr >= d->sg[i].addr &&
740                          addr < d->sg[i].addr + this_size)
741                         size += d->sg[i].addr + this_size - addr;
742         }
743         return size;
744 }
745 
746 /*
747  * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is
748  * read before the DMA controller finished disabling the channel.
749  */
750 static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg)
751 {
752         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
753         uint32_t val;
754 
755         val = omap_dma_chan_read(c, reg);
756         if (val == 0 && od->plat->errata & DMA_ERRATA_3_3)
757                 val = omap_dma_chan_read(c, reg);
758 
759         return val;
760 }
761 
762 static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c)
763 {
764         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
765         dma_addr_t addr, cdac;
766 
767         if (__dma_omap15xx(od->plat->dma_attr)) {
768                 addr = omap_dma_chan_read(c, CPC);
769         } else {
770                 addr = omap_dma_chan_read_3_3(c, CSAC);
771                 cdac = omap_dma_chan_read_3_3(c, CDAC);
772 
773                 /*
774                  * CDAC == 0 indicates that the DMA transfer on the channel has
775                  * not been started (no data has been transferred so far).
776                  * Return the programmed source start address in this case.
777                  */
778                 if (cdac == 0)
779                         addr = omap_dma_chan_read(c, CSSA);
780         }
781 
782         if (dma_omap1())
783                 addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000;
784 
785         return addr;
786 }
787 
788 static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c)
789 {
790         struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device);
791         dma_addr_t addr;
792 
793         if (__dma_omap15xx(od->plat->dma_attr)) {
794                 addr = omap_dma_chan_read(c, CPC);
795         } else {
796                 addr = omap_dma_chan_read_3_3(c, CDAC);
797 
798                 /*
799                  * CDAC == 0 indicates that the DMA transfer on the channel
800                  * has not been started (no data has been transferred so
801                  * far).  Return the programmed destination start address in
802                  * this case.
803                  */
804                 if (addr == 0)
805                         addr = omap_dma_chan_read(c, CDSA);
806         }
807 
808         if (dma_omap1())
809                 addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000;
810 
811         return addr;
812 }
813 
814 static enum dma_status omap_dma_tx_status(struct dma_chan *chan,
815         dma_cookie_t cookie, struct dma_tx_state *txstate)
816 {
817         struct omap_chan *c = to_omap_dma_chan(chan);
818         struct virt_dma_desc *vd;
819         enum dma_status ret;
820         unsigned long flags;
821 
822         ret = dma_cookie_status(chan, cookie, txstate);
823 
824         if (!c->paused && c->running) {
825                 uint32_t ccr = omap_dma_chan_read(c, CCR);
826                 /*
827                  * The channel is no longer active, set the return value
828                  * accordingly
829                  */
830                 if (!(ccr & CCR_ENABLE))
831                         ret = DMA_COMPLETE;
832         }
833 
834         if (ret == DMA_COMPLETE || !txstate)
835                 return ret;
836 
837         spin_lock_irqsave(&c->vc.lock, flags);
838         vd = vchan_find_desc(&c->vc, cookie);
839         if (vd) {
840                 txstate->residue = omap_dma_desc_size(to_omap_dma_desc(&vd->tx));
841         } else if (c->desc && c->desc->vd.tx.cookie == cookie) {
842                 struct omap_desc *d = c->desc;
843                 dma_addr_t pos;
844 
845                 if (d->dir == DMA_MEM_TO_DEV)
846                         pos = omap_dma_get_src_pos(c);
847                 else if (d->dir == DMA_DEV_TO_MEM  || d->dir == DMA_MEM_TO_MEM)
848                         pos = omap_dma_get_dst_pos(c);
849                 else
850                         pos = 0;
851 
852                 txstate->residue = omap_dma_desc_size_pos(d, pos);
853         } else {
854                 txstate->residue = 0;
855         }
856         if (ret == DMA_IN_PROGRESS && c->paused)
857                 ret = DMA_PAUSED;
858         spin_unlock_irqrestore(&c->vc.lock, flags);
859 
860         return ret;
861 }
862 
863 static void omap_dma_issue_pending(struct dma_chan *chan)
864 {
865         struct omap_chan *c = to_omap_dma_chan(chan);
866         unsigned long flags;
867 
868         spin_lock_irqsave(&c->vc.lock, flags);
869         if (vchan_issue_pending(&c->vc) && !c->desc)
870                 omap_dma_start_desc(c);
871         spin_unlock_irqrestore(&c->vc.lock, flags);
872 }
873 
874 static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg(
875         struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen,
876         enum dma_transfer_direction dir, unsigned long tx_flags, void *context)
877 {
878         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
879         struct omap_chan *c = to_omap_dma_chan(chan);
880         enum dma_slave_buswidth dev_width;
881         struct scatterlist *sgent;
882         struct omap_desc *d;
883         dma_addr_t dev_addr;
884         unsigned i, es, en, frame_bytes;
885         bool ll_failed = false;
886         u32 burst;
887         u32 port_window, port_window_bytes;
888 
889         if (dir == DMA_DEV_TO_MEM) {
890                 dev_addr = c->cfg.src_addr;
891                 dev_width = c->cfg.src_addr_width;
892                 burst = c->cfg.src_maxburst;
893                 port_window = c->cfg.src_port_window_size;
894         } else if (dir == DMA_MEM_TO_DEV) {
895                 dev_addr = c->cfg.dst_addr;
896                 dev_width = c->cfg.dst_addr_width;
897                 burst = c->cfg.dst_maxburst;
898                 port_window = c->cfg.dst_port_window_size;
899         } else {
900                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
901                 return NULL;
902         }
903 
904         /* Bus width translates to the element size (ES) */
905         switch (dev_width) {
906         case DMA_SLAVE_BUSWIDTH_1_BYTE:
907                 es = CSDP_DATA_TYPE_8;
908                 break;
909         case DMA_SLAVE_BUSWIDTH_2_BYTES:
910                 es = CSDP_DATA_TYPE_16;
911                 break;
912         case DMA_SLAVE_BUSWIDTH_4_BYTES:
913                 es = CSDP_DATA_TYPE_32;
914                 break;
915         default: /* not reached */
916                 return NULL;
917         }
918 
919         /* When the port_window is used, one frame must cover the window */
920         if (port_window) {
921                 burst = port_window;
922                 port_window_bytes = port_window * es_bytes[es];
923         }
924 
925         /* Now allocate and setup the descriptor. */
926         d = kzalloc(sizeof(*d) + sglen * sizeof(d->sg[0]), GFP_ATOMIC);
927         if (!d)
928                 return NULL;
929 
930         d->dir = dir;
931         d->dev_addr = dev_addr;
932         d->es = es;
933 
934         d->ccr = c->ccr | CCR_SYNC_FRAME;
935         if (dir == DMA_DEV_TO_MEM) {
936                 d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED;
937 
938                 d->ccr |= CCR_DST_AMODE_POSTINC;
939                 if (port_window) {
940                         d->ccr |= CCR_SRC_AMODE_DBLIDX;
941 
942                         if (port_window_bytes >= 64)
943                                 d->csdp |= CSDP_SRC_BURST_64;
944                         else if (port_window_bytes >= 32)
945                                 d->csdp |= CSDP_SRC_BURST_32;
946                         else if (port_window_bytes >= 16)
947                                 d->csdp |= CSDP_SRC_BURST_16;
948 
949                 } else {
950                         d->ccr |= CCR_SRC_AMODE_CONSTANT;
951                 }
952         } else {
953                 d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED;
954 
955                 d->ccr |= CCR_SRC_AMODE_POSTINC;
956                 if (port_window) {
957                         d->ccr |= CCR_DST_AMODE_DBLIDX;
958                         d->ei = 1;
959                         /*
960                          * One frame covers the port_window and by  configure
961                          * the source frame index to be -1 * (port_window - 1)
962                          * we instruct the sDMA that after a frame is processed
963                          * it should move back to the start of the window.
964                          */
965                         d->fi = -(port_window_bytes - 1);
966 
967                         if (port_window_bytes >= 64)
968                                 d->csdp |= CSDP_DST_BURST_64;
969                         else if (port_window_bytes >= 32)
970                                 d->csdp |= CSDP_DST_BURST_32;
971                         else if (port_window_bytes >= 16)
972                                 d->csdp |= CSDP_DST_BURST_16;
973                 } else {
974                         d->ccr |= CCR_DST_AMODE_CONSTANT;
975                 }
976         }
977 
978         d->cicr = CICR_DROP_IE | CICR_BLOCK_IE;
979         d->csdp |= es;
980 
981         if (dma_omap1()) {
982                 d->cicr |= CICR_TOUT_IE;
983 
984                 if (dir == DMA_DEV_TO_MEM)
985                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB;
986                 else
987                         d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF;
988         } else {
989                 if (dir == DMA_DEV_TO_MEM)
990                         d->ccr |= CCR_TRIGGER_SRC;
991 
992                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
993 
994                 if (port_window)
995                         d->csdp |= CSDP_WRITE_LAST_NON_POSTED;
996         }
997         if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS)
998                 d->clnk_ctrl = c->dma_ch;
999 
1000         /*
1001          * Build our scatterlist entries: each contains the address,
1002          * the number of elements (EN) in each frame, and the number of
1003          * frames (FN).  Number of bytes for this entry = ES * EN * FN.
1004          *
1005          * Burst size translates to number of elements with frame sync.
1006          * Note: DMA engine defines burst to be the number of dev-width
1007          * transfers.
1008          */
1009         en = burst;
1010         frame_bytes = es_bytes[es] * en;
1011 
1012         if (sglen >= 2)
1013                 d->using_ll = od->ll123_supported;
1014 
1015         for_each_sg(sgl, sgent, sglen, i) {
1016                 struct omap_sg *osg = &d->sg[i];
1017 
1018                 osg->addr = sg_dma_address(sgent);
1019                 osg->en = en;
1020                 osg->fn = sg_dma_len(sgent) / frame_bytes;
1021                 if (port_window && dir == DMA_DEV_TO_MEM) {
1022                         osg->ei = 1;
1023                         /*
1024                          * One frame covers the port_window and by  configure
1025                          * the source frame index to be -1 * (port_window - 1)
1026                          * we instruct the sDMA that after a frame is processed
1027                          * it should move back to the start of the window.
1028                          */
1029                         osg->fi = -(port_window_bytes - 1);
1030                 }
1031 
1032                 if (d->using_ll) {
1033                         osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC,
1034                                                       &osg->t2_desc_paddr);
1035                         if (!osg->t2_desc) {
1036                                 dev_err(chan->device->dev,
1037                                         "t2_desc[%d] allocation failed\n", i);
1038                                 ll_failed = true;
1039                                 d->using_ll = false;
1040                                 continue;
1041                         }
1042 
1043                         omap_dma_fill_type2_desc(d, i, dir, (i == sglen - 1));
1044                 }
1045         }
1046 
1047         d->sglen = sglen;
1048 
1049         /* Release the dma_pool entries if one allocation failed */
1050         if (ll_failed) {
1051                 for (i = 0; i < d->sglen; i++) {
1052                         struct omap_sg *osg = &d->sg[i];
1053 
1054                         if (osg->t2_desc) {
1055                                 dma_pool_free(od->desc_pool, osg->t2_desc,
1056                                               osg->t2_desc_paddr);
1057                                 osg->t2_desc = NULL;
1058                         }
1059                 }
1060         }
1061 
1062         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1063 }
1064 
1065 static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic(
1066         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1067         size_t period_len, enum dma_transfer_direction dir, unsigned long flags)
1068 {
1069         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1070         struct omap_chan *c = to_omap_dma_chan(chan);
1071         enum dma_slave_buswidth dev_width;
1072         struct omap_desc *d;
1073         dma_addr_t dev_addr;
1074         unsigned es;
1075         u32 burst;
1076 
1077         if (dir == DMA_DEV_TO_MEM) {
1078                 dev_addr = c->cfg.src_addr;
1079                 dev_width = c->cfg.src_addr_width;
1080                 burst = c->cfg.src_maxburst;
1081         } else if (dir == DMA_MEM_TO_DEV) {
1082                 dev_addr = c->cfg.dst_addr;
1083                 dev_width = c->cfg.dst_addr_width;
1084                 burst = c->cfg.dst_maxburst;
1085         } else {
1086                 dev_err(chan->device->dev, "%s: bad direction?\n", __func__);
1087                 return NULL;
1088         }
1089 
1090         /* Bus width translates to the element size (ES) */
1091         switch (dev_width) {
1092         case DMA_SLAVE_BUSWIDTH_1_BYTE:
1093                 es = CSDP_DATA_TYPE_8;
1094                 break;
1095         case DMA_SLAVE_BUSWIDTH_2_BYTES:
1096                 es = CSDP_DATA_TYPE_16;
1097                 break;
1098         case DMA_SLAVE_BUSWIDTH_4_BYTES:
1099                 es = CSDP_DATA_TYPE_32;
1100                 break;
1101         default: /* not reached */
1102                 return NULL;
1103         }
1104 
1105         /* Now allocate and setup the descriptor. */
1106         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1107         if (!d)
1108                 return NULL;
1109 
1110         d->dir = dir;
1111         d->dev_addr = dev_addr;
1112         d->fi = burst;
1113         d->es = es;
1114         d->sg[0].addr = buf_addr;
1115         d->sg[0].en = period_len / es_bytes[es];
1116         d->sg[0].fn = buf_len / period_len;
1117         d->sglen = 1;
1118 
1119         d->ccr = c->ccr;
1120         if (dir == DMA_DEV_TO_MEM)
1121                 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT;
1122         else
1123                 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC;
1124 
1125         d->cicr = CICR_DROP_IE;
1126         if (flags & DMA_PREP_INTERRUPT)
1127                 d->cicr |= CICR_FRAME_IE;
1128 
1129         d->csdp = es;
1130 
1131         if (dma_omap1()) {
1132                 d->cicr |= CICR_TOUT_IE;
1133 
1134                 if (dir == DMA_DEV_TO_MEM)
1135                         d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI;
1136                 else
1137                         d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF;
1138         } else {
1139                 if (burst)
1140                         d->ccr |= CCR_SYNC_PACKET;
1141                 else
1142                         d->ccr |= CCR_SYNC_ELEMENT;
1143 
1144                 if (dir == DMA_DEV_TO_MEM) {
1145                         d->ccr |= CCR_TRIGGER_SRC;
1146                         d->csdp |= CSDP_DST_PACKED;
1147                 } else {
1148                         d->csdp |= CSDP_SRC_PACKED;
1149                 }
1150 
1151                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1152 
1153                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1154         }
1155 
1156         if (__dma_omap15xx(od->plat->dma_attr))
1157                 d->ccr |= CCR_AUTO_INIT | CCR_REPEAT;
1158         else
1159                 d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK;
1160 
1161         c->cyclic = true;
1162 
1163         return vchan_tx_prep(&c->vc, &d->vd, flags);
1164 }
1165 
1166 static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy(
1167         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1168         size_t len, unsigned long tx_flags)
1169 {
1170         struct omap_chan *c = to_omap_dma_chan(chan);
1171         struct omap_desc *d;
1172         uint8_t data_type;
1173 
1174         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1175         if (!d)
1176                 return NULL;
1177 
1178         data_type = __ffs((src | dest | len));
1179         if (data_type > CSDP_DATA_TYPE_32)
1180                 data_type = CSDP_DATA_TYPE_32;
1181 
1182         d->dir = DMA_MEM_TO_MEM;
1183         d->dev_addr = src;
1184         d->fi = 0;
1185         d->es = data_type;
1186         d->sg[0].en = len / BIT(data_type);
1187         d->sg[0].fn = 1;
1188         d->sg[0].addr = dest;
1189         d->sglen = 1;
1190         d->ccr = c->ccr;
1191         d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC;
1192 
1193         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1194 
1195         d->csdp = data_type;
1196 
1197         if (dma_omap1()) {
1198                 d->cicr |= CICR_TOUT_IE;
1199                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1200         } else {
1201                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1202                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1203                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1204         }
1205 
1206         return vchan_tx_prep(&c->vc, &d->vd, tx_flags);
1207 }
1208 
1209 static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved(
1210         struct dma_chan *chan, struct dma_interleaved_template *xt,
1211         unsigned long flags)
1212 {
1213         struct omap_chan *c = to_omap_dma_chan(chan);
1214         struct omap_desc *d;
1215         struct omap_sg *sg;
1216         uint8_t data_type;
1217         size_t src_icg, dst_icg;
1218 
1219         /* Slave mode is not supported */
1220         if (is_slave_direction(xt->dir))
1221                 return NULL;
1222 
1223         if (xt->frame_size != 1 || xt->numf == 0)
1224                 return NULL;
1225 
1226         d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC);
1227         if (!d)
1228                 return NULL;
1229 
1230         data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size));
1231         if (data_type > CSDP_DATA_TYPE_32)
1232                 data_type = CSDP_DATA_TYPE_32;
1233 
1234         sg = &d->sg[0];
1235         d->dir = DMA_MEM_TO_MEM;
1236         d->dev_addr = xt->src_start;
1237         d->es = data_type;
1238         sg->en = xt->sgl[0].size / BIT(data_type);
1239         sg->fn = xt->numf;
1240         sg->addr = xt->dst_start;
1241         d->sglen = 1;
1242         d->ccr = c->ccr;
1243 
1244         src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]);
1245         dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]);
1246         if (src_icg) {
1247                 d->ccr |= CCR_SRC_AMODE_DBLIDX;
1248                 d->ei = 1;
1249                 d->fi = src_icg;
1250         } else if (xt->src_inc) {
1251                 d->ccr |= CCR_SRC_AMODE_POSTINC;
1252                 d->fi = 0;
1253         } else {
1254                 dev_err(chan->device->dev,
1255                         "%s: SRC constant addressing is not supported\n",
1256                         __func__);
1257                 kfree(d);
1258                 return NULL;
1259         }
1260 
1261         if (dst_icg) {
1262                 d->ccr |= CCR_DST_AMODE_DBLIDX;
1263                 sg->ei = 1;
1264                 sg->fi = dst_icg;
1265         } else if (xt->dst_inc) {
1266                 d->ccr |= CCR_DST_AMODE_POSTINC;
1267                 sg->fi = 0;
1268         } else {
1269                 dev_err(chan->device->dev,
1270                         "%s: DST constant addressing is not supported\n",
1271                         __func__);
1272                 kfree(d);
1273                 return NULL;
1274         }
1275 
1276         d->cicr = CICR_DROP_IE | CICR_FRAME_IE;
1277 
1278         d->csdp = data_type;
1279 
1280         if (dma_omap1()) {
1281                 d->cicr |= CICR_TOUT_IE;
1282                 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF;
1283         } else {
1284                 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED;
1285                 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE;
1286                 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64;
1287         }
1288 
1289         return vchan_tx_prep(&c->vc, &d->vd, flags);
1290 }
1291 
1292 static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg)
1293 {
1294         struct omap_chan *c = to_omap_dma_chan(chan);
1295 
1296         if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1297             cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
1298                 return -EINVAL;
1299 
1300         memcpy(&c->cfg, cfg, sizeof(c->cfg));
1301 
1302         return 0;
1303 }
1304 
1305 static int omap_dma_terminate_all(struct dma_chan *chan)
1306 {
1307         struct omap_chan *c = to_omap_dma_chan(chan);
1308         unsigned long flags;
1309         LIST_HEAD(head);
1310 
1311         spin_lock_irqsave(&c->vc.lock, flags);
1312 
1313         /*
1314          * Stop DMA activity: we assume the callback will not be called
1315          * after omap_dma_stop() returns (even if it does, it will see
1316          * c->desc is NULL and exit.)
1317          */
1318         if (c->desc) {
1319                 omap_dma_desc_free(&c->desc->vd);
1320                 c->desc = NULL;
1321                 /* Avoid stopping the dma twice */
1322                 if (!c->paused)
1323                         omap_dma_stop(c);
1324         }
1325 
1326         c->cyclic = false;
1327         c->paused = false;
1328 
1329         vchan_get_all_descriptors(&c->vc, &head);
1330         spin_unlock_irqrestore(&c->vc.lock, flags);
1331         vchan_dma_desc_free_list(&c->vc, &head);
1332 
1333         return 0;
1334 }
1335 
1336 static void omap_dma_synchronize(struct dma_chan *chan)
1337 {
1338         struct omap_chan *c = to_omap_dma_chan(chan);
1339 
1340         vchan_synchronize(&c->vc);
1341 }
1342 
1343 static int omap_dma_pause(struct dma_chan *chan)
1344 {
1345         struct omap_chan *c = to_omap_dma_chan(chan);
1346         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1347         unsigned long flags;
1348         int ret = -EINVAL;
1349         bool can_pause = false;
1350 
1351         spin_lock_irqsave(&od->irq_lock, flags);
1352 
1353         if (!c->desc)
1354                 goto out;
1355 
1356         if (c->cyclic)
1357                 can_pause = true;
1358 
1359         /*
1360          * We do not allow DMA_MEM_TO_DEV transfers to be paused.
1361          * From the AM572x TRM, 16.1.4.18 Disabling a Channel During Transfer:
1362          * "When a channel is disabled during a transfer, the channel undergoes
1363          * an abort, unless it is hardware-source-synchronized …".
1364          * A source-synchronised channel is one where the fetching of data is
1365          * under control of the device. In other words, a device-to-memory
1366          * transfer. So, a destination-synchronised channel (which would be a
1367          * memory-to-device transfer) undergoes an abort if the the CCR_ENABLE
1368          * bit is cleared.
1369          * From 16.1.4.20.4.6.2 Abort: "If an abort trigger occurs, the channel
1370          * aborts immediately after completion of current read/write
1371          * transactions and then the FIFO is cleaned up." The term "cleaned up"
1372          * is not defined. TI recommends to check that RD_ACTIVE and WR_ACTIVE
1373          * are both clear _before_ disabling the channel, otherwise data loss
1374          * will occur.
1375          * The problem is that if the channel is active, then device activity
1376          * can result in DMA activity starting between reading those as both
1377          * clear and the write to DMA_CCR to clear the enable bit hitting the
1378          * hardware. If the DMA hardware can't drain the data in its FIFO to the
1379          * destination, then data loss "might" occur (say if we write to an UART
1380          * and the UART is not accepting any further data).
1381          */
1382         else if (c->desc->dir == DMA_DEV_TO_MEM)
1383                 can_pause = true;
1384 
1385         if (can_pause && !c->paused) {
1386                 ret = omap_dma_stop(c);
1387                 if (!ret)
1388                         c->paused = true;
1389         }
1390 out:
1391         spin_unlock_irqrestore(&od->irq_lock, flags);
1392 
1393         return ret;
1394 }
1395 
1396 static int omap_dma_resume(struct dma_chan *chan)
1397 {
1398         struct omap_chan *c = to_omap_dma_chan(chan);
1399         struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1400         unsigned long flags;
1401         int ret = -EINVAL;
1402 
1403         spin_lock_irqsave(&od->irq_lock, flags);
1404 
1405         if (c->paused && c->desc) {
1406                 mb();
1407 
1408                 /* Restore channel link register */
1409                 omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl);
1410 
1411                 omap_dma_start(c, c->desc);
1412                 c->paused = false;
1413                 ret = 0;
1414         }
1415         spin_unlock_irqrestore(&od->irq_lock, flags);
1416 
1417         return ret;
1418 }
1419 
1420 static int omap_dma_chan_init(struct omap_dmadev *od)
1421 {
1422         struct omap_chan *c;
1423 
1424         c = kzalloc(sizeof(*c), GFP_KERNEL);
1425         if (!c)
1426                 return -ENOMEM;
1427 
1428         c->reg_map = od->reg_map;
1429         c->vc.desc_free = omap_dma_desc_free;
1430         vchan_init(&c->vc, &od->ddev);
1431 
1432         return 0;
1433 }
1434 
1435 static void omap_dma_free(struct omap_dmadev *od)
1436 {
1437         while (!list_empty(&od->ddev.channels)) {
1438                 struct omap_chan *c = list_first_entry(&od->ddev.channels,
1439                         struct omap_chan, vc.chan.device_node);
1440 
1441                 list_del(&c->vc.chan.device_node);
1442                 tasklet_kill(&c->vc.task);
1443                 kfree(c);
1444         }
1445 }
1446 
1447 #define OMAP_DMA_BUSWIDTHS      (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1448                                  BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
1449                                  BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1450 
1451 static int omap_dma_probe(struct platform_device *pdev)
1452 {
1453         struct omap_dmadev *od;
1454         struct resource *res;
1455         int rc, i, irq;
1456         u32 lch_count;
1457 
1458         od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL);
1459         if (!od)
1460                 return -ENOMEM;
1461 
1462         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1463         od->base = devm_ioremap_resource(&pdev->dev, res);
1464         if (IS_ERR(od->base))
1465                 return PTR_ERR(od->base);
1466 
1467         od->plat = omap_get_plat_info();
1468         if (!od->plat)
1469                 return -EPROBE_DEFER;
1470 
1471         od->reg_map = od->plat->reg_map;
1472 
1473         dma_cap_set(DMA_SLAVE, od->ddev.cap_mask);
1474         dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask);
1475         dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask);
1476         dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask);
1477         od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources;
1478         od->ddev.device_free_chan_resources = omap_dma_free_chan_resources;
1479         od->ddev.device_tx_status = omap_dma_tx_status;
1480         od->ddev.device_issue_pending = omap_dma_issue_pending;
1481         od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg;
1482         od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic;
1483         od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy;
1484         od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved;
1485         od->ddev.device_config = omap_dma_slave_config;
1486         od->ddev.device_pause = omap_dma_pause;
1487         od->ddev.device_resume = omap_dma_resume;
1488         od->ddev.device_terminate_all = omap_dma_terminate_all;
1489         od->ddev.device_synchronize = omap_dma_synchronize;
1490         od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS;
1491         od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS;
1492         od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1493         od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1494         od->ddev.dev = &pdev->dev;
1495         INIT_LIST_HEAD(&od->ddev.channels);
1496         spin_lock_init(&od->lock);
1497         spin_lock_init(&od->irq_lock);
1498 
1499         /* Number of DMA requests */
1500         od->dma_requests = OMAP_SDMA_REQUESTS;
1501         if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
1502                                                       "dma-requests",
1503                                                       &od->dma_requests)) {
1504                 dev_info(&pdev->dev,
1505                          "Missing dma-requests property, using %u.\n",
1506                          OMAP_SDMA_REQUESTS);
1507         }
1508 
1509         /* Number of available logical channels */
1510         if (!pdev->dev.of_node) {
1511                 lch_count = od->plat->dma_attr->lch_count;
1512                 if (unlikely(!lch_count))
1513                         lch_count = OMAP_SDMA_CHANNELS;
1514         } else if (of_property_read_u32(pdev->dev.of_node, "dma-channels",
1515                                         &lch_count)) {
1516                 dev_info(&pdev->dev,
1517                          "Missing dma-channels property, using %u.\n",
1518                          OMAP_SDMA_CHANNELS);
1519                 lch_count = OMAP_SDMA_CHANNELS;
1520         }
1521 
1522         od->lch_map = devm_kcalloc(&pdev->dev, lch_count, sizeof(*od->lch_map),
1523                                    GFP_KERNEL);
1524         if (!od->lch_map)
1525                 return -ENOMEM;
1526 
1527         for (i = 0; i < od->dma_requests; i++) {
1528                 rc = omap_dma_chan_init(od);
1529                 if (rc) {
1530                         omap_dma_free(od);
1531                         return rc;
1532                 }
1533         }
1534 
1535         irq = platform_get_irq(pdev, 1);
1536         if (irq <= 0) {
1537                 dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq);
1538                 od->legacy = true;
1539         } else {
1540                 /* Disable all interrupts */
1541                 od->irq_enable_mask = 0;
1542                 omap_dma_glbl_write(od, IRQENABLE_L1, 0);
1543 
1544                 rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq,
1545                                       IRQF_SHARED, "omap-dma-engine", od);
1546                 if (rc)
1547                         return rc;
1548         }
1549 
1550         if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123)
1551                 od->ll123_supported = true;
1552 
1553         od->ddev.filter.map = od->plat->slave_map;
1554         od->ddev.filter.mapcnt = od->plat->slavecnt;
1555         od->ddev.filter.fn = omap_dma_filter_fn;
1556 
1557         if (od->ll123_supported) {
1558                 od->desc_pool = dma_pool_create(dev_name(&pdev->dev),
1559                                                 &pdev->dev,
1560                                                 sizeof(struct omap_type2_desc),
1561                                                 4, 0);
1562                 if (!od->desc_pool) {
1563                         dev_err(&pdev->dev,
1564                                 "unable to allocate descriptor pool\n");
1565                         od->ll123_supported = false;
1566                 }
1567         }
1568 
1569         rc = dma_async_device_register(&od->ddev);
1570         if (rc) {
1571                 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n",
1572                         rc);
1573                 omap_dma_free(od);
1574                 return rc;
1575         }
1576 
1577         platform_set_drvdata(pdev, od);
1578 
1579         if (pdev->dev.of_node) {
1580                 omap_dma_info.dma_cap = od->ddev.cap_mask;
1581 
1582                 /* Device-tree DMA controller registration */
1583                 rc = of_dma_controller_register(pdev->dev.of_node,
1584                                 of_dma_simple_xlate, &omap_dma_info);
1585                 if (rc) {
1586                         pr_warn("OMAP-DMA: failed to register DMA controller\n");
1587                         dma_async_device_unregister(&od->ddev);
1588                         omap_dma_free(od);
1589                 }
1590         }
1591 
1592         dev_info(&pdev->dev, "OMAP DMA engine driver%s\n",
1593                  od->ll123_supported ? " (LinkedList1/2/3 supported)" : "");
1594 
1595         return rc;
1596 }
1597 
1598 static int omap_dma_remove(struct platform_device *pdev)
1599 {
1600         struct omap_dmadev *od = platform_get_drvdata(pdev);
1601         int irq;
1602 
1603         if (pdev->dev.of_node)
1604                 of_dma_controller_free(pdev->dev.of_node);
1605 
1606         irq = platform_get_irq(pdev, 1);
1607         devm_free_irq(&pdev->dev, irq, od);
1608 
1609         dma_async_device_unregister(&od->ddev);
1610 
1611         if (!od->legacy) {
1612                 /* Disable all interrupts */
1613                 omap_dma_glbl_write(od, IRQENABLE_L0, 0);
1614         }
1615 
1616         if (od->ll123_supported)
1617                 dma_pool_destroy(od->desc_pool);
1618 
1619         omap_dma_free(od);
1620 
1621         return 0;
1622 }
1623 
1624 static const struct of_device_id omap_dma_match[] = {
1625         { .compatible = "ti,omap2420-sdma", },
1626         { .compatible = "ti,omap2430-sdma", },
1627         { .compatible = "ti,omap3430-sdma", },
1628         { .compatible = "ti,omap3630-sdma", },
1629         { .compatible = "ti,omap4430-sdma", },
1630         {},
1631 };
1632 MODULE_DEVICE_TABLE(of, omap_dma_match);
1633 
1634 static struct platform_driver omap_dma_driver = {
1635         .probe  = omap_dma_probe,
1636         .remove = omap_dma_remove,
1637         .driver = {
1638                 .name = "omap-dma-engine",
1639                 .of_match_table = of_match_ptr(omap_dma_match),
1640         },
1641 };
1642 
1643 bool omap_dma_filter_fn(struct dma_chan *chan, void *param)
1644 {
1645         if (chan->device->dev->driver == &omap_dma_driver.driver) {
1646                 struct omap_dmadev *od = to_omap_dma_dev(chan->device);
1647                 struct omap_chan *c = to_omap_dma_chan(chan);
1648                 unsigned req = *(unsigned *)param;
1649 
1650                 if (req <= od->dma_requests) {
1651                         c->dma_sig = req;
1652                         return true;
1653                 }
1654         }
1655         return false;
1656 }
1657 EXPORT_SYMBOL_GPL(omap_dma_filter_fn);
1658 
1659 static int omap_dma_init(void)
1660 {
1661         return platform_driver_register(&omap_dma_driver);
1662 }
1663 subsys_initcall(omap_dma_init);
1664 
1665 static void __exit omap_dma_exit(void)
1666 {
1667         platform_driver_unregister(&omap_dma_driver);
1668 }
1669 module_exit(omap_dma_exit);
1670 
1671 MODULE_AUTHOR("Russell King");
1672 MODULE_LICENSE("GPL");
1673 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us