Version:  2.0.40 2.2.26 2.4.37 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1

Linux/drivers/dma/sa11x0-dma.c

  1 /*
  2  * SA11x0 DMAengine support
  3  *
  4  * Copyright (C) 2012 Russell King
  5  *   Derived in part from arch/arm/mach-sa1100/dma.c,
  6  *   Copyright (C) 2000, 2001 by Nicolas Pitre
  7  *
  8  * This program is free software; you can redistribute it and/or modify
  9  * it under the terms of the GNU General Public License version 2 as
 10  * published by the Free Software Foundation.
 11  */
 12 #include <linux/sched.h>
 13 #include <linux/device.h>
 14 #include <linux/dmaengine.h>
 15 #include <linux/init.h>
 16 #include <linux/interrupt.h>
 17 #include <linux/kernel.h>
 18 #include <linux/module.h>
 19 #include <linux/platform_device.h>
 20 #include <linux/sa11x0-dma.h>
 21 #include <linux/slab.h>
 22 #include <linux/spinlock.h>
 23 
 24 #include "virt-dma.h"
 25 
 26 #define NR_PHY_CHAN     6
 27 #define DMA_ALIGN       3
 28 #define DMA_MAX_SIZE    0x1fff
 29 #define DMA_CHUNK_SIZE  0x1000
 30 
 31 #define DMA_DDAR        0x00
 32 #define DMA_DCSR_S      0x04
 33 #define DMA_DCSR_C      0x08
 34 #define DMA_DCSR_R      0x0c
 35 #define DMA_DBSA        0x10
 36 #define DMA_DBTA        0x14
 37 #define DMA_DBSB        0x18
 38 #define DMA_DBTB        0x1c
 39 #define DMA_SIZE        0x20
 40 
 41 #define DCSR_RUN        (1 << 0)
 42 #define DCSR_IE         (1 << 1)
 43 #define DCSR_ERROR      (1 << 2)
 44 #define DCSR_DONEA      (1 << 3)
 45 #define DCSR_STRTA      (1 << 4)
 46 #define DCSR_DONEB      (1 << 5)
 47 #define DCSR_STRTB      (1 << 6)
 48 #define DCSR_BIU        (1 << 7)
 49 
 50 #define DDAR_RW         (1 << 0)        /* 0 = W, 1 = R */
 51 #define DDAR_E          (1 << 1)        /* 0 = LE, 1 = BE */
 52 #define DDAR_BS         (1 << 2)        /* 0 = BS4, 1 = BS8 */
 53 #define DDAR_DW         (1 << 3)        /* 0 = 8b, 1 = 16b */
 54 #define DDAR_Ser0UDCTr  (0x0 << 4)
 55 #define DDAR_Ser0UDCRc  (0x1 << 4)
 56 #define DDAR_Ser1SDLCTr (0x2 << 4)
 57 #define DDAR_Ser1SDLCRc (0x3 << 4)
 58 #define DDAR_Ser1UARTTr (0x4 << 4)
 59 #define DDAR_Ser1UARTRc (0x5 << 4)
 60 #define DDAR_Ser2ICPTr  (0x6 << 4)
 61 #define DDAR_Ser2ICPRc  (0x7 << 4)
 62 #define DDAR_Ser3UARTTr (0x8 << 4)
 63 #define DDAR_Ser3UARTRc (0x9 << 4)
 64 #define DDAR_Ser4MCP0Tr (0xa << 4)
 65 #define DDAR_Ser4MCP0Rc (0xb << 4)
 66 #define DDAR_Ser4MCP1Tr (0xc << 4)
 67 #define DDAR_Ser4MCP1Rc (0xd << 4)
 68 #define DDAR_Ser4SSPTr  (0xe << 4)
 69 #define DDAR_Ser4SSPRc  (0xf << 4)
 70 
 71 struct sa11x0_dma_sg {
 72         u32                     addr;
 73         u32                     len;
 74 };
 75 
 76 struct sa11x0_dma_desc {
 77         struct virt_dma_desc    vd;
 78 
 79         u32                     ddar;
 80         size_t                  size;
 81         unsigned                period;
 82         bool                    cyclic;
 83 
 84         unsigned                sglen;
 85         struct sa11x0_dma_sg    sg[0];
 86 };
 87 
 88 struct sa11x0_dma_phy;
 89 
 90 struct sa11x0_dma_chan {
 91         struct virt_dma_chan    vc;
 92 
 93         /* protected by c->vc.lock */
 94         struct sa11x0_dma_phy   *phy;
 95         enum dma_status         status;
 96 
 97         /* protected by d->lock */
 98         struct list_head        node;
 99 
100         u32                     ddar;
101         const char              *name;
102 };
103 
104 struct sa11x0_dma_phy {
105         void __iomem            *base;
106         struct sa11x0_dma_dev   *dev;
107         unsigned                num;
108 
109         struct sa11x0_dma_chan  *vchan;
110 
111         /* Protected by c->vc.lock */
112         unsigned                sg_load;
113         struct sa11x0_dma_desc  *txd_load;
114         unsigned                sg_done;
115         struct sa11x0_dma_desc  *txd_done;
116         u32                     dbs[2];
117         u32                     dbt[2];
118         u32                     dcsr;
119 };
120 
121 struct sa11x0_dma_dev {
122         struct dma_device       slave;
123         void __iomem            *base;
124         spinlock_t              lock;
125         struct tasklet_struct   task;
126         struct list_head        chan_pending;
127         struct sa11x0_dma_phy   phy[NR_PHY_CHAN];
128 };
129 
130 static struct sa11x0_dma_chan *to_sa11x0_dma_chan(struct dma_chan *chan)
131 {
132         return container_of(chan, struct sa11x0_dma_chan, vc.chan);
133 }
134 
135 static struct sa11x0_dma_dev *to_sa11x0_dma(struct dma_device *dmadev)
136 {
137         return container_of(dmadev, struct sa11x0_dma_dev, slave);
138 }
139 
140 static struct sa11x0_dma_desc *sa11x0_dma_next_desc(struct sa11x0_dma_chan *c)
141 {
142         struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
143 
144         return vd ? container_of(vd, struct sa11x0_dma_desc, vd) : NULL;
145 }
146 
147 static void sa11x0_dma_free_desc(struct virt_dma_desc *vd)
148 {
149         kfree(container_of(vd, struct sa11x0_dma_desc, vd));
150 }
151 
152 static void sa11x0_dma_start_desc(struct sa11x0_dma_phy *p, struct sa11x0_dma_desc *txd)
153 {
154         list_del(&txd->vd.node);
155         p->txd_load = txd;
156         p->sg_load = 0;
157 
158         dev_vdbg(p->dev->slave.dev, "pchan %u: txd %p[%x]: starting: DDAR:%x\n",
159                 p->num, &txd->vd, txd->vd.tx.cookie, txd->ddar);
160 }
161 
162 static void noinline sa11x0_dma_start_sg(struct sa11x0_dma_phy *p,
163         struct sa11x0_dma_chan *c)
164 {
165         struct sa11x0_dma_desc *txd = p->txd_load;
166         struct sa11x0_dma_sg *sg;
167         void __iomem *base = p->base;
168         unsigned dbsx, dbtx;
169         u32 dcsr;
170 
171         if (!txd)
172                 return;
173 
174         dcsr = readl_relaxed(base + DMA_DCSR_R);
175 
176         /* Don't try to load the next transfer if both buffers are started */
177         if ((dcsr & (DCSR_STRTA | DCSR_STRTB)) == (DCSR_STRTA | DCSR_STRTB))
178                 return;
179 
180         if (p->sg_load == txd->sglen) {
181                 if (!txd->cyclic) {
182                         struct sa11x0_dma_desc *txn = sa11x0_dma_next_desc(c);
183 
184                         /*
185                          * We have reached the end of the current descriptor.
186                          * Peek at the next descriptor, and if compatible with
187                          * the current, start processing it.
188                          */
189                         if (txn && txn->ddar == txd->ddar) {
190                                 txd = txn;
191                                 sa11x0_dma_start_desc(p, txn);
192                         } else {
193                                 p->txd_load = NULL;
194                                 return;
195                         }
196                 } else {
197                         /* Cyclic: reset back to beginning */
198                         p->sg_load = 0;
199                 }
200         }
201 
202         sg = &txd->sg[p->sg_load++];
203 
204         /* Select buffer to load according to channel status */
205         if (((dcsr & (DCSR_BIU | DCSR_STRTB)) == (DCSR_BIU | DCSR_STRTB)) ||
206             ((dcsr & (DCSR_BIU | DCSR_STRTA)) == 0)) {
207                 dbsx = DMA_DBSA;
208                 dbtx = DMA_DBTA;
209                 dcsr = DCSR_STRTA | DCSR_IE | DCSR_RUN;
210         } else {
211                 dbsx = DMA_DBSB;
212                 dbtx = DMA_DBTB;
213                 dcsr = DCSR_STRTB | DCSR_IE | DCSR_RUN;
214         }
215 
216         writel_relaxed(sg->addr, base + dbsx);
217         writel_relaxed(sg->len, base + dbtx);
218         writel(dcsr, base + DMA_DCSR_S);
219 
220         dev_dbg(p->dev->slave.dev, "pchan %u: load: DCSR:%02x DBS%c:%08x DBT%c:%08x\n",
221                 p->num, dcsr,
222                 'A' + (dbsx == DMA_DBSB), sg->addr,
223                 'A' + (dbtx == DMA_DBTB), sg->len);
224 }
225 
226 static void noinline sa11x0_dma_complete(struct sa11x0_dma_phy *p,
227         struct sa11x0_dma_chan *c)
228 {
229         struct sa11x0_dma_desc *txd = p->txd_done;
230 
231         if (++p->sg_done == txd->sglen) {
232                 if (!txd->cyclic) {
233                         vchan_cookie_complete(&txd->vd);
234 
235                         p->sg_done = 0;
236                         p->txd_done = p->txd_load;
237 
238                         if (!p->txd_done)
239                                 tasklet_schedule(&p->dev->task);
240                 } else {
241                         if ((p->sg_done % txd->period) == 0)
242                                 vchan_cyclic_callback(&txd->vd);
243 
244                         /* Cyclic: reset back to beginning */
245                         p->sg_done = 0;
246                 }
247         }
248 
249         sa11x0_dma_start_sg(p, c);
250 }
251 
252 static irqreturn_t sa11x0_dma_irq(int irq, void *dev_id)
253 {
254         struct sa11x0_dma_phy *p = dev_id;
255         struct sa11x0_dma_dev *d = p->dev;
256         struct sa11x0_dma_chan *c;
257         u32 dcsr;
258 
259         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
260         if (!(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB)))
261                 return IRQ_NONE;
262 
263         /* Clear reported status bits */
264         writel_relaxed(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB),
265                 p->base + DMA_DCSR_C);
266 
267         dev_dbg(d->slave.dev, "pchan %u: irq: DCSR:%02x\n", p->num, dcsr);
268 
269         if (dcsr & DCSR_ERROR) {
270                 dev_err(d->slave.dev, "pchan %u: error. DCSR:%02x DDAR:%08x DBSA:%08x DBTA:%08x DBSB:%08x DBTB:%08x\n",
271                         p->num, dcsr,
272                         readl_relaxed(p->base + DMA_DDAR),
273                         readl_relaxed(p->base + DMA_DBSA),
274                         readl_relaxed(p->base + DMA_DBTA),
275                         readl_relaxed(p->base + DMA_DBSB),
276                         readl_relaxed(p->base + DMA_DBTB));
277         }
278 
279         c = p->vchan;
280         if (c) {
281                 unsigned long flags;
282 
283                 spin_lock_irqsave(&c->vc.lock, flags);
284                 /*
285                  * Now that we're holding the lock, check that the vchan
286                  * really is associated with this pchan before touching the
287                  * hardware.  This should always succeed, because we won't
288                  * change p->vchan or c->phy while the channel is actively
289                  * transferring.
290                  */
291                 if (c->phy == p) {
292                         if (dcsr & DCSR_DONEA)
293                                 sa11x0_dma_complete(p, c);
294                         if (dcsr & DCSR_DONEB)
295                                 sa11x0_dma_complete(p, c);
296                 }
297                 spin_unlock_irqrestore(&c->vc.lock, flags);
298         }
299 
300         return IRQ_HANDLED;
301 }
302 
303 static void sa11x0_dma_start_txd(struct sa11x0_dma_chan *c)
304 {
305         struct sa11x0_dma_desc *txd = sa11x0_dma_next_desc(c);
306 
307         /* If the issued list is empty, we have no further txds to process */
308         if (txd) {
309                 struct sa11x0_dma_phy *p = c->phy;
310 
311                 sa11x0_dma_start_desc(p, txd);
312                 p->txd_done = txd;
313                 p->sg_done = 0;
314 
315                 /* The channel should not have any transfers started */
316                 WARN_ON(readl_relaxed(p->base + DMA_DCSR_R) &
317                                       (DCSR_STRTA | DCSR_STRTB));
318 
319                 /* Clear the run and start bits before changing DDAR */
320                 writel_relaxed(DCSR_RUN | DCSR_STRTA | DCSR_STRTB,
321                                p->base + DMA_DCSR_C);
322                 writel_relaxed(txd->ddar, p->base + DMA_DDAR);
323 
324                 /* Try to start both buffers */
325                 sa11x0_dma_start_sg(p, c);
326                 sa11x0_dma_start_sg(p, c);
327         }
328 }
329 
330 static void sa11x0_dma_tasklet(unsigned long arg)
331 {
332         struct sa11x0_dma_dev *d = (struct sa11x0_dma_dev *)arg;
333         struct sa11x0_dma_phy *p;
334         struct sa11x0_dma_chan *c;
335         unsigned pch, pch_alloc = 0;
336 
337         dev_dbg(d->slave.dev, "tasklet enter\n");
338 
339         list_for_each_entry(c, &d->slave.channels, vc.chan.device_node) {
340                 spin_lock_irq(&c->vc.lock);
341                 p = c->phy;
342                 if (p && !p->txd_done) {
343                         sa11x0_dma_start_txd(c);
344                         if (!p->txd_done) {
345                                 /* No current txd associated with this channel */
346                                 dev_dbg(d->slave.dev, "pchan %u: free\n", p->num);
347 
348                                 /* Mark this channel free */
349                                 c->phy = NULL;
350                                 p->vchan = NULL;
351                         }
352                 }
353                 spin_unlock_irq(&c->vc.lock);
354         }
355 
356         spin_lock_irq(&d->lock);
357         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
358                 p = &d->phy[pch];
359 
360                 if (p->vchan == NULL && !list_empty(&d->chan_pending)) {
361                         c = list_first_entry(&d->chan_pending,
362                                 struct sa11x0_dma_chan, node);
363                         list_del_init(&c->node);
364 
365                         pch_alloc |= 1 << pch;
366 
367                         /* Mark this channel allocated */
368                         p->vchan = c;
369 
370                         dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc);
371                 }
372         }
373         spin_unlock_irq(&d->lock);
374 
375         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
376                 if (pch_alloc & (1 << pch)) {
377                         p = &d->phy[pch];
378                         c = p->vchan;
379 
380                         spin_lock_irq(&c->vc.lock);
381                         c->phy = p;
382 
383                         sa11x0_dma_start_txd(c);
384                         spin_unlock_irq(&c->vc.lock);
385                 }
386         }
387 
388         dev_dbg(d->slave.dev, "tasklet exit\n");
389 }
390 
391 
392 static void sa11x0_dma_free_chan_resources(struct dma_chan *chan)
393 {
394         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
395         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
396         unsigned long flags;
397 
398         spin_lock_irqsave(&d->lock, flags);
399         list_del_init(&c->node);
400         spin_unlock_irqrestore(&d->lock, flags);
401 
402         vchan_free_chan_resources(&c->vc);
403 }
404 
405 static dma_addr_t sa11x0_dma_pos(struct sa11x0_dma_phy *p)
406 {
407         unsigned reg;
408         u32 dcsr;
409 
410         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
411 
412         if ((dcsr & (DCSR_BIU | DCSR_STRTA)) == DCSR_STRTA ||
413             (dcsr & (DCSR_BIU | DCSR_STRTB)) == DCSR_BIU)
414                 reg = DMA_DBSA;
415         else
416                 reg = DMA_DBSB;
417 
418         return readl_relaxed(p->base + reg);
419 }
420 
421 static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,
422         dma_cookie_t cookie, struct dma_tx_state *state)
423 {
424         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
425         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
426         struct sa11x0_dma_phy *p;
427         struct virt_dma_desc *vd;
428         unsigned long flags;
429         enum dma_status ret;
430 
431         ret = dma_cookie_status(&c->vc.chan, cookie, state);
432         if (ret == DMA_COMPLETE)
433                 return ret;
434 
435         if (!state)
436                 return c->status;
437 
438         spin_lock_irqsave(&c->vc.lock, flags);
439         p = c->phy;
440 
441         /*
442          * If the cookie is on our issue queue, then the residue is
443          * its total size.
444          */
445         vd = vchan_find_desc(&c->vc, cookie);
446         if (vd) {
447                 state->residue = container_of(vd, struct sa11x0_dma_desc, vd)->size;
448         } else if (!p) {
449                 state->residue = 0;
450         } else {
451                 struct sa11x0_dma_desc *txd;
452                 size_t bytes = 0;
453 
454                 if (p->txd_done && p->txd_done->vd.tx.cookie == cookie)
455                         txd = p->txd_done;
456                 else if (p->txd_load && p->txd_load->vd.tx.cookie == cookie)
457                         txd = p->txd_load;
458                 else
459                         txd = NULL;
460 
461                 ret = c->status;
462                 if (txd) {
463                         dma_addr_t addr = sa11x0_dma_pos(p);
464                         unsigned i;
465 
466                         dev_vdbg(d->slave.dev, "tx_status: addr:%x\n", addr);
467 
468                         for (i = 0; i < txd->sglen; i++) {
469                                 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n",
470                                         i, txd->sg[i].addr, txd->sg[i].len);
471                                 if (addr >= txd->sg[i].addr &&
472                                     addr < txd->sg[i].addr + txd->sg[i].len) {
473                                         unsigned len;
474 
475                                         len = txd->sg[i].len -
476                                                 (addr - txd->sg[i].addr);
477                                         dev_vdbg(d->slave.dev, "tx_status: [%u] +%x\n",
478                                                 i, len);
479                                         bytes += len;
480                                         i++;
481                                         break;
482                                 }
483                         }
484                         for (; i < txd->sglen; i++) {
485                                 dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x ++\n",
486                                         i, txd->sg[i].addr, txd->sg[i].len);
487                                 bytes += txd->sg[i].len;
488                         }
489                 }
490                 state->residue = bytes;
491         }
492         spin_unlock_irqrestore(&c->vc.lock, flags);
493 
494         dev_vdbg(d->slave.dev, "tx_status: bytes 0x%zx\n", state->residue);
495 
496         return ret;
497 }
498 
499 /*
500  * Move pending txds to the issued list, and re-init pending list.
501  * If not already pending, add this channel to the list of pending
502  * channels and trigger the tasklet to run.
503  */
504 static void sa11x0_dma_issue_pending(struct dma_chan *chan)
505 {
506         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
507         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
508         unsigned long flags;
509 
510         spin_lock_irqsave(&c->vc.lock, flags);
511         if (vchan_issue_pending(&c->vc)) {
512                 if (!c->phy) {
513                         spin_lock(&d->lock);
514                         if (list_empty(&c->node)) {
515                                 list_add_tail(&c->node, &d->chan_pending);
516                                 tasklet_schedule(&d->task);
517                                 dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
518                         }
519                         spin_unlock(&d->lock);
520                 }
521         } else
522                 dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
523         spin_unlock_irqrestore(&c->vc.lock, flags);
524 }
525 
526 static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(
527         struct dma_chan *chan, struct scatterlist *sg, unsigned int sglen,
528         enum dma_transfer_direction dir, unsigned long flags, void *context)
529 {
530         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
531         struct sa11x0_dma_desc *txd;
532         struct scatterlist *sgent;
533         unsigned i, j = sglen;
534         size_t size = 0;
535 
536         /* SA11x0 channels can only operate in their native direction */
537         if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
538                 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
539                         &c->vc, c->ddar, dir);
540                 return NULL;
541         }
542 
543         /* Do not allow zero-sized txds */
544         if (sglen == 0)
545                 return NULL;
546 
547         for_each_sg(sg, sgent, sglen, i) {
548                 dma_addr_t addr = sg_dma_address(sgent);
549                 unsigned int len = sg_dma_len(sgent);
550 
551                 if (len > DMA_MAX_SIZE)
552                         j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1;
553                 if (addr & DMA_ALIGN) {
554                         dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %08x\n",
555                                 &c->vc, addr);
556                         return NULL;
557                 }
558         }
559 
560         txd = kzalloc(sizeof(*txd) + j * sizeof(txd->sg[0]), GFP_ATOMIC);
561         if (!txd) {
562                 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
563                 return NULL;
564         }
565 
566         j = 0;
567         for_each_sg(sg, sgent, sglen, i) {
568                 dma_addr_t addr = sg_dma_address(sgent);
569                 unsigned len = sg_dma_len(sgent);
570 
571                 size += len;
572 
573                 do {
574                         unsigned tlen = len;
575 
576                         /*
577                          * Check whether the transfer will fit.  If not, try
578                          * to split the transfer up such that we end up with
579                          * equal chunks - but make sure that we preserve the
580                          * alignment.  This avoids small segments.
581                          */
582                         if (tlen > DMA_MAX_SIZE) {
583                                 unsigned mult = DIV_ROUND_UP(tlen,
584                                         DMA_MAX_SIZE & ~DMA_ALIGN);
585 
586                                 tlen = (tlen / mult) & ~DMA_ALIGN;
587                         }
588 
589                         txd->sg[j].addr = addr;
590                         txd->sg[j].len = tlen;
591 
592                         addr += tlen;
593                         len -= tlen;
594                         j++;
595                 } while (len);
596         }
597 
598         txd->ddar = c->ddar;
599         txd->size = size;
600         txd->sglen = j;
601 
602         dev_dbg(chan->device->dev, "vchan %p: txd %p: size %u nr %u\n",
603                 &c->vc, &txd->vd, txd->size, txd->sglen);
604 
605         return vchan_tx_prep(&c->vc, &txd->vd, flags);
606 }
607 
608 static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic(
609         struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
610         enum dma_transfer_direction dir, unsigned long flags)
611 {
612         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
613         struct sa11x0_dma_desc *txd;
614         unsigned i, j, k, sglen, sgperiod;
615 
616         /* SA11x0 channels can only operate in their native direction */
617         if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
618                 dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
619                         &c->vc, c->ddar, dir);
620                 return NULL;
621         }
622 
623         sgperiod = DIV_ROUND_UP(period, DMA_MAX_SIZE & ~DMA_ALIGN);
624         sglen = size * sgperiod / period;
625 
626         /* Do not allow zero-sized txds */
627         if (sglen == 0)
628                 return NULL;
629 
630         txd = kzalloc(sizeof(*txd) + sglen * sizeof(txd->sg[0]), GFP_ATOMIC);
631         if (!txd) {
632                 dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
633                 return NULL;
634         }
635 
636         for (i = k = 0; i < size / period; i++) {
637                 size_t tlen, len = period;
638 
639                 for (j = 0; j < sgperiod; j++, k++) {
640                         tlen = len;
641 
642                         if (tlen > DMA_MAX_SIZE) {
643                                 unsigned mult = DIV_ROUND_UP(tlen, DMA_MAX_SIZE & ~DMA_ALIGN);
644                                 tlen = (tlen / mult) & ~DMA_ALIGN;
645                         }
646 
647                         txd->sg[k].addr = addr;
648                         txd->sg[k].len = tlen;
649                         addr += tlen;
650                         len -= tlen;
651                 }
652 
653                 WARN_ON(len != 0);
654         }
655 
656         WARN_ON(k != sglen);
657 
658         txd->ddar = c->ddar;
659         txd->size = size;
660         txd->sglen = sglen;
661         txd->cyclic = 1;
662         txd->period = sgperiod;
663 
664         return vchan_tx_prep(&c->vc, &txd->vd, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
665 }
666 
667 static int sa11x0_dma_device_config(struct dma_chan *chan,
668                                     struct dma_slave_config *cfg)
669 {
670         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
671         u32 ddar = c->ddar & ((0xf << 4) | DDAR_RW);
672         dma_addr_t addr;
673         enum dma_slave_buswidth width;
674         u32 maxburst;
675 
676         if (ddar & DDAR_RW) {
677                 addr = cfg->src_addr;
678                 width = cfg->src_addr_width;
679                 maxburst = cfg->src_maxburst;
680         } else {
681                 addr = cfg->dst_addr;
682                 width = cfg->dst_addr_width;
683                 maxburst = cfg->dst_maxburst;
684         }
685 
686         if ((width != DMA_SLAVE_BUSWIDTH_1_BYTE &&
687              width != DMA_SLAVE_BUSWIDTH_2_BYTES) ||
688             (maxburst != 4 && maxburst != 8))
689                 return -EINVAL;
690 
691         if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
692                 ddar |= DDAR_DW;
693         if (maxburst == 8)
694                 ddar |= DDAR_BS;
695 
696         dev_dbg(c->vc.chan.device->dev, "vchan %p: dma_slave_config addr %x width %u burst %u\n",
697                 &c->vc, addr, width, maxburst);
698 
699         c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6;
700 
701         return 0;
702 }
703 
704 static int sa11x0_dma_device_pause(struct dma_chan *chan)
705 {
706         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
707         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
708         struct sa11x0_dma_phy *p;
709         LIST_HEAD(head);
710         unsigned long flags;
711 
712         dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc);
713         spin_lock_irqsave(&c->vc.lock, flags);
714         if (c->status == DMA_IN_PROGRESS) {
715                 c->status = DMA_PAUSED;
716 
717                 p = c->phy;
718                 if (p) {
719                         writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
720                 } else {
721                         spin_lock(&d->lock);
722                         list_del_init(&c->node);
723                         spin_unlock(&d->lock);
724                 }
725         }
726         spin_unlock_irqrestore(&c->vc.lock, flags);
727 
728         return 0;
729 }
730 
731 static int sa11x0_dma_device_resume(struct dma_chan *chan)
732 {
733         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
734         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
735         struct sa11x0_dma_phy *p;
736         LIST_HEAD(head);
737         unsigned long flags;
738 
739         dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc);
740         spin_lock_irqsave(&c->vc.lock, flags);
741         if (c->status == DMA_PAUSED) {
742                 c->status = DMA_IN_PROGRESS;
743 
744                 p = c->phy;
745                 if (p) {
746                         writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_S);
747                 } else if (!list_empty(&c->vc.desc_issued)) {
748                         spin_lock(&d->lock);
749                         list_add_tail(&c->node, &d->chan_pending);
750                         spin_unlock(&d->lock);
751                 }
752         }
753         spin_unlock_irqrestore(&c->vc.lock, flags);
754 
755         return 0;
756 }
757 
758 static int sa11x0_dma_device_terminate_all(struct dma_chan *chan)
759 {
760         struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
761         struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
762         struct sa11x0_dma_phy *p;
763         LIST_HEAD(head);
764         unsigned long flags;
765 
766         dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
767         /* Clear the tx descriptor lists */
768         spin_lock_irqsave(&c->vc.lock, flags);
769         vchan_get_all_descriptors(&c->vc, &head);
770 
771         p = c->phy;
772         if (p) {
773                 dev_dbg(d->slave.dev, "pchan %u: terminating\n", p->num);
774                 /* vchan is assigned to a pchan - stop the channel */
775                 writel(DCSR_RUN | DCSR_IE |
776                        DCSR_STRTA | DCSR_DONEA |
777                        DCSR_STRTB | DCSR_DONEB,
778                        p->base + DMA_DCSR_C);
779 
780                 if (p->txd_load) {
781                         if (p->txd_load != p->txd_done)
782                                 list_add_tail(&p->txd_load->vd.node, &head);
783                         p->txd_load = NULL;
784                 }
785                 if (p->txd_done) {
786                         list_add_tail(&p->txd_done->vd.node, &head);
787                         p->txd_done = NULL;
788                 }
789                 c->phy = NULL;
790                 spin_lock(&d->lock);
791                 p->vchan = NULL;
792                 spin_unlock(&d->lock);
793                 tasklet_schedule(&d->task);
794         }
795         spin_unlock_irqrestore(&c->vc.lock, flags);
796         vchan_dma_desc_free_list(&c->vc, &head);
797 
798         return 0;
799 }
800 
801 struct sa11x0_dma_channel_desc {
802         u32 ddar;
803         const char *name;
804 };
805 
806 #define CD(d1, d2) { .ddar = DDAR_##d1 | d2, .name = #d1 }
807 static const struct sa11x0_dma_channel_desc chan_desc[] = {
808         CD(Ser0UDCTr, 0),
809         CD(Ser0UDCRc, DDAR_RW),
810         CD(Ser1SDLCTr, 0),
811         CD(Ser1SDLCRc, DDAR_RW),
812         CD(Ser1UARTTr, 0),
813         CD(Ser1UARTRc, DDAR_RW),
814         CD(Ser2ICPTr, 0),
815         CD(Ser2ICPRc, DDAR_RW),
816         CD(Ser3UARTTr, 0),
817         CD(Ser3UARTRc, DDAR_RW),
818         CD(Ser4MCP0Tr, 0),
819         CD(Ser4MCP0Rc, DDAR_RW),
820         CD(Ser4MCP1Tr, 0),
821         CD(Ser4MCP1Rc, DDAR_RW),
822         CD(Ser4SSPTr, 0),
823         CD(Ser4SSPRc, DDAR_RW),
824 };
825 
826 static int sa11x0_dma_init_dmadev(struct dma_device *dmadev,
827         struct device *dev)
828 {
829         unsigned i;
830 
831         INIT_LIST_HEAD(&dmadev->channels);
832         dmadev->dev = dev;
833         dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources;
834         dmadev->device_config = sa11x0_dma_device_config;
835         dmadev->device_pause = sa11x0_dma_device_pause;
836         dmadev->device_resume = sa11x0_dma_device_resume;
837         dmadev->device_terminate_all = sa11x0_dma_device_terminate_all;
838         dmadev->device_tx_status = sa11x0_dma_tx_status;
839         dmadev->device_issue_pending = sa11x0_dma_issue_pending;
840 
841         for (i = 0; i < ARRAY_SIZE(chan_desc); i++) {
842                 struct sa11x0_dma_chan *c;
843 
844                 c = kzalloc(sizeof(*c), GFP_KERNEL);
845                 if (!c) {
846                         dev_err(dev, "no memory for channel %u\n", i);
847                         return -ENOMEM;
848                 }
849 
850                 c->status = DMA_IN_PROGRESS;
851                 c->ddar = chan_desc[i].ddar;
852                 c->name = chan_desc[i].name;
853                 INIT_LIST_HEAD(&c->node);
854 
855                 c->vc.desc_free = sa11x0_dma_free_desc;
856                 vchan_init(&c->vc, dmadev);
857         }
858 
859         return dma_async_device_register(dmadev);
860 }
861 
862 static int sa11x0_dma_request_irq(struct platform_device *pdev, int nr,
863         void *data)
864 {
865         int irq = platform_get_irq(pdev, nr);
866 
867         if (irq <= 0)
868                 return -ENXIO;
869 
870         return request_irq(irq, sa11x0_dma_irq, 0, dev_name(&pdev->dev), data);
871 }
872 
873 static void sa11x0_dma_free_irq(struct platform_device *pdev, int nr,
874         void *data)
875 {
876         int irq = platform_get_irq(pdev, nr);
877         if (irq > 0)
878                 free_irq(irq, data);
879 }
880 
881 static void sa11x0_dma_free_channels(struct dma_device *dmadev)
882 {
883         struct sa11x0_dma_chan *c, *cn;
884 
885         list_for_each_entry_safe(c, cn, &dmadev->channels, vc.chan.device_node) {
886                 list_del(&c->vc.chan.device_node);
887                 tasklet_kill(&c->vc.task);
888                 kfree(c);
889         }
890 }
891 
892 static int sa11x0_dma_probe(struct platform_device *pdev)
893 {
894         struct sa11x0_dma_dev *d;
895         struct resource *res;
896         unsigned i;
897         int ret;
898 
899         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
900         if (!res)
901                 return -ENXIO;
902 
903         d = kzalloc(sizeof(*d), GFP_KERNEL);
904         if (!d) {
905                 ret = -ENOMEM;
906                 goto err_alloc;
907         }
908 
909         spin_lock_init(&d->lock);
910         INIT_LIST_HEAD(&d->chan_pending);
911 
912         d->base = ioremap(res->start, resource_size(res));
913         if (!d->base) {
914                 ret = -ENOMEM;
915                 goto err_ioremap;
916         }
917 
918         tasklet_init(&d->task, sa11x0_dma_tasklet, (unsigned long)d);
919 
920         for (i = 0; i < NR_PHY_CHAN; i++) {
921                 struct sa11x0_dma_phy *p = &d->phy[i];
922 
923                 p->dev = d;
924                 p->num = i;
925                 p->base = d->base + i * DMA_SIZE;
926                 writel_relaxed(DCSR_RUN | DCSR_IE | DCSR_ERROR |
927                         DCSR_DONEA | DCSR_STRTA | DCSR_DONEB | DCSR_STRTB,
928                         p->base + DMA_DCSR_C);
929                 writel_relaxed(0, p->base + DMA_DDAR);
930 
931                 ret = sa11x0_dma_request_irq(pdev, i, p);
932                 if (ret) {
933                         while (i) {
934                                 i--;
935                                 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
936                         }
937                         goto err_irq;
938                 }
939         }
940 
941         dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
942         dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
943         d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg;
944         d->slave.device_prep_dma_cyclic = sa11x0_dma_prep_dma_cyclic;
945         d->slave.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
946         d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
947         d->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
948                                    BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);
949         d->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
950                                    BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);
951         ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev);
952         if (ret) {
953                 dev_warn(d->slave.dev, "failed to register slave async device: %d\n",
954                         ret);
955                 goto err_slave_reg;
956         }
957 
958         platform_set_drvdata(pdev, d);
959         return 0;
960 
961  err_slave_reg:
962         sa11x0_dma_free_channels(&d->slave);
963         for (i = 0; i < NR_PHY_CHAN; i++)
964                 sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
965  err_irq:
966         tasklet_kill(&d->task);
967         iounmap(d->base);
968  err_ioremap:
969         kfree(d);
970  err_alloc:
971         return ret;
972 }
973 
974 static int sa11x0_dma_remove(struct platform_device *pdev)
975 {
976         struct sa11x0_dma_dev *d = platform_get_drvdata(pdev);
977         unsigned pch;
978 
979         dma_async_device_unregister(&d->slave);
980 
981         sa11x0_dma_free_channels(&d->slave);
982         for (pch = 0; pch < NR_PHY_CHAN; pch++)
983                 sa11x0_dma_free_irq(pdev, pch, &d->phy[pch]);
984         tasklet_kill(&d->task);
985         iounmap(d->base);
986         kfree(d);
987 
988         return 0;
989 }
990 
991 static int sa11x0_dma_suspend(struct device *dev)
992 {
993         struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
994         unsigned pch;
995 
996         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
997                 struct sa11x0_dma_phy *p = &d->phy[pch];
998                 u32 dcsr, saved_dcsr;
999 
1000                 dcsr = saved_dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1001                 if (dcsr & DCSR_RUN) {
1002                         writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
1003                         dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1004                 }
1005 
1006                 saved_dcsr &= DCSR_RUN | DCSR_IE;
1007                 if (dcsr & DCSR_BIU) {
1008                         p->dbs[0] = readl_relaxed(p->base + DMA_DBSB);
1009                         p->dbt[0] = readl_relaxed(p->base + DMA_DBTB);
1010                         p->dbs[1] = readl_relaxed(p->base + DMA_DBSA);
1011                         p->dbt[1] = readl_relaxed(p->base + DMA_DBTA);
1012                         saved_dcsr |= (dcsr & DCSR_STRTA ? DCSR_STRTB : 0) |
1013                                       (dcsr & DCSR_STRTB ? DCSR_STRTA : 0);
1014                 } else {
1015                         p->dbs[0] = readl_relaxed(p->base + DMA_DBSA);
1016                         p->dbt[0] = readl_relaxed(p->base + DMA_DBTA);
1017                         p->dbs[1] = readl_relaxed(p->base + DMA_DBSB);
1018                         p->dbt[1] = readl_relaxed(p->base + DMA_DBTB);
1019                         saved_dcsr |= dcsr & (DCSR_STRTA | DCSR_STRTB);
1020                 }
1021                 p->dcsr = saved_dcsr;
1022 
1023                 writel(DCSR_STRTA | DCSR_STRTB, p->base + DMA_DCSR_C);
1024         }
1025 
1026         return 0;
1027 }
1028 
1029 static int sa11x0_dma_resume(struct device *dev)
1030 {
1031         struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1032         unsigned pch;
1033 
1034         for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1035                 struct sa11x0_dma_phy *p = &d->phy[pch];
1036                 struct sa11x0_dma_desc *txd = NULL;
1037                 u32 dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1038 
1039                 WARN_ON(dcsr & (DCSR_BIU | DCSR_STRTA | DCSR_STRTB | DCSR_RUN));
1040 
1041                 if (p->txd_done)
1042                         txd = p->txd_done;
1043                 else if (p->txd_load)
1044                         txd = p->txd_load;
1045 
1046                 if (!txd)
1047                         continue;
1048 
1049                 writel_relaxed(txd->ddar, p->base + DMA_DDAR);
1050 
1051                 writel_relaxed(p->dbs[0], p->base + DMA_DBSA);
1052                 writel_relaxed(p->dbt[0], p->base + DMA_DBTA);
1053                 writel_relaxed(p->dbs[1], p->base + DMA_DBSB);
1054                 writel_relaxed(p->dbt[1], p->base + DMA_DBTB);
1055                 writel_relaxed(p->dcsr, p->base + DMA_DCSR_S);
1056         }
1057 
1058         return 0;
1059 }
1060 
1061 static const struct dev_pm_ops sa11x0_dma_pm_ops = {
1062         .suspend_noirq = sa11x0_dma_suspend,
1063         .resume_noirq = sa11x0_dma_resume,
1064         .freeze_noirq = sa11x0_dma_suspend,
1065         .thaw_noirq = sa11x0_dma_resume,
1066         .poweroff_noirq = sa11x0_dma_suspend,
1067         .restore_noirq = sa11x0_dma_resume,
1068 };
1069 
1070 static struct platform_driver sa11x0_dma_driver = {
1071         .driver = {
1072                 .name   = "sa11x0-dma",
1073                 .pm     = &sa11x0_dma_pm_ops,
1074         },
1075         .probe          = sa11x0_dma_probe,
1076         .remove         = sa11x0_dma_remove,
1077 };
1078 
1079 bool sa11x0_dma_filter_fn(struct dma_chan *chan, void *param)
1080 {
1081         if (chan->device->dev->driver == &sa11x0_dma_driver.driver) {
1082                 struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
1083                 const char *p = param;
1084 
1085                 return !strcmp(c->name, p);
1086         }
1087         return false;
1088 }
1089 EXPORT_SYMBOL(sa11x0_dma_filter_fn);
1090 
1091 static int __init sa11x0_dma_init(void)
1092 {
1093         return platform_driver_register(&sa11x0_dma_driver);
1094 }
1095 subsys_initcall(sa11x0_dma_init);
1096 
1097 static void __exit sa11x0_dma_exit(void)
1098 {
1099         platform_driver_unregister(&sa11x0_dma_driver);
1100 }
1101 module_exit(sa11x0_dma_exit);
1102 
1103 MODULE_AUTHOR("Russell King");
1104 MODULE_DESCRIPTION("SA-11x0 DMA driver");
1105 MODULE_LICENSE("GPL v2");
1106 MODULE_ALIAS("platform:sa11x0-dma");
1107 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us