Version:  2.0.40 2.2.26 2.4.37 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17

Linux/drivers/dma/iop-adma.c

  1 /*
  2  * offload engine driver for the Intel Xscale series of i/o processors
  3  * Copyright © 2006, Intel Corporation.
  4  *
  5  * This program is free software; you can redistribute it and/or modify it
  6  * under the terms and conditions of the GNU General Public License,
  7  * version 2, as published by the Free Software Foundation.
  8  *
  9  * This program is distributed in the hope it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 12  * more details.
 13  *
 14  * You should have received a copy of the GNU General Public License along with
 15  * this program; if not, write to the Free Software Foundation, Inc.,
 16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 17  *
 18  */
 19 
 20 /*
 21  * This driver supports the asynchrounous DMA copy and RAID engines available
 22  * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
 23  */
 24 
 25 #include <linux/init.h>
 26 #include <linux/module.h>
 27 #include <linux/delay.h>
 28 #include <linux/dma-mapping.h>
 29 #include <linux/spinlock.h>
 30 #include <linux/interrupt.h>
 31 #include <linux/platform_device.h>
 32 #include <linux/memory.h>
 33 #include <linux/ioport.h>
 34 #include <linux/raid/pq.h>
 35 #include <linux/slab.h>
 36 
 37 #include <mach/adma.h>
 38 
 39 #include "dmaengine.h"
 40 
 41 #define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common)
 42 #define to_iop_adma_device(dev) \
 43         container_of(dev, struct iop_adma_device, common)
 44 #define tx_to_iop_adma_slot(tx) \
 45         container_of(tx, struct iop_adma_desc_slot, async_tx)
 46 
 47 /**
 48  * iop_adma_free_slots - flags descriptor slots for reuse
 49  * @slot: Slot to free
 50  * Caller must hold &iop_chan->lock while calling this function
 51  */
 52 static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
 53 {
 54         int stride = slot->slots_per_op;
 55 
 56         while (stride--) {
 57                 slot->slots_per_op = 0;
 58                 slot = list_entry(slot->slot_node.next,
 59                                 struct iop_adma_desc_slot,
 60                                 slot_node);
 61         }
 62 }
 63 
 64 static dma_cookie_t
 65 iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
 66         struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
 67 {
 68         struct dma_async_tx_descriptor *tx = &desc->async_tx;
 69 
 70         BUG_ON(tx->cookie < 0);
 71         if (tx->cookie > 0) {
 72                 cookie = tx->cookie;
 73                 tx->cookie = 0;
 74 
 75                 /* call the callback (must not sleep or submit new
 76                  * operations to this channel)
 77                  */
 78                 if (tx->callback)
 79                         tx->callback(tx->callback_param);
 80 
 81                 dma_descriptor_unmap(tx);
 82                 if (desc->group_head)
 83                         desc->group_head = NULL;
 84         }
 85 
 86         /* run dependent operations */
 87         dma_run_dependencies(tx);
 88 
 89         return cookie;
 90 }
 91 
 92 static int
 93 iop_adma_clean_slot(struct iop_adma_desc_slot *desc,
 94         struct iop_adma_chan *iop_chan)
 95 {
 96         /* the client is allowed to attach dependent operations
 97          * until 'ack' is set
 98          */
 99         if (!async_tx_test_ack(&desc->async_tx))
100                 return 0;
101 
102         /* leave the last descriptor in the chain
103          * so we can append to it
104          */
105         if (desc->chain_node.next == &iop_chan->chain)
106                 return 1;
107 
108         dev_dbg(iop_chan->device->common.dev,
109                 "\tfree slot: %d slots_per_op: %d\n",
110                 desc->idx, desc->slots_per_op);
111 
112         list_del(&desc->chain_node);
113         iop_adma_free_slots(desc);
114 
115         return 0;
116 }
117 
118 static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
119 {
120         struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL;
121         dma_cookie_t cookie = 0;
122         u32 current_desc = iop_chan_get_current_descriptor(iop_chan);
123         int busy = iop_chan_is_busy(iop_chan);
124         int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
125 
126         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
127         /* free completed slots from the chain starting with
128          * the oldest descriptor
129          */
130         list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
131                                         chain_node) {
132                 pr_debug("\tcookie: %d slot: %d busy: %d "
133                         "this_desc: %#x next_desc: %#x ack: %d\n",
134                         iter->async_tx.cookie, iter->idx, busy,
135                         iter->async_tx.phys, iop_desc_get_next_desc(iter),
136                         async_tx_test_ack(&iter->async_tx));
137                 prefetch(_iter);
138                 prefetch(&_iter->async_tx);
139 
140                 /* do not advance past the current descriptor loaded into the
141                  * hardware channel, subsequent descriptors are either in
142                  * process or have not been submitted
143                  */
144                 if (seen_current)
145                         break;
146 
147                 /* stop the search if we reach the current descriptor and the
148                  * channel is busy, or if it appears that the current descriptor
149                  * needs to be re-read (i.e. has been appended to)
150                  */
151                 if (iter->async_tx.phys == current_desc) {
152                         BUG_ON(seen_current++);
153                         if (busy || iop_desc_get_next_desc(iter))
154                                 break;
155                 }
156 
157                 /* detect the start of a group transaction */
158                 if (!slot_cnt && !slots_per_op) {
159                         slot_cnt = iter->slot_cnt;
160                         slots_per_op = iter->slots_per_op;
161                         if (slot_cnt <= slots_per_op) {
162                                 slot_cnt = 0;
163                                 slots_per_op = 0;
164                         }
165                 }
166 
167                 if (slot_cnt) {
168                         pr_debug("\tgroup++\n");
169                         if (!grp_start)
170                                 grp_start = iter;
171                         slot_cnt -= slots_per_op;
172                 }
173 
174                 /* all the members of a group are complete */
175                 if (slots_per_op != 0 && slot_cnt == 0) {
176                         struct iop_adma_desc_slot *grp_iter, *_grp_iter;
177                         int end_of_chain = 0;
178                         pr_debug("\tgroup end\n");
179 
180                         /* collect the total results */
181                         if (grp_start->xor_check_result) {
182                                 u32 zero_sum_result = 0;
183                                 slot_cnt = grp_start->slot_cnt;
184                                 grp_iter = grp_start;
185 
186                                 list_for_each_entry_from(grp_iter,
187                                         &iop_chan->chain, chain_node) {
188                                         zero_sum_result |=
189                                             iop_desc_get_zero_result(grp_iter);
190                                             pr_debug("\titer%d result: %d\n",
191                                             grp_iter->idx, zero_sum_result);
192                                         slot_cnt -= slots_per_op;
193                                         if (slot_cnt == 0)
194                                                 break;
195                                 }
196                                 pr_debug("\tgrp_start->xor_check_result: %p\n",
197                                         grp_start->xor_check_result);
198                                 *grp_start->xor_check_result = zero_sum_result;
199                         }
200 
201                         /* clean up the group */
202                         slot_cnt = grp_start->slot_cnt;
203                         grp_iter = grp_start;
204                         list_for_each_entry_safe_from(grp_iter, _grp_iter,
205                                 &iop_chan->chain, chain_node) {
206                                 cookie = iop_adma_run_tx_complete_actions(
207                                         grp_iter, iop_chan, cookie);
208 
209                                 slot_cnt -= slots_per_op;
210                                 end_of_chain = iop_adma_clean_slot(grp_iter,
211                                         iop_chan);
212 
213                                 if (slot_cnt == 0 || end_of_chain)
214                                         break;
215                         }
216 
217                         /* the group should be complete at this point */
218                         BUG_ON(slot_cnt);
219 
220                         slots_per_op = 0;
221                         grp_start = NULL;
222                         if (end_of_chain)
223                                 break;
224                         else
225                                 continue;
226                 } else if (slots_per_op) /* wait for group completion */
227                         continue;
228 
229                 /* write back zero sum results (single descriptor case) */
230                 if (iter->xor_check_result && iter->async_tx.cookie)
231                         *iter->xor_check_result =
232                                 iop_desc_get_zero_result(iter);
233 
234                 cookie = iop_adma_run_tx_complete_actions(
235                                         iter, iop_chan, cookie);
236 
237                 if (iop_adma_clean_slot(iter, iop_chan))
238                         break;
239         }
240 
241         if (cookie > 0) {
242                 iop_chan->common.completed_cookie = cookie;
243                 pr_debug("\tcompleted cookie %d\n", cookie);
244         }
245 }
246 
247 static void
248 iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
249 {
250         spin_lock_bh(&iop_chan->lock);
251         __iop_adma_slot_cleanup(iop_chan);
252         spin_unlock_bh(&iop_chan->lock);
253 }
254 
255 static void iop_adma_tasklet(unsigned long data)
256 {
257         struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data;
258 
259         /* lockdep will flag depedency submissions as potentially
260          * recursive locking, this is not the case as a dependency
261          * submission will never recurse a channels submit routine.
262          * There are checks in async_tx.c to prevent this.
263          */
264         spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING);
265         __iop_adma_slot_cleanup(iop_chan);
266         spin_unlock(&iop_chan->lock);
267 }
268 
269 static struct iop_adma_desc_slot *
270 iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots,
271                         int slots_per_op)
272 {
273         struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL;
274         LIST_HEAD(chain);
275         int slots_found, retry = 0;
276 
277         /* start search from the last allocated descrtiptor
278          * if a contiguous allocation can not be found start searching
279          * from the beginning of the list
280          */
281 retry:
282         slots_found = 0;
283         if (retry == 0)
284                 iter = iop_chan->last_used;
285         else
286                 iter = list_entry(&iop_chan->all_slots,
287                         struct iop_adma_desc_slot,
288                         slot_node);
289 
290         list_for_each_entry_safe_continue(
291                 iter, _iter, &iop_chan->all_slots, slot_node) {
292                 prefetch(_iter);
293                 prefetch(&_iter->async_tx);
294                 if (iter->slots_per_op) {
295                         /* give up after finding the first busy slot
296                          * on the second pass through the list
297                          */
298                         if (retry)
299                                 break;
300 
301                         slots_found = 0;
302                         continue;
303                 }
304 
305                 /* start the allocation if the slot is correctly aligned */
306                 if (!slots_found++) {
307                         if (iop_desc_is_aligned(iter, slots_per_op))
308                                 alloc_start = iter;
309                         else {
310                                 slots_found = 0;
311                                 continue;
312                         }
313                 }
314 
315                 if (slots_found == num_slots) {
316                         struct iop_adma_desc_slot *alloc_tail = NULL;
317                         struct iop_adma_desc_slot *last_used = NULL;
318                         iter = alloc_start;
319                         while (num_slots) {
320                                 int i;
321                                 dev_dbg(iop_chan->device->common.dev,
322                                         "allocated slot: %d "
323                                         "(desc %p phys: %#x) slots_per_op %d\n",
324                                         iter->idx, iter->hw_desc,
325                                         iter->async_tx.phys, slots_per_op);
326 
327                                 /* pre-ack all but the last descriptor */
328                                 if (num_slots != slots_per_op)
329                                         async_tx_ack(&iter->async_tx);
330 
331                                 list_add_tail(&iter->chain_node, &chain);
332                                 alloc_tail = iter;
333                                 iter->async_tx.cookie = 0;
334                                 iter->slot_cnt = num_slots;
335                                 iter->xor_check_result = NULL;
336                                 for (i = 0; i < slots_per_op; i++) {
337                                         iter->slots_per_op = slots_per_op - i;
338                                         last_used = iter;
339                                         iter = list_entry(iter->slot_node.next,
340                                                 struct iop_adma_desc_slot,
341                                                 slot_node);
342                                 }
343                                 num_slots -= slots_per_op;
344                         }
345                         alloc_tail->group_head = alloc_start;
346                         alloc_tail->async_tx.cookie = -EBUSY;
347                         list_splice(&chain, &alloc_tail->tx_list);
348                         iop_chan->last_used = last_used;
349                         iop_desc_clear_next_desc(alloc_start);
350                         iop_desc_clear_next_desc(alloc_tail);
351                         return alloc_tail;
352                 }
353         }
354         if (!retry++)
355                 goto retry;
356 
357         /* perform direct reclaim if the allocation fails */
358         __iop_adma_slot_cleanup(iop_chan);
359 
360         return NULL;
361 }
362 
363 static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan)
364 {
365         dev_dbg(iop_chan->device->common.dev, "pending: %d\n",
366                 iop_chan->pending);
367 
368         if (iop_chan->pending >= IOP_ADMA_THRESHOLD) {
369                 iop_chan->pending = 0;
370                 iop_chan_append(iop_chan);
371         }
372 }
373 
374 static dma_cookie_t
375 iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
376 {
377         struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx);
378         struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan);
379         struct iop_adma_desc_slot *grp_start, *old_chain_tail;
380         int slot_cnt;
381         int slots_per_op;
382         dma_cookie_t cookie;
383         dma_addr_t next_dma;
384 
385         grp_start = sw_desc->group_head;
386         slot_cnt = grp_start->slot_cnt;
387         slots_per_op = grp_start->slots_per_op;
388 
389         spin_lock_bh(&iop_chan->lock);
390         cookie = dma_cookie_assign(tx);
391 
392         old_chain_tail = list_entry(iop_chan->chain.prev,
393                 struct iop_adma_desc_slot, chain_node);
394         list_splice_init(&sw_desc->tx_list,
395                          &old_chain_tail->chain_node);
396 
397         /* fix up the hardware chain */
398         next_dma = grp_start->async_tx.phys;
399         iop_desc_set_next_desc(old_chain_tail, next_dma);
400         BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */
401 
402         /* check for pre-chained descriptors */
403         iop_paranoia(iop_desc_get_next_desc(sw_desc));
404 
405         /* increment the pending count by the number of slots
406          * memcpy operations have a 1:1 (slot:operation) relation
407          * other operations are heavier and will pop the threshold
408          * more often.
409          */
410         iop_chan->pending += slot_cnt;
411         iop_adma_check_threshold(iop_chan);
412         spin_unlock_bh(&iop_chan->lock);
413 
414         dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n",
415                 __func__, sw_desc->async_tx.cookie, sw_desc->idx);
416 
417         return cookie;
418 }
419 
420 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan);
421 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan);
422 
423 /**
424  * iop_adma_alloc_chan_resources -  returns the number of allocated descriptors
425  * @chan - allocate descriptor resources for this channel
426  * @client - current client requesting the channel be ready for requests
427  *
428  * Note: We keep the slots for 1 operation on iop_chan->chain at all times.  To
429  * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be
430  * greater than 2x the number slots needed to satisfy a device->max_xor
431  * request.
432  * */
433 static int iop_adma_alloc_chan_resources(struct dma_chan *chan)
434 {
435         char *hw_desc;
436         int idx;
437         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
438         struct iop_adma_desc_slot *slot = NULL;
439         int init = iop_chan->slots_allocated ? 0 : 1;
440         struct iop_adma_platform_data *plat_data =
441                 dev_get_platdata(&iop_chan->device->pdev->dev);
442         int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE;
443 
444         /* Allocate descriptor slots */
445         do {
446                 idx = iop_chan->slots_allocated;
447                 if (idx == num_descs_in_pool)
448                         break;
449 
450                 slot = kzalloc(sizeof(*slot), GFP_KERNEL);
451                 if (!slot) {
452                         printk(KERN_INFO "IOP ADMA Channel only initialized"
453                                 " %d descriptor slots", idx);
454                         break;
455                 }
456                 hw_desc = (char *) iop_chan->device->dma_desc_pool_virt;
457                 slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
458 
459                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
460                 slot->async_tx.tx_submit = iop_adma_tx_submit;
461                 INIT_LIST_HEAD(&slot->tx_list);
462                 INIT_LIST_HEAD(&slot->chain_node);
463                 INIT_LIST_HEAD(&slot->slot_node);
464                 hw_desc = (char *) iop_chan->device->dma_desc_pool;
465                 slot->async_tx.phys =
466                         (dma_addr_t) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
467                 slot->idx = idx;
468 
469                 spin_lock_bh(&iop_chan->lock);
470                 iop_chan->slots_allocated++;
471                 list_add_tail(&slot->slot_node, &iop_chan->all_slots);
472                 spin_unlock_bh(&iop_chan->lock);
473         } while (iop_chan->slots_allocated < num_descs_in_pool);
474 
475         if (idx && !iop_chan->last_used)
476                 iop_chan->last_used = list_entry(iop_chan->all_slots.next,
477                                         struct iop_adma_desc_slot,
478                                         slot_node);
479 
480         dev_dbg(iop_chan->device->common.dev,
481                 "allocated %d descriptor slots last_used: %p\n",
482                 iop_chan->slots_allocated, iop_chan->last_used);
483 
484         /* initialize the channel and the chain with a null operation */
485         if (init) {
486                 if (dma_has_cap(DMA_MEMCPY,
487                         iop_chan->device->common.cap_mask))
488                         iop_chan_start_null_memcpy(iop_chan);
489                 else if (dma_has_cap(DMA_XOR,
490                         iop_chan->device->common.cap_mask))
491                         iop_chan_start_null_xor(iop_chan);
492                 else
493                         BUG();
494         }
495 
496         return (idx > 0) ? idx : -ENOMEM;
497 }
498 
499 static struct dma_async_tx_descriptor *
500 iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
501 {
502         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
503         struct iop_adma_desc_slot *sw_desc, *grp_start;
504         int slot_cnt, slots_per_op;
505 
506         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
507 
508         spin_lock_bh(&iop_chan->lock);
509         slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan);
510         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
511         if (sw_desc) {
512                 grp_start = sw_desc->group_head;
513                 iop_desc_init_interrupt(grp_start, iop_chan);
514                 sw_desc->async_tx.flags = flags;
515         }
516         spin_unlock_bh(&iop_chan->lock);
517 
518         return sw_desc ? &sw_desc->async_tx : NULL;
519 }
520 
521 static struct dma_async_tx_descriptor *
522 iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest,
523                          dma_addr_t dma_src, size_t len, unsigned long flags)
524 {
525         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
526         struct iop_adma_desc_slot *sw_desc, *grp_start;
527         int slot_cnt, slots_per_op;
528 
529         if (unlikely(!len))
530                 return NULL;
531         BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT);
532 
533         dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
534                 __func__, len);
535 
536         spin_lock_bh(&iop_chan->lock);
537         slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op);
538         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
539         if (sw_desc) {
540                 grp_start = sw_desc->group_head;
541                 iop_desc_init_memcpy(grp_start, flags);
542                 iop_desc_set_byte_count(grp_start, iop_chan, len);
543                 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
544                 iop_desc_set_memcpy_src_addr(grp_start, dma_src);
545                 sw_desc->async_tx.flags = flags;
546         }
547         spin_unlock_bh(&iop_chan->lock);
548 
549         return sw_desc ? &sw_desc->async_tx : NULL;
550 }
551 
552 static struct dma_async_tx_descriptor *
553 iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest,
554                       dma_addr_t *dma_src, unsigned int src_cnt, size_t len,
555                       unsigned long flags)
556 {
557         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
558         struct iop_adma_desc_slot *sw_desc, *grp_start;
559         int slot_cnt, slots_per_op;
560 
561         if (unlikely(!len))
562                 return NULL;
563         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
564 
565         dev_dbg(iop_chan->device->common.dev,
566                 "%s src_cnt: %d len: %u flags: %lx\n",
567                 __func__, src_cnt, len, flags);
568 
569         spin_lock_bh(&iop_chan->lock);
570         slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op);
571         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
572         if (sw_desc) {
573                 grp_start = sw_desc->group_head;
574                 iop_desc_init_xor(grp_start, src_cnt, flags);
575                 iop_desc_set_byte_count(grp_start, iop_chan, len);
576                 iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest);
577                 sw_desc->async_tx.flags = flags;
578                 while (src_cnt--)
579                         iop_desc_set_xor_src_addr(grp_start, src_cnt,
580                                                   dma_src[src_cnt]);
581         }
582         spin_unlock_bh(&iop_chan->lock);
583 
584         return sw_desc ? &sw_desc->async_tx : NULL;
585 }
586 
587 static struct dma_async_tx_descriptor *
588 iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src,
589                           unsigned int src_cnt, size_t len, u32 *result,
590                           unsigned long flags)
591 {
592         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
593         struct iop_adma_desc_slot *sw_desc, *grp_start;
594         int slot_cnt, slots_per_op;
595 
596         if (unlikely(!len))
597                 return NULL;
598 
599         dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
600                 __func__, src_cnt, len);
601 
602         spin_lock_bh(&iop_chan->lock);
603         slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op);
604         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
605         if (sw_desc) {
606                 grp_start = sw_desc->group_head;
607                 iop_desc_init_zero_sum(grp_start, src_cnt, flags);
608                 iop_desc_set_zero_sum_byte_count(grp_start, len);
609                 grp_start->xor_check_result = result;
610                 pr_debug("\t%s: grp_start->xor_check_result: %p\n",
611                         __func__, grp_start->xor_check_result);
612                 sw_desc->async_tx.flags = flags;
613                 while (src_cnt--)
614                         iop_desc_set_zero_sum_src_addr(grp_start, src_cnt,
615                                                        dma_src[src_cnt]);
616         }
617         spin_unlock_bh(&iop_chan->lock);
618 
619         return sw_desc ? &sw_desc->async_tx : NULL;
620 }
621 
622 static struct dma_async_tx_descriptor *
623 iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
624                      unsigned int src_cnt, const unsigned char *scf, size_t len,
625                      unsigned long flags)
626 {
627         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
628         struct iop_adma_desc_slot *sw_desc, *g;
629         int slot_cnt, slots_per_op;
630         int continue_srcs;
631 
632         if (unlikely(!len))
633                 return NULL;
634         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
635 
636         dev_dbg(iop_chan->device->common.dev,
637                 "%s src_cnt: %d len: %u flags: %lx\n",
638                 __func__, src_cnt, len, flags);
639 
640         if (dmaf_p_disabled_continue(flags))
641                 continue_srcs = 1+src_cnt;
642         else if (dmaf_continue(flags))
643                 continue_srcs = 3+src_cnt;
644         else
645                 continue_srcs = 0+src_cnt;
646 
647         spin_lock_bh(&iop_chan->lock);
648         slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op);
649         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
650         if (sw_desc) {
651                 int i;
652 
653                 g = sw_desc->group_head;
654                 iop_desc_set_byte_count(g, iop_chan, len);
655 
656                 /* even if P is disabled its destination address (bits
657                  * [3:0]) must match Q.  It is ok if P points to an
658                  * invalid address, it won't be written.
659                  */
660                 if (flags & DMA_PREP_PQ_DISABLE_P)
661                         dst[0] = dst[1] & 0x7;
662 
663                 iop_desc_set_pq_addr(g, dst);
664                 sw_desc->async_tx.flags = flags;
665                 for (i = 0; i < src_cnt; i++)
666                         iop_desc_set_pq_src_addr(g, i, src[i], scf[i]);
667 
668                 /* if we are continuing a previous operation factor in
669                  * the old p and q values, see the comment for dma_maxpq
670                  * in include/linux/dmaengine.h
671                  */
672                 if (dmaf_p_disabled_continue(flags))
673                         iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
674                 else if (dmaf_continue(flags)) {
675                         iop_desc_set_pq_src_addr(g, i++, dst[0], 0);
676                         iop_desc_set_pq_src_addr(g, i++, dst[1], 1);
677                         iop_desc_set_pq_src_addr(g, i++, dst[1], 0);
678                 }
679                 iop_desc_init_pq(g, i, flags);
680         }
681         spin_unlock_bh(&iop_chan->lock);
682 
683         return sw_desc ? &sw_desc->async_tx : NULL;
684 }
685 
686 static struct dma_async_tx_descriptor *
687 iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
688                          unsigned int src_cnt, const unsigned char *scf,
689                          size_t len, enum sum_check_flags *pqres,
690                          unsigned long flags)
691 {
692         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
693         struct iop_adma_desc_slot *sw_desc, *g;
694         int slot_cnt, slots_per_op;
695 
696         if (unlikely(!len))
697                 return NULL;
698         BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT);
699 
700         dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
701                 __func__, src_cnt, len);
702 
703         spin_lock_bh(&iop_chan->lock);
704         slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op);
705         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
706         if (sw_desc) {
707                 /* for validate operations p and q are tagged onto the
708                  * end of the source list
709                  */
710                 int pq_idx = src_cnt;
711 
712                 g = sw_desc->group_head;
713                 iop_desc_init_pq_zero_sum(g, src_cnt+2, flags);
714                 iop_desc_set_pq_zero_sum_byte_count(g, len);
715                 g->pq_check_result = pqres;
716                 pr_debug("\t%s: g->pq_check_result: %p\n",
717                         __func__, g->pq_check_result);
718                 sw_desc->async_tx.flags = flags;
719                 while (src_cnt--)
720                         iop_desc_set_pq_zero_sum_src_addr(g, src_cnt,
721                                                           src[src_cnt],
722                                                           scf[src_cnt]);
723                 iop_desc_set_pq_zero_sum_addr(g, pq_idx, src);
724         }
725         spin_unlock_bh(&iop_chan->lock);
726 
727         return sw_desc ? &sw_desc->async_tx : NULL;
728 }
729 
730 static void iop_adma_free_chan_resources(struct dma_chan *chan)
731 {
732         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
733         struct iop_adma_desc_slot *iter, *_iter;
734         int in_use_descs = 0;
735 
736         iop_adma_slot_cleanup(iop_chan);
737 
738         spin_lock_bh(&iop_chan->lock);
739         list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
740                                         chain_node) {
741                 in_use_descs++;
742                 list_del(&iter->chain_node);
743         }
744         list_for_each_entry_safe_reverse(
745                 iter, _iter, &iop_chan->all_slots, slot_node) {
746                 list_del(&iter->slot_node);
747                 kfree(iter);
748                 iop_chan->slots_allocated--;
749         }
750         iop_chan->last_used = NULL;
751 
752         dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n",
753                 __func__, iop_chan->slots_allocated);
754         spin_unlock_bh(&iop_chan->lock);
755 
756         /* one is ok since we left it on there on purpose */
757         if (in_use_descs > 1)
758                 printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n",
759                         in_use_descs - 1);
760 }
761 
762 /**
763  * iop_adma_status - poll the status of an ADMA transaction
764  * @chan: ADMA channel handle
765  * @cookie: ADMA transaction identifier
766  * @txstate: a holder for the current state of the channel or NULL
767  */
768 static enum dma_status iop_adma_status(struct dma_chan *chan,
769                                         dma_cookie_t cookie,
770                                         struct dma_tx_state *txstate)
771 {
772         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
773         int ret;
774 
775         ret = dma_cookie_status(chan, cookie, txstate);
776         if (ret == DMA_COMPLETE)
777                 return ret;
778 
779         iop_adma_slot_cleanup(iop_chan);
780 
781         return dma_cookie_status(chan, cookie, txstate);
782 }
783 
784 static irqreturn_t iop_adma_eot_handler(int irq, void *data)
785 {
786         struct iop_adma_chan *chan = data;
787 
788         dev_dbg(chan->device->common.dev, "%s\n", __func__);
789 
790         tasklet_schedule(&chan->irq_tasklet);
791 
792         iop_adma_device_clear_eot_status(chan);
793 
794         return IRQ_HANDLED;
795 }
796 
797 static irqreturn_t iop_adma_eoc_handler(int irq, void *data)
798 {
799         struct iop_adma_chan *chan = data;
800 
801         dev_dbg(chan->device->common.dev, "%s\n", __func__);
802 
803         tasklet_schedule(&chan->irq_tasklet);
804 
805         iop_adma_device_clear_eoc_status(chan);
806 
807         return IRQ_HANDLED;
808 }
809 
810 static irqreturn_t iop_adma_err_handler(int irq, void *data)
811 {
812         struct iop_adma_chan *chan = data;
813         unsigned long status = iop_chan_get_status(chan);
814 
815         dev_err(chan->device->common.dev,
816                 "error ( %s%s%s%s%s%s%s)\n",
817                 iop_is_err_int_parity(status, chan) ? "int_parity " : "",
818                 iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "",
819                 iop_is_err_int_tabort(status, chan) ? "int_tabort " : "",
820                 iop_is_err_int_mabort(status, chan) ? "int_mabort " : "",
821                 iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "",
822                 iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "",
823                 iop_is_err_split_tx(status, chan) ? "split_tx " : "");
824 
825         iop_adma_device_clear_err_status(chan);
826 
827         BUG();
828 
829         return IRQ_HANDLED;
830 }
831 
832 static void iop_adma_issue_pending(struct dma_chan *chan)
833 {
834         struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
835 
836         if (iop_chan->pending) {
837                 iop_chan->pending = 0;
838                 iop_chan_append(iop_chan);
839         }
840 }
841 
842 /*
843  * Perform a transaction to verify the HW works.
844  */
845 #define IOP_ADMA_TEST_SIZE 2000
846 
847 static int iop_adma_memcpy_self_test(struct iop_adma_device *device)
848 {
849         int i;
850         void *src, *dest;
851         dma_addr_t src_dma, dest_dma;
852         struct dma_chan *dma_chan;
853         dma_cookie_t cookie;
854         struct dma_async_tx_descriptor *tx;
855         int err = 0;
856         struct iop_adma_chan *iop_chan;
857 
858         dev_dbg(device->common.dev, "%s\n", __func__);
859 
860         src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
861         if (!src)
862                 return -ENOMEM;
863         dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL);
864         if (!dest) {
865                 kfree(src);
866                 return -ENOMEM;
867         }
868 
869         /* Fill in src buffer */
870         for (i = 0; i < IOP_ADMA_TEST_SIZE; i++)
871                 ((u8 *) src)[i] = (u8)i;
872 
873         /* Start copy, using first DMA channel */
874         dma_chan = container_of(device->common.channels.next,
875                                 struct dma_chan,
876                                 device_node);
877         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
878                 err = -ENODEV;
879                 goto out;
880         }
881 
882         dest_dma = dma_map_single(dma_chan->device->dev, dest,
883                                 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
884         src_dma = dma_map_single(dma_chan->device->dev, src,
885                                 IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE);
886         tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
887                                       IOP_ADMA_TEST_SIZE,
888                                       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
889 
890         cookie = iop_adma_tx_submit(tx);
891         iop_adma_issue_pending(dma_chan);
892         msleep(1);
893 
894         if (iop_adma_status(dma_chan, cookie, NULL) !=
895                         DMA_COMPLETE) {
896                 dev_err(dma_chan->device->dev,
897                         "Self-test copy timed out, disabling\n");
898                 err = -ENODEV;
899                 goto free_resources;
900         }
901 
902         iop_chan = to_iop_adma_chan(dma_chan);
903         dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
904                 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
905         if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) {
906                 dev_err(dma_chan->device->dev,
907                         "Self-test copy failed compare, disabling\n");
908                 err = -ENODEV;
909                 goto free_resources;
910         }
911 
912 free_resources:
913         iop_adma_free_chan_resources(dma_chan);
914 out:
915         kfree(src);
916         kfree(dest);
917         return err;
918 }
919 
920 #define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */
921 static int
922 iop_adma_xor_val_self_test(struct iop_adma_device *device)
923 {
924         int i, src_idx;
925         struct page *dest;
926         struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST];
927         struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
928         dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
929         dma_addr_t dest_dma;
930         struct dma_async_tx_descriptor *tx;
931         struct dma_chan *dma_chan;
932         dma_cookie_t cookie;
933         u8 cmp_byte = 0;
934         u32 cmp_word;
935         u32 zero_sum_result;
936         int err = 0;
937         struct iop_adma_chan *iop_chan;
938 
939         dev_dbg(device->common.dev, "%s\n", __func__);
940 
941         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
942                 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
943                 if (!xor_srcs[src_idx]) {
944                         while (src_idx--)
945                                 __free_page(xor_srcs[src_idx]);
946                         return -ENOMEM;
947                 }
948         }
949 
950         dest = alloc_page(GFP_KERNEL);
951         if (!dest) {
952                 while (src_idx--)
953                         __free_page(xor_srcs[src_idx]);
954                 return -ENOMEM;
955         }
956 
957         /* Fill in src buffers */
958         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
959                 u8 *ptr = page_address(xor_srcs[src_idx]);
960                 for (i = 0; i < PAGE_SIZE; i++)
961                         ptr[i] = (1 << src_idx);
962         }
963 
964         for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++)
965                 cmp_byte ^= (u8) (1 << src_idx);
966 
967         cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
968                         (cmp_byte << 8) | cmp_byte;
969 
970         memset(page_address(dest), 0, PAGE_SIZE);
971 
972         dma_chan = container_of(device->common.channels.next,
973                                 struct dma_chan,
974                                 device_node);
975         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
976                 err = -ENODEV;
977                 goto out;
978         }
979 
980         /* test xor */
981         dest_dma = dma_map_page(dma_chan->device->dev, dest, 0,
982                                 PAGE_SIZE, DMA_FROM_DEVICE);
983         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
984                 dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
985                                            0, PAGE_SIZE, DMA_TO_DEVICE);
986         tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
987                                    IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE,
988                                    DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
989 
990         cookie = iop_adma_tx_submit(tx);
991         iop_adma_issue_pending(dma_chan);
992         msleep(8);
993 
994         if (iop_adma_status(dma_chan, cookie, NULL) !=
995                 DMA_COMPLETE) {
996                 dev_err(dma_chan->device->dev,
997                         "Self-test xor timed out, disabling\n");
998                 err = -ENODEV;
999                 goto free_resources;
1000         }
1001 
1002         iop_chan = to_iop_adma_chan(dma_chan);
1003         dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
1004                 PAGE_SIZE, DMA_FROM_DEVICE);
1005         for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
1006                 u32 *ptr = page_address(dest);
1007                 if (ptr[i] != cmp_word) {
1008                         dev_err(dma_chan->device->dev,
1009                                 "Self-test xor failed compare, disabling\n");
1010                         err = -ENODEV;
1011                         goto free_resources;
1012                 }
1013         }
1014         dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma,
1015                 PAGE_SIZE, DMA_TO_DEVICE);
1016 
1017         /* skip zero sum if the capability is not present */
1018         if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
1019                 goto free_resources;
1020 
1021         /* zero sum the sources with the destintation page */
1022         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1023                 zero_sum_srcs[i] = xor_srcs[i];
1024         zero_sum_srcs[i] = dest;
1025 
1026         zero_sum_result = 1;
1027 
1028         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1029                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1030                                            zero_sum_srcs[i], 0, PAGE_SIZE,
1031                                            DMA_TO_DEVICE);
1032         tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1033                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1034                                        &zero_sum_result,
1035                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1036 
1037         cookie = iop_adma_tx_submit(tx);
1038         iop_adma_issue_pending(dma_chan);
1039         msleep(8);
1040 
1041         if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
1042                 dev_err(dma_chan->device->dev,
1043                         "Self-test zero sum timed out, disabling\n");
1044                 err = -ENODEV;
1045                 goto free_resources;
1046         }
1047 
1048         if (zero_sum_result != 0) {
1049                 dev_err(dma_chan->device->dev,
1050                         "Self-test zero sum failed compare, disabling\n");
1051                 err = -ENODEV;
1052                 goto free_resources;
1053         }
1054 
1055         /* test for non-zero parity sum */
1056         zero_sum_result = 0;
1057         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++)
1058                 dma_srcs[i] = dma_map_page(dma_chan->device->dev,
1059                                            zero_sum_srcs[i], 0, PAGE_SIZE,
1060                                            DMA_TO_DEVICE);
1061         tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs,
1062                                        IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE,
1063                                        &zero_sum_result,
1064                                        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1065 
1066         cookie = iop_adma_tx_submit(tx);
1067         iop_adma_issue_pending(dma_chan);
1068         msleep(8);
1069 
1070         if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) {
1071                 dev_err(dma_chan->device->dev,
1072                         "Self-test non-zero sum timed out, disabling\n");
1073                 err = -ENODEV;
1074                 goto free_resources;
1075         }
1076 
1077         if (zero_sum_result != 1) {
1078                 dev_err(dma_chan->device->dev,
1079                         "Self-test non-zero sum failed compare, disabling\n");
1080                 err = -ENODEV;
1081                 goto free_resources;
1082         }
1083 
1084 free_resources:
1085         iop_adma_free_chan_resources(dma_chan);
1086 out:
1087         src_idx = IOP_ADMA_NUM_SRC_TEST;
1088         while (src_idx--)
1089                 __free_page(xor_srcs[src_idx]);
1090         __free_page(dest);
1091         return err;
1092 }
1093 
1094 #ifdef CONFIG_RAID6_PQ
1095 static int
1096 iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device)
1097 {
1098         /* combined sources, software pq results, and extra hw pq results */
1099         struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2];
1100         /* ptr to the extra hw pq buffers defined above */
1101         struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2];
1102         /* address conversion buffers (dma_map / page_address) */
1103         void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2];
1104         dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST+2];
1105         dma_addr_t *pq_dest = &pq_src[IOP_ADMA_NUM_SRC_TEST];
1106 
1107         int i;
1108         struct dma_async_tx_descriptor *tx;
1109         struct dma_chan *dma_chan;
1110         dma_cookie_t cookie;
1111         u32 zero_sum_result;
1112         int err = 0;
1113         struct device *dev;
1114 
1115         dev_dbg(device->common.dev, "%s\n", __func__);
1116 
1117         for (i = 0; i < ARRAY_SIZE(pq); i++) {
1118                 pq[i] = alloc_page(GFP_KERNEL);
1119                 if (!pq[i]) {
1120                         while (i--)
1121                                 __free_page(pq[i]);
1122                         return -ENOMEM;
1123                 }
1124         }
1125 
1126         /* Fill in src buffers */
1127         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) {
1128                 pq_sw[i] = page_address(pq[i]);
1129                 memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE);
1130         }
1131         pq_sw[i] = page_address(pq[i]);
1132         pq_sw[i+1] = page_address(pq[i+1]);
1133 
1134         dma_chan = container_of(device->common.channels.next,
1135                                 struct dma_chan,
1136                                 device_node);
1137         if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
1138                 err = -ENODEV;
1139                 goto out;
1140         }
1141 
1142         dev = dma_chan->device->dev;
1143 
1144         /* initialize the dests */
1145         memset(page_address(pq_hw[0]), 0 , PAGE_SIZE);
1146         memset(page_address(pq_hw[1]), 0 , PAGE_SIZE);
1147 
1148         /* test pq */
1149         pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1150         pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE);
1151         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1152                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1153                                          DMA_TO_DEVICE);
1154 
1155         tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src,
1156                                   IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp,
1157                                   PAGE_SIZE,
1158                                   DMA_PREP_INTERRUPT |
1159                                   DMA_CTRL_ACK);
1160 
1161         cookie = iop_adma_tx_submit(tx);
1162         iop_adma_issue_pending(dma_chan);
1163         msleep(8);
1164 
1165         if (iop_adma_status(dma_chan, cookie, NULL) !=
1166                 DMA_COMPLETE) {
1167                 dev_err(dev, "Self-test pq timed out, disabling\n");
1168                 err = -ENODEV;
1169                 goto free_resources;
1170         }
1171 
1172         raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw);
1173 
1174         if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST],
1175                    page_address(pq_hw[0]), PAGE_SIZE) != 0) {
1176                 dev_err(dev, "Self-test p failed compare, disabling\n");
1177                 err = -ENODEV;
1178                 goto free_resources;
1179         }
1180         if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1],
1181                    page_address(pq_hw[1]), PAGE_SIZE) != 0) {
1182                 dev_err(dev, "Self-test q failed compare, disabling\n");
1183                 err = -ENODEV;
1184                 goto free_resources;
1185         }
1186 
1187         /* test correct zero sum using the software generated pq values */
1188         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1189                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1190                                          DMA_TO_DEVICE);
1191 
1192         zero_sum_result = ~0;
1193         tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1194                                       pq_src, IOP_ADMA_NUM_SRC_TEST,
1195                                       raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1196                                       DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1197 
1198         cookie = iop_adma_tx_submit(tx);
1199         iop_adma_issue_pending(dma_chan);
1200         msleep(8);
1201 
1202         if (iop_adma_status(dma_chan, cookie, NULL) !=
1203                 DMA_COMPLETE) {
1204                 dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n");
1205                 err = -ENODEV;
1206                 goto free_resources;
1207         }
1208 
1209         if (zero_sum_result != 0) {
1210                 dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n",
1211                         zero_sum_result);
1212                 err = -ENODEV;
1213                 goto free_resources;
1214         }
1215 
1216         /* test incorrect zero sum */
1217         i = IOP_ADMA_NUM_SRC_TEST;
1218         memset(pq_sw[i] + 100, 0, 100);
1219         memset(pq_sw[i+1] + 200, 0, 200);
1220         for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++)
1221                 pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE,
1222                                          DMA_TO_DEVICE);
1223 
1224         zero_sum_result = 0;
1225         tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST],
1226                                       pq_src, IOP_ADMA_NUM_SRC_TEST,
1227                                       raid6_gfexp, PAGE_SIZE, &zero_sum_result,
1228                                       DMA_PREP_INTERRUPT|DMA_CTRL_ACK);
1229 
1230         cookie = iop_adma_tx_submit(tx);
1231         iop_adma_issue_pending(dma_chan);
1232         msleep(8);
1233 
1234         if (iop_adma_status(dma_chan, cookie, NULL) !=
1235                 DMA_COMPLETE) {
1236                 dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n");
1237                 err = -ENODEV;
1238                 goto free_resources;
1239         }
1240 
1241         if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) {
1242                 dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n",
1243                         zero_sum_result);
1244                 err = -ENODEV;
1245                 goto free_resources;
1246         }
1247 
1248 free_resources:
1249         iop_adma_free_chan_resources(dma_chan);
1250 out:
1251         i = ARRAY_SIZE(pq);
1252         while (i--)
1253                 __free_page(pq[i]);
1254         return err;
1255 }
1256 #endif
1257 
1258 static int iop_adma_remove(struct platform_device *dev)
1259 {
1260         struct iop_adma_device *device = platform_get_drvdata(dev);
1261         struct dma_chan *chan, *_chan;
1262         struct iop_adma_chan *iop_chan;
1263         struct iop_adma_platform_data *plat_data = dev_get_platdata(&dev->dev);
1264 
1265         dma_async_device_unregister(&device->common);
1266 
1267         dma_free_coherent(&dev->dev, plat_data->pool_size,
1268                         device->dma_desc_pool_virt, device->dma_desc_pool);
1269 
1270         list_for_each_entry_safe(chan, _chan, &device->common.channels,
1271                                 device_node) {
1272                 iop_chan = to_iop_adma_chan(chan);
1273                 list_del(&chan->device_node);
1274                 kfree(iop_chan);
1275         }
1276         kfree(device);
1277 
1278         return 0;
1279 }
1280 
1281 static int iop_adma_probe(struct platform_device *pdev)
1282 {
1283         struct resource *res;
1284         int ret = 0, i;
1285         struct iop_adma_device *adev;
1286         struct iop_adma_chan *iop_chan;
1287         struct dma_device *dma_dev;
1288         struct iop_adma_platform_data *plat_data = dev_get_platdata(&pdev->dev);
1289 
1290         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1291         if (!res)
1292                 return -ENODEV;
1293 
1294         if (!devm_request_mem_region(&pdev->dev, res->start,
1295                                 resource_size(res), pdev->name))
1296                 return -EBUSY;
1297 
1298         adev = kzalloc(sizeof(*adev), GFP_KERNEL);
1299         if (!adev)
1300                 return -ENOMEM;
1301         dma_dev = &adev->common;
1302 
1303         /* allocate coherent memory for hardware descriptors
1304          * note: writecombine gives slightly better performance, but
1305          * requires that we explicitly flush the writes
1306          */
1307         if ((adev->dma_desc_pool_virt = dma_alloc_writecombine(&pdev->dev,
1308                                         plat_data->pool_size,
1309                                         &adev->dma_desc_pool,
1310                                         GFP_KERNEL)) == NULL) {
1311                 ret = -ENOMEM;
1312                 goto err_free_adev;
1313         }
1314 
1315         dev_dbg(&pdev->dev, "%s: allocated descriptor pool virt %p phys %p\n",
1316                 __func__, adev->dma_desc_pool_virt,
1317                 (void *) adev->dma_desc_pool);
1318 
1319         adev->id = plat_data->hw_id;
1320 
1321         /* discover transaction capabilites from the platform data */
1322         dma_dev->cap_mask = plat_data->cap_mask;
1323 
1324         adev->pdev = pdev;
1325         platform_set_drvdata(pdev, adev);
1326 
1327         INIT_LIST_HEAD(&dma_dev->channels);
1328 
1329         /* set base routines */
1330         dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources;
1331         dma_dev->device_free_chan_resources = iop_adma_free_chan_resources;
1332         dma_dev->device_tx_status = iop_adma_status;
1333         dma_dev->device_issue_pending = iop_adma_issue_pending;
1334         dma_dev->dev = &pdev->dev;
1335 
1336         /* set prep routines based on capability */
1337         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1338                 dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy;
1339         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1340                 dma_dev->max_xor = iop_adma_get_max_xor();
1341                 dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor;
1342         }
1343         if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask))
1344                 dma_dev->device_prep_dma_xor_val =
1345                         iop_adma_prep_dma_xor_val;
1346         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
1347                 dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0);
1348                 dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq;
1349         }
1350         if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask))
1351                 dma_dev->device_prep_dma_pq_val =
1352                         iop_adma_prep_dma_pq_val;
1353         if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1354                 dma_dev->device_prep_dma_interrupt =
1355                         iop_adma_prep_dma_interrupt;
1356 
1357         iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL);
1358         if (!iop_chan) {
1359                 ret = -ENOMEM;
1360                 goto err_free_dma;
1361         }
1362         iop_chan->device = adev;
1363 
1364         iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start,
1365                                         resource_size(res));
1366         if (!iop_chan->mmr_base) {
1367                 ret = -ENOMEM;
1368                 goto err_free_iop_chan;
1369         }
1370         tasklet_init(&iop_chan->irq_tasklet, iop_adma_tasklet, (unsigned long)
1371                 iop_chan);
1372 
1373         /* clear errors before enabling interrupts */
1374         iop_adma_device_clear_err_status(iop_chan);
1375 
1376         for (i = 0; i < 3; i++) {
1377                 irq_handler_t handler[] = { iop_adma_eot_handler,
1378                                         iop_adma_eoc_handler,
1379                                         iop_adma_err_handler };
1380                 int irq = platform_get_irq(pdev, i);
1381                 if (irq < 0) {
1382                         ret = -ENXIO;
1383                         goto err_free_iop_chan;
1384                 } else {
1385                         ret = devm_request_irq(&pdev->dev, irq,
1386                                         handler[i], 0, pdev->name, iop_chan);
1387                         if (ret)
1388                                 goto err_free_iop_chan;
1389                 }
1390         }
1391 
1392         spin_lock_init(&iop_chan->lock);
1393         INIT_LIST_HEAD(&iop_chan->chain);
1394         INIT_LIST_HEAD(&iop_chan->all_slots);
1395         iop_chan->common.device = dma_dev;
1396         dma_cookie_init(&iop_chan->common);
1397         list_add_tail(&iop_chan->common.device_node, &dma_dev->channels);
1398 
1399         if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1400                 ret = iop_adma_memcpy_self_test(adev);
1401                 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1402                 if (ret)
1403                         goto err_free_iop_chan;
1404         }
1405 
1406         if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1407                 ret = iop_adma_xor_val_self_test(adev);
1408                 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1409                 if (ret)
1410                         goto err_free_iop_chan;
1411         }
1412 
1413         if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) &&
1414             dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) {
1415                 #ifdef CONFIG_RAID6_PQ
1416                 ret = iop_adma_pq_zero_sum_self_test(adev);
1417                 dev_dbg(&pdev->dev, "pq self test returned %d\n", ret);
1418                 #else
1419                 /* can not test raid6, so do not publish capability */
1420                 dma_cap_clear(DMA_PQ, dma_dev->cap_mask);
1421                 dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask);
1422                 ret = 0;
1423                 #endif
1424                 if (ret)
1425                         goto err_free_iop_chan;
1426         }
1427 
1428         dev_info(&pdev->dev, "Intel(R) IOP: ( %s%s%s%s%s%s)\n",
1429                  dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "",
1430                  dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "",
1431                  dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1432                  dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "",
1433                  dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1434                  dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1435 
1436         dma_async_device_register(dma_dev);
1437         goto out;
1438 
1439  err_free_iop_chan:
1440         kfree(iop_chan);
1441  err_free_dma:
1442         dma_free_coherent(&adev->pdev->dev, plat_data->pool_size,
1443                         adev->dma_desc_pool_virt, adev->dma_desc_pool);
1444  err_free_adev:
1445         kfree(adev);
1446  out:
1447         return ret;
1448 }
1449 
1450 static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan)
1451 {
1452         struct iop_adma_desc_slot *sw_desc, *grp_start;
1453         dma_cookie_t cookie;
1454         int slot_cnt, slots_per_op;
1455 
1456         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1457 
1458         spin_lock_bh(&iop_chan->lock);
1459         slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op);
1460         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1461         if (sw_desc) {
1462                 grp_start = sw_desc->group_head;
1463 
1464                 list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1465                 async_tx_ack(&sw_desc->async_tx);
1466                 iop_desc_init_memcpy(grp_start, 0);
1467                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1468                 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1469                 iop_desc_set_memcpy_src_addr(grp_start, 0);
1470 
1471                 cookie = dma_cookie_assign(&sw_desc->async_tx);
1472 
1473                 /* initialize the completed cookie to be less than
1474                  * the most recently used cookie
1475                  */
1476                 iop_chan->common.completed_cookie = cookie - 1;
1477 
1478                 /* channel should not be busy */
1479                 BUG_ON(iop_chan_is_busy(iop_chan));
1480 
1481                 /* clear any prior error-status bits */
1482                 iop_adma_device_clear_err_status(iop_chan);
1483 
1484                 /* disable operation */
1485                 iop_chan_disable(iop_chan);
1486 
1487                 /* set the descriptor address */
1488                 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1489 
1490                 /* 1/ don't add pre-chained descriptors
1491                  * 2/ dummy read to flush next_desc write
1492                  */
1493                 BUG_ON(iop_desc_get_next_desc(sw_desc));
1494 
1495                 /* run the descriptor */
1496                 iop_chan_enable(iop_chan);
1497         } else
1498                 dev_err(iop_chan->device->common.dev,
1499                         "failed to allocate null descriptor\n");
1500         spin_unlock_bh(&iop_chan->lock);
1501 }
1502 
1503 static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
1504 {
1505         struct iop_adma_desc_slot *sw_desc, *grp_start;
1506         dma_cookie_t cookie;
1507         int slot_cnt, slots_per_op;
1508 
1509         dev_dbg(iop_chan->device->common.dev, "%s\n", __func__);
1510 
1511         spin_lock_bh(&iop_chan->lock);
1512         slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op);
1513         sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1514         if (sw_desc) {
1515                 grp_start = sw_desc->group_head;
1516                 list_splice_init(&sw_desc->tx_list, &iop_chan->chain);
1517                 async_tx_ack(&sw_desc->async_tx);
1518                 iop_desc_init_null_xor(grp_start, 2, 0);
1519                 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1520                 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1521                 iop_desc_set_xor_src_addr(grp_start, 0, 0);
1522                 iop_desc_set_xor_src_addr(grp_start, 1, 0);
1523 
1524                 cookie = dma_cookie_assign(&sw_desc->async_tx);
1525 
1526                 /* initialize the completed cookie to be less than
1527                  * the most recently used cookie
1528                  */
1529                 iop_chan->common.completed_cookie = cookie - 1;
1530 
1531                 /* channel should not be busy */
1532                 BUG_ON(iop_chan_is_busy(iop_chan));
1533 
1534                 /* clear any prior error-status bits */
1535                 iop_adma_device_clear_err_status(iop_chan);
1536 
1537                 /* disable operation */
1538                 iop_chan_disable(iop_chan);
1539 
1540                 /* set the descriptor address */
1541                 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1542 
1543                 /* 1/ don't add pre-chained descriptors
1544                  * 2/ dummy read to flush next_desc write
1545                  */
1546                 BUG_ON(iop_desc_get_next_desc(sw_desc));
1547 
1548                 /* run the descriptor */
1549                 iop_chan_enable(iop_chan);
1550         } else
1551                 dev_err(iop_chan->device->common.dev,
1552                         "failed to allocate null descriptor\n");
1553         spin_unlock_bh(&iop_chan->lock);
1554 }
1555 
1556 static struct platform_driver iop_adma_driver = {
1557         .probe          = iop_adma_probe,
1558         .remove         = iop_adma_remove,
1559         .driver         = {
1560                 .owner  = THIS_MODULE,
1561                 .name   = "iop-adma",
1562         },
1563 };
1564 
1565 module_platform_driver(iop_adma_driver);
1566 
1567 MODULE_AUTHOR("Intel Corporation");
1568 MODULE_DESCRIPTION("IOP ADMA Engine Driver");
1569 MODULE_LICENSE("GPL");
1570 MODULE_ALIAS("platform:iop-adma");
1571 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us