Version:  2.0.40 2.2.26 2.4.37 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16

Linux/drivers/dma/ppc4xx/adma.c

  1 /*
  2  * Copyright (C) 2006-2009 DENX Software Engineering.
  3  *
  4  * Author: Yuri Tikhonov <yur@emcraft.com>
  5  *
  6  * Further porting to arch/powerpc by
  7  *      Anatolij Gustschin <agust@denx.de>
  8  *
  9  * This program is free software; you can redistribute it and/or modify it
 10  * under the terms of the GNU General Public License as published by the Free
 11  * Software Foundation; either version 2 of the License, or (at your option)
 12  * any later version.
 13  *
 14  * This program is distributed in the hope that it will be useful, but WITHOUT
 15  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 16  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 17  * more details.
 18  *
 19  * You should have received a copy of the GNU General Public License along with
 20  * this program; if not, write to the Free Software Foundation, Inc., 59
 21  * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 22  *
 23  * The full GNU General Public License is included in this distribution in the
 24  * file called COPYING.
 25  */
 26 
 27 /*
 28  * This driver supports the asynchrounous DMA copy and RAID engines available
 29  * on the AMCC PPC440SPe Processors.
 30  * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
 31  * ADMA driver written by D.Williams.
 32  */
 33 
 34 #include <linux/init.h>
 35 #include <linux/module.h>
 36 #include <linux/async_tx.h>
 37 #include <linux/delay.h>
 38 #include <linux/dma-mapping.h>
 39 #include <linux/spinlock.h>
 40 #include <linux/interrupt.h>
 41 #include <linux/slab.h>
 42 #include <linux/uaccess.h>
 43 #include <linux/proc_fs.h>
 44 #include <linux/of.h>
 45 #include <linux/of_address.h>
 46 #include <linux/of_irq.h>
 47 #include <linux/of_platform.h>
 48 #include <asm/dcr.h>
 49 #include <asm/dcr-regs.h>
 50 #include "adma.h"
 51 #include "../dmaengine.h"
 52 
 53 enum ppc_adma_init_code {
 54         PPC_ADMA_INIT_OK = 0,
 55         PPC_ADMA_INIT_MEMRES,
 56         PPC_ADMA_INIT_MEMREG,
 57         PPC_ADMA_INIT_ALLOC,
 58         PPC_ADMA_INIT_COHERENT,
 59         PPC_ADMA_INIT_CHANNEL,
 60         PPC_ADMA_INIT_IRQ1,
 61         PPC_ADMA_INIT_IRQ2,
 62         PPC_ADMA_INIT_REGISTER
 63 };
 64 
 65 static char *ppc_adma_errors[] = {
 66         [PPC_ADMA_INIT_OK] = "ok",
 67         [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
 68         [PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
 69         [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
 70                                 "structure",
 71         [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
 72                                    "hardware descriptors",
 73         [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
 74         [PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
 75         [PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
 76         [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
 77 };
 78 
 79 static enum ppc_adma_init_code
 80 ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
 81 
 82 struct ppc_dma_chan_ref {
 83         struct dma_chan *chan;
 84         struct list_head node;
 85 };
 86 
 87 /* The list of channels exported by ppc440spe ADMA */
 88 struct list_head
 89 ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
 90 
 91 /* This flag is set when want to refetch the xor chain in the interrupt
 92  * handler
 93  */
 94 static u32 do_xor_refetch;
 95 
 96 /* Pointer to DMA0, DMA1 CP/CS FIFO */
 97 static void *ppc440spe_dma_fifo_buf;
 98 
 99 /* Pointers to last submitted to DMA0, DMA1 CDBs */
100 static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
101 static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
102 
103 /* Pointer to last linked and submitted xor CB */
104 static struct ppc440spe_adma_desc_slot *xor_last_linked;
105 static struct ppc440spe_adma_desc_slot *xor_last_submit;
106 
107 /* This array is used in data-check operations for storing a pattern */
108 static char ppc440spe_qword[16];
109 
110 static atomic_t ppc440spe_adma_err_irq_ref;
111 static dcr_host_t ppc440spe_mq_dcr_host;
112 static unsigned int ppc440spe_mq_dcr_len;
113 
114 /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
115  * the block size in transactions, then we do not allow to activate more than
116  * only one RXOR transactions simultaneously. So use this var to store
117  * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
118  * set) or not (PPC440SPE_RXOR_RUN is clear).
119  */
120 static unsigned long ppc440spe_rxor_state;
121 
122 /* These are used in enable & check routines
123  */
124 static u32 ppc440spe_r6_enabled;
125 static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
126 static struct completion ppc440spe_r6_test_comp;
127 
128 static int ppc440spe_adma_dma2rxor_prep_src(
129                 struct ppc440spe_adma_desc_slot *desc,
130                 struct ppc440spe_rxor *cursor, int index,
131                 int src_cnt, u32 addr);
132 static void ppc440spe_adma_dma2rxor_set_src(
133                 struct ppc440spe_adma_desc_slot *desc,
134                 int index, dma_addr_t addr);
135 static void ppc440spe_adma_dma2rxor_set_mult(
136                 struct ppc440spe_adma_desc_slot *desc,
137                 int index, u8 mult);
138 
139 #ifdef ADMA_LL_DEBUG
140 #define ADMA_LL_DBG(x) ({ if (1) x; 0; })
141 #else
142 #define ADMA_LL_DBG(x) ({ if (0) x; 0; })
143 #endif
144 
145 static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
146 {
147         struct dma_cdb *cdb;
148         struct xor_cb *cb;
149         int i;
150 
151         switch (chan->device->id) {
152         case 0:
153         case 1:
154                 cdb = block;
155 
156                 pr_debug("CDB at %p [%d]:\n"
157                         "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
158                         "\t sg1u 0x%08x sg1l 0x%08x\n"
159                         "\t sg2u 0x%08x sg2l 0x%08x\n"
160                         "\t sg3u 0x%08x sg3l 0x%08x\n",
161                         cdb, chan->device->id,
162                         cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
163                         le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
164                         le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
165                         le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
166                 );
167                 break;
168         case 2:
169                 cb = block;
170 
171                 pr_debug("CB at %p [%d]:\n"
172                         "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
173                         "\t cbtah 0x%08x cbtal 0x%08x\n"
174                         "\t cblah 0x%08x cblal 0x%08x\n",
175                         cb, chan->device->id,
176                         cb->cbc, cb->cbbc, cb->cbs,
177                         cb->cbtah, cb->cbtal,
178                         cb->cblah, cb->cblal);
179                 for (i = 0; i < 16; i++) {
180                         if (i && !cb->ops[i].h && !cb->ops[i].l)
181                                 continue;
182                         pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
183                                 i, cb->ops[i].h, cb->ops[i].l);
184                 }
185                 break;
186         }
187 }
188 
189 static void print_cb_list(struct ppc440spe_adma_chan *chan,
190                           struct ppc440spe_adma_desc_slot *iter)
191 {
192         for (; iter; iter = iter->hw_next)
193                 print_cb(chan, iter->hw_desc);
194 }
195 
196 static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
197                              unsigned int src_cnt)
198 {
199         int i;
200 
201         pr_debug("\n%s(%d):\nsrc: ", __func__, id);
202         for (i = 0; i < src_cnt; i++)
203                 pr_debug("\t0x%016llx ", src[i]);
204         pr_debug("dst:\n\t0x%016llx\n", dst);
205 }
206 
207 static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
208                             unsigned int src_cnt)
209 {
210         int i;
211 
212         pr_debug("\n%s(%d):\nsrc: ", __func__, id);
213         for (i = 0; i < src_cnt; i++)
214                 pr_debug("\t0x%016llx ", src[i]);
215         pr_debug("dst: ");
216         for (i = 0; i < 2; i++)
217                 pr_debug("\t0x%016llx ", dst[i]);
218 }
219 
220 static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
221                                     unsigned int src_cnt,
222                                     const unsigned char *scf)
223 {
224         int i;
225 
226         pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
227         if (scf) {
228                 for (i = 0; i < src_cnt; i++)
229                         pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
230         } else {
231                 for (i = 0; i < src_cnt; i++)
232                         pr_debug("\t0x%016llx(no) ", src[i]);
233         }
234 
235         pr_debug("dst: ");
236         for (i = 0; i < 2; i++)
237                 pr_debug("\t0x%016llx ", src[src_cnt + i]);
238 }
239 
240 /******************************************************************************
241  * Command (Descriptor) Blocks low-level routines
242  ******************************************************************************/
243 /**
244  * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
245  * pseudo operation
246  */
247 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
248                                           struct ppc440spe_adma_chan *chan)
249 {
250         struct xor_cb *p;
251 
252         switch (chan->device->id) {
253         case PPC440SPE_XOR_ID:
254                 p = desc->hw_desc;
255                 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
256                 /* NOP with Command Block Complete Enable */
257                 p->cbc = XOR_CBCR_CBCE_BIT;
258                 break;
259         case PPC440SPE_DMA0_ID:
260         case PPC440SPE_DMA1_ID:
261                 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
262                 /* NOP with interrupt */
263                 set_bit(PPC440SPE_DESC_INT, &desc->flags);
264                 break;
265         default:
266                 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
267                                 __func__);
268                 break;
269         }
270 }
271 
272 /**
273  * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
274  * pseudo operation
275  */
276 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
277 {
278         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
279         desc->hw_next = NULL;
280         desc->src_cnt = 0;
281         desc->dst_cnt = 1;
282 }
283 
284 /**
285  * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
286  */
287 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
288                                          int src_cnt, unsigned long flags)
289 {
290         struct xor_cb *hw_desc = desc->hw_desc;
291 
292         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
293         desc->hw_next = NULL;
294         desc->src_cnt = src_cnt;
295         desc->dst_cnt = 1;
296 
297         hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
298         if (flags & DMA_PREP_INTERRUPT)
299                 /* Enable interrupt on completion */
300                 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
301 }
302 
303 /**
304  * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
305  * operation in DMA2 controller
306  */
307 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
308                 int dst_cnt, int src_cnt, unsigned long flags)
309 {
310         struct xor_cb *hw_desc = desc->hw_desc;
311 
312         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
313         desc->hw_next = NULL;
314         desc->src_cnt = src_cnt;
315         desc->dst_cnt = dst_cnt;
316         memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
317         desc->descs_per_op = 0;
318 
319         hw_desc->cbc = XOR_CBCR_TGT_BIT;
320         if (flags & DMA_PREP_INTERRUPT)
321                 /* Enable interrupt on completion */
322                 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
323 }
324 
325 #define DMA_CTRL_FLAGS_LAST     DMA_PREP_FENCE
326 #define DMA_PREP_ZERO_P         (DMA_CTRL_FLAGS_LAST << 1)
327 #define DMA_PREP_ZERO_Q         (DMA_PREP_ZERO_P << 1)
328 
329 /**
330  * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
331  * with DMA0/1
332  */
333 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
334                                 int dst_cnt, int src_cnt, unsigned long flags,
335                                 unsigned long op)
336 {
337         struct dma_cdb *hw_desc;
338         struct ppc440spe_adma_desc_slot *iter;
339         u8 dopc;
340 
341         /* Common initialization of a PQ descriptors chain */
342         set_bits(op, &desc->flags);
343         desc->src_cnt = src_cnt;
344         desc->dst_cnt = dst_cnt;
345 
346         /* WXOR MULTICAST if both P and Q are being computed
347          * MV_SG1_SG2 if Q only
348          */
349         dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
350                 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
351 
352         list_for_each_entry(iter, &desc->group_list, chain_node) {
353                 hw_desc = iter->hw_desc;
354                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
355 
356                 if (likely(!list_is_last(&iter->chain_node,
357                                 &desc->group_list))) {
358                         /* set 'next' pointer */
359                         iter->hw_next = list_entry(iter->chain_node.next,
360                                 struct ppc440spe_adma_desc_slot, chain_node);
361                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
362                 } else {
363                         /* this is the last descriptor.
364                          * this slot will be pasted from ADMA level
365                          * each time it wants to configure parameters
366                          * of the transaction (src, dst, ...)
367                          */
368                         iter->hw_next = NULL;
369                         if (flags & DMA_PREP_INTERRUPT)
370                                 set_bit(PPC440SPE_DESC_INT, &iter->flags);
371                         else
372                                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
373                 }
374         }
375 
376         /* Set OPS depending on WXOR/RXOR type of operation */
377         if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
378                 /* This is a WXOR only chain:
379                  * - first descriptors are for zeroing destinations
380                  *   if PPC440SPE_ZERO_P/Q set;
381                  * - descriptors remained are for GF-XOR operations.
382                  */
383                 iter = list_first_entry(&desc->group_list,
384                                         struct ppc440spe_adma_desc_slot,
385                                         chain_node);
386 
387                 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
388                         hw_desc = iter->hw_desc;
389                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
390                         iter = list_first_entry(&iter->chain_node,
391                                         struct ppc440spe_adma_desc_slot,
392                                         chain_node);
393                 }
394 
395                 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
396                         hw_desc = iter->hw_desc;
397                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
398                         iter = list_first_entry(&iter->chain_node,
399                                         struct ppc440spe_adma_desc_slot,
400                                         chain_node);
401                 }
402 
403                 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
404                         hw_desc = iter->hw_desc;
405                         hw_desc->opc = dopc;
406                 }
407         } else {
408                 /* This is either RXOR-only or mixed RXOR/WXOR */
409 
410                 /* The first 1 or 2 slots in chain are always RXOR,
411                  * if need to calculate P & Q, then there are two
412                  * RXOR slots; if only P or only Q, then there is one
413                  */
414                 iter = list_first_entry(&desc->group_list,
415                                         struct ppc440spe_adma_desc_slot,
416                                         chain_node);
417                 hw_desc = iter->hw_desc;
418                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
419 
420                 if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
421                         iter = list_first_entry(&iter->chain_node,
422                                                 struct ppc440spe_adma_desc_slot,
423                                                 chain_node);
424                         hw_desc = iter->hw_desc;
425                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
426                 }
427 
428                 /* The remaining descs (if any) are WXORs */
429                 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
430                         iter = list_first_entry(&iter->chain_node,
431                                                 struct ppc440spe_adma_desc_slot,
432                                                 chain_node);
433                         list_for_each_entry_from(iter, &desc->group_list,
434                                                 chain_node) {
435                                 hw_desc = iter->hw_desc;
436                                 hw_desc->opc = dopc;
437                         }
438                 }
439         }
440 }
441 
442 /**
443  * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
444  * for PQ_ZERO_SUM operation
445  */
446 static void ppc440spe_desc_init_dma01pqzero_sum(
447                                 struct ppc440spe_adma_desc_slot *desc,
448                                 int dst_cnt, int src_cnt)
449 {
450         struct dma_cdb *hw_desc;
451         struct ppc440spe_adma_desc_slot *iter;
452         int i = 0;
453         u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
454                                    DMA_CDB_OPC_MV_SG1_SG2;
455         /*
456          * Initialize starting from 2nd or 3rd descriptor dependent
457          * on dst_cnt. First one or two slots are for cloning P
458          * and/or Q to chan->pdest and/or chan->qdest as we have
459          * to preserve original P/Q.
460          */
461         iter = list_first_entry(&desc->group_list,
462                                 struct ppc440spe_adma_desc_slot, chain_node);
463         iter = list_entry(iter->chain_node.next,
464                           struct ppc440spe_adma_desc_slot, chain_node);
465 
466         if (dst_cnt > 1) {
467                 iter = list_entry(iter->chain_node.next,
468                                   struct ppc440spe_adma_desc_slot, chain_node);
469         }
470         /* initialize each source descriptor in chain */
471         list_for_each_entry_from(iter, &desc->group_list, chain_node) {
472                 hw_desc = iter->hw_desc;
473                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
474                 iter->src_cnt = 0;
475                 iter->dst_cnt = 0;
476 
477                 /* This is a ZERO_SUM operation:
478                  * - <src_cnt> descriptors starting from 2nd or 3rd
479                  *   descriptor are for GF-XOR operations;
480                  * - remaining <dst_cnt> descriptors are for checking the result
481                  */
482                 if (i++ < src_cnt)
483                         /* MV_SG1_SG2 if only Q is being verified
484                          * MULTICAST if both P and Q are being verified
485                          */
486                         hw_desc->opc = dopc;
487                 else
488                         /* DMA_CDB_OPC_DCHECK128 operation */
489                         hw_desc->opc = DMA_CDB_OPC_DCHECK128;
490 
491                 if (likely(!list_is_last(&iter->chain_node,
492                                          &desc->group_list))) {
493                         /* set 'next' pointer */
494                         iter->hw_next = list_entry(iter->chain_node.next,
495                                                 struct ppc440spe_adma_desc_slot,
496                                                 chain_node);
497                 } else {
498                         /* this is the last descriptor.
499                          * this slot will be pasted from ADMA level
500                          * each time it wants to configure parameters
501                          * of the transaction (src, dst, ...)
502                          */
503                         iter->hw_next = NULL;
504                         /* always enable interrupt generation since we get
505                          * the status of pqzero from the handler
506                          */
507                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
508                 }
509         }
510         desc->src_cnt = src_cnt;
511         desc->dst_cnt = dst_cnt;
512 }
513 
514 /**
515  * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
516  */
517 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
518                                         unsigned long flags)
519 {
520         struct dma_cdb *hw_desc = desc->hw_desc;
521 
522         memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
523         desc->hw_next = NULL;
524         desc->src_cnt = 1;
525         desc->dst_cnt = 1;
526 
527         if (flags & DMA_PREP_INTERRUPT)
528                 set_bit(PPC440SPE_DESC_INT, &desc->flags);
529         else
530                 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
531 
532         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
533 }
534 
535 /**
536  * ppc440spe_desc_set_src_addr - set source address into the descriptor
537  */
538 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
539                                         struct ppc440spe_adma_chan *chan,
540                                         int src_idx, dma_addr_t addrh,
541                                         dma_addr_t addrl)
542 {
543         struct dma_cdb *dma_hw_desc;
544         struct xor_cb *xor_hw_desc;
545         phys_addr_t addr64, tmplow, tmphi;
546 
547         switch (chan->device->id) {
548         case PPC440SPE_DMA0_ID:
549         case PPC440SPE_DMA1_ID:
550                 if (!addrh) {
551                         addr64 = addrl;
552                         tmphi = (addr64 >> 32);
553                         tmplow = (addr64 & 0xFFFFFFFF);
554                 } else {
555                         tmphi = addrh;
556                         tmplow = addrl;
557                 }
558                 dma_hw_desc = desc->hw_desc;
559                 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
560                 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
561                 break;
562         case PPC440SPE_XOR_ID:
563                 xor_hw_desc = desc->hw_desc;
564                 xor_hw_desc->ops[src_idx].l = addrl;
565                 xor_hw_desc->ops[src_idx].h |= addrh;
566                 break;
567         }
568 }
569 
570 /**
571  * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
572  */
573 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
574                         struct ppc440spe_adma_chan *chan, u32 mult_index,
575                         int sg_index, unsigned char mult_value)
576 {
577         struct dma_cdb *dma_hw_desc;
578         struct xor_cb *xor_hw_desc;
579         u32 *psgu;
580 
581         switch (chan->device->id) {
582         case PPC440SPE_DMA0_ID:
583         case PPC440SPE_DMA1_ID:
584                 dma_hw_desc = desc->hw_desc;
585 
586                 switch (sg_index) {
587                 /* for RXOR operations set multiplier
588                  * into source cued address
589                  */
590                 case DMA_CDB_SG_SRC:
591                         psgu = &dma_hw_desc->sg1u;
592                         break;
593                 /* for WXOR operations set multiplier
594                  * into destination cued address(es)
595                  */
596                 case DMA_CDB_SG_DST1:
597                         psgu = &dma_hw_desc->sg2u;
598                         break;
599                 case DMA_CDB_SG_DST2:
600                         psgu = &dma_hw_desc->sg3u;
601                         break;
602                 default:
603                         BUG();
604                 }
605 
606                 *psgu |= cpu_to_le32(mult_value << mult_index);
607                 break;
608         case PPC440SPE_XOR_ID:
609                 xor_hw_desc = desc->hw_desc;
610                 break;
611         default:
612                 BUG();
613         }
614 }
615 
616 /**
617  * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
618  */
619 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
620                                 struct ppc440spe_adma_chan *chan,
621                                 dma_addr_t addrh, dma_addr_t addrl,
622                                 u32 dst_idx)
623 {
624         struct dma_cdb *dma_hw_desc;
625         struct xor_cb *xor_hw_desc;
626         phys_addr_t addr64, tmphi, tmplow;
627         u32 *psgu, *psgl;
628 
629         switch (chan->device->id) {
630         case PPC440SPE_DMA0_ID:
631         case PPC440SPE_DMA1_ID:
632                 if (!addrh) {
633                         addr64 = addrl;
634                         tmphi = (addr64 >> 32);
635                         tmplow = (addr64 & 0xFFFFFFFF);
636                 } else {
637                         tmphi = addrh;
638                         tmplow = addrl;
639                 }
640                 dma_hw_desc = desc->hw_desc;
641 
642                 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
643                 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
644 
645                 *psgl = cpu_to_le32((u32)tmplow);
646                 *psgu |= cpu_to_le32((u32)tmphi);
647                 break;
648         case PPC440SPE_XOR_ID:
649                 xor_hw_desc = desc->hw_desc;
650                 xor_hw_desc->cbtal = addrl;
651                 xor_hw_desc->cbtah |= addrh;
652                 break;
653         }
654 }
655 
656 /**
657  * ppc440spe_desc_set_byte_count - set number of data bytes involved
658  * into the operation
659  */
660 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
661                                 struct ppc440spe_adma_chan *chan,
662                                 u32 byte_count)
663 {
664         struct dma_cdb *dma_hw_desc;
665         struct xor_cb *xor_hw_desc;
666 
667         switch (chan->device->id) {
668         case PPC440SPE_DMA0_ID:
669         case PPC440SPE_DMA1_ID:
670                 dma_hw_desc = desc->hw_desc;
671                 dma_hw_desc->cnt = cpu_to_le32(byte_count);
672                 break;
673         case PPC440SPE_XOR_ID:
674                 xor_hw_desc = desc->hw_desc;
675                 xor_hw_desc->cbbc = byte_count;
676                 break;
677         }
678 }
679 
680 /**
681  * ppc440spe_desc_set_rxor_block_size - set RXOR block size
682  */
683 static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
684 {
685         /* assume that byte_count is aligned on the 512-boundary;
686          * thus write it directly to the register (bits 23:31 are
687          * reserved there).
688          */
689         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
690 }
691 
692 /**
693  * ppc440spe_desc_set_dcheck - set CHECK pattern
694  */
695 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
696                                 struct ppc440spe_adma_chan *chan, u8 *qword)
697 {
698         struct dma_cdb *dma_hw_desc;
699 
700         switch (chan->device->id) {
701         case PPC440SPE_DMA0_ID:
702         case PPC440SPE_DMA1_ID:
703                 dma_hw_desc = desc->hw_desc;
704                 iowrite32(qword[0], &dma_hw_desc->sg3l);
705                 iowrite32(qword[4], &dma_hw_desc->sg3u);
706                 iowrite32(qword[8], &dma_hw_desc->sg2l);
707                 iowrite32(qword[12], &dma_hw_desc->sg2u);
708                 break;
709         default:
710                 BUG();
711         }
712 }
713 
714 /**
715  * ppc440spe_xor_set_link - set link address in xor CB
716  */
717 static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
718                                 struct ppc440spe_adma_desc_slot *next_desc)
719 {
720         struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
721 
722         if (unlikely(!next_desc || !(next_desc->phys))) {
723                 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
724                         __func__, next_desc,
725                         next_desc ? next_desc->phys : 0);
726                 BUG();
727         }
728 
729         xor_hw_desc->cbs = 0;
730         xor_hw_desc->cblal = next_desc->phys;
731         xor_hw_desc->cblah = 0;
732         xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
733 }
734 
735 /**
736  * ppc440spe_desc_set_link - set the address of descriptor following this
737  * descriptor in chain
738  */
739 static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
740                                 struct ppc440spe_adma_desc_slot *prev_desc,
741                                 struct ppc440spe_adma_desc_slot *next_desc)
742 {
743         unsigned long flags;
744         struct ppc440spe_adma_desc_slot *tail = next_desc;
745 
746         if (unlikely(!prev_desc || !next_desc ||
747                 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
748                 /* If previous next is overwritten something is wrong.
749                  * though we may refetch from append to initiate list
750                  * processing; in this case - it's ok.
751                  */
752                 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
753                         "prev->hw_next=0x%p\n", __func__, prev_desc,
754                         next_desc, prev_desc ? prev_desc->hw_next : 0);
755                 BUG();
756         }
757 
758         local_irq_save(flags);
759 
760         /* do s/w chaining both for DMA and XOR descriptors */
761         prev_desc->hw_next = next_desc;
762 
763         switch (chan->device->id) {
764         case PPC440SPE_DMA0_ID:
765         case PPC440SPE_DMA1_ID:
766                 break;
767         case PPC440SPE_XOR_ID:
768                 /* bind descriptor to the chain */
769                 while (tail->hw_next)
770                         tail = tail->hw_next;
771                 xor_last_linked = tail;
772 
773                 if (prev_desc == xor_last_submit)
774                         /* do not link to the last submitted CB */
775                         break;
776                 ppc440spe_xor_set_link(prev_desc, next_desc);
777                 break;
778         }
779 
780         local_irq_restore(flags);
781 }
782 
783 /**
784  * ppc440spe_desc_get_link - get the address of the descriptor that
785  * follows this one
786  */
787 static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
788                                         struct ppc440spe_adma_chan *chan)
789 {
790         if (!desc->hw_next)
791                 return 0;
792 
793         return desc->hw_next->phys;
794 }
795 
796 /**
797  * ppc440spe_desc_is_aligned - check alignment
798  */
799 static inline int ppc440spe_desc_is_aligned(
800         struct ppc440spe_adma_desc_slot *desc, int num_slots)
801 {
802         return (desc->idx & (num_slots - 1)) ? 0 : 1;
803 }
804 
805 /**
806  * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
807  * XOR operation
808  */
809 static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
810                         int *slots_per_op)
811 {
812         int slot_cnt;
813 
814         /* each XOR descriptor provides up to 16 source operands */
815         slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
816 
817         if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
818                 return slot_cnt;
819 
820         printk(KERN_ERR "%s: len %d > max %d !!\n",
821                 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
822         BUG();
823         return slot_cnt;
824 }
825 
826 /**
827  * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
828  * DMA2 PQ operation
829  */
830 static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
831                 int src_cnt, size_t len)
832 {
833         signed long long order = 0;
834         int state = 0;
835         int addr_count = 0;
836         int i;
837         for (i = 1; i < src_cnt; i++) {
838                 dma_addr_t cur_addr = srcs[i];
839                 dma_addr_t old_addr = srcs[i-1];
840                 switch (state) {
841                 case 0:
842                         if (cur_addr == old_addr + len) {
843                                 /* direct RXOR */
844                                 order = 1;
845                                 state = 1;
846                                 if (i == src_cnt-1)
847                                         addr_count++;
848                         } else if (old_addr == cur_addr + len) {
849                                 /* reverse RXOR */
850                                 order = -1;
851                                 state = 1;
852                                 if (i == src_cnt-1)
853                                         addr_count++;
854                         } else {
855                                 state = 3;
856                         }
857                         break;
858                 case 1:
859                         if (i == src_cnt-2 || (order == -1
860                                 && cur_addr != old_addr - len)) {
861                                 order = 0;
862                                 state = 0;
863                                 addr_count++;
864                         } else if (cur_addr == old_addr + len*order) {
865                                 state = 2;
866                                 if (i == src_cnt-1)
867                                         addr_count++;
868                         } else if (cur_addr == old_addr + 2*len) {
869                                 state = 2;
870                                 if (i == src_cnt-1)
871                                         addr_count++;
872                         } else if (cur_addr == old_addr + 3*len) {
873                                 state = 2;
874                                 if (i == src_cnt-1)
875                                         addr_count++;
876                         } else {
877                                 order = 0;
878                                 state = 0;
879                                 addr_count++;
880                         }
881                         break;
882                 case 2:
883                         order = 0;
884                         state = 0;
885                         addr_count++;
886                                 break;
887                 }
888                 if (state == 3)
889                         break;
890         }
891         if (src_cnt <= 1 || (state != 1 && state != 2)) {
892                 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
893                         __func__, src_cnt, state, addr_count, order);
894                 for (i = 0; i < src_cnt; i++)
895                         pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
896                 BUG();
897         }
898 
899         return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
900 }
901 
902 
903 /******************************************************************************
904  * ADMA channel low-level routines
905  ******************************************************************************/
906 
907 static u32
908 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
909 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
910 
911 /**
912  * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
913  */
914 static void ppc440spe_adma_device_clear_eot_status(
915                                         struct ppc440spe_adma_chan *chan)
916 {
917         struct dma_regs *dma_reg;
918         struct xor_regs *xor_reg;
919         u8 *p = chan->device->dma_desc_pool_virt;
920         struct dma_cdb *cdb;
921         u32 rv, i;
922 
923         switch (chan->device->id) {
924         case PPC440SPE_DMA0_ID:
925         case PPC440SPE_DMA1_ID:
926                 /* read FIFO to ack */
927                 dma_reg = chan->device->dma_reg;
928                 while ((rv = ioread32(&dma_reg->csfpl))) {
929                         i = rv & DMA_CDB_ADDR_MSK;
930                         cdb = (struct dma_cdb *)&p[i -
931                             (u32)chan->device->dma_desc_pool];
932 
933                         /* Clear opcode to ack. This is necessary for
934                          * ZeroSum operations only
935                          */
936                         cdb->opc = 0;
937 
938                         if (test_bit(PPC440SPE_RXOR_RUN,
939                             &ppc440spe_rxor_state)) {
940                                 /* probably this is a completed RXOR op,
941                                  * get pointer to CDB using the fact that
942                                  * physical and virtual addresses of CDB
943                                  * in pools have the same offsets
944                                  */
945                                 if (le32_to_cpu(cdb->sg1u) &
946                                     DMA_CUED_XOR_BASE) {
947                                         /* this is a RXOR */
948                                         clear_bit(PPC440SPE_RXOR_RUN,
949                                                   &ppc440spe_rxor_state);
950                                 }
951                         }
952 
953                         if (rv & DMA_CDB_STATUS_MSK) {
954                                 /* ZeroSum check failed
955                                  */
956                                 struct ppc440spe_adma_desc_slot *iter;
957                                 dma_addr_t phys = rv & ~DMA_CDB_MSK;
958 
959                                 /*
960                                  * Update the status of corresponding
961                                  * descriptor.
962                                  */
963                                 list_for_each_entry(iter, &chan->chain,
964                                     chain_node) {
965                                         if (iter->phys == phys)
966                                                 break;
967                                 }
968                                 /*
969                                  * if cannot find the corresponding
970                                  * slot it's a bug
971                                  */
972                                 BUG_ON(&iter->chain_node == &chan->chain);
973 
974                                 if (iter->xor_check_result) {
975                                         if (test_bit(PPC440SPE_DESC_PCHECK,
976                                                      &iter->flags)) {
977                                                 *iter->xor_check_result |=
978                                                         SUM_CHECK_P_RESULT;
979                                         } else
980                                         if (test_bit(PPC440SPE_DESC_QCHECK,
981                                                      &iter->flags)) {
982                                                 *iter->xor_check_result |=
983                                                         SUM_CHECK_Q_RESULT;
984                                         } else
985                                                 BUG();
986                                 }
987                         }
988                 }
989 
990                 rv = ioread32(&dma_reg->dsts);
991                 if (rv) {
992                         pr_err("DMA%d err status: 0x%x\n",
993                                chan->device->id, rv);
994                         /* write back to clear */
995                         iowrite32(rv, &dma_reg->dsts);
996                 }
997                 break;
998         case PPC440SPE_XOR_ID:
999                 /* reset status bits to ack */
1000                 xor_reg = chan->device->xor_reg;
1001                 rv = ioread32be(&xor_reg->sr);
1002                 iowrite32be(rv, &xor_reg->sr);
1003 
1004                 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
1005                         if (rv & XOR_IE_RPTIE_BIT) {
1006                                 /* Read PLB Timeout Error.
1007                                  * Try to resubmit the CB
1008                                  */
1009                                 u32 val = ioread32be(&xor_reg->ccbalr);
1010 
1011                                 iowrite32be(val, &xor_reg->cblalr);
1012 
1013                                 val = ioread32be(&xor_reg->crsr);
1014                                 iowrite32be(val | XOR_CRSR_XAE_BIT,
1015                                             &xor_reg->crsr);
1016                         } else
1017                                 pr_err("XOR ERR 0x%x status\n", rv);
1018                         break;
1019                 }
1020 
1021                 /*  if the XORcore is idle, but there are unprocessed CBs
1022                  * then refetch the s/w chain here
1023                  */
1024                 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
1025                     do_xor_refetch)
1026                         ppc440spe_chan_append(chan);
1027                 break;
1028         }
1029 }
1030 
1031 /**
1032  * ppc440spe_chan_is_busy - get the channel status
1033  */
1034 static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
1035 {
1036         struct dma_regs *dma_reg;
1037         struct xor_regs *xor_reg;
1038         int busy = 0;
1039 
1040         switch (chan->device->id) {
1041         case PPC440SPE_DMA0_ID:
1042         case PPC440SPE_DMA1_ID:
1043                 dma_reg = chan->device->dma_reg;
1044                 /*  if command FIFO's head and tail pointers are equal and
1045                  * status tail is the same as command, then channel is free
1046                  */
1047                 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
1048                     ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
1049                         busy = 1;
1050                 break;
1051         case PPC440SPE_XOR_ID:
1052                 /* use the special status bit for the XORcore
1053                  */
1054                 xor_reg = chan->device->xor_reg;
1055                 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
1056                 break;
1057         }
1058 
1059         return busy;
1060 }
1061 
1062 /**
1063  * ppc440spe_chan_set_first_xor_descriptor -  init XORcore chain
1064  */
1065 static void ppc440spe_chan_set_first_xor_descriptor(
1066                                 struct ppc440spe_adma_chan *chan,
1067                                 struct ppc440spe_adma_desc_slot *next_desc)
1068 {
1069         struct xor_regs *xor_reg = chan->device->xor_reg;
1070 
1071         if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
1072                 printk(KERN_INFO "%s: Warn: XORcore is running "
1073                         "when try to set the first CDB!\n",
1074                         __func__);
1075 
1076         xor_last_submit = xor_last_linked = next_desc;
1077 
1078         iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
1079 
1080         iowrite32be(next_desc->phys, &xor_reg->cblalr);
1081         iowrite32be(0, &xor_reg->cblahr);
1082         iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
1083                     &xor_reg->cbcr);
1084 
1085         chan->hw_chain_inited = 1;
1086 }
1087 
1088 /**
1089  * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1090  * called with irqs disabled
1091  */
1092 static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
1093                 struct ppc440spe_adma_desc_slot *desc)
1094 {
1095         u32 pcdb;
1096         struct dma_regs *dma_reg = chan->device->dma_reg;
1097 
1098         pcdb = desc->phys;
1099         if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
1100                 pcdb |= DMA_CDB_NO_INT;
1101 
1102         chan_last_sub[chan->device->id] = desc;
1103 
1104         ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
1105 
1106         iowrite32(pcdb, &dma_reg->cpfpl);
1107 }
1108 
1109 /**
1110  * ppc440spe_chan_append - update the h/w chain in the channel
1111  */
1112 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
1113 {
1114         struct xor_regs *xor_reg;
1115         struct ppc440spe_adma_desc_slot *iter;
1116         struct xor_cb *xcb;
1117         u32 cur_desc;
1118         unsigned long flags;
1119 
1120         local_irq_save(flags);
1121 
1122         switch (chan->device->id) {
1123         case PPC440SPE_DMA0_ID:
1124         case PPC440SPE_DMA1_ID:
1125                 cur_desc = ppc440spe_chan_get_current_descriptor(chan);
1126 
1127                 if (likely(cur_desc)) {
1128                         iter = chan_last_sub[chan->device->id];
1129                         BUG_ON(!iter);
1130                 } else {
1131                         /* first peer */
1132                         iter = chan_first_cdb[chan->device->id];
1133                         BUG_ON(!iter);
1134                         ppc440spe_dma_put_desc(chan, iter);
1135                         chan->hw_chain_inited = 1;
1136                 }
1137 
1138                 /* is there something new to append */
1139                 if (!iter->hw_next)
1140                         break;
1141 
1142                 /* flush descriptors from the s/w queue to fifo */
1143                 list_for_each_entry_continue(iter, &chan->chain, chain_node) {
1144                         ppc440spe_dma_put_desc(chan, iter);
1145                         if (!iter->hw_next)
1146                                 break;
1147                 }
1148                 break;
1149         case PPC440SPE_XOR_ID:
1150                 /* update h/w links and refetch */
1151                 if (!xor_last_submit->hw_next)
1152                         break;
1153 
1154                 xor_reg = chan->device->xor_reg;
1155                 /* the last linked CDB has to generate an interrupt
1156                  * that we'd be able to append the next lists to h/w
1157                  * regardless of the XOR engine state at the moment of
1158                  * appending of these next lists
1159                  */
1160                 xcb = xor_last_linked->hw_desc;
1161                 xcb->cbc |= XOR_CBCR_CBCE_BIT;
1162 
1163                 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
1164                         /* XORcore is idle. Refetch now */
1165                         do_xor_refetch = 0;
1166                         ppc440spe_xor_set_link(xor_last_submit,
1167                                 xor_last_submit->hw_next);
1168 
1169                         ADMA_LL_DBG(print_cb_list(chan,
1170                                 xor_last_submit->hw_next));
1171 
1172                         xor_last_submit = xor_last_linked;
1173                         iowrite32be(ioread32be(&xor_reg->crsr) |
1174                                     XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
1175                                     &xor_reg->crsr);
1176                 } else {
1177                         /* XORcore is running. Refetch later in the handler */
1178                         do_xor_refetch = 1;
1179                 }
1180 
1181                 break;
1182         }
1183 
1184         local_irq_restore(flags);
1185 }
1186 
1187 /**
1188  * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1189  */
1190 static u32
1191 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
1192 {
1193         struct dma_regs *dma_reg;
1194         struct xor_regs *xor_reg;
1195 
1196         if (unlikely(!chan->hw_chain_inited))
1197                 /* h/w descriptor chain is not initialized yet */
1198                 return 0;
1199 
1200         switch (chan->device->id) {
1201         case PPC440SPE_DMA0_ID:
1202         case PPC440SPE_DMA1_ID:
1203                 dma_reg = chan->device->dma_reg;
1204                 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
1205         case PPC440SPE_XOR_ID:
1206                 xor_reg = chan->device->xor_reg;
1207                 return ioread32be(&xor_reg->ccbalr);
1208         }
1209         return 0;
1210 }
1211 
1212 /**
1213  * ppc440spe_chan_run - enable the channel
1214  */
1215 static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
1216 {
1217         struct xor_regs *xor_reg;
1218 
1219         switch (chan->device->id) {
1220         case PPC440SPE_DMA0_ID:
1221         case PPC440SPE_DMA1_ID:
1222                 /* DMAs are always enabled, do nothing */
1223                 break;
1224         case PPC440SPE_XOR_ID:
1225                 /* drain write buffer */
1226                 xor_reg = chan->device->xor_reg;
1227 
1228                 /* fetch descriptor pointed to in <link> */
1229                 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
1230                             &xor_reg->crsr);
1231                 break;
1232         }
1233 }
1234 
1235 /******************************************************************************
1236  * ADMA device level
1237  ******************************************************************************/
1238 
1239 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
1240 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
1241 
1242 static dma_cookie_t
1243 ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
1244 
1245 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
1246                                     dma_addr_t addr, int index);
1247 static void
1248 ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
1249                                   dma_addr_t addr, int index);
1250 
1251 static void
1252 ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
1253                            dma_addr_t *paddr, unsigned long flags);
1254 static void
1255 ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
1256                           dma_addr_t addr, int index);
1257 static void
1258 ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
1259                                unsigned char mult, int index, int dst_pos);
1260 static void
1261 ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
1262                                    dma_addr_t paddr, dma_addr_t qaddr);
1263 
1264 static struct page *ppc440spe_rxor_srcs[32];
1265 
1266 /**
1267  * ppc440spe_can_rxor - check if the operands may be processed with RXOR
1268  */
1269 static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
1270 {
1271         int i, order = 0, state = 0;
1272         int idx = 0;
1273 
1274         if (unlikely(!(src_cnt > 1)))
1275                 return 0;
1276 
1277         BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
1278 
1279         /* Skip holes in the source list before checking */
1280         for (i = 0; i < src_cnt; i++) {
1281                 if (!srcs[i])
1282                         continue;
1283                 ppc440spe_rxor_srcs[idx++] = srcs[i];
1284         }
1285         src_cnt = idx;
1286 
1287         for (i = 1; i < src_cnt; i++) {
1288                 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
1289                 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
1290 
1291                 switch (state) {
1292                 case 0:
1293                         if (cur_addr == old_addr + len) {
1294                                 /* direct RXOR */
1295                                 order = 1;
1296                                 state = 1;
1297                         } else if (old_addr == cur_addr + len) {
1298                                 /* reverse RXOR */
1299                                 order = -1;
1300                                 state = 1;
1301                         } else
1302                                 goto out;
1303                         break;
1304                 case 1:
1305                         if ((i == src_cnt - 2) ||
1306                             (order == -1 && cur_addr != old_addr - len)) {
1307                                 order = 0;
1308                                 state = 0;
1309                         } else if ((cur_addr == old_addr + len * order) ||
1310                                    (cur_addr == old_addr + 2 * len) ||
1311                                    (cur_addr == old_addr + 3 * len)) {
1312                                 state = 2;
1313                         } else {
1314                                 order = 0;
1315                                 state = 0;
1316                         }
1317                         break;
1318                 case 2:
1319                         order = 0;
1320                         state = 0;
1321                         break;
1322                 }
1323         }
1324 
1325 out:
1326         if (state == 1 || state == 2)
1327                 return 1;
1328 
1329         return 0;
1330 }
1331 
1332 /**
1333  * ppc440spe_adma_device_estimate - estimate the efficiency of processing
1334  *      the operation given on this channel. It's assumed that 'chan' is
1335  *      capable to process 'cap' type of operation.
1336  * @chan: channel to use
1337  * @cap: type of transaction
1338  * @dst_lst: array of destination pointers
1339  * @dst_cnt: number of destination operands
1340  * @src_lst: array of source pointers
1341  * @src_cnt: number of source operands
1342  * @src_sz: size of each source operand
1343  */
1344 static int ppc440spe_adma_estimate(struct dma_chan *chan,
1345         enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
1346         struct page **src_lst, int src_cnt, size_t src_sz)
1347 {
1348         int ef = 1;
1349 
1350         if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
1351                 /* If RAID-6 capabilities were not activated don't try
1352                  * to use them
1353                  */
1354                 if (unlikely(!ppc440spe_r6_enabled))
1355                         return -1;
1356         }
1357         /*  In the current implementation of ppc440spe ADMA driver it
1358          * makes sense to pick out only pq case, because it may be
1359          * processed:
1360          * (1) either using Biskup method on DMA2;
1361          * (2) or on DMA0/1.
1362          *  Thus we give a favour to (1) if the sources are suitable;
1363          * else let it be processed on one of the DMA0/1 engines.
1364          *  In the sum_product case where destination is also the
1365          * source process it on DMA0/1 only.
1366          */
1367         if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
1368 
1369                 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
1370                         ef = 0; /* sum_product case, process on DMA0/1 */
1371                 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz))
1372                         ef = 3; /* override (DMA0/1 + idle) */
1373                 else
1374                         ef = 0; /* can't process on DMA2 if !rxor */
1375         }
1376 
1377         /* channel idleness increases the priority */
1378         if (likely(ef) &&
1379             !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
1380                 ef++;
1381 
1382         return ef;
1383 }
1384 
1385 struct dma_chan *
1386 ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
1387         struct page **dst_lst, int dst_cnt, struct page **src_lst,
1388         int src_cnt, size_t src_sz)
1389 {
1390         struct dma_chan *best_chan = NULL;
1391         struct ppc_dma_chan_ref *ref;
1392         int best_rank = -1;
1393 
1394         if (unlikely(!src_sz))
1395                 return NULL;
1396         if (src_sz > PAGE_SIZE) {
1397                 /*
1398                  * should a user of the api ever pass > PAGE_SIZE requests
1399                  * we sort out cases where temporary page-sized buffers
1400                  * are used.
1401                  */
1402                 switch (cap) {
1403                 case DMA_PQ:
1404                         if (src_cnt == 1 && dst_lst[1] == src_lst[0])
1405                                 return NULL;
1406                         if (src_cnt == 2 && dst_lst[1] == src_lst[1])
1407                                 return NULL;
1408                         break;
1409                 case DMA_PQ_VAL:
1410                 case DMA_XOR_VAL:
1411                         return NULL;
1412                 default:
1413                         break;
1414                 }
1415         }
1416 
1417         list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
1418                 if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
1419                         int rank;
1420 
1421                         rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst,
1422                                         dst_cnt, src_lst, src_cnt, src_sz);
1423                         if (rank > best_rank) {
1424                                 best_rank = rank;
1425                                 best_chan = ref->chan;
1426                         }
1427                 }
1428         }
1429 
1430         return best_chan;
1431 }
1432 EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
1433 
1434 /**
1435  * ppc440spe_get_group_entry - get group entry with index idx
1436  * @tdesc: is the last allocated slot in the group.
1437  */
1438 static struct ppc440spe_adma_desc_slot *
1439 ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
1440 {
1441         struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
1442         int i = 0;
1443 
1444         if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
1445                 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1446                         __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
1447                 BUG();
1448         }
1449 
1450         list_for_each_entry(iter, &tdesc->group_list, chain_node) {
1451                 if (i++ == entry_idx)
1452                         break;
1453         }
1454         return iter;
1455 }
1456 
1457 /**
1458  * ppc440spe_adma_free_slots - flags descriptor slots for reuse
1459  * @slot: Slot to free
1460  * Caller must hold &ppc440spe_chan->lock while calling this function
1461  */
1462 static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1463                                       struct ppc440spe_adma_chan *chan)
1464 {
1465         int stride = slot->slots_per_op;
1466 
1467         while (stride--) {
1468                 slot->slots_per_op = 0;
1469                 slot = list_entry(slot->slot_node.next,
1470                                 struct ppc440spe_adma_desc_slot,
1471                                 slot_node);
1472         }
1473 }
1474 
1475 /**
1476  * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1477  * upon completion
1478  */
1479 static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1480                 struct ppc440spe_adma_desc_slot *desc,
1481                 struct ppc440spe_adma_chan *chan,
1482                 dma_cookie_t cookie)
1483 {
1484         BUG_ON(desc->async_tx.cookie < 0);
1485         if (desc->async_tx.cookie > 0) {
1486                 cookie = desc->async_tx.cookie;
1487                 desc->async_tx.cookie = 0;
1488 
1489                 /* call the callback (must not sleep or submit new
1490                  * operations to this channel)
1491                  */
1492                 if (desc->async_tx.callback)
1493                         desc->async_tx.callback(
1494                                 desc->async_tx.callback_param);
1495 
1496                 dma_descriptor_unmap(&desc->async_tx);
1497         }
1498 
1499         /* run dependent operations */
1500         dma_run_dependencies(&desc->async_tx);
1501 
1502         return cookie;
1503 }
1504 
1505 /**
1506  * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1507  */
1508 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
1509                 struct ppc440spe_adma_chan *chan)
1510 {
1511         /* the client is allowed to attach dependent operations
1512          * until 'ack' is set
1513          */
1514         if (!async_tx_test_ack(&desc->async_tx))
1515                 return 0;
1516 
1517         /* leave the last descriptor in the chain
1518          * so we can append to it
1519          */
1520         if (list_is_last(&desc->chain_node, &chan->chain) ||
1521             desc->phys == ppc440spe_chan_get_current_descriptor(chan))
1522                 return 1;
1523 
1524         if (chan->device->id != PPC440SPE_XOR_ID) {
1525                 /* our DMA interrupt handler clears opc field of
1526                  * each processed descriptor. For all types of
1527                  * operations except for ZeroSum we do not actually
1528                  * need ack from the interrupt handler. ZeroSum is a
1529                  * special case since the result of this operation
1530                  * is available from the handler only, so if we see
1531                  * such type of descriptor (which is unprocessed yet)
1532                  * then leave it in chain.
1533                  */
1534                 struct dma_cdb *cdb = desc->hw_desc;
1535                 if (cdb->opc == DMA_CDB_OPC_DCHECK128)
1536                         return 1;
1537         }
1538 
1539         dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
1540                 desc->phys, desc->idx, desc->slots_per_op);
1541 
1542         list_del(&desc->chain_node);
1543         ppc440spe_adma_free_slots(desc, chan);
1544         return 0;
1545 }
1546 
1547 /**
1548  * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1549  *      which runs through the channel CDBs list until reach the descriptor
1550  *      currently processed. When routine determines that all CDBs of group
1551  *      are completed then corresponding callbacks (if any) are called and slots
1552  *      are freed.
1553  */
1554 static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1555 {
1556         struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
1557         dma_cookie_t cookie = 0;
1558         u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
1559         int busy = ppc440spe_chan_is_busy(chan);
1560         int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
1561 
1562         dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
1563                 chan->device->id, __func__);
1564 
1565         if (!current_desc) {
1566                 /*  There were no transactions yet, so
1567                  * nothing to clean
1568                  */
1569                 return;
1570         }
1571 
1572         /* free completed slots from the chain starting with
1573          * the oldest descriptor
1574          */
1575         list_for_each_entry_safe(iter, _iter, &chan->chain,
1576                                         chain_node) {
1577                 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
1578                     "busy: %d this_desc: %#llx next_desc: %#x "
1579                     "cur: %#x ack: %d\n",
1580                     iter->async_tx.cookie, iter->idx, busy, iter->phys,
1581                     ppc440spe_desc_get_link(iter, chan), current_desc,
1582                     async_tx_test_ack(&iter->async_tx));
1583                 prefetch(_iter);
1584                 prefetch(&_iter->async_tx);
1585 
1586                 /* do not advance past the current descriptor loaded into the
1587                  * hardware channel,subsequent descriptors are either in process
1588                  * or have not been submitted
1589                  */
1590                 if (seen_current)
1591                         break;
1592 
1593                 /* stop the search if we reach the current descriptor and the
1594                  * channel is busy, or if it appears that the current descriptor
1595                  * needs to be re-read (i.e. has been appended to)
1596                  */
1597                 if (iter->phys == current_desc) {
1598                         BUG_ON(seen_current++);
1599                         if (busy || ppc440spe_desc_get_link(iter, chan)) {
1600                                 /* not all descriptors of the group have
1601                                  * been completed; exit.
1602                                  */
1603                                 break;
1604                         }
1605                 }
1606 
1607                 /* detect the start of a group transaction */
1608                 if (!slot_cnt && !slots_per_op) {
1609                         slot_cnt = iter->slot_cnt;
1610                         slots_per_op = iter->slots_per_op;
1611                         if (slot_cnt <= slots_per_op) {
1612                                 slot_cnt = 0;
1613                                 slots_per_op = 0;
1614                         }
1615                 }
1616 
1617                 if (slot_cnt) {
1618                         if (!group_start)
1619                                 group_start = iter;
1620                         slot_cnt -= slots_per_op;
1621                 }
1622 
1623                 /* all the members of a group are complete */
1624                 if (slots_per_op != 0 && slot_cnt == 0) {
1625                         struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
1626                         int end_of_chain = 0;
1627 
1628                         /* clean up the group */
1629                         slot_cnt = group_start->slot_cnt;
1630                         grp_iter = group_start;
1631                         list_for_each_entry_safe_from(grp_iter, _grp_iter,
1632                                 &chan->chain, chain_node) {
1633 
1634                                 cookie = ppc440spe_adma_run_tx_complete_actions(
1635                                         grp_iter, chan, cookie);
1636 
1637                                 slot_cnt -= slots_per_op;
1638                                 end_of_chain = ppc440spe_adma_clean_slot(
1639                                     grp_iter, chan);
1640                                 if (end_of_chain && slot_cnt) {
1641                                         /* Should wait for ZeroSum completion */
1642                                         if (cookie > 0)
1643                                                 chan->common.completed_cookie = cookie;
1644                                         return;
1645                                 }
1646 
1647                                 if (slot_cnt == 0 || end_of_chain)
1648                                         break;
1649                         }
1650 
1651                         /* the group should be complete at this point */
1652                         BUG_ON(slot_cnt);
1653 
1654                         slots_per_op = 0;
1655                         group_start = NULL;
1656                         if (end_of_chain)
1657                                 break;
1658                         else
1659                                 continue;
1660                 } else if (slots_per_op) /* wait for group completion */
1661                         continue;
1662 
1663                 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan,
1664                     cookie);
1665 
1666                 if (ppc440spe_adma_clean_slot(iter, chan))
1667                         break;
1668         }
1669 
1670         BUG_ON(!seen_current);
1671 
1672         if (cookie > 0) {
1673                 chan->common.completed_cookie = cookie;
1674                 pr_debug("\tcompleted cookie %d\n", cookie);
1675         }
1676 
1677 }
1678 
1679 /**
1680  * ppc440spe_adma_tasklet - clean up watch-dog initiator
1681  */
1682 static void ppc440spe_adma_tasklet(unsigned long data)
1683 {
1684         struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data;
1685 
1686         spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
1687         __ppc440spe_adma_slot_cleanup(chan);
1688         spin_unlock(&chan->lock);
1689 }
1690 
1691 /**
1692  * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1693  */
1694 static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1695 {
1696         spin_lock_bh(&chan->lock);
1697         __ppc440spe_adma_slot_cleanup(chan);
1698         spin_unlock_bh(&chan->lock);
1699 }
1700 
1701 /**
1702  * ppc440spe_adma_alloc_slots - allocate free slots (if any)
1703  */
1704 static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
1705                 struct ppc440spe_adma_chan *chan, int num_slots,
1706                 int slots_per_op)
1707 {
1708         struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
1709         struct ppc440spe_adma_desc_slot *alloc_start = NULL;
1710         struct list_head chain = LIST_HEAD_INIT(chain);
1711         int slots_found, retry = 0;
1712 
1713 
1714         BUG_ON(!num_slots || !slots_per_op);
1715         /* start search from the last allocated descrtiptor
1716          * if a contiguous allocation can not be found start searching
1717          * from the beginning of the list
1718          */
1719 retry:
1720         slots_found = 0;
1721         if (retry == 0)
1722                 iter = chan->last_used;
1723         else
1724                 iter = list_entry(&chan->all_slots,
1725                                   struct ppc440spe_adma_desc_slot,
1726                                   slot_node);
1727         list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
1728             slot_node) {
1729                 prefetch(_iter);
1730                 prefetch(&_iter->async_tx);
1731                 if (iter->slots_per_op) {
1732                         slots_found = 0;
1733                         continue;
1734                 }
1735 
1736                 /* start the allocation if the slot is correctly aligned */
1737                 if (!slots_found++)
1738                         alloc_start = iter;
1739 
1740                 if (slots_found == num_slots) {
1741                         struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
1742                         struct ppc440spe_adma_desc_slot *last_used = NULL;
1743 
1744                         iter = alloc_start;
1745                         while (num_slots) {
1746                                 int i;
1747                                 /* pre-ack all but the last descriptor */
1748                                 if (num_slots != slots_per_op)
1749                                         async_tx_ack(&iter->async_tx);
1750 
1751                                 list_add_tail(&iter->chain_node, &chain);
1752                                 alloc_tail = iter;
1753                                 iter->async_tx.cookie = 0;
1754                                 iter->hw_next = NULL;
1755                                 iter->flags = 0;
1756                                 iter->slot_cnt = num_slots;
1757                                 iter->xor_check_result = NULL;
1758                                 for (i = 0; i < slots_per_op; i++) {
1759                                         iter->slots_per_op = slots_per_op - i;
1760                                         last_used = iter;
1761                                         iter = list_entry(iter->slot_node.next,
1762                                                 struct ppc440spe_adma_desc_slot,
1763                                                 slot_node);
1764                                 }
1765                                 num_slots -= slots_per_op;
1766                         }
1767                         alloc_tail->group_head = alloc_start;
1768                         alloc_tail->async_tx.cookie = -EBUSY;
1769                         list_splice(&chain, &alloc_tail->group_list);
1770                         chan->last_used = last_used;
1771                         return alloc_tail;
1772                 }
1773         }
1774         if (!retry++)
1775                 goto retry;
1776 
1777         /* try to free some slots if the allocation fails */
1778         tasklet_schedule(&chan->irq_tasklet);
1779         return NULL;
1780 }
1781 
1782 /**
1783  * ppc440spe_adma_alloc_chan_resources -  allocate pools for CDB slots
1784  */
1785 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
1786 {
1787         struct ppc440spe_adma_chan *ppc440spe_chan;
1788         struct ppc440spe_adma_desc_slot *slot = NULL;
1789         char *hw_desc;
1790         int i, db_sz;
1791         int init;
1792 
1793         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1794         init = ppc440spe_chan->slots_allocated ? 0 : 1;
1795         chan->chan_id = ppc440spe_chan->device->id;
1796 
1797         /* Allocate descriptor slots */
1798         i = ppc440spe_chan->slots_allocated;
1799         if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
1800                 db_sz = sizeof(struct dma_cdb);
1801         else
1802                 db_sz = sizeof(struct xor_cb);
1803 
1804         for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
1805                 slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot),
1806                                GFP_KERNEL);
1807                 if (!slot) {
1808                         printk(KERN_INFO "SPE ADMA Channel only initialized"
1809                                 " %d descriptor slots", i--);
1810                         break;
1811                 }
1812 
1813                 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
1814                 slot->hw_desc = (void *) &hw_desc[i * db_sz];
1815                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
1816                 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
1817                 INIT_LIST_HEAD(&slot->chain_node);
1818                 INIT_LIST_HEAD(&slot->slot_node);
1819                 INIT_LIST_HEAD(&slot->group_list);
1820                 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
1821                 slot->idx = i;
1822 
1823                 spin_lock_bh(&ppc440spe_chan->lock);
1824                 ppc440spe_chan->slots_allocated++;
1825                 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots);
1826                 spin_unlock_bh(&ppc440spe_chan->lock);
1827         }
1828 
1829         if (i && !ppc440spe_chan->last_used) {
1830                 ppc440spe_chan->last_used =
1831                         list_entry(ppc440spe_chan->all_slots.next,
1832                                 struct ppc440spe_adma_desc_slot,
1833                                 slot_node);
1834         }
1835 
1836         dev_dbg(ppc440spe_chan->device->common.dev,
1837                 "ppc440spe adma%d: allocated %d descriptor slots\n",
1838                 ppc440spe_chan->device->id, i);
1839 
1840         /* initialize the channel and the chain with a null operation */
1841         if (init) {
1842                 switch (ppc440spe_chan->device->id) {
1843                 case PPC440SPE_DMA0_ID:
1844                 case PPC440SPE_DMA1_ID:
1845                         ppc440spe_chan->hw_chain_inited = 0;
1846                         /* Use WXOR for self-testing */
1847                         if (!ppc440spe_r6_tchan)
1848                                 ppc440spe_r6_tchan = ppc440spe_chan;
1849                         break;
1850                 case PPC440SPE_XOR_ID:
1851                         ppc440spe_chan_start_null_xor(ppc440spe_chan);
1852                         break;
1853                 default:
1854                         BUG();
1855                 }
1856                 ppc440spe_chan->needs_unmap = 1;
1857         }
1858 
1859         return (i > 0) ? i : -ENOMEM;
1860 }
1861 
1862 /**
1863  * ppc440spe_rxor_set_region_data -
1864  */
1865 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
1866         u8 xor_arg_no, u32 mask)
1867 {
1868         struct xor_cb *xcb = desc->hw_desc;
1869 
1870         xcb->ops[xor_arg_no].h |= mask;
1871 }
1872 
1873 /**
1874  * ppc440spe_rxor_set_src -
1875  */
1876 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
1877         u8 xor_arg_no, dma_addr_t addr)
1878 {
1879         struct xor_cb *xcb = desc->hw_desc;
1880 
1881         xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
1882         xcb->ops[xor_arg_no].l = addr;
1883 }
1884 
1885 /**
1886  * ppc440spe_rxor_set_mult -
1887  */
1888 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
1889         u8 xor_arg_no, u8 idx, u8 mult)
1890 {
1891         struct xor_cb *xcb = desc->hw_desc;
1892 
1893         xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
1894 }
1895 
1896 /**
1897  * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
1898  *      has been achieved
1899  */
1900 static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
1901 {
1902         dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
1903                 chan->device->id, chan->pending);
1904 
1905         if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
1906                 chan->pending = 0;
1907                 ppc440spe_chan_append(chan);
1908         }
1909 }
1910 
1911 /**
1912  * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
1913  *      (it's not necessary that descriptors will be submitted to the h/w
1914  *      chains too right now)
1915  */
1916 static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
1917 {
1918         struct ppc440spe_adma_desc_slot *sw_desc;
1919         struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
1920         struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
1921         int slot_cnt;
1922         int slots_per_op;
1923         dma_cookie_t cookie;
1924 
1925         sw_desc = tx_to_ppc440spe_adma_slot(tx);
1926 
1927         group_start = sw_desc->group_head;
1928         slot_cnt = group_start->slot_cnt;
1929         slots_per_op = group_start->slots_per_op;
1930 
1931         spin_lock_bh(&chan->lock);
1932         cookie = dma_cookie_assign(tx);
1933 
1934         if (unlikely(list_empty(&chan->chain))) {
1935                 /* first peer */
1936                 list_splice_init(&sw_desc->group_list, &chan->chain);
1937                 chan_first_cdb[chan->device->id] = group_start;
1938         } else {
1939                 /* isn't first peer, bind CDBs to chain */
1940                 old_chain_tail = list_entry(chan->chain.prev,
1941                                         struct ppc440spe_adma_desc_slot,
1942                                         chain_node);
1943                 list_splice_init(&sw_desc->group_list,
1944                     &old_chain_tail->chain_node);
1945                 /* fix up the hardware chain */
1946                 ppc440spe_desc_set_link(chan, old_chain_tail, group_start);
1947         }
1948 
1949         /* increment the pending count by the number of operations */
1950         chan->pending += slot_cnt / slots_per_op;
1951         ppc440spe_adma_check_threshold(chan);
1952         spin_unlock_bh(&chan->lock);
1953 
1954         dev_dbg(chan->device->common.dev,
1955                 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
1956                 chan->device->id, __func__,
1957                 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
1958 
1959         return cookie;
1960 }
1961 
1962 /**
1963  * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
1964  */
1965 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
1966                 struct dma_chan *chan, unsigned long flags)
1967 {
1968         struct ppc440spe_adma_chan *ppc440spe_chan;
1969         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1970         int slot_cnt, slots_per_op;
1971 
1972         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1973 
1974         dev_dbg(ppc440spe_chan->device->common.dev,
1975                 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
1976                 __func__);
1977 
1978         spin_lock_bh(&ppc440spe_chan->lock);
1979         slot_cnt = slots_per_op = 1;
1980         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
1981                         slots_per_op);
1982         if (sw_desc) {
1983                 group_start = sw_desc->group_head;
1984                 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan);
1985                 group_start->unmap_len = 0;
1986                 sw_desc->async_tx.flags = flags;
1987         }
1988         spin_unlock_bh(&ppc440spe_chan->lock);
1989 
1990         return sw_desc ? &sw_desc->async_tx : NULL;
1991 }
1992 
1993 /**
1994  * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
1995  */
1996 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
1997                 struct dma_chan *chan, dma_addr_t dma_dest,
1998                 dma_addr_t dma_src, size_t len, unsigned long flags)
1999 {
2000         struct ppc440spe_adma_chan *ppc440spe_chan;
2001         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2002         int slot_cnt, slots_per_op;
2003 
2004         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2005 
2006         if (unlikely(!len))
2007                 return NULL;
2008 
2009         BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT);
2010 
2011         spin_lock_bh(&ppc440spe_chan->lock);
2012 
2013         dev_dbg(ppc440spe_chan->device->common.dev,
2014                 "ppc440spe adma%d: %s len: %u int_en %d\n",
2015                 ppc440spe_chan->device->id, __func__, len,
2016                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2017         slot_cnt = slots_per_op = 1;
2018         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2019                 slots_per_op);
2020         if (sw_desc) {
2021                 group_start = sw_desc->group_head;
2022                 ppc440spe_desc_init_memcpy(group_start, flags);
2023                 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2024                 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0);
2025                 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2026                 sw_desc->unmap_len = len;
2027                 sw_desc->async_tx.flags = flags;
2028         }
2029         spin_unlock_bh(&ppc440spe_chan->lock);
2030 
2031         return sw_desc ? &sw_desc->async_tx : NULL;
2032 }
2033 
2034 /**
2035  * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2036  */
2037 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
2038                 struct dma_chan *chan, dma_addr_t dma_dest,
2039                 dma_addr_t *dma_src, u32 src_cnt, size_t len,
2040                 unsigned long flags)
2041 {
2042         struct ppc440spe_adma_chan *ppc440spe_chan;
2043         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2044         int slot_cnt, slots_per_op;
2045 
2046         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2047 
2048         ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
2049                                      dma_dest, dma_src, src_cnt));
2050         if (unlikely(!len))
2051                 return NULL;
2052         BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2053 
2054         dev_dbg(ppc440spe_chan->device->common.dev,
2055                 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2056                 ppc440spe_chan->device->id, __func__, src_cnt, len,
2057                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2058 
2059         spin_lock_bh(&ppc440spe_chan->lock);
2060         slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op);
2061         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2062                         slots_per_op);
2063         if (sw_desc) {
2064                 group_start = sw_desc->group_head;
2065                 ppc440spe_desc_init_xor(group_start, src_cnt, flags);
2066                 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2067                 while (src_cnt--)
2068                         ppc440spe_adma_memcpy_xor_set_src(group_start,
2069                                 dma_src[src_cnt], src_cnt);
2070                 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2071                 sw_desc->unmap_len = len;
2072                 sw_desc->async_tx.flags = flags;
2073         }
2074         spin_unlock_bh(&ppc440spe_chan->lock);
2075 
2076         return sw_desc ? &sw_desc->async_tx : NULL;
2077 }
2078 
2079 static inline void
2080 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2081                                 int src_cnt);
2082 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
2083 
2084 /**
2085  * ppc440spe_adma_init_dma2rxor_slot -
2086  */
2087 static void ppc440spe_adma_init_dma2rxor_slot(
2088                 struct ppc440spe_adma_desc_slot *desc,
2089                 dma_addr_t *src, int src_cnt)
2090 {
2091         int i;
2092 
2093         /* initialize CDB */
2094         for (i = 0; i < src_cnt; i++) {
2095                 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,
2096                                                  desc->src_cnt, (u32)src[i]);
2097         }
2098 }
2099 
2100 /**
2101  * ppc440spe_dma01_prep_mult -
2102  * for Q operation where destination is also the source
2103  */
2104 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
2105                 struct ppc440spe_adma_chan *ppc440spe_chan,
2106                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2107                 const unsigned char *scf, size_t len, unsigned long flags)
2108 {
2109         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2110         unsigned long op = 0;
2111         int slot_cnt;
2112 
2113         set_bit(PPC440SPE_DESC_WXOR, &op);
2114         slot_cnt = 2;
2115 
2116         spin_lock_bh(&ppc440spe_chan->lock);
2117 
2118         /* use WXOR, each descriptor occupies one slot */
2119         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2120         if (sw_desc) {
2121                 struct ppc440spe_adma_chan *chan;
2122                 struct ppc440spe_adma_desc_slot *iter;
2123                 struct dma_cdb *hw_desc;
2124 
2125                 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2126                 set_bits(op, &sw_desc->flags);
2127                 sw_desc->src_cnt = src_cnt;
2128                 sw_desc->dst_cnt = dst_cnt;
2129                 /* First descriptor, zero data in the destination and copy it
2130                  * to q page using MULTICAST transfer.
2131                  */
2132                 iter = list_first_entry(&sw_desc->group_list,
2133                                         struct ppc440spe_adma_desc_slot,
2134                                         chain_node);
2135                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2136                 /* set 'next' pointer */
2137                 iter->hw_next = list_entry(iter->chain_node.next,
2138                                            struct ppc440spe_adma_desc_slot,
2139                                            chain_node);
2140                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2141                 hw_desc = iter->hw_desc;
2142                 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2143 
2144                 ppc440spe_desc_set_dest_addr(iter, chan,
2145                                              DMA_CUED_XOR_BASE, dst[0], 0);
2146                 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1);
2147                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2148                                             src[0]);
2149                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2150                 iter->unmap_len = len;
2151 
2152                 /*
2153                  * Second descriptor, multiply data from the q page
2154                  * and store the result in real destination.
2155                  */
2156                 iter = list_first_entry(&iter->chain_node,
2157                                         struct ppc440spe_adma_desc_slot,
2158                                         chain_node);
2159                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2160                 iter->hw_next = NULL;
2161                 if (flags & DMA_PREP_INTERRUPT)
2162                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2163                 else
2164                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2165 
2166                 hw_desc = iter->hw_desc;
2167                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2168                 ppc440spe_desc_set_src_addr(iter, chan, 0,
2169                                             DMA_CUED_XOR_HB, dst[1]);
2170                 ppc440spe_desc_set_dest_addr(iter, chan,
2171                                              DMA_CUED_XOR_BASE, dst[0], 0);
2172 
2173                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2174                                             DMA_CDB_SG_DST1, scf[0]);
2175                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2176                 iter->unmap_len = len;
2177                 sw_desc->async_tx.flags = flags;
2178         }
2179 
2180         spin_unlock_bh(&ppc440spe_chan->lock);
2181 
2182         return sw_desc;
2183 }
2184 
2185 /**
2186  * ppc440spe_dma01_prep_sum_product -
2187  * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2188  * the source.
2189  */
2190 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
2191                 struct ppc440spe_adma_chan *ppc440spe_chan,
2192                 dma_addr_t *dst, dma_addr_t *src, int src_cnt,
2193                 const unsigned char *scf, size_t len, unsigned long flags)
2194 {
2195         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2196         unsigned long op = 0;
2197         int slot_cnt;
2198 
2199         set_bit(PPC440SPE_DESC_WXOR, &op);
2200         slot_cnt = 3;
2201 
2202         spin_lock_bh(&ppc440spe_chan->lock);
2203 
2204         /* WXOR, each descriptor occupies one slot */
2205         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2206         if (sw_desc) {
2207                 struct ppc440spe_adma_chan *chan;
2208                 struct ppc440spe_adma_desc_slot *iter;
2209                 struct dma_cdb *hw_desc;
2210 
2211                 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2212                 set_bits(op, &sw_desc->flags);
2213                 sw_desc->src_cnt = src_cnt;
2214                 sw_desc->dst_cnt = 1;
2215                 /* 1st descriptor, src[1] data to q page and zero destination */
2216                 iter = list_first_entry(&sw_desc->group_list,
2217                                         struct ppc440spe_adma_desc_slot,
2218                                         chain_node);
2219                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2220                 iter->hw_next = list_entry(iter->chain_node.next,
2221                                            struct ppc440spe_adma_desc_slot,
2222                                            chain_node);
2223                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2224                 hw_desc = iter->hw_desc;
2225                 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2226 
2227                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2228                                              *dst, 0);
2229                 ppc440spe_desc_set_dest_addr(iter, chan, 0,
2230                                              ppc440spe_chan->qdest, 1);
2231                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2232                                             src[1]);
2233                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2234                 iter->unmap_len = len;
2235 
2236                 /* 2nd descriptor, multiply src[1] data and store the
2237                  * result in destination */
2238                 iter = list_first_entry(&iter->chain_node,
2239                                         struct ppc440spe_adma_desc_slot,
2240                                         chain_node);
2241                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2242                 /* set 'next' pointer */
2243                 iter->hw_next = list_entry(iter->chain_node.next,
2244                                            struct ppc440spe_adma_desc_slot,
2245                                            chain_node);
2246                 if (flags & DMA_PREP_INTERRUPT)
2247                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2248                 else
2249                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2250 
2251                 hw_desc = iter->hw_desc;
2252                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2253                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2254                                             ppc440spe_chan->qdest);
2255                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2256                                              *dst, 0);
2257                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2258                                             DMA_CDB_SG_DST1, scf[1]);
2259                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2260                 iter->unmap_len = len;
2261 
2262                 /*
2263                  * 3rd descriptor, multiply src[0] data and xor it
2264                  * with destination
2265                  */
2266                 iter = list_first_entry(&iter->chain_node,
2267                                         struct ppc440spe_adma_desc_slot,
2268                                         chain_node);
2269                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2270                 iter->hw_next = NULL;
2271                 if (flags & DMA_PREP_INTERRUPT)
2272                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2273                 else
2274                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2275 
2276                 hw_desc = iter->hw_desc;
2277                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2278                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2279                                             src[0]);
2280                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2281                                              *dst, 0);
2282                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2283                                             DMA_CDB_SG_DST1, scf[0]);
2284                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2285                 iter->unmap_len = len;
2286                 sw_desc->async_tx.flags = flags;
2287         }
2288 
2289         spin_unlock_bh(&ppc440spe_chan->lock);
2290 
2291         return sw_desc;
2292 }
2293 
2294 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
2295                 struct ppc440spe_adma_chan *ppc440spe_chan,
2296                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2297                 const unsigned char *scf, size_t len, unsigned long flags)
2298 {
2299         int slot_cnt;
2300         struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2301         unsigned long op = 0;
2302         unsigned char mult = 1;
2303 
2304         pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2305                  __func__, dst_cnt, src_cnt, len);
2306         /*  select operations WXOR/RXOR depending on the
2307          * source addresses of operators and the number
2308          * of destinations (RXOR support only Q-parity calculations)
2309          */
2310         set_bit(PPC440SPE_DESC_WXOR, &op);
2311         if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) {
2312                 /* no active RXOR;
2313                  * do RXOR if:
2314                  * - there are more than 1 source,
2315                  * - len is aligned on 512-byte boundary,
2316                  * - source addresses fit to one of 4 possible regions.
2317                  */
2318                 if (src_cnt > 1 &&
2319                     !(len & MQ0_CF2H_RXOR_BS_MASK) &&
2320                     (src[0] + len) == src[1]) {
2321                         /* may do RXOR R1 R2 */
2322                         set_bit(PPC440SPE_DESC_RXOR, &op);
2323                         if (src_cnt != 2) {
2324                                 /* may try to enhance region of RXOR */
2325                                 if ((src[1] + len) == src[2]) {
2326                                         /* do RXOR R1 R2 R3 */
2327                                         set_bit(PPC440SPE_DESC_RXOR123,
2328                                                 &op);
2329                                 } else if ((src[1] + len * 2) == src[2]) {
2330                                         /* do RXOR R1 R2 R4 */
2331                                         set_bit(PPC440SPE_DESC_RXOR124, &op);
2332                                 } else if ((src[1] + len * 3) == src[2]) {
2333                                         /* do RXOR R1 R2 R5 */
2334                                         set_bit(PPC440SPE_DESC_RXOR125,
2335                                                 &op);
2336                                 } else {
2337                                         /* do RXOR R1 R2 */
2338                                         set_bit(PPC440SPE_DESC_RXOR12,
2339                                                 &op);
2340                                 }
2341                         } else {
2342                                 /* do RXOR R1 R2 */
2343                                 set_bit(PPC440SPE_DESC_RXOR12, &op);
2344                         }
2345                 }
2346 
2347                 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2348                         /* can not do this operation with RXOR */
2349                         clear_bit(PPC440SPE_RXOR_RUN,
2350                                 &ppc440spe_rxor_state);
2351                 } else {
2352                         /* can do; set block size right now */
2353                         ppc440spe_desc_set_rxor_block_size(len);
2354                 }
2355         }
2356 
2357         /* Number of necessary slots depends on operation type selected */
2358         if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2359                 /*  This is a WXOR only chain. Need descriptors for each
2360                  * source to GF-XOR them with WXOR, and need descriptors
2361                  * for each destination to zero them with WXOR
2362                  */
2363                 slot_cnt = src_cnt;
2364 
2365                 if (flags & DMA_PREP_ZERO_P) {
2366                         slot_cnt++;
2367                         set_bit(PPC440SPE_ZERO_P, &op);
2368                 }
2369                 if (flags & DMA_PREP_ZERO_Q) {
2370                         slot_cnt++;
2371                         set_bit(PPC440SPE_ZERO_Q, &op);
2372                 }
2373         } else {
2374                 /*  Need 1/2 descriptor for RXOR operation, and
2375                  * need (src_cnt - (2 or 3)) for WXOR of sources
2376                  * remained (if any)
2377                  */
2378                 slot_cnt = dst_cnt;
2379 
2380                 if (flags & DMA_PREP_ZERO_P)
2381                         set_bit(PPC440SPE_ZERO_P, &op);
2382                 if (flags & DMA_PREP_ZERO_Q)
2383                         set_bit(PPC440SPE_ZERO_Q, &op);
2384 
2385                 if (test_bit(PPC440SPE_DESC_RXOR12, &op))
2386                         slot_cnt += src_cnt - 2;
2387                 else
2388                         slot_cnt += src_cnt - 3;
2389 
2390                 /*  Thus we have either RXOR only chain or
2391                  * mixed RXOR/WXOR
2392                  */
2393                 if (slot_cnt == dst_cnt)
2394                         /* RXOR only chain */
2395                         clear_bit(PPC440SPE_DESC_WXOR, &op);
2396         }
2397 
2398         spin_lock_bh(&ppc440spe_chan->lock);
2399         /* for both RXOR/WXOR each descriptor occupies one slot */
2400         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2401         if (sw_desc) {
2402                 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
2403                                 flags, op);
2404 
2405                 /* setup dst/src/mult */
2406                 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2407                          __func__, dst[0], dst[1]);
2408                 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2409                 while (src_cnt--) {
2410                         ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2411                                                   src_cnt);
2412 
2413                         /* NOTE: "Multi = 0 is equivalent to = 1" as it
2414                          * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2415                          * doesn't work for RXOR with DMA0/1! Instead, multi=0
2416                          * leads to zeroing source data after RXOR.
2417                          * So, for P case set-up mult=1 explicitly.
2418                          */
2419                         if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2420                                 mult = scf[src_cnt];
2421                         ppc440spe_adma_pq_set_src_mult(sw_desc,
2422                                 mult, src_cnt,  dst_cnt - 1);
2423                 }
2424 
2425                 /* Setup byte count foreach slot just allocated */
2426                 sw_desc->async_tx.flags = flags;
2427                 list_for_each_entry(iter, &sw_desc->group_list,
2428                                 chain_node) {
2429                         ppc440spe_desc_set_byte_count(iter,
2430                                 ppc440spe_chan, len);
2431                         iter->unmap_len = len;
2432                 }
2433         }
2434         spin_unlock_bh(&ppc440spe_chan->lock);
2435 
2436         return sw_desc;
2437 }
2438 
2439 static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
2440                 struct ppc440spe_adma_chan *ppc440spe_chan,
2441                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2442                 const unsigned char *scf, size_t len, unsigned long flags)
2443 {
2444         int slot_cnt, descs_per_op;
2445         struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2446         unsigned long op = 0;
2447         unsigned char mult = 1;
2448 
2449         BUG_ON(!dst_cnt);
2450         /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2451                  __func__, dst_cnt, src_cnt, len);*/
2452 
2453         spin_lock_bh(&ppc440spe_chan->lock);
2454         descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len);
2455         if (descs_per_op < 0) {
2456                 spin_unlock_bh(&ppc440spe_chan->lock);
2457                 return NULL;
2458         }
2459 
2460         /* depending on number of sources we have 1 or 2 RXOR chains */
2461         slot_cnt = descs_per_op * dst_cnt;
2462 
2463         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2464         if (sw_desc) {
2465                 op = slot_cnt;
2466                 sw_desc->async_tx.flags = flags;
2467                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2468                         ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt,
2469                                 --op ? 0 : flags);
2470                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2471                                 len);
2472                         iter->unmap_len = len;
2473 
2474                         ppc440spe_init_rxor_cursor(&(iter->rxor_cursor));
2475                         iter->rxor_cursor.len = len;
2476                         iter->descs_per_op = descs_per_op;
2477                 }
2478                 op = 0;
2479                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2480                         op++;
2481                         if (op % descs_per_op == 0)
2482                                 ppc440spe_adma_init_dma2rxor_slot(iter, src,
2483                                                                   src_cnt);
2484                         if (likely(!list_is_last(&iter->chain_node,
2485                                                  &sw_desc->group_list))) {
2486                                 /* set 'next' pointer */
2487                                 iter->hw_next =
2488                                         list_entry(iter->chain_node.next,
2489                                                 struct ppc440spe_adma_desc_slot,
2490                                                 chain_node);
2491                                 ppc440spe_xor_set_link(iter, iter->hw_next);
2492                         } else {
2493                                 /* this is the last descriptor. */
2494                                 iter->hw_next = NULL;
2495                         }
2496                 }
2497 
2498                 /* fixup head descriptor */
2499                 sw_desc->dst_cnt = dst_cnt;
2500                 if (flags & DMA_PREP_ZERO_P)
2501                         set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
2502                 if (flags & DMA_PREP_ZERO_Q)
2503                         set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
2504 
2505                 /* setup dst/src/mult */
2506                 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2507 
2508                 while (src_cnt--) {
2509                         /* handle descriptors (if dst_cnt == 2) inside
2510                          * the ppc440spe_adma_pq_set_srcxxx() functions
2511                          */
2512                         ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2513                                                   src_cnt);
2514                         if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2515                                 mult = scf[src_cnt];
2516                         ppc440spe_adma_pq_set_src_mult(sw_desc,
2517                                         mult, src_cnt, dst_cnt - 1);
2518                 }
2519         }
2520         spin_unlock_bh(&ppc440spe_chan->lock);
2521         ppc440spe_desc_set_rxor_block_size(len);
2522         return sw_desc;
2523 }
2524 
2525 /**
2526  * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2527  */
2528 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
2529                 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
2530                 unsigned int src_cnt, const unsigned char *scf,
2531                 size_t len, unsigned long flags)
2532 {
2533         struct ppc440spe_adma_chan *ppc440spe_chan;
2534         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2535         int dst_cnt = 0;
2536 
2537         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2538 
2539         ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
2540                                     dst, src, src_cnt));
2541         BUG_ON(!len);
2542         BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2543         BUG_ON(!src_cnt);
2544 
2545         if (src_cnt == 1 && dst[1] == src[0]) {
2546                 dma_addr_t dest[2];
2547 
2548                 /* dst[1] is real destination (Q) */
2549                 dest[0] = dst[1];
2550                 /* this is the page to multicast source data to */
2551                 dest[1] = ppc440spe_chan->qdest;
2552                 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2553                                 dest, 2, src, src_cnt, scf, len, flags);
2554                 return sw_desc ? &sw_desc->async_tx : NULL;
2555         }
2556 
2557         if (src_cnt == 2 && dst[1] == src[1]) {
2558                 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2559                                         &dst[1], src, 2, scf, len, flags);
2560                 return sw_desc ? &sw_desc->async_tx : NULL;
2561         }
2562 
2563         if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
2564                 BUG_ON(!dst[0]);
2565                 dst_cnt++;
2566                 flags |= DMA_PREP_ZERO_P;
2567         }
2568 
2569         if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
2570                 BUG_ON(!dst[1]);
2571                 dst_cnt++;
2572                 flags |= DMA_PREP_ZERO_Q;
2573         }
2574 
2575         BUG_ON(!dst_cnt);
2576 
2577         dev_dbg(ppc440spe_chan->device->common.dev,
2578                 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2579                 ppc440spe_chan->device->id, __func__, src_cnt, len,
2580                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2581 
2582         switch (ppc440spe_chan->device->id) {
2583         case PPC440SPE_DMA0_ID:
2584         case PPC440SPE_DMA1_ID:
2585                 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2586                                 dst, dst_cnt, src, src_cnt, scf,
2587                                 len, flags);
2588                 break;
2589 
2590         case PPC440SPE_XOR_ID:
2591                 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2592                                 dst, dst_cnt, src, src_cnt, scf,
2593                                 len, flags);
2594                 break;
2595         }
2596 
2597         return sw_desc ? &sw_desc->async_tx : NULL;
2598 }
2599 
2600 /**
2601  * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2602  * a PQ_ZERO_SUM operation
2603  */
2604 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
2605                 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
2606                 unsigned int src_cnt, const unsigned char *scf, size_t len,
2607                 enum sum_check_flags *pqres, unsigned long flags)
2608 {
2609         struct ppc440spe_adma_chan *ppc440spe_chan;
2610         struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2611         dma_addr_t pdest, qdest;
2612         int slot_cnt, slots_per_op, idst, dst_cnt;
2613 
2614         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2615 
2616         if (flags & DMA_PREP_PQ_DISABLE_P)
2617                 pdest = 0;
2618         else
2619                 pdest = pq[0];
2620 
2621         if (flags & DMA_PREP_PQ_DISABLE_Q)
2622                 qdest = 0;
2623         else
2624                 qdest = pq[1];
2625 
2626         ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
2627                                             src, src_cnt, scf));
2628 
2629         /* Always use WXOR for P/Q calculations (two destinations).
2630          * Need 1 or 2 extra slots to verify results are zero.
2631          */
2632         idst = dst_cnt = (pdest && qdest) ? 2 : 1;
2633 
2634         /* One additional slot per destination to clone P/Q
2635          * before calculation (we have to preserve destinations).
2636          */
2637         slot_cnt = src_cnt + dst_cnt * 2;
2638         slots_per_op = 1;
2639 
2640         spin_lock_bh(&ppc440spe_chan->lock);
2641         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2642                                              slots_per_op);
2643         if (sw_desc) {
2644                 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
2645 
2646                 /* Setup byte count for each slot just allocated */
2647                 sw_desc->async_tx.flags = flags;
2648                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2649                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2650                                                       len);
2651                         iter->unmap_len = len;
2652                 }
2653 
2654                 if (pdest) {
2655                         struct dma_cdb *hw_desc;
2656                         struct ppc440spe_adma_chan *chan;
2657 
2658                         iter = sw_desc->group_head;
2659                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2660                         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2661                         iter->hw_next = list_entry(iter->chain_node.next,
2662                                                 struct ppc440spe_adma_desc_slot,
2663                                                 chain_node);
2664                         hw_desc = iter->hw_desc;
2665                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2666                         iter->src_cnt = 0;
2667                         iter->dst_cnt = 0;
2668                         ppc440spe_desc_set_dest_addr(iter, chan, 0,
2669                                                      ppc440spe_chan->pdest, 0);
2670                         ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest);
2671                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2672                                                       len);
2673                         iter->unmap_len = 0;
2674                         /* override pdest to preserve original P */
2675                         pdest = ppc440spe_chan->pdest;
2676                 }
2677                 if (qdest) {
2678                         struct dma_cdb *hw_desc;
2679                         struct ppc440spe_adma_chan *chan;
2680 
2681                         iter = list_first_entry(&sw_desc->group_list,
2682                                                 struct ppc440spe_adma_desc_slot,
2683                                                 chain_node);
2684                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2685 
2686                         if (pdest) {
2687                                 iter = list_entry(iter->chain_node.next,
2688                                                 struct ppc440spe_adma_desc_slot,
2689                                                 chain_node);
2690                         }
2691 
2692                         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2693                         iter->hw_next = list_entry(iter->chain_node.next,
2694                                                 struct ppc440spe_adma_desc_slot,
2695                                                 chain_node);
2696                         hw_desc = iter->hw_desc;
2697                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2698                         iter->src_cnt = 0;
2699                         iter->dst_cnt = 0;
2700                         ppc440spe_desc_set_dest_addr(iter, chan, 0,
2701                                                      ppc440spe_chan->qdest, 0);
2702                         ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest);
2703                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2704                                                       len);
2705                         iter->unmap_len = 0;
2706                         /* override qdest to preserve original Q */
2707                         qdest = ppc440spe_chan->qdest;
2708                 }
2709 
2710                 /* Setup destinations for P/Q ops */
2711                 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
2712 
2713                 /* Setup zero QWORDs into DCHECK CDBs */
2714                 idst = dst_cnt;
2715                 list_for_each_entry_reverse(iter, &sw_desc->group_list,
2716                                             chain_node) {
2717                         /*
2718                          * The last CDB corresponds to Q-parity check,
2719                          * the one before last CDB corresponds
2720                          * P-parity check
2721                          */
2722                         if (idst == DMA_DEST_MAX_NUM) {
2723                                 if (idst == dst_cnt) {
2724                                         set_bit(PPC440SPE_DESC_QCHECK,
2725                                                 &iter->flags);
2726                                 } else {
2727                                         set_bit(PPC440SPE_DESC_PCHECK,
2728                                                 &iter->flags);
2729                                 }
2730                         } else {
2731                                 if (qdest) {
2732                                         set_bit(PPC440SPE_DESC_QCHECK,
2733                                                 &iter->flags);
2734                                 } else {
2735                                         set_bit(PPC440SPE_DESC_PCHECK,
2736                                                 &iter->flags);
2737                                 }
2738                         }
2739                         iter->xor_check_result = pqres;
2740 
2741                         /*
2742                          * set it to zero, if check fail then result will
2743                          * be updated
2744                          */
2745                         *iter->xor_check_result = 0;
2746                         ppc440spe_desc_set_dcheck(iter, ppc440spe_chan,
2747                                 ppc440spe_qword);
2748 
2749                         if (!(--dst_cnt))
2750                                 break;
2751                 }
2752 
2753                 /* Setup sources and mults for P/Q ops */
2754                 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
2755                                                      chain_node) {
2756                         struct ppc440spe_adma_chan *chan;
2757                         u32 mult_dst;
2758 
2759                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2760                         ppc440spe_desc_set_src_addr(iter, chan, 0,
2761                                                     DMA_CUED_XOR_HB,
2762                                                     src[src_cnt - 1]);
2763                         if (qdest) {
2764                                 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
2765                                                            DMA_CDB_SG_DST1;
2766                                 ppc440spe_desc_set_src_mult(iter, chan,
2767                                                             DMA_CUED_MULT1_OFF,
2768                                                             mult_dst,
2769                                                             scf[src_cnt - 1]);
2770                         }
2771                         if (!(--src_cnt))
2772                                 break;
2773                 }
2774         }
2775         spin_unlock_bh(&ppc440spe_chan->lock);
2776         return sw_desc ? &sw_desc->async_tx : NULL;
2777 }
2778 
2779 /**
2780  * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
2781  * XOR ZERO_SUM operation
2782  */
2783 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
2784                 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
2785                 size_t len, enum sum_check_flags *result, unsigned long flags)
2786 {
2787         struct dma_async_tx_descriptor *tx;
2788         dma_addr_t pq[2];
2789 
2790         /* validate P, disable Q */
2791         pq[0] = src[0];
2792         pq[1] = 0;
2793         flags |= DMA_PREP_PQ_DISABLE_Q;
2794 
2795         tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1],
2796                                                 src_cnt - 1, 0, len,
2797                                                 result, flags);
2798         return tx;
2799 }
2800 
2801 /**
2802  * ppc440spe_adma_set_dest - set destination address into descriptor
2803  */
2804 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2805                 dma_addr_t addr, int index)
2806 {
2807         struct ppc440spe_adma_chan *chan;
2808 
2809         BUG_ON(index >= sw_desc->dst_cnt);
2810 
2811         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2812 
2813         switch (chan->device->id) {
2814         case PPC440SPE_DMA0_ID:
2815         case PPC440SPE_DMA1_ID:
2816                 /* to do: support transfers lengths >
2817                  * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
2818                  */
2819                 ppc440spe_desc_set_dest_addr(sw_desc->group_head,
2820                         chan, 0, addr, index);
2821                 break;
2822         case PPC440SPE_XOR_ID:
2823                 sw_desc = ppc440spe_get_group_entry(sw_desc, index);
2824                 ppc440spe_desc_set_dest_addr(sw_desc,
2825                         chan, 0, addr, index);
2826                 break;
2827         }
2828 }
2829 
2830 static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
2831                 struct ppc440spe_adma_chan *chan, dma_addr_t addr)
2832 {
2833         /*  To clear destinations update the descriptor
2834          * (P or Q depending on index) as follows:
2835          * addr is destination (0 corresponds to SG2):
2836          */
2837         ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0);
2838 
2839         /* ... and the addr is source: */
2840         ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr);
2841 
2842         /* addr is always SG2 then the mult is always DST1 */
2843         ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2844                                     DMA_CDB_SG_DST1, 1);
2845 }
2846 
2847 /**
2848  * ppc440spe_adma_pq_set_dest - set destination address into descriptor
2849  * for the PQXOR operation
2850  */
2851 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2852                 dma_addr_t *addrs, unsigned long flags)
2853 {
2854         struct ppc440spe_adma_desc_slot *iter;
2855         struct ppc440spe_adma_chan *chan;
2856         dma_addr_t paddr, qaddr;
2857         dma_addr_t addr = 0, ppath, qpath;
2858         int index = 0, i;
2859 
2860         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2861 
2862         if (flags & DMA_PREP_PQ_DISABLE_P)
2863                 paddr = 0;
2864         else
2865                 paddr = addrs[0];
2866 
2867         if (flags & DMA_PREP_PQ_DISABLE_Q)
2868                 qaddr = 0;
2869         else
2870                 qaddr = addrs[1];
2871 
2872         if (!paddr || !qaddr)
2873                 addr = paddr ? paddr : qaddr;
2874 
2875         switch (chan->device->id) {
2876         case PPC440SPE_DMA0_ID:
2877         case PPC440SPE_DMA1_ID:
2878                 /* walk through the WXOR source list and set P/Q-destinations
2879                  * for each slot:
2880                  */
2881                 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
2882                         /* This is WXOR-only chain; may have 1/2 zero descs */
2883                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
2884                                 index++;
2885                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
2886                                 index++;
2887 
2888                         iter = ppc440spe_get_group_entry(sw_desc, index);
2889                         if (addr) {
2890                                 /* one destination */
2891                                 list_for_each_entry_from(iter,
2892                                         &sw_desc->group_list, chain_node)
2893                                         ppc440spe_desc_set_dest_addr(iter, chan,
2894                                                 DMA_CUED_XOR_BASE, addr, 0);
2895                         } else {
2896                                 /* two destinations */
2897                                 list_for_each_entry_from(iter,
2898                                         &sw_desc->group_list, chain_node) {
2899                                         ppc440spe_desc_set_dest_addr(iter, chan,
2900                                                 DMA_CUED_XOR_BASE, paddr, 0);
2901                                         ppc440spe_desc_set_dest_addr(iter, chan,
2902                                                 DMA_CUED_XOR_BASE, qaddr, 1);
2903                                 }
2904                         }
2905 
2906                         if (index) {
2907                                 /*  To clear destinations update the descriptor
2908                                  * (1st,2nd, or both depending on flags)
2909                                  */
2910                                 index = 0;
2911                                 if (test_bit(PPC440SPE_ZERO_P,
2912                                                 &sw_desc->flags)) {
2913                                         iter = ppc440spe_get_group_entry(
2914                                                         sw_desc, index++);
2915                                         ppc440spe_adma_pq_zero_op(iter, chan,
2916                                                         paddr);
2917                                 }
2918 
2919                                 if (test_bit(PPC440SPE_ZERO_Q,
2920                                                 &sw_desc->flags)) {
2921                                         iter = ppc440spe_get_group_entry(
2922                                                         sw_desc, index++);
2923                                         ppc440spe_adma_pq_zero_op(iter, chan,
2924                                                         qaddr);
2925                                 }
2926 
2927                                 return;
2928                         }
2929                 } else {
2930                         /* This is RXOR-only or RXOR/WXOR mixed chain */
2931 
2932                         /* If we want to include destination into calculations,
2933                          * then make dest addresses cued with mult=1 (XOR).
2934                          */
2935                         ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2936                                         DMA_CUED_XOR_HB :
2937                                         DMA_CUED_XOR_BASE |
2938                                                 (1 << DMA_CUED_MULT1_OFF);
2939                         qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2940                                         DMA_CUED_XOR_HB :
2941                                         DMA_CUED_XOR_BASE |
2942                                                 (1 << DMA_CUED_MULT1_OFF);
2943 
2944                         /* Setup destination(s) in RXOR slot(s) */
2945                         iter = ppc440spe_get_group_entry(sw_desc, index++);
2946                         ppc440spe_desc_set_dest_addr(iter, chan,
2947                                                 paddr ? ppath : qpath,
2948                                                 paddr ? paddr : qaddr, 0);
2949                         if (!addr) {
2950                                 /* two destinations */
2951                                 iter = ppc440spe_get_group_entry(sw_desc,
2952                                                                  index++);
2953                                 ppc440spe_desc_set_dest_addr(iter, chan,
2954                                                 qpath, qaddr, 0);
2955                         }
2956 
2957                         if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
2958                                 /* Setup destination(s) in remaining WXOR
2959                                  * slots
2960                                  */
2961                                 iter = ppc440spe_get_group_entry(sw_desc,
2962                                                                  index);
2963                                 if (addr) {
2964                                         /* one destination */
2965                                         list_for_each_entry_from(iter,
2966                                             &sw_desc->group_list,
2967                                             chain_node)
2968                                                 ppc440spe_desc_set_dest_addr(
2969                                                         iter, chan,
2970                                                         DMA_CUED_XOR_BASE,
2971                                                         addr, 0);
2972 
2973                                 } else {
2974                                         /* two destinations */
2975                                         list_for_each_entry_from(iter,
2976                                             &sw_desc->group_list,
2977                                             chain_node) {
2978                                                 ppc440spe_desc_set_dest_addr(
2979                                                         iter, chan,
2980                                                         DMA_CUED_XOR_BASE,
2981                                                         paddr, 0);
2982                                                 ppc440spe_desc_set_dest_addr(
2983                                                         iter, chan,
2984                                                         DMA_CUED_XOR_BASE,
2985                                                         qaddr, 1);
2986                                         }
2987                                 }
2988                         }
2989 
2990                 }
2991                 break;
2992 
2993         case PPC440SPE_XOR_ID:
2994                 /* DMA2 descriptors have only 1 destination, so there are
2995                  * two chains - one for each dest.
2996                  * If we want to include destination into calculations,
2997                  * then make dest addresses cued with mult=1 (XOR).
2998                  */
2999                 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
3000                                 DMA_CUED_XOR_HB :
3001                                 DMA_CUED_XOR_BASE |
3002                                         (1 << DMA_CUED_MULT1_OFF);
3003 
3004                 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
3005                                 DMA_CUED_XOR_HB :
3006                                 DMA_CUED_XOR_BASE |
3007                                         (1 << DMA_CUED_MULT1_OFF);
3008 
3009                 iter = ppc440spe_get_group_entry(sw_desc, 0);
3010                 for (i = 0; i < sw_desc->descs_per_op; i++) {
3011                         ppc440spe_desc_set_dest_addr(iter, chan,
3012                                 paddr ? ppath : qpath,
3013                                 paddr ? paddr : qaddr, 0);
3014                         iter = list_entry(iter->chain_node.next,
3015                                           struct ppc440spe_adma_desc_slot,
3016                                           chain_node);
3017                 }
3018 
3019                 if (!addr) {
3020                         /* Two destinations; setup Q here */
3021                         iter = ppc440spe_get_group_entry(sw_desc,
3022                                 sw_desc->descs_per_op);
3023                         for (i = 0; i < sw_desc->descs_per_op; i++) {
3024                                 ppc440spe_desc_set_dest_addr(iter,
3025                                         chan, qpath, qaddr, 0);
3026                                 iter = list_entry(iter->chain_node.next,
3027                                                 struct ppc440spe_adma_desc_slot,
3028                                                 chain_node);
3029                         }
3030                 }
3031 
3032                 break;
3033         }
3034 }
3035 
3036 /**
3037  * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3038  * for the PQ_ZERO_SUM operation
3039  */
3040 static void ppc440spe_adma_pqzero_sum_set_dest(
3041                 struct ppc440spe_adma_desc_slot *sw_desc,
3042                 dma_addr_t paddr, dma_addr_t qaddr)
3043 {
3044         struct ppc440spe_adma_desc_slot *iter, *end;
3045         struct ppc440spe_adma_chan *chan;
3046         dma_addr_t addr = 0;
3047         int idx;
3048 
3049         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3050 
3051         /* walk through the WXOR source list and set P/Q-destinations
3052          * for each slot
3053          */
3054         idx = (paddr && qaddr) ? 2 : 1;
3055         /* set end */
3056         list_for_each_entry_reverse(end, &sw_desc->group_list,
3057                                     chain_node) {
3058                 if (!(--idx))
3059                         break;
3060         }
3061         /* set start */
3062         idx = (paddr && qaddr) ? 2 : 1;
3063         iter = ppc440spe_get_group_entry(sw_desc, idx);
3064 
3065         if (paddr && qaddr) {
3066                 /* two destinations */
3067                 list_for_each_entry_from(iter, &sw_desc->group_list,
3068                                          chain_node) {
3069                         if (unlikely(iter == end))
3070                                 break;
3071                         ppc440spe_desc_set_dest_addr(iter, chan,
3072                                                 DMA_CUED_XOR_BASE, paddr, 0);
3073                         ppc440spe_desc_set_dest_addr(iter, chan,
3074                                                 DMA_CUED_XOR_BASE, qaddr, 1);
3075                 }
3076         } else {
3077                 /* one destination */
3078                 addr = paddr ? paddr : qaddr;
3079                 list_for_each_entry_from(iter, &sw_desc->group_list,
3080                                          chain_node) {
3081                         if (unlikely(iter == end))
3082                                 break;
3083                         ppc440spe_desc_set_dest_addr(iter, chan,
3084                                                 DMA_CUED_XOR_BASE, addr, 0);
3085                 }
3086         }
3087 
3088         /*  The remaining descriptors are DATACHECK. These have no need in
3089          * destination. Actually, these destinations are used there
3090          * as sources for check operation. So, set addr as source.
3091          */
3092         ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr);
3093 
3094         if (!addr) {
3095                 end = list_entry(end->chain_node.next,
3096                                  struct ppc440spe_adma_desc_slot, chain_node);
3097                 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr);
3098         }
3099 }
3100 
3101 /**
3102  * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3103  */
3104 static inline void ppc440spe_desc_set_xor_src_cnt(
3105                         struct ppc440spe_adma_desc_slot *desc,
3106                         int src_cnt)
3107 {
3108         struct xor_cb *hw_desc = desc->hw_desc;
3109 
3110         hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
3111         hw_desc->cbc |= src_cnt;
3112 }
3113 
3114 /**
3115  * ppc440spe_adma_pq_set_src - set source address into descriptor
3116  */
3117 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3118                 dma_addr_t addr, int index)
3119 {
3120         struct ppc440spe_adma_chan *chan;
3121         dma_addr_t haddr = 0;
3122         struct ppc440spe_adma_desc_slot *iter = NULL;
3123 
3124         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3125 
3126         switch (chan->device->id) {
3127         case PPC440SPE_DMA0_ID:
3128         case PPC440SPE_DMA1_ID:
3129                 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3130                  */
3131                 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3132                         /* RXOR-only or RXOR/WXOR operation */
3133                         int iskip = test_bit(PPC440SPE_DESC_RXOR12,
3134                                 &sw_desc->flags) ?  2 : 3;
3135 
3136                         if (index == 0) {
3137                                 /* 1st slot (RXOR) */
3138                                 /* setup sources region (R1-2-3, R1-2-4,
3139                                  * or R1-2-5)
3140                                  */
3141                                 if (test_bit(PPC440SPE_DESC_RXOR12,
3142                                                 &sw_desc->flags))
3143                                         haddr = DMA_RXOR12 <<
3144                                                 DMA_CUED_REGION_OFF;
3145                                 else if (test_bit(PPC440SPE_DESC_RXOR123,
3146                                     &sw_desc->flags))
3147                                         haddr = DMA_RXOR123 <<
3148                                                 DMA_CUED_REGION_OFF;
3149                                 else if (test_bit(PPC440SPE_DESC_RXOR124,
3150                                     &sw_desc->flags))
3151                                         haddr = DMA_RXOR124 <<
3152                                                 DMA_CUED_REGION_OFF;
3153                                 else if (test_bit(PPC440SPE_DESC_RXOR125,
3154                                     &sw_desc->flags))
3155                                         haddr = DMA_RXOR125 <<
3156                                                 DMA_CUED_REGION_OFF;
3157                                 else
3158                                         BUG();
3159                                 haddr |= DMA_CUED_XOR_BASE;
3160                                 iter = ppc440spe_get_group_entry(sw_desc, 0);
3161                         } else if (index < iskip) {
3162                                 /* 1st slot (RXOR)
3163                                  * shall actually set source address only once
3164                                  * instead of first <iskip>
3165                                  */
3166                                 iter = NULL;
3167                         } else {
3168                                 /* 2nd/3d and next slots (WXOR);
3169                                  * skip first slot with RXOR
3170                                  */
3171                                 haddr = DMA_CUED_XOR_HB;
3172                                 iter = ppc440spe_get_group_entry(sw_desc,
3173                                     index - iskip + sw_desc->dst_cnt);
3174                         }
3175                 } else {
3176                         int znum = 0;
3177 
3178                         /* WXOR-only operation; skip first slots with
3179                          * zeroing destinations
3180                          */
3181                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3182                                 znum++;
3183                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3184                                 znum++;
3185 
3186                         haddr = DMA_CUED_XOR_HB;
3187                         iter = ppc440spe_get_group_entry(sw_desc,
3188                                         index + znum);
3189                 }
3190 
3191                 if (likely(iter)) {
3192                         ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr);
3193 
3194                         if (!index &&
3195                             test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3196                             sw_desc->dst_cnt == 2) {
3197                                 /* if we have two destinations for RXOR, then
3198                                  * setup source in the second descr too
3199                                  */
3200                                 iter = ppc440spe_get_group_entry(sw_desc, 1);
3201                                 ppc440spe_desc_set_src_addr(iter, chan, 0,
3202                                         haddr, addr);
3203                         }
3204                 }
3205                 break;
3206 
3207         case PPC440SPE_XOR_ID:
3208                 /* DMA2 may do Biskup */
3209                 iter = sw_desc->group_head;
3210                 if (iter->dst_cnt == 2) {
3211                         /* both P & Q calculations required; set P src here */
3212                         ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3213 
3214                         /* this is for Q */
3215                         iter = ppc440spe_get_group_entry(sw_desc,
3216                                 sw_desc->descs_per_op);
3217                 }
3218                 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3219                 break;
3220         }
3221 }
3222 
3223 /**
3224  * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3225  */
3226 static void ppc440spe_adma_memcpy_xor_set_src(
3227                 struct ppc440spe_adma_desc_slot *sw_desc,
3228                 dma_addr_t addr, int index)
3229 {
3230         struct ppc440spe_adma_chan *chan;
3231 
3232         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3233         sw_desc = sw_desc->group_head;
3234 
3235         if (likely(sw_desc))
3236                 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
3237 }
3238 
3239 /**
3240  * ppc440spe_adma_dma2rxor_inc_addr  -
3241  */
3242 static void ppc440spe_adma_dma2rxor_inc_addr(
3243                 struct ppc440spe_adma_desc_slot *desc,
3244                 struct ppc440spe_rxor *cursor, int index, int src_cnt)
3245 {
3246         cursor->addr_count++;
3247         if (index == src_cnt - 1) {
3248                 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3249         } else if (cursor->addr_count == XOR_MAX_OPS) {
3250                 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3251                 cursor->addr_count = 0;
3252                 cursor->desc_count++;
3253         }
3254 }
3255 
3256 /**
3257  * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3258  */
3259 static int ppc440spe_adma_dma2rxor_prep_src(
3260                 struct ppc440spe_adma_desc_slot *hdesc,
3261                 struct ppc440spe_rxor *cursor, int index,
3262                 int src_cnt, u32 addr)
3263 {
3264         int rval = 0;
3265         u32 sign;
3266         struct ppc440spe_adma_desc_slot *desc = hdesc;
3267         int i;
3268 
3269         for (i = 0; i < cursor->desc_count; i++) {
3270                 desc = list_entry(hdesc->chain_node.next,
3271                                   struct ppc440spe_adma_desc_slot,
3272                                   chain_node);
3273         }
3274 
3275         switch (cursor->state) {
3276         case 0:
3277                 if (addr == cursor->addrl + cursor->len) {
3278                         /* direct RXOR */
3279                         cursor->state = 1;
3280                         cursor->xor_count++;
3281                         if (index == src_cnt-1) {
3282                                 ppc440spe_rxor_set_region(desc,
3283                                         cursor->addr_count,
3284                                         DMA_RXOR12 << DMA_CUED_REGION_OFF);
3285                                 ppc440spe_adma_dma2rxor_inc_addr(
3286                                         desc, cursor, index, src_cnt);
3287                         }
3288                 } else if (cursor->addrl == addr + cursor->len) {
3289                         /* reverse RXOR */
3290                         cursor->state = 1;
3291                         cursor->xor_count++;
3292                         set_bit(cursor->addr_count, &desc->reverse_flags[0]);
3293                         if (index == src_cnt-1) {
3294                                 ppc440spe_rxor_set_region(desc,
3295                                         cursor->addr_count,
3296                                         DMA_RXOR12 << DMA_CUED_REGION_OFF);
3297                                 ppc440spe_adma_dma2rxor_inc_addr(
3298                                         desc, cursor, index, src_cnt);
3299                         }
3300                 } else {
3301                         printk(KERN_ERR "Cannot build "
3302                                 "DMA2 RXOR command block.\n");
3303                         BUG();
3304                 }
3305                 break;
3306         case 1:
3307                 sign = test_bit(cursor->addr_count,
3308                                 desc->reverse_flags)
3309                         ? -1 : 1;
3310                 if (index == src_cnt-2 || (sign == -1
3311                         && addr != cursor->addrl - 2*cursor->len)) {
3312                         cursor->state = 0;
3313                         cursor->xor_count = 1;
3314                         cursor->addrl = addr;
3315                         ppc440spe_rxor_set_region(desc,
3316                                 cursor->addr_count,
3317                                 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3318                         ppc440spe_adma_dma2rxor_inc_addr(
3319                                 desc, cursor, index, src_cnt);
3320                 } else if (addr == cursor->addrl + 2*sign*cursor->len) {
3321                         cursor->state = 2;
3322                         cursor->xor_count = 0;
3323                         ppc440spe_rxor_set_region(desc,
3324                                 cursor->addr_count,
3325                                 DMA_RXOR123 << DMA_CUED_REGION_OFF);
3326                         if (index == src_cnt-1) {
3327                                 ppc440spe_adma_dma2rxor_inc_addr(
3328                                         desc, cursor, index, src_cnt);
3329                         }
3330                 } else if (addr == cursor->addrl + 3*cursor->len) {
3331                         cursor->state = 2;
3332                         cursor->xor_count = 0;
3333                         ppc440spe_rxor_set_region(desc,
3334                                 cursor->addr_count,
3335                                 DMA_RXOR124 << DMA_CUED_REGION_OFF);
3336                         if (index == src_cnt-1) {
3337                                 ppc440spe_adma_dma2rxor_inc_addr(
3338                                         desc, cursor, index, src_cnt);
3339                         }
3340                 } else if (addr == cursor->addrl + 4*cursor->len) {
3341                         cursor->state = 2;
3342                         cursor->xor_count = 0;
3343                         ppc440spe_rxor_set_region(desc,
3344                                 cursor->addr_count,
3345                                 DMA_RXOR125 << DMA_CUED_REGION_OFF);
3346                         if (index == src_cnt-1) {
3347                                 ppc440spe_adma_dma2rxor_inc_addr(
3348                                         desc, cursor, index, src_cnt);
3349                         }
3350                 } else {
3351                         cursor->state = 0;
3352                         cursor->xor_count = 1;
3353                         cursor->addrl = addr;
3354                         ppc440spe_rxor_set_region(desc,
3355                                 cursor->addr_count,
3356                                 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3357                         ppc440spe_adma_dma2rxor_inc_addr(
3358                                 desc, cursor, index, src_cnt);
3359                 }
3360                 break;
3361         case 2:
3362                 cursor->state = 0;
3363                 cursor->addrl = addr;
3364                 cursor->xor_count++;
3365                 if (index) {
3366                         ppc440spe_adma_dma2rxor_inc_addr(
3367                                 desc, cursor, index, src_cnt);
3368                 }
3369                 break;
3370         }
3371 
3372         return rval;
3373 }
3374 
3375 /**
3376  * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3377  *      ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3378  */
3379 static void ppc440spe_adma_dma2rxor_set_src(
3380                 struct ppc440spe_adma_desc_slot *desc,
3381                 int index, dma_addr_t addr)
3382 {
3383         struct xor_cb *xcb = desc->hw_desc;
3384         int k = 0, op = 0, lop = 0;
3385 
3386         /* get the RXOR operand which corresponds to index addr */
3387         while (op <= index) {
3388                 lop = op;
3389                 if (k == XOR_MAX_OPS) {
3390                         k = 0;
3391                         desc = list_entry(desc->chain_node.next,
3392                                 struct ppc440spe_adma_desc_slot, chain_node);
3393                         xcb = desc->hw_desc;
3394 
3395                 }
3396                 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3397                     (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3398                         op += 2;
3399                 else
3400                         op += 3;
3401         }
3402 
3403         BUG_ON(k < 1);
3404 
3405         if (test_bit(k-1, desc->reverse_flags)) {
3406                 /* reverse operand order; put last op in RXOR group */
3407                 if (index == op - 1)
3408                         ppc440spe_rxor_set_src(desc, k - 1, addr);
3409         } else {
3410                 /* direct operand order; put first op in RXOR group */
3411                 if (index == lop)
3412                         ppc440spe_rxor_set_src(desc, k - 1, addr);
3413         }
3414 }
3415 
3416 /**
3417  * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3418  *      ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3419  */
3420 static void ppc440spe_adma_dma2rxor_set_mult(
3421                 struct ppc440spe_adma_desc_slot *desc,
3422                 int index, u8 mult)
3423 {
3424         struct xor_cb *xcb = desc->hw_desc;
3425         int k = 0, op = 0, lop = 0;
3426 
3427         /* get the RXOR operand which corresponds to index mult */
3428         while (op <= index) {
3429                 lop = op;
3430                 if (k == XOR_MAX_OPS) {
3431                         k = 0;
3432                         desc = list_entry(desc->chain_node.next,
3433                                           struct ppc440spe_adma_desc_slot,
3434                                           chain_node);
3435                         xcb = desc->hw_desc;
3436 
3437                 }
3438                 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3439                     (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3440                         op += 2;
3441                 else
3442                         op += 3;
3443         }
3444 
3445         BUG_ON(k < 1);
3446         if (test_bit(k-1, desc->reverse_flags)) {
3447                 /* reverse order */
3448                 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);
3449         } else {
3450                 /* direct order */
3451                 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);
3452         }
3453 }
3454 
3455 /**
3456  * ppc440spe_init_rxor_cursor -
3457  */
3458 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
3459 {
3460         memset(cursor, 0, sizeof(struct ppc440spe_rxor));
3461         cursor->state = 2;
3462 }
3463 
3464 /**
3465  * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3466  * descriptor for the PQXOR operation
3467  */
3468 static void ppc440spe_adma_pq_set_src_mult(
3469                 struct ppc440spe_adma_desc_slot *sw_desc,
3470                 unsigned char mult, int index, int dst_pos)
3471 {
3472         struct ppc440spe_adma_chan *chan;
3473         u32 mult_idx, mult_dst;
3474         struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
3475 
3476         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3477 
3478         switch (chan->device->id) {
3479         case PPC440SPE_DMA0_ID:
3480         case PPC440SPE_DMA1_ID:
3481                 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3482                         int region = test_bit(PPC440SPE_DESC_RXOR12,
3483                                         &sw_desc->flags) ? 2 : 3;
3484 
3485                         if (index < region) {
3486                                 /* RXOR multipliers */
3487                                 iter = ppc440spe_get_group_entry(sw_desc,
3488                                         sw_desc->dst_cnt - 1);
3489                                 if (sw_desc->dst_cnt == 2)
3490                                         iter1 = ppc440spe_get_group_entry(
3491                                                         sw_desc, 0);
3492 
3493                                 mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
3494                                 mult_dst = DMA_CDB_SG_SRC;
3495                         } else {
3496                                 /* WXOR multiplier */
3497                                 iter = ppc440spe_get_group_entry(sw_desc,
3498                                                         index - region +
3499                                                         sw_desc->dst_cnt);
3500                                 mult_idx = DMA_CUED_MULT1_OFF;
3501                                 mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
3502                                                      DMA_CDB_SG_DST1;
3503                         }
3504                 } else {
3505                         int znum = 0;
3506 
3507                         /* WXOR-only;
3508                          * skip first slots with destinations (if ZERO_DST has
3509                          * place)
3510                          */
3511                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3512                                 znum++;
3513                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3514                                 znum++;
3515 
3516                         iter = ppc440spe_get_group_entry(sw_desc, index + znum);
3517                         mult_idx = DMA_CUED_MULT1_OFF;
3518                         mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
3519                 }
3520 
3521                 if (likely(iter)) {
3522                         ppc440spe_desc_set_src_mult(iter, chan,
3523                                 mult_idx, mult_dst, mult);
3524 
3525                         if (unlikely(iter1)) {
3526                                 /* if we have two destinations for RXOR, then
3527                                  * we've just set Q mult. Set-up P now.
3528                                  */
3529                                 ppc440spe_desc_set_src_mult(iter1, chan,
3530                                         mult_idx, mult_dst, 1);
3531                         }
3532 
3533                 }
3534                 break;
3535 
3536         case PPC440SPE_XOR_ID:
3537                 iter = sw_desc->group_head;
3538                 if (sw_desc->dst_cnt == 2) {
3539                         /* both P & Q calculations required; set P mult here */
3540                         ppc440spe_adma_dma2rxor_set_mult(iter, index, 1);
3541 
3542                         /* and then set Q mult */
3543                         iter = ppc440spe_get_group_entry(sw_desc,
3544                                sw_desc->descs_per_op);
3545                 }
3546                 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult);
3547                 break;
3548         }
3549 }
3550 
3551 /**
3552  * ppc440spe_adma_free_chan_resources - free the resources allocated
3553  */
3554 static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
3555 {
3556         struct ppc440spe_adma_chan *ppc440spe_chan;
3557         struct ppc440spe_adma_desc_slot *iter, *_iter;
3558         int in_use_descs = 0;
3559 
3560         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3561         ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3562 
3563         spin_lock_bh(&ppc440spe_chan->lock);
3564         list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
3565                                         chain_node) {
3566                 in_use_descs++;
3567                 list_del(&iter->chain_node);
3568         }
3569         list_for_each_entry_safe_reverse(iter, _iter,
3570                         &ppc440spe_chan->all_slots, slot_node) {
3571                 list_del(&iter->slot_node);
3572                 kfree(iter);
3573                 ppc440spe_chan->slots_allocated--;
3574         }
3575         ppc440spe_chan->last_used = NULL;
3576 
3577         dev_dbg(ppc440spe_chan->device->common.dev,
3578                 "ppc440spe adma%d %s slots_allocated %d\n",
3579                 ppc440spe_chan->device->id,
3580                 __func__, ppc440spe_chan->slots_allocated);
3581         spin_unlock_bh(&ppc440spe_chan->lock);
3582 
3583         /* one is ok since we left it on there on purpose */
3584         if (in_use_descs > 1)
3585                 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
3586                         in_use_descs - 1);
3587 }
3588 
3589 /**
3590  * ppc440spe_adma_tx_status - poll the status of an ADMA transaction
3591  * @chan: ADMA channel handle
3592  * @cookie: ADMA transaction identifier
3593  * @txstate: a holder for the current state of the channel
3594  */
3595 static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan,
3596                         dma_cookie_t cookie, struct dma_tx_state *txstate)
3597 {
3598         struct ppc440spe_adma_chan *ppc440spe_chan;
3599         enum dma_status ret;
3600 
3601         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3602         ret = dma_cookie_status(chan, cookie, txstate);
3603         if (ret == DMA_COMPLETE)
3604                 return ret;
3605 
3606         ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3607 
3608         return dma_cookie_status(chan, cookie, txstate);
3609 }
3610 
3611 /**
3612  * ppc440spe_adma_eot_handler - end of transfer interrupt handler
3613  */
3614 static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
3615 {
3616         struct ppc440spe_adma_chan *chan = data;
3617 
3618         dev_dbg(chan->device->common.dev,
3619                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3620 
3621         tasklet_schedule(&chan->irq_tasklet);
3622         ppc440spe_adma_device_clear_eot_status(chan);
3623 
3624         return IRQ_HANDLED;
3625 }
3626 
3627 /**
3628  * ppc440spe_adma_err_handler - DMA error interrupt handler;
3629  *      do the same things as a eot handler
3630  */
3631 static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
3632 {
3633         struct ppc440spe_adma_chan *chan = data;
3634 
3635         dev_dbg(chan->device->common.dev,
3636                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3637 
3638         tasklet_schedule(&chan->irq_tasklet);
3639         ppc440spe_adma_device_clear_eot_status(chan);
3640 
3641         return IRQ_HANDLED;
3642 }
3643 
3644 /**
3645  * ppc440spe_test_callback - called when test operation has been done
3646  */
3647 static void ppc440spe_test_callback(void *unused)
3648 {
3649         complete(&ppc440spe_r6_test_comp);
3650 }
3651 
3652 /**
3653  * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
3654  */
3655 static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
3656 {
3657         struct ppc440spe_adma_chan *ppc440spe_chan;
3658 
3659         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3660         dev_dbg(ppc440spe_chan->device->common.dev,
3661                 "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
3662                 __func__, ppc440spe_chan->pending);
3663 
3664         if (ppc440spe_chan->pending) {
3665                 ppc440spe_chan->pending = 0;
3666                 ppc440spe_chan_append(ppc440spe_chan);
3667         }
3668 }
3669 
3670 /**
3671  * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
3672  *      use FIFOs (as opposite to chains used in XOR) so this is a XOR
3673  *      specific operation)
3674  */
3675 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
3676 {
3677         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
3678         dma_cookie_t cookie;
3679         int slot_cnt, slots_per_op;
3680 
3681         dev_dbg(chan->device->common.dev,
3682                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3683 
3684         spin_lock_bh(&chan->lock);
3685         slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op);
3686         sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
3687         if (sw_desc) {
3688                 group_start = sw_desc->group_head;
3689                 list_splice_init(&sw_desc->group_list, &chan->chain);
3690                 async_tx_ack(&sw_desc->async_tx);
3691                 ppc440spe_desc_init_null_xor(group_start);
3692 
3693                 cookie = dma_cookie_assign(&sw_desc->async_tx);
3694 
3695                 /* initialize the completed cookie to be less than
3696                  * the most recently used cookie
3697                  */
3698                 chan->common.completed_cookie = cookie - 1;
3699 
3700                 /* channel should not be busy */
3701                 BUG_ON(ppc440spe_chan_is_busy(chan));
3702 
3703                 /* set the descriptor address */
3704                 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
3705 
3706                 /* run the descriptor */
3707                 ppc440spe_chan_run(chan);
3708         } else
3709                 printk(KERN_ERR "ppc440spe adma%d"
3710                         " failed to allocate null descriptor\n",
3711                         chan->device->id);
3712         spin_unlock_bh(&chan->lock);
3713 }
3714 
3715 /**
3716  * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
3717  *      For this we just perform one WXOR operation with the same source
3718  *      and destination addresses, the GF-multiplier is 1; so if RAID-6
3719  *      capabilities are enabled then we'll get src/dst filled with zero.
3720  */
3721 static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
3722 {
3723         struct ppc440spe_adma_desc_slot *sw_desc, *iter;
3724         struct page *pg;
3725         char *a;
3726         dma_addr_t dma_addr, addrs[2];
3727         unsigned long op = 0;
3728         int rval = 0;
3729 
3730         set_bit(PPC440SPE_DESC_WXOR, &op);
3731 
3732         pg = alloc_page(GFP_KERNEL);
3733         if (!pg)
3734                 return -ENOMEM;
3735 
3736         spin_lock_bh(&chan->lock);
3737         sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
3738         if (sw_desc) {
3739                 /* 1 src, 1 dsr, int_ena, WXOR */
3740                 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
3741                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
3742                         ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE);
3743                         iter->unmap_len = PAGE_SIZE;
3744                 }
3745         } else {
3746                 rval = -EFAULT;
3747                 spin_unlock_bh(&chan->lock);
3748                 goto exit;
3749         }
3750         spin_unlock_bh(&chan->lock);
3751 
3752         /* Fill the test page with ones */
3753         memset(page_address(pg), 0xFF, PAGE_SIZE);
3754         dma_addr = dma_map_page(chan->device->dev, pg, 0,
3755                                 PAGE_SIZE, DMA_BIDIRECTIONAL);
3756 
3757         /* Setup addresses */
3758         ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
3759         ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
3760         addrs[0] = dma_addr;
3761         addrs[1] = 0;
3762         ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
3763 
3764         async_tx_ack(&sw_desc->async_tx);
3765         sw_desc->async_tx.callback = ppc440spe_test_callback;
3766         sw_desc->async_tx.callback_param = NULL;
3767 
3768         init_completion(&ppc440spe_r6_test_comp);
3769 
3770         ppc440spe_adma_tx_submit(&sw_desc->async_tx);
3771         ppc440spe_adma_issue_pending(&chan->common);
3772 
3773         wait_for_completion(&ppc440spe_r6_test_comp);
3774 
3775         /* Now check if the test page is zeroed */
3776         a = page_address(pg);
3777         if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) {
3778                 /* page is zero - RAID-6 enabled */
3779                 rval = 0;
3780         } else {
3781                 /* RAID-6 was not enabled */
3782                 rval = -EINVAL;
3783         }
3784 exit:
3785         __free_page(pg);
3786         return rval;
3787 }
3788 
3789 static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
3790 {
3791         switch (adev->id) {
3792         case PPC440SPE_DMA0_ID:
3793         case PPC440SPE_DMA1_ID:
3794                 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
3795                 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3796                 dma_cap_set(DMA_PQ, adev->common.cap_mask);
3797                 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
3798                 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
3799                 break;
3800         case PPC440SPE_XOR_ID:
3801                 dma_cap_set(DMA_XOR, adev->common.cap_mask);
3802                 dma_cap_set(DMA_PQ, adev->common.cap_mask);
3803                 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3804                 adev->common.cap_mask = adev->common.cap_mask;
3805                 break;
3806         }
3807 
3808         /* Set base routines */
3809         adev->common.device_alloc_chan_resources =
3810                                 ppc440spe_adma_alloc_chan_resources;
3811         adev->common.device_free_chan_resources =
3812                                 ppc440spe_adma_free_chan_resources;
3813         adev->common.device_tx_status = ppc440spe_adma_tx_status;
3814         adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
3815 
3816         /* Set prep routines based on capability */
3817         if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
3818                 adev->common.device_prep_dma_memcpy =
3819                         ppc440spe_adma_prep_dma_memcpy;
3820         }
3821         if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
3822                 adev->common.max_xor = XOR_MAX_OPS;
3823                 adev->common.device_prep_dma_xor =
3824                         ppc440spe_adma_prep_dma_xor;
3825         }
3826         if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
3827                 switch (adev->id) {
3828                 case PPC440SPE_DMA0_ID:
3829                         dma_set_maxpq(&adev->common,
3830                                 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3831                         break;
3832                 case PPC440SPE_DMA1_ID:
3833                         dma_set_maxpq(&adev->common,
3834                                 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3835                         break;
3836                 case PPC440SPE_XOR_ID:
3837                         adev->common.max_pq = XOR_MAX_OPS * 3;
3838                         break;
3839                 }
3840                 adev->common.device_prep_dma_pq =
3841                         ppc440spe_adma_prep_dma_pq;
3842         }
3843         if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
3844                 switch (adev->id) {
3845                 case PPC440SPE_DMA0_ID:
3846                         adev->common.max_pq = DMA0_FIFO_SIZE /
3847                                                 sizeof(struct dma_cdb);
3848                         break;
3849                 case PPC440SPE_DMA1_ID:
3850                         adev->common.max_pq = DMA1_FIFO_SIZE /
3851                                                 sizeof(struct dma_cdb);
3852                         break;
3853                 }
3854                 adev->common.device_prep_dma_pq_val =
3855                         ppc440spe_adma_prep_dma_pqzero_sum;
3856         }
3857         if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
3858                 switch (adev->id) {
3859                 case PPC440SPE_DMA0_ID:
3860                         adev->common.max_xor = DMA0_FIFO_SIZE /
3861                                                 sizeof(struct dma_cdb);
3862                         break;
3863                 case PPC440SPE_DMA1_ID:
3864                         adev->common.max_xor = DMA1_FIFO_SIZE /
3865                                                 sizeof(struct dma_cdb);
3866                         break;
3867                 }
3868                 adev->common.device_prep_dma_xor_val =
3869                         ppc440spe_adma_prep_dma_xor_zero_sum;
3870         }
3871         if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
3872                 adev->common.device_prep_dma_interrupt =
3873                         ppc440spe_adma_prep_dma_interrupt;
3874         }
3875         pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
3876           "( %s%s%s%s%s%s)\n",
3877           dev_name(adev->dev),
3878           dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
3879           dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
3880           dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
3881           dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
3882           dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
3883           dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
3884 }
3885 
3886 static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
3887                                      struct ppc440spe_adma_chan *chan,
3888                                      int *initcode)
3889 {
3890         struct platform_device *ofdev;
3891         struct device_node *np;
3892         int ret;
3893 
3894         ofdev = container_of(adev->dev, struct platform_device, dev);
3895         np = ofdev->dev.of_node;
3896         if (adev->id != PPC440SPE_XOR_ID) {
3897                 adev->err_irq = irq_of_parse_and_map(np, 1);
3898                 if (adev->err_irq == NO_IRQ) {
3899                         dev_warn(adev->dev, "no err irq resource?\n");
3900                         *initcode = PPC_ADMA_INIT_IRQ2;
3901                         adev->err_irq = -ENXIO;
3902                 } else
3903                         atomic_inc(&ppc440spe_adma_err_irq_ref);
3904         } else {
3905                 adev->err_irq = -ENXIO;
3906         }
3907 
3908         adev->irq = irq_of_parse_and_map(np, 0);
3909         if (adev->irq == NO_IRQ) {
3910                 dev_err(adev->dev, "no irq resource\n");
3911                 *initcode = PPC_ADMA_INIT_IRQ1;
3912                 ret = -ENXIO;
3913                 goto err_irq_map;
3914         }
3915         dev_dbg(adev->dev, "irq %d, err irq %d\n",
3916                 adev->irq, adev->err_irq);
3917 
3918         ret = request_irq(adev->irq, ppc440spe_adma_eot_handler,
3919                           0, dev_driver_string(adev->dev), chan);
3920         if (ret) {
3921                 dev_err(adev->dev, "can't request irq %d\n",
3922                         adev->irq);
3923                 *initcode = PPC_ADMA_INIT_IRQ1;
3924                 ret = -EIO;
3925                 goto err_req1;
3926         }
3927 
3928         /* only DMA engines have a separate error IRQ
3929          * so it's Ok if err_irq < 0 in XOR engine case.
3930          */
3931         if (adev->err_irq > 0) {
3932                 /* both DMA engines share common error IRQ */
3933                 ret = request_irq(adev->err_irq,
3934                                   ppc440spe_adma_err_handler,
3935                                   IRQF_SHARED,
3936                                   dev_driver_string(adev->dev),
3937                                   chan);
3938                 if (ret) {
3939                         dev_err(adev->dev, "can't request irq %d\n",
3940                                 adev->err_irq);
3941                         *initcode = PPC_ADMA_INIT_IRQ2;
3942                         ret = -EIO;
3943                         goto err_req2;
3944                 }
3945         }
3946 
3947         if (adev->id == PPC440SPE_XOR_ID) {
3948                 /* enable XOR engine interrupts */
3949                 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3950                             XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
3951                             &adev->xor_reg->ier);
3952         } else {
3953                 u32 mask, enable;
3954 
3955                 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
3956                 if (!np) {
3957                         pr_err("%s: can't find I2O device tree node\n",
3958                                 __func__);
3959                         ret = -ENODEV;
3960                         goto err_req2;
3961                 }
3962                 adev->i2o_reg = of_iomap(np, 0);
3963                 if (!adev->i2o_reg) {
3964                         pr_err("%s: failed to map I2O registers\n", __func__);
3965                         of_node_put(np);
3966                         ret = -EINVAL;
3967                         goto err_req2;
3968                 }
3969                 of_node_put(np);
3970                 /* Unmask 'CS FIFO Attention' interrupts and
3971                  * enable generating interrupts on errors
3972                  */
3973                 enable = (adev->id == PPC440SPE_DMA0_ID) ?
3974                          ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3975                          ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3976                 mask = ioread32(&adev->i2o_reg->iopim) & enable;
3977                 iowrite32(mask, &adev->i2o_reg->iopim);
3978         }
3979         return 0;
3980 
3981 err_req2:
3982         free_irq(adev->irq, chan);
3983 err_req1:
3984         irq_dispose_mapping(adev->irq);
3985 err_irq_map:
3986         if (adev->err_irq > 0) {
3987                 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref))
3988                         irq_dispose_mapping(adev->err_irq);
3989         }
3990         return ret;
3991 }
3992 
3993 static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
3994                                         struct ppc440spe_adma_chan *chan)
3995 {
3996         u32 mask, disable;
3997 
3998         if (adev->id == PPC440SPE_XOR_ID) {
3999                 /* disable XOR engine interrupts */
4000                 mask = ioread32be(&adev->xor_reg->ier);
4001                 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
4002                           XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
4003                 iowrite32be(mask, &adev->xor_reg->ier);
4004         } else {
4005                 /* disable DMAx engine interrupts */
4006                 disable = (adev->id == PPC440SPE_DMA0_ID) ?
4007                           (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
4008                           (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
4009                 mask = ioread32(&adev->i2o_reg->iopim) | disable;
4010                 iowrite32(mask, &adev->i2o_reg->iopim);
4011         }
4012         free_irq(adev->irq, chan);
4013         irq_dispose_mapping(adev->irq);
4014         if (adev->err_irq > 0) {
4015                 free_irq(adev->err_irq, chan);
4016                 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) {
4017                         irq_dispose_mapping(adev->err_irq);
4018                         iounmap(adev->i2o_reg);
4019                 }
4020         }
4021 }
4022 
4023 /**
4024  * ppc440spe_adma_probe - probe the asynch device
4025  */
4026 static int ppc440spe_adma_probe(struct platform_device *ofdev)
4027 {
4028         struct device_node *np = ofdev->dev.of_node;
4029         struct resource res;
4030         struct ppc440spe_adma_device *adev;
4031         struct ppc440spe_adma_chan *chan;
4032         struct ppc_dma_chan_ref *ref, *_ref;
4033         int ret = 0, initcode = PPC_ADMA_INIT_OK;
4034         const u32 *idx;
4035         int len;
4036         void *regs;
4037         u32 id, pool_size;
4038 
4039         if (of_device_is_compatible(np, "amcc,xor-accelerator")) {
4040                 id = PPC440SPE_XOR_ID;
4041                 /* As far as the XOR engine is concerned, it does not
4042                  * use FIFOs but uses linked list. So there is no dependency
4043                  * between pool size to allocate and the engine configuration.
4044                  */
4045                 pool_size = PAGE_SIZE << 1;
4046         } else {
4047                 /* it is DMA0 or DMA1 */
4048                 idx = of_get_property(np, "cell-index", &len);
4049                 if (!idx || (len != sizeof(u32))) {
4050                         dev_err(&ofdev->dev, "Device node %s has missing "
4051                                 "or invalid cell-index property\n",
4052                                 np->full_name);
4053                         return -EINVAL;
4054                 }
4055                 id = *idx;
4056                 /* DMA0,1 engines use FIFO to maintain CDBs, so we
4057                  * should allocate the pool accordingly to size of this
4058                  * FIFO. Thus, the pool size depends on the FIFO depth:
4059                  * how much CDBs pointers the FIFO may contain then so
4060                  * much CDBs we should provide in the pool.
4061                  * That is
4062                  *   CDB size = 32B;
4063                  *   CDBs number = (DMA0_FIFO_SIZE >> 3);
4064                  *   Pool size = CDBs number * CDB size =
4065                  *      = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4066                  */
4067                 pool_size = (id == PPC440SPE_DMA0_ID) ?
4068                             DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4069                 pool_size <<= 2;
4070         }
4071 
4072         if (of_address_to_resource(np, 0, &res)) {
4073                 dev_err(&ofdev->dev, "failed to get memory resource\n");
4074                 initcode = PPC_ADMA_INIT_MEMRES;
4075                 ret = -ENODEV;
4076                 goto out;
4077         }
4078 
4079         if (!request_mem_region(res.start, resource_size(&res),
4080                                 dev_driver_string(&ofdev->dev))) {
4081                 dev_err(&ofdev->dev, "failed to request memory region %pR\n",
4082                         &res);
4083                 initcode = PPC_ADMA_INIT_MEMREG;
4084                 ret = -EBUSY;
4085                 goto out;
4086         }
4087 
4088         /* create a device */
4089         adev = kzalloc(sizeof(*adev), GFP_KERNEL);
4090         if (!adev) {
4091                 dev_err(&ofdev->dev, "failed to allocate device\n");
4092                 initcode = PPC_ADMA_INIT_ALLOC;
4093                 ret = -ENOMEM;
4094                 goto err_adev_alloc;
4095         }
4096 
4097         adev->id = id;
4098         adev->pool_size = pool_size;
4099         /* allocate coherent memory for hardware descriptors */
4100         adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev,
4101                                         adev->pool_size, &adev->dma_desc_pool,
4102                                         GFP_KERNEL);
4103         if (adev->dma_desc_pool_virt == NULL) {
4104                 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
4105                         "memory for hardware descriptors\n",
4106                         adev->pool_size);
4107                 initcode = PPC_ADMA_INIT_COHERENT;
4108                 ret = -ENOMEM;
4109                 goto err_dma_alloc;
4110         }
4111         dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n",
4112                 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
4113 
4114         regs = ioremap(res.start, resource_size(&res));
4115         if (!regs) {
4116                 dev_err(&ofdev->dev, "failed to ioremap regs!\n");
4117                 ret = -ENOMEM;
4118                 goto err_regs_alloc;
4119         }
4120 
4121         if (adev->id == PPC440SPE_XOR_ID) {
4122                 adev->xor_reg = regs;
4123                 /* Reset XOR */
4124                 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
4125                 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
4126         } else {
4127                 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
4128                                    DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4129                 adev->dma_reg = regs;
4130                 /* DMAx_FIFO_SIZE is defined in bytes,
4131                  * <fsiz> - is defined in number of CDB pointers (8byte).
4132                  * DMA FIFO Length = CSlength + CPlength, where
4133                  * CSlength = CPlength = (fsiz + 1) * 8.
4134                  */
4135                 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
4136                           &adev->dma_reg->fsiz);
4137                 /* Configure DMA engine */
4138                 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
4139                           &adev->dma_reg->cfg);
4140                 /* Clear Status */
4141                 iowrite32(~0, &adev->dma_reg->dsts);
4142         }
4143 
4144         adev->dev = &ofdev->dev;
4145         adev->common.dev = &ofdev->dev;
4146         INIT_LIST_HEAD(&adev->common.channels);
4147         platform_set_drvdata(ofdev, adev);
4148 
4149         /* create a channel */
4150         chan = kzalloc(sizeof(*chan), GFP_KERNEL);
4151         if (!chan) {
4152                 dev_err(&ofdev->dev, "can't allocate channel structure\n");
4153                 initcode = PPC_ADMA_INIT_CHANNEL;
4154                 ret = -ENOMEM;
4155                 goto err_chan_alloc;
4156         }
4157 
4158         spin_lock_init(&chan->lock);
4159         INIT_LIST_HEAD(&chan->chain);
4160         INIT_LIST_HEAD(&chan->all_slots);
4161         chan->device = adev;
4162         chan->common.device = &adev->common;
4163         dma_cookie_init(&chan->common);
4164         list_add_tail(&chan->common.device_node, &adev->common.channels);
4165         tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet,
4166                      (unsigned long)chan);
4167 
4168         /* allocate and map helper pages for async validation or
4169          * async_mult/async_sum_product operations on DMA0/1.
4170          */
4171         if (adev->id != PPC440SPE_XOR_ID) {
4172                 chan->pdest_page = alloc_page(GFP_KERNEL);
4173                 chan->qdest_page = alloc_page(GFP_KERNEL);
4174                 if (!chan->pdest_page ||
4175                     !chan->qdest_page) {
4176                         if (chan->pdest_page)
4177                                 __free_page(chan->pdest_page);
4178                         if (chan->qdest_page)
4179                                 __free_page(chan->qdest_page);
4180                         ret = -ENOMEM;
4181                         goto err_page_alloc;
4182                 }
4183                 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
4184                                            PAGE_SIZE, DMA_BIDIRECTIONAL);
4185                 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
4186                                            PAGE_SIZE, DMA_BIDIRECTIONAL);
4187         }
4188 
4189         ref = kmalloc(sizeof(*ref), GFP_KERNEL);
4190         if (ref) {
4191                 ref->chan = &chan->common;
4192                 INIT_LIST_HEAD(&ref->node);
4193                 list_add_tail(&ref->node, &ppc440spe_adma_chan_list);
4194         } else {
4195                 dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
4196                 ret = -ENOMEM;
4197                 goto err_ref_alloc;
4198         }
4199 
4200         ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode);
4201         if (ret)
4202                 goto err_irq;
4203 
4204         ppc440spe_adma_init_capabilities(adev);
4205 
4206         ret = dma_async_device_register(&adev->common);
4207         if (ret) {
4208                 initcode = PPC_ADMA_INIT_REGISTER;
4209                 dev_err(&ofdev->dev, "failed to register dma device\n");
4210                 goto err_dev_reg;
4211         }
4212 
4213         goto out;
4214 
4215 err_dev_reg:
4216         ppc440spe_adma_release_irqs(adev, chan);
4217 err_irq:
4218         list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
4219                 if (chan == to_ppc440spe_adma_chan(ref->chan)) {
4220                         list_del(&ref->node);
4221                         kfree(ref);
4222                 }
4223         }
4224 err_ref_alloc:
4225         if (adev->id != PPC440SPE_XOR_ID) {
4226                 dma_unmap_page(&ofdev->dev, chan->pdest,
4227                                PAGE_SIZE, DMA_BIDIRECTIONAL);
4228                 dma_unmap_page(&ofdev->dev, chan->qdest,
4229                                PAGE_SIZE, DMA_BIDIRECTIONAL);
4230                 __free_page(chan->pdest_page);
4231                 __free_page(chan->qdest_page);
4232         }
4233 err_page_alloc:
4234         kfree(chan);
4235 err_chan_alloc:
4236         if (adev->id == PPC440SPE_XOR_ID)
4237                 iounmap(adev->xor_reg);
4238         else
4239                 iounmap(adev->dma_reg);
4240 err_regs_alloc:
4241         dma_free_coherent(adev->dev, adev->pool_size,
4242                           adev->dma_desc_pool_virt,
4243                           adev->dma_desc_pool);
4244 err_dma_alloc:
4245         kfree(adev);
4246 err_adev_alloc:
4247         release_mem_region(res.start, resource_size(&res));
4248 out:
4249         if (id < PPC440SPE_ADMA_ENGINES_NUM)
4250                 ppc440spe_adma_devices[id] = initcode;
4251 
4252         return ret;
4253 }
4254 
4255 /**
4256  * ppc440spe_adma_remove - remove the asynch device
4257  */
4258 static int ppc440spe_adma_remove(struct platform_device *ofdev)
4259 {
4260         struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev);
4261         struct device_node *np = ofdev->dev.of_node;
4262         struct resource res;
4263         struct dma_chan *chan, *_chan;
4264         struct ppc_dma_chan_ref *ref, *_ref;
4265         struct ppc440spe_adma_chan *ppc440spe_chan;
4266 
4267         if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
4268                 ppc440spe_adma_devices[adev->id] = -1;
4269 
4270         dma_async_device_unregister(&adev->common);
4271 
4272         list_for_each_entry_safe(chan, _chan, &adev->common.channels,
4273                                  device_node) {
4274                 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4275                 ppc440spe_adma_release_irqs(adev, ppc440spe_chan);
4276                 tasklet_kill(&ppc440spe_chan->irq_tasklet);
4277                 if (adev->id != PPC440SPE_XOR_ID) {
4278                         dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
4279                                         PAGE_SIZE, DMA_BIDIRECTIONAL);
4280                         dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
4281                                         PAGE_SIZE, DMA_BIDIRECTIONAL);
4282                         __free_page(ppc440spe_chan->pdest_page);
4283                         __free_page(ppc440spe_chan->qdest_page);
4284                 }
4285                 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
4286                                          node) {
4287                         if (ppc440spe_chan ==
4288                             to_ppc440spe_adma_chan(ref->chan)) {
4289                                 list_del(&ref->node);
4290                                 kfree(ref);
4291                         }
4292                 }
4293                 list_del(&chan->device_node);
4294                 kfree(ppc440spe_chan);
4295         }
4296 
4297         dma_free_coherent(adev->dev, adev->pool_size,
4298                           adev->dma_desc_pool_virt, adev->dma_desc_pool);
4299         if (adev->id == PPC440SPE_XOR_ID)
4300                 iounmap(adev->xor_reg);
4301         else
4302                 iounmap(adev->dma_reg);
4303         of_address_to_resource(np, 0, &res);
4304         release_mem_region(res.start, resource_size(&res));
4305         kfree(adev);
4306         return 0;
4307 }
4308 
4309 /*
4310  * /sys driver interface to enable h/w RAID-6 capabilities
4311  * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4312  * directory are "devices", "enable" and "poly".
4313  * "devices" shows available engines.
4314  * "enable" is used to enable RAID-6 capabilities or to check
4315  * whether these has been activated.
4316  * "poly" allows setting/checking used polynomial (for PPC440SPe only).
4317  */
4318 
4319 static ssize_t show_ppc440spe_devices(struct device_driver *dev, char *buf)
4320 {
4321         ssize_t size = 0;
4322         int i;
4323 
4324         for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
4325                 if (ppc440spe_adma_devices[i] == -1)
4326                         continue;
4327                 size += snprintf(buf + size, PAGE_SIZE - size,
4328                                  "PPC440SP(E)-ADMA.%d: %s\n", i,
4329                                  ppc_adma_errors[ppc440spe_adma_devices[i]]);
4330         }
4331         return size;
4332 }
4333 
4334 static ssize_t show_ppc440spe_r6enable(struct device_driver *dev, char *buf)
4335 {
4336         return snprintf(buf, PAGE_SIZE,
4337                         "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4338                         ppc440spe_r6_enabled ? "EN" : "DIS");
4339 }
4340 
4341 static ssize_t store_ppc440spe_r6enable(struct device_driver *dev,
4342                                         const char *buf, size_t count)
4343 {
4344         unsigned long val;
4345 
4346         if (!count || count > 11)
4347                 return -EINVAL;
4348 
4349         if (!ppc440spe_r6_tchan)
4350                 return -EFAULT;
4351 
4352         /* Write a key */
4353         sscanf(buf, "%lx", &val);
4354         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
4355         isync();
4356 
4357         /* Verify whether it really works now */
4358         if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) {
4359                 pr_info("PPC440SP(e) RAID-6 has been activated "
4360                         "successfully\n");
4361                 ppc440spe_r6_enabled = 1;
4362         } else {
4363                 pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4364                         " Error key ?\n");
4365                 ppc440spe_r6_enabled = 0;
4366         }
4367         return count;
4368 }
4369 
4370 static ssize_t show_ppc440spe_r6poly(struct device_driver *dev, char *buf)
4371 {
4372         ssize_t size = 0;
4373         u32 reg;
4374 
4375 #ifdef CONFIG_440SP
4376         /* 440SP has fixed polynomial */
4377         reg = 0x4d;
4378 #else
4379         reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4380         reg >>= MQ0_CFBHL_POLY;
4381         reg &= 0xFF;
4382 #endif
4383 
4384         size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver "
4385                         "uses 0x1%02x polynomial.\n", reg);
4386         return size;
4387 }
4388 
4389 static ssize_t store_ppc440spe_r6poly(struct device_driver *dev,
4390                                       const char *buf, size_t count)
4391 {
4392         unsigned long reg, val;
4393 
4394 #ifdef CONFIG_440SP
4395         /* 440SP uses default 0x14D polynomial only */
4396         return -EINVAL;
4397 #endif
4398 
4399         if (!count || count > 6)
4400                 return -EINVAL;
4401 
4402         /* e.g., 0x14D or 0x11D */
4403         sscanf(buf, "%lx", &val);
4404 
4405         if (val & ~0x1FF)
4406                 return -EINVAL;
4407 
4408         val &= 0xFF;
4409         reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4410         reg &= ~(0xFF << MQ0_CFBHL_POLY);
4411         reg |= val << MQ0_CFBHL_POLY;
4412         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
4413 
4414         return count;
4415 }
4416 
4417 static DRIVER_ATTR(devices, S_IRUGO, show_ppc440spe_devices, NULL);
4418 static DRIVER_ATTR(enable, S_IRUGO | S_IWUSR, show_ppc440spe_r6enable,
4419                    store_ppc440spe_r6enable);
4420 static DRIVER_ATTR(poly, S_IRUGO | S_IWUSR, show_ppc440spe_r6poly,
4421                    store_ppc440spe_r6poly);
4422 
4423 /*
4424  * Common initialisation for RAID engines; allocate memory for
4425  * DMAx FIFOs, perform configuration common for all DMA engines.
4426  * Further DMA engine specific configuration is done at probe time.
4427  */
4428 static int ppc440spe_configure_raid_devices(void)
4429 {
4430         struct device_node *np;
4431         struct resource i2o_res;
4432         struct i2o_regs __iomem *i2o_reg;
4433         dcr_host_t i2o_dcr_host;
4434         unsigned int dcr_base, dcr_len;
4435         int i, ret;
4436 
4437         np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4438         if (!np) {
4439                 pr_err("%s: can't find I2O device tree node\n",
4440                         __func__);
4441                 return -ENODEV;
4442         }
4443 
4444         if (of_address_to_resource(np, 0, &i2o_res)) {
4445                 of_node_put(np);
4446                 return -EINVAL;
4447         }
4448 
4449         i2o_reg = of_iomap(np, 0);
4450         if (!i2o_reg) {
4451                 pr_err("%s: failed to map I2O registers\n", __func__);
4452                 of_node_put(np);
4453                 return -EINVAL;
4454         }
4455 
4456         /* Get I2O DCRs base */
4457         dcr_base = dcr_resource_start(np, 0);
4458         dcr_len = dcr_resource_len(np, 0);
4459         if (!dcr_base && !dcr_len) {
4460                 pr_err("%s: can't get DCR registers base/len!\n",
4461                         np->full_name);
4462                 of_node_put(np);
4463                 iounmap(i2o_reg);
4464                 return -ENODEV;
4465         }
4466 
4467         i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
4468         if (!DCR_MAP_OK(i2o_dcr_host)) {
4469                 pr_err("%s: failed to map DCRs!\n", np->full_name);
4470                 of_node_put(np);
4471                 iounmap(i2o_reg);
4472                 return -ENODEV;
4473         }
4474         of_node_put(np);
4475 
4476         /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4477          * the base address of FIFO memory space.
4478          * Actually we need twice more physical memory than programmed in the
4479          * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4480          */
4481         ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
4482                                          GFP_KERNEL);
4483         if (!ppc440spe_dma_fifo_buf) {
4484                 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
4485                 iounmap(i2o_reg);
4486                 dcr_unmap(i2o_dcr_host, dcr_len);
4487                 return -ENOMEM;
4488         }
4489 
4490         /*
4491          * Configure h/w
4492          */
4493         /* Reset I2O/DMA */
4494         mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
4495         mtdcri(SDR0, DCRN_SDR0_SRST, 0);
4496 
4497         /* Setup the base address of mmaped registers */
4498         dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
4499         dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
4500                                                 I2O_REG_ENABLE);
4501         dcr_unmap(i2o_dcr_host, dcr_len);
4502 
4503         /* Setup FIFO memory space base address */
4504         iowrite32(0, &i2o_reg->ifbah);
4505         iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
4506 
4507         /* set zero FIFO size for I2O, so the whole
4508          * ppc440spe_dma_fifo_buf is used by DMAs.
4509          * DMAx_FIFOs will be configured while probe.
4510          */
4511         iowrite32(0, &i2o_reg->ifsiz);
4512         iounmap(i2o_reg);
4513 
4514         /* To prepare WXOR/RXOR functionality we need access to
4515          * Memory Queue Module DCRs (finally it will be enabled
4516          * via /sys interface of the ppc440spe ADMA driver).
4517          */
4518         np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe");
4519         if (!np) {
4520                 pr_err("%s: can't find MQ device tree node\n",
4521                         __func__);
4522                 ret = -ENODEV;
4523                 goto out_free;
4524         }
4525 
4526         /* Get MQ DCRs base */
4527         dcr_base = dcr_resource_start(np, 0);
4528         dcr_len = dcr_resource_len(np, 0);
4529         if (!dcr_base && !dcr_len) {
4530                 pr_err("%s: can't get DCR registers base/len!\n",
4531                         np->full_name);
4532                 ret = -ENODEV;
4533                 goto out_mq;
4534         }
4535 
4536         ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
4537         if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
4538                 pr_err("%s: failed to map DCRs!\n", np->full_name);
4539                 ret = -ENODEV;
4540                 goto out_mq;
4541         }
4542         of_node_put(np);
4543         ppc440spe_mq_dcr_len = dcr_len;
4544 
4545         /* Set HB alias */
4546         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
4547 
4548         /* Set:
4549          * - LL transaction passing limit to 1;
4550          * - Memory controller cycle limit to 1;
4551          * - Galois Polynomial to 0x14d (default)
4552          */
4553         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
4554                   (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
4555                   (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
4556 
4557         atomic_set(&ppc440spe_adma_err_irq_ref, 0);
4558         for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
4559                 ppc440spe_adma_devices[i] = -1;
4560 
4561         return 0;
4562 
4563 out_mq:
4564         of_node_put(np);
4565 out_free:
4566         kfree(ppc440spe_dma_fifo_buf);
4567         return ret;
4568 }
4569 
4570 static const struct of_device_id ppc440spe_adma_of_match[] = {
4571         { .compatible   = "ibm,dma-440spe", },
4572         { .compatible   = "amcc,xor-accelerator", },
4573         {},
4574 };
4575 MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
4576 
4577 static struct platform_driver ppc440spe_adma_driver = {
4578         .probe = ppc440spe_adma_probe,
4579         .remove = ppc440spe_adma_remove,
4580         .driver = {
4581                 .name = "PPC440SP(E)-ADMA",
4582                 .owner = THIS_MODULE,
4583                 .of_match_table = ppc440spe_adma_of_match,
4584         },
4585 };
4586 
4587 static __init int ppc440spe_adma_init(void)
4588 {
4589         int ret;
4590 
4591         ret = ppc440spe_configure_raid_devices();
4592         if (ret)
4593                 return ret;
4594 
4595         ret = platform_driver_register(&ppc440spe_adma_driver);
4596         if (ret) {
4597                 pr_err("%s: failed to register platform driver\n",
4598                         __func__);
4599                 goto out_reg;
4600         }
4601 
4602         /* Initialization status */
4603         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4604                                  &driver_attr_devices);
4605         if (ret)
4606                 goto out_dev;
4607 
4608         /* RAID-6 h/w enable entry */
4609         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4610                                  &driver_attr_enable);
4611         if (ret)
4612                 goto out_en;
4613 
4614         /* GF polynomial to use */
4615         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4616                                  &driver_attr_poly);
4617         if (!ret)
4618                 return ret;
4619 
4620         driver_remove_file(&ppc440spe_adma_driver.driver,
4621                            &driver_attr_enable);
4622 out_en:
4623         driver_remove_file(&ppc440spe_adma_driver.driver,
4624                            &driver_attr_devices);
4625 out_dev:
4626         /* User will not be able to enable h/w RAID-6 */
4627         pr_err("%s: failed to create RAID-6 driver interface\n",
4628                 __func__);
4629         platform_driver_unregister(&ppc440spe_adma_driver);
4630 out_reg:
4631         dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4632         kfree(ppc440spe_dma_fifo_buf);
4633         return ret;
4634 }
4635 
4636 static void __exit ppc440spe_adma_exit(void)
4637 {
4638         driver_remove_file(&ppc440spe_adma_driver.driver,
4639                            &driver_attr_poly);
4640         driver_remove_file(&ppc440spe_adma_driver.driver,
4641                            &driver_attr_enable);
4642         driver_remove_file(&ppc440spe_adma_driver.driver,
4643                            &driver_attr_devices);
4644         platform_driver_unregister(&ppc440spe_adma_driver);
4645         dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4646         kfree(ppc440spe_dma_fifo_buf);
4647 }
4648 
4649 arch_initcall(ppc440spe_adma_init);
4650 module_exit(ppc440spe_adma_exit);
4651 
4652 MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
4653 MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
4654 MODULE_LICENSE("GPL");
4655 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us