Version:  2.0.40 2.2.26 2.4.37 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7

Linux/drivers/crypto/talitos.c

  1 /*
  2  * talitos - Freescale Integrated Security Engine (SEC) device driver
  3  *
  4  * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
  5  *
  6  * Scatterlist Crypto API glue code copied from files with the following:
  7  * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
  8  *
  9  * Crypto algorithm registration code copied from hifn driver:
 10  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
 11  * All rights reserved.
 12  *
 13  * This program is free software; you can redistribute it and/or modify
 14  * it under the terms of the GNU General Public License as published by
 15  * the Free Software Foundation; either version 2 of the License, or
 16  * (at your option) any later version.
 17  *
 18  * This program is distributed in the hope that it will be useful,
 19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 21  * GNU General Public License for more details.
 22  *
 23  * You should have received a copy of the GNU General Public License
 24  * along with this program; if not, write to the Free Software
 25  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 26  */
 27 
 28 #include <linux/kernel.h>
 29 #include <linux/module.h>
 30 #include <linux/mod_devicetable.h>
 31 #include <linux/device.h>
 32 #include <linux/interrupt.h>
 33 #include <linux/crypto.h>
 34 #include <linux/hw_random.h>
 35 #include <linux/of_address.h>
 36 #include <linux/of_irq.h>
 37 #include <linux/of_platform.h>
 38 #include <linux/dma-mapping.h>
 39 #include <linux/io.h>
 40 #include <linux/spinlock.h>
 41 #include <linux/rtnetlink.h>
 42 #include <linux/slab.h>
 43 
 44 #include <crypto/algapi.h>
 45 #include <crypto/aes.h>
 46 #include <crypto/des.h>
 47 #include <crypto/sha.h>
 48 #include <crypto/md5.h>
 49 #include <crypto/internal/aead.h>
 50 #include <crypto/authenc.h>
 51 #include <crypto/skcipher.h>
 52 #include <crypto/hash.h>
 53 #include <crypto/internal/hash.h>
 54 #include <crypto/scatterwalk.h>
 55 
 56 #include "talitos.h"
 57 
 58 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
 59                            bool is_sec1)
 60 {
 61         ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
 62         if (!is_sec1)
 63                 ptr->eptr = upper_32_bits(dma_addr);
 64 }
 65 
 66 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
 67                              struct talitos_ptr *src_ptr, bool is_sec1)
 68 {
 69         dst_ptr->ptr = src_ptr->ptr;
 70         if (!is_sec1)
 71                 dst_ptr->eptr = src_ptr->eptr;
 72 }
 73 
 74 static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len,
 75                                bool is_sec1)
 76 {
 77         if (is_sec1) {
 78                 ptr->res = 0;
 79                 ptr->len1 = cpu_to_be16(len);
 80         } else {
 81                 ptr->len = cpu_to_be16(len);
 82         }
 83 }
 84 
 85 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
 86                                            bool is_sec1)
 87 {
 88         if (is_sec1)
 89                 return be16_to_cpu(ptr->len1);
 90         else
 91                 return be16_to_cpu(ptr->len);
 92 }
 93 
 94 static void to_talitos_ptr_extent_clear(struct talitos_ptr *ptr, bool is_sec1)
 95 {
 96         if (!is_sec1)
 97                 ptr->j_extent = 0;
 98 }
 99 
100 /*
101  * map virtual single (contiguous) pointer to h/w descriptor pointer
102  */
103 static void map_single_talitos_ptr(struct device *dev,
104                                    struct talitos_ptr *ptr,
105                                    unsigned int len, void *data,
106                                    enum dma_data_direction dir)
107 {
108         dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
109         struct talitos_private *priv = dev_get_drvdata(dev);
110         bool is_sec1 = has_ftr_sec1(priv);
111 
112         to_talitos_ptr_len(ptr, len, is_sec1);
113         to_talitos_ptr(ptr, dma_addr, is_sec1);
114         to_talitos_ptr_extent_clear(ptr, is_sec1);
115 }
116 
117 /*
118  * unmap bus single (contiguous) h/w descriptor pointer
119  */
120 static void unmap_single_talitos_ptr(struct device *dev,
121                                      struct talitos_ptr *ptr,
122                                      enum dma_data_direction dir)
123 {
124         struct talitos_private *priv = dev_get_drvdata(dev);
125         bool is_sec1 = has_ftr_sec1(priv);
126 
127         dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
128                          from_talitos_ptr_len(ptr, is_sec1), dir);
129 }
130 
131 static int reset_channel(struct device *dev, int ch)
132 {
133         struct talitos_private *priv = dev_get_drvdata(dev);
134         unsigned int timeout = TALITOS_TIMEOUT;
135         bool is_sec1 = has_ftr_sec1(priv);
136 
137         if (is_sec1) {
138                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
139                           TALITOS1_CCCR_LO_RESET);
140 
141                 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
142                         TALITOS1_CCCR_LO_RESET) && --timeout)
143                         cpu_relax();
144         } else {
145                 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
146                           TALITOS2_CCCR_RESET);
147 
148                 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
149                         TALITOS2_CCCR_RESET) && --timeout)
150                         cpu_relax();
151         }
152 
153         if (timeout == 0) {
154                 dev_err(dev, "failed to reset channel %d\n", ch);
155                 return -EIO;
156         }
157 
158         /* set 36-bit addressing, done writeback enable and done IRQ enable */
159         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
160                   TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
161 
162         /* and ICCR writeback, if available */
163         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
164                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
165                           TALITOS_CCCR_LO_IWSE);
166 
167         return 0;
168 }
169 
170 static int reset_device(struct device *dev)
171 {
172         struct talitos_private *priv = dev_get_drvdata(dev);
173         unsigned int timeout = TALITOS_TIMEOUT;
174         bool is_sec1 = has_ftr_sec1(priv);
175         u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
176 
177         setbits32(priv->reg + TALITOS_MCR, mcr);
178 
179         while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
180                && --timeout)
181                 cpu_relax();
182 
183         if (priv->irq[1]) {
184                 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
185                 setbits32(priv->reg + TALITOS_MCR, mcr);
186         }
187 
188         if (timeout == 0) {
189                 dev_err(dev, "failed to reset device\n");
190                 return -EIO;
191         }
192 
193         return 0;
194 }
195 
196 /*
197  * Reset and initialize the device
198  */
199 static int init_device(struct device *dev)
200 {
201         struct talitos_private *priv = dev_get_drvdata(dev);
202         int ch, err;
203         bool is_sec1 = has_ftr_sec1(priv);
204 
205         /*
206          * Master reset
207          * errata documentation: warning: certain SEC interrupts
208          * are not fully cleared by writing the MCR:SWR bit,
209          * set bit twice to completely reset
210          */
211         err = reset_device(dev);
212         if (err)
213                 return err;
214 
215         err = reset_device(dev);
216         if (err)
217                 return err;
218 
219         /* reset channels */
220         for (ch = 0; ch < priv->num_channels; ch++) {
221                 err = reset_channel(dev, ch);
222                 if (err)
223                         return err;
224         }
225 
226         /* enable channel done and error interrupts */
227         if (is_sec1) {
228                 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
229                 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
230                 /* disable parity error check in DEU (erroneous? test vect.) */
231                 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
232         } else {
233                 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
234                 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
235         }
236 
237         /* disable integrity check error interrupts (use writeback instead) */
238         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
239                 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
240                           TALITOS_MDEUICR_LO_ICE);
241 
242         return 0;
243 }
244 
245 /**
246  * talitos_submit - submits a descriptor to the device for processing
247  * @dev:        the SEC device to be used
248  * @ch:         the SEC device channel to be used
249  * @desc:       the descriptor to be processed by the device
250  * @callback:   whom to call when processing is complete
251  * @context:    a handle for use by caller (optional)
252  *
253  * desc must contain valid dma-mapped (bus physical) address pointers.
254  * callback must check err and feedback in descriptor header
255  * for device processing status.
256  */
257 int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
258                    void (*callback)(struct device *dev,
259                                     struct talitos_desc *desc,
260                                     void *context, int error),
261                    void *context)
262 {
263         struct talitos_private *priv = dev_get_drvdata(dev);
264         struct talitos_request *request;
265         unsigned long flags;
266         int head;
267         bool is_sec1 = has_ftr_sec1(priv);
268 
269         spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
270 
271         if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
272                 /* h/w fifo is full */
273                 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
274                 return -EAGAIN;
275         }
276 
277         head = priv->chan[ch].head;
278         request = &priv->chan[ch].fifo[head];
279 
280         /* map descriptor and save caller data */
281         if (is_sec1) {
282                 desc->hdr1 = desc->hdr;
283                 desc->next_desc = 0;
284                 request->dma_desc = dma_map_single(dev, &desc->hdr1,
285                                                    TALITOS_DESC_SIZE,
286                                                    DMA_BIDIRECTIONAL);
287         } else {
288                 request->dma_desc = dma_map_single(dev, desc,
289                                                    TALITOS_DESC_SIZE,
290                                                    DMA_BIDIRECTIONAL);
291         }
292         request->callback = callback;
293         request->context = context;
294 
295         /* increment fifo head */
296         priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
297 
298         smp_wmb();
299         request->desc = desc;
300 
301         /* GO! */
302         wmb();
303         out_be32(priv->chan[ch].reg + TALITOS_FF,
304                  upper_32_bits(request->dma_desc));
305         out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
306                  lower_32_bits(request->dma_desc));
307 
308         spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
309 
310         return -EINPROGRESS;
311 }
312 EXPORT_SYMBOL(talitos_submit);
313 
314 /*
315  * process what was done, notify callback of error if not
316  */
317 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
318 {
319         struct talitos_private *priv = dev_get_drvdata(dev);
320         struct talitos_request *request, saved_req;
321         unsigned long flags;
322         int tail, status;
323         bool is_sec1 = has_ftr_sec1(priv);
324 
325         spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
326 
327         tail = priv->chan[ch].tail;
328         while (priv->chan[ch].fifo[tail].desc) {
329                 __be32 hdr;
330 
331                 request = &priv->chan[ch].fifo[tail];
332 
333                 /* descriptors with their done bits set don't get the error */
334                 rmb();
335                 hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr;
336 
337                 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
338                         status = 0;
339                 else
340                         if (!error)
341                                 break;
342                         else
343                                 status = error;
344 
345                 dma_unmap_single(dev, request->dma_desc,
346                                  TALITOS_DESC_SIZE,
347                                  DMA_BIDIRECTIONAL);
348 
349                 /* copy entries so we can call callback outside lock */
350                 saved_req.desc = request->desc;
351                 saved_req.callback = request->callback;
352                 saved_req.context = request->context;
353 
354                 /* release request entry in fifo */
355                 smp_wmb();
356                 request->desc = NULL;
357 
358                 /* increment fifo tail */
359                 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
360 
361                 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
362 
363                 atomic_dec(&priv->chan[ch].submit_count);
364 
365                 saved_req.callback(dev, saved_req.desc, saved_req.context,
366                                    status);
367                 /* channel may resume processing in single desc error case */
368                 if (error && !reset_ch && status == error)
369                         return;
370                 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
371                 tail = priv->chan[ch].tail;
372         }
373 
374         spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
375 }
376 
377 /*
378  * process completed requests for channels that have done status
379  */
380 #define DEF_TALITOS1_DONE(name, ch_done_mask)                           \
381 static void talitos1_done_##name(unsigned long data)                    \
382 {                                                                       \
383         struct device *dev = (struct device *)data;                     \
384         struct talitos_private *priv = dev_get_drvdata(dev);            \
385         unsigned long flags;                                            \
386                                                                         \
387         if (ch_done_mask & 0x10000000)                                  \
388                 flush_channel(dev, 0, 0, 0);                    \
389         if (priv->num_channels == 1)                                    \
390                 goto out;                                               \
391         if (ch_done_mask & 0x40000000)                                  \
392                 flush_channel(dev, 1, 0, 0);                    \
393         if (ch_done_mask & 0x00010000)                                  \
394                 flush_channel(dev, 2, 0, 0);                    \
395         if (ch_done_mask & 0x00040000)                                  \
396                 flush_channel(dev, 3, 0, 0);                    \
397                                                                         \
398 out:                                                                    \
399         /* At this point, all completed channels have been processed */ \
400         /* Unmask done interrupts for channels completed later on. */   \
401         spin_lock_irqsave(&priv->reg_lock, flags);                      \
402         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
403         clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);    \
404         spin_unlock_irqrestore(&priv->reg_lock, flags);                 \
405 }
406 
407 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
408 
409 #define DEF_TALITOS2_DONE(name, ch_done_mask)                           \
410 static void talitos2_done_##name(unsigned long data)                    \
411 {                                                                       \
412         struct device *dev = (struct device *)data;                     \
413         struct talitos_private *priv = dev_get_drvdata(dev);            \
414         unsigned long flags;                                            \
415                                                                         \
416         if (ch_done_mask & 1)                                           \
417                 flush_channel(dev, 0, 0, 0);                            \
418         if (priv->num_channels == 1)                                    \
419                 goto out;                                               \
420         if (ch_done_mask & (1 << 2))                                    \
421                 flush_channel(dev, 1, 0, 0);                            \
422         if (ch_done_mask & (1 << 4))                                    \
423                 flush_channel(dev, 2, 0, 0);                            \
424         if (ch_done_mask & (1 << 6))                                    \
425                 flush_channel(dev, 3, 0, 0);                            \
426                                                                         \
427 out:                                                                    \
428         /* At this point, all completed channels have been processed */ \
429         /* Unmask done interrupts for channels completed later on. */   \
430         spin_lock_irqsave(&priv->reg_lock, flags);                      \
431         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
432         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);    \
433         spin_unlock_irqrestore(&priv->reg_lock, flags);                 \
434 }
435 
436 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
437 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
438 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
439 
440 /*
441  * locate current (offending) descriptor
442  */
443 static u32 current_desc_hdr(struct device *dev, int ch)
444 {
445         struct talitos_private *priv = dev_get_drvdata(dev);
446         int tail, iter;
447         dma_addr_t cur_desc;
448 
449         cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
450         cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
451 
452         if (!cur_desc) {
453                 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
454                 return 0;
455         }
456 
457         tail = priv->chan[ch].tail;
458 
459         iter = tail;
460         while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) {
461                 iter = (iter + 1) & (priv->fifo_len - 1);
462                 if (iter == tail) {
463                         dev_err(dev, "couldn't locate current descriptor\n");
464                         return 0;
465                 }
466         }
467 
468         return priv->chan[ch].fifo[iter].desc->hdr;
469 }
470 
471 /*
472  * user diagnostics; report root cause of error based on execution unit status
473  */
474 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
475 {
476         struct talitos_private *priv = dev_get_drvdata(dev);
477         int i;
478 
479         if (!desc_hdr)
480                 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
481 
482         switch (desc_hdr & DESC_HDR_SEL0_MASK) {
483         case DESC_HDR_SEL0_AFEU:
484                 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
485                         in_be32(priv->reg_afeu + TALITOS_EUISR),
486                         in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
487                 break;
488         case DESC_HDR_SEL0_DEU:
489                 dev_err(dev, "DEUISR 0x%08x_%08x\n",
490                         in_be32(priv->reg_deu + TALITOS_EUISR),
491                         in_be32(priv->reg_deu + TALITOS_EUISR_LO));
492                 break;
493         case DESC_HDR_SEL0_MDEUA:
494         case DESC_HDR_SEL0_MDEUB:
495                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
496                         in_be32(priv->reg_mdeu + TALITOS_EUISR),
497                         in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
498                 break;
499         case DESC_HDR_SEL0_RNG:
500                 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
501                         in_be32(priv->reg_rngu + TALITOS_ISR),
502                         in_be32(priv->reg_rngu + TALITOS_ISR_LO));
503                 break;
504         case DESC_HDR_SEL0_PKEU:
505                 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
506                         in_be32(priv->reg_pkeu + TALITOS_EUISR),
507                         in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
508                 break;
509         case DESC_HDR_SEL0_AESU:
510                 dev_err(dev, "AESUISR 0x%08x_%08x\n",
511                         in_be32(priv->reg_aesu + TALITOS_EUISR),
512                         in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
513                 break;
514         case DESC_HDR_SEL0_CRCU:
515                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
516                         in_be32(priv->reg_crcu + TALITOS_EUISR),
517                         in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
518                 break;
519         case DESC_HDR_SEL0_KEU:
520                 dev_err(dev, "KEUISR 0x%08x_%08x\n",
521                         in_be32(priv->reg_pkeu + TALITOS_EUISR),
522                         in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
523                 break;
524         }
525 
526         switch (desc_hdr & DESC_HDR_SEL1_MASK) {
527         case DESC_HDR_SEL1_MDEUA:
528         case DESC_HDR_SEL1_MDEUB:
529                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
530                         in_be32(priv->reg_mdeu + TALITOS_EUISR),
531                         in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
532                 break;
533         case DESC_HDR_SEL1_CRCU:
534                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
535                         in_be32(priv->reg_crcu + TALITOS_EUISR),
536                         in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
537                 break;
538         }
539 
540         for (i = 0; i < 8; i++)
541                 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
542                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
543                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
544 }
545 
546 /*
547  * recover from error interrupts
548  */
549 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
550 {
551         struct talitos_private *priv = dev_get_drvdata(dev);
552         unsigned int timeout = TALITOS_TIMEOUT;
553         int ch, error, reset_dev = 0;
554         u32 v_lo;
555         bool is_sec1 = has_ftr_sec1(priv);
556         int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
557 
558         for (ch = 0; ch < priv->num_channels; ch++) {
559                 /* skip channels without errors */
560                 if (is_sec1) {
561                         /* bits 29, 31, 17, 19 */
562                         if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
563                                 continue;
564                 } else {
565                         if (!(isr & (1 << (ch * 2 + 1))))
566                                 continue;
567                 }
568 
569                 error = -EINVAL;
570 
571                 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
572 
573                 if (v_lo & TALITOS_CCPSR_LO_DOF) {
574                         dev_err(dev, "double fetch fifo overflow error\n");
575                         error = -EAGAIN;
576                         reset_ch = 1;
577                 }
578                 if (v_lo & TALITOS_CCPSR_LO_SOF) {
579                         /* h/w dropped descriptor */
580                         dev_err(dev, "single fetch fifo overflow error\n");
581                         error = -EAGAIN;
582                 }
583                 if (v_lo & TALITOS_CCPSR_LO_MDTE)
584                         dev_err(dev, "master data transfer error\n");
585                 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
586                         dev_err(dev, is_sec1 ? "pointeur not complete error\n"
587                                              : "s/g data length zero error\n");
588                 if (v_lo & TALITOS_CCPSR_LO_FPZ)
589                         dev_err(dev, is_sec1 ? "parity error\n"
590                                              : "fetch pointer zero error\n");
591                 if (v_lo & TALITOS_CCPSR_LO_IDH)
592                         dev_err(dev, "illegal descriptor header error\n");
593                 if (v_lo & TALITOS_CCPSR_LO_IEU)
594                         dev_err(dev, is_sec1 ? "static assignment error\n"
595                                              : "invalid exec unit error\n");
596                 if (v_lo & TALITOS_CCPSR_LO_EU)
597                         report_eu_error(dev, ch, current_desc_hdr(dev, ch));
598                 if (!is_sec1) {
599                         if (v_lo & TALITOS_CCPSR_LO_GB)
600                                 dev_err(dev, "gather boundary error\n");
601                         if (v_lo & TALITOS_CCPSR_LO_GRL)
602                                 dev_err(dev, "gather return/length error\n");
603                         if (v_lo & TALITOS_CCPSR_LO_SB)
604                                 dev_err(dev, "scatter boundary error\n");
605                         if (v_lo & TALITOS_CCPSR_LO_SRL)
606                                 dev_err(dev, "scatter return/length error\n");
607                 }
608 
609                 flush_channel(dev, ch, error, reset_ch);
610 
611                 if (reset_ch) {
612                         reset_channel(dev, ch);
613                 } else {
614                         setbits32(priv->chan[ch].reg + TALITOS_CCCR,
615                                   TALITOS2_CCCR_CONT);
616                         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
617                         while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
618                                TALITOS2_CCCR_CONT) && --timeout)
619                                 cpu_relax();
620                         if (timeout == 0) {
621                                 dev_err(dev, "failed to restart channel %d\n",
622                                         ch);
623                                 reset_dev = 1;
624                         }
625                 }
626         }
627         if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
628             (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
629                 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
630                         dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
631                                 isr, isr_lo);
632                 else
633                         dev_err(dev, "done overflow, internal time out, or "
634                                 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
635 
636                 /* purge request queues */
637                 for (ch = 0; ch < priv->num_channels; ch++)
638                         flush_channel(dev, ch, -EIO, 1);
639 
640                 /* reset and reinitialize the device */
641                 init_device(dev);
642         }
643 }
644 
645 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
646 static irqreturn_t talitos1_interrupt_##name(int irq, void *data)              \
647 {                                                                              \
648         struct device *dev = data;                                             \
649         struct talitos_private *priv = dev_get_drvdata(dev);                   \
650         u32 isr, isr_lo;                                                       \
651         unsigned long flags;                                                   \
652                                                                                \
653         spin_lock_irqsave(&priv->reg_lock, flags);                             \
654         isr = in_be32(priv->reg + TALITOS_ISR);                                \
655         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
656         /* Acknowledge interrupt */                                            \
657         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
658         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
659                                                                                \
660         if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) {    \
661                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
662                 talitos_error(dev, isr & ch_err_mask, isr_lo);                 \
663         }                                                                      \
664         else {                                                                 \
665                 if (likely(isr & ch_done_mask)) {                              \
666                         /* mask further done interrupts. */                    \
667                         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
668                         /* done_task will unmask done interrupts at exit */    \
669                         tasklet_schedule(&priv->done_task[tlet]);              \
670                 }                                                              \
671                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
672         }                                                                      \
673                                                                                \
674         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
675                                                                 IRQ_NONE;      \
676 }
677 
678 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
679 
680 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
681 static irqreturn_t talitos2_interrupt_##name(int irq, void *data)              \
682 {                                                                              \
683         struct device *dev = data;                                             \
684         struct talitos_private *priv = dev_get_drvdata(dev);                   \
685         u32 isr, isr_lo;                                                       \
686         unsigned long flags;                                                   \
687                                                                                \
688         spin_lock_irqsave(&priv->reg_lock, flags);                             \
689         isr = in_be32(priv->reg + TALITOS_ISR);                                \
690         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
691         /* Acknowledge interrupt */                                            \
692         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
693         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
694                                                                                \
695         if (unlikely(isr & ch_err_mask || isr_lo)) {                           \
696                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
697                 talitos_error(dev, isr & ch_err_mask, isr_lo);                 \
698         }                                                                      \
699         else {                                                                 \
700                 if (likely(isr & ch_done_mask)) {                              \
701                         /* mask further done interrupts. */                    \
702                         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
703                         /* done_task will unmask done interrupts at exit */    \
704                         tasklet_schedule(&priv->done_task[tlet]);              \
705                 }                                                              \
706                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
707         }                                                                      \
708                                                                                \
709         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
710                                                                 IRQ_NONE;      \
711 }
712 
713 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
714 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
715                        0)
716 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
717                        1)
718 
719 /*
720  * hwrng
721  */
722 static int talitos_rng_data_present(struct hwrng *rng, int wait)
723 {
724         struct device *dev = (struct device *)rng->priv;
725         struct talitos_private *priv = dev_get_drvdata(dev);
726         u32 ofl;
727         int i;
728 
729         for (i = 0; i < 20; i++) {
730                 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
731                       TALITOS_RNGUSR_LO_OFL;
732                 if (ofl || !wait)
733                         break;
734                 udelay(10);
735         }
736 
737         return !!ofl;
738 }
739 
740 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
741 {
742         struct device *dev = (struct device *)rng->priv;
743         struct talitos_private *priv = dev_get_drvdata(dev);
744 
745         /* rng fifo requires 64-bit accesses */
746         *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
747         *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
748 
749         return sizeof(u32);
750 }
751 
752 static int talitos_rng_init(struct hwrng *rng)
753 {
754         struct device *dev = (struct device *)rng->priv;
755         struct talitos_private *priv = dev_get_drvdata(dev);
756         unsigned int timeout = TALITOS_TIMEOUT;
757 
758         setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
759         while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
760                  & TALITOS_RNGUSR_LO_RD)
761                && --timeout)
762                 cpu_relax();
763         if (timeout == 0) {
764                 dev_err(dev, "failed to reset rng hw\n");
765                 return -ENODEV;
766         }
767 
768         /* start generating */
769         setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
770 
771         return 0;
772 }
773 
774 static int talitos_register_rng(struct device *dev)
775 {
776         struct talitos_private *priv = dev_get_drvdata(dev);
777         int err;
778 
779         priv->rng.name          = dev_driver_string(dev),
780         priv->rng.init          = talitos_rng_init,
781         priv->rng.data_present  = talitos_rng_data_present,
782         priv->rng.data_read     = talitos_rng_data_read,
783         priv->rng.priv          = (unsigned long)dev;
784 
785         err = hwrng_register(&priv->rng);
786         if (!err)
787                 priv->rng_registered = true;
788 
789         return err;
790 }
791 
792 static void talitos_unregister_rng(struct device *dev)
793 {
794         struct talitos_private *priv = dev_get_drvdata(dev);
795 
796         if (!priv->rng_registered)
797                 return;
798 
799         hwrng_unregister(&priv->rng);
800         priv->rng_registered = false;
801 }
802 
803 /*
804  * crypto alg
805  */
806 #define TALITOS_CRA_PRIORITY            3000
807 #define TALITOS_MAX_KEY_SIZE            96
808 #define TALITOS_MAX_IV_LENGTH           16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
809 
810 struct talitos_ctx {
811         struct device *dev;
812         int ch;
813         __be32 desc_hdr_template;
814         u8 key[TALITOS_MAX_KEY_SIZE];
815         u8 iv[TALITOS_MAX_IV_LENGTH];
816         unsigned int keylen;
817         unsigned int enckeylen;
818         unsigned int authkeylen;
819 };
820 
821 #define HASH_MAX_BLOCK_SIZE             SHA512_BLOCK_SIZE
822 #define TALITOS_MDEU_MAX_CONTEXT_SIZE   TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
823 
824 struct talitos_ahash_req_ctx {
825         u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
826         unsigned int hw_context_size;
827         u8 buf[HASH_MAX_BLOCK_SIZE];
828         u8 bufnext[HASH_MAX_BLOCK_SIZE];
829         unsigned int swinit;
830         unsigned int first;
831         unsigned int last;
832         unsigned int to_hash_later;
833         unsigned int nbuf;
834         struct scatterlist bufsl[2];
835         struct scatterlist *psrc;
836 };
837 
838 struct talitos_export_state {
839         u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
840         u8 buf[HASH_MAX_BLOCK_SIZE];
841         unsigned int swinit;
842         unsigned int first;
843         unsigned int last;
844         unsigned int to_hash_later;
845         unsigned int nbuf;
846 };
847 
848 static int aead_setkey(struct crypto_aead *authenc,
849                        const u8 *key, unsigned int keylen)
850 {
851         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
852         struct crypto_authenc_keys keys;
853 
854         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
855                 goto badkey;
856 
857         if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
858                 goto badkey;
859 
860         memcpy(ctx->key, keys.authkey, keys.authkeylen);
861         memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
862 
863         ctx->keylen = keys.authkeylen + keys.enckeylen;
864         ctx->enckeylen = keys.enckeylen;
865         ctx->authkeylen = keys.authkeylen;
866 
867         return 0;
868 
869 badkey:
870         crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
871         return -EINVAL;
872 }
873 
874 /*
875  * talitos_edesc - s/w-extended descriptor
876  * @src_nents: number of segments in input scatterlist
877  * @dst_nents: number of segments in output scatterlist
878  * @icv_ool: whether ICV is out-of-line
879  * @iv_dma: dma address of iv for checking continuity and link table
880  * @dma_len: length of dma mapped link_tbl space
881  * @dma_link_tbl: bus physical address of link_tbl/buf
882  * @desc: h/w descriptor
883  * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
884  * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
885  *
886  * if decrypting (with authcheck), or either one of src_nents or dst_nents
887  * is greater than 1, an integrity check value is concatenated to the end
888  * of link_tbl data
889  */
890 struct talitos_edesc {
891         int src_nents;
892         int dst_nents;
893         bool icv_ool;
894         dma_addr_t iv_dma;
895         int dma_len;
896         dma_addr_t dma_link_tbl;
897         struct talitos_desc desc;
898         union {
899                 struct talitos_ptr link_tbl[0];
900                 u8 buf[0];
901         };
902 };
903 
904 static void talitos_sg_unmap(struct device *dev,
905                              struct talitos_edesc *edesc,
906                              struct scatterlist *src,
907                              struct scatterlist *dst)
908 {
909         unsigned int src_nents = edesc->src_nents ? : 1;
910         unsigned int dst_nents = edesc->dst_nents ? : 1;
911 
912         if (src != dst) {
913                 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
914 
915                 if (dst) {
916                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
917                 }
918         } else
919                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
920 }
921 
922 static void ipsec_esp_unmap(struct device *dev,
923                             struct talitos_edesc *edesc,
924                             struct aead_request *areq)
925 {
926         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
927         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
928         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
929         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
930 
931         talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
932 
933         if (edesc->dma_len)
934                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
935                                  DMA_BIDIRECTIONAL);
936 }
937 
938 /*
939  * ipsec_esp descriptor callbacks
940  */
941 static void ipsec_esp_encrypt_done(struct device *dev,
942                                    struct talitos_desc *desc, void *context,
943                                    int err)
944 {
945         struct aead_request *areq = context;
946         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
947         unsigned int authsize = crypto_aead_authsize(authenc);
948         struct talitos_edesc *edesc;
949         struct scatterlist *sg;
950         void *icvdata;
951 
952         edesc = container_of(desc, struct talitos_edesc, desc);
953 
954         ipsec_esp_unmap(dev, edesc, areq);
955 
956         /* copy the generated ICV to dst */
957         if (edesc->icv_ool) {
958                 icvdata = &edesc->link_tbl[edesc->src_nents +
959                                            edesc->dst_nents + 2];
960                 sg = sg_last(areq->dst, edesc->dst_nents);
961                 memcpy((char *)sg_virt(sg) + sg->length - authsize,
962                        icvdata, authsize);
963         }
964 
965         kfree(edesc);
966 
967         aead_request_complete(areq, err);
968 }
969 
970 static void ipsec_esp_decrypt_swauth_done(struct device *dev,
971                                           struct talitos_desc *desc,
972                                           void *context, int err)
973 {
974         struct aead_request *req = context;
975         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
976         unsigned int authsize = crypto_aead_authsize(authenc);
977         struct talitos_edesc *edesc;
978         struct scatterlist *sg;
979         char *oicv, *icv;
980 
981         edesc = container_of(desc, struct talitos_edesc, desc);
982 
983         ipsec_esp_unmap(dev, edesc, req);
984 
985         if (!err) {
986                 /* auth check */
987                 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
988                 icv = (char *)sg_virt(sg) + sg->length - authsize;
989 
990                 if (edesc->dma_len) {
991                         oicv = (char *)&edesc->link_tbl[edesc->src_nents +
992                                                         edesc->dst_nents + 2];
993                         if (edesc->icv_ool)
994                                 icv = oicv + authsize;
995                 } else
996                         oicv = (char *)&edesc->link_tbl[0];
997 
998                 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
999         }
1000 
1001         kfree(edesc);
1002 
1003         aead_request_complete(req, err);
1004 }
1005 
1006 static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
1007                                           struct talitos_desc *desc,
1008                                           void *context, int err)
1009 {
1010         struct aead_request *req = context;
1011         struct talitos_edesc *edesc;
1012 
1013         edesc = container_of(desc, struct talitos_edesc, desc);
1014 
1015         ipsec_esp_unmap(dev, edesc, req);
1016 
1017         /* check ICV auth status */
1018         if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1019                      DESC_HDR_LO_ICCR1_PASS))
1020                 err = -EBADMSG;
1021 
1022         kfree(edesc);
1023 
1024         aead_request_complete(req, err);
1025 }
1026 
1027 /*
1028  * convert scatterlist to SEC h/w link table format
1029  * stop at cryptlen bytes
1030  */
1031 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1032                                  unsigned int offset, int cryptlen,
1033                                  struct talitos_ptr *link_tbl_ptr)
1034 {
1035         int n_sg = sg_count;
1036         int count = 0;
1037 
1038         while (cryptlen && sg && n_sg--) {
1039                 unsigned int len = sg_dma_len(sg);
1040 
1041                 if (offset >= len) {
1042                         offset -= len;
1043                         goto next;
1044                 }
1045 
1046                 len -= offset;
1047 
1048                 if (len > cryptlen)
1049                         len = cryptlen;
1050 
1051                 to_talitos_ptr(link_tbl_ptr + count,
1052                                sg_dma_address(sg) + offset, 0);
1053                 link_tbl_ptr[count].len = cpu_to_be16(len);
1054                 link_tbl_ptr[count].j_extent = 0;
1055                 count++;
1056                 cryptlen -= len;
1057                 offset = 0;
1058 
1059 next:
1060                 sg = sg_next(sg);
1061         }
1062 
1063         /* tag end of link table */
1064         if (count > 0)
1065                 link_tbl_ptr[count - 1].j_extent = DESC_PTR_LNKTBL_RETURN;
1066 
1067         return count;
1068 }
1069 
1070 static inline int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
1071                                  int cryptlen,
1072                                  struct talitos_ptr *link_tbl_ptr)
1073 {
1074         return sg_to_link_tbl_offset(sg, sg_count, 0, cryptlen,
1075                                      link_tbl_ptr);
1076 }
1077 
1078 /*
1079  * fill in and submit ipsec_esp descriptor
1080  */
1081 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1082                      void (*callback)(struct device *dev,
1083                                       struct talitos_desc *desc,
1084                                       void *context, int error))
1085 {
1086         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1087         unsigned int authsize = crypto_aead_authsize(aead);
1088         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1089         struct device *dev = ctx->dev;
1090         struct talitos_desc *desc = &edesc->desc;
1091         unsigned int cryptlen = areq->cryptlen;
1092         unsigned int ivsize = crypto_aead_ivsize(aead);
1093         int tbl_off = 0;
1094         int sg_count, ret;
1095         int sg_link_tbl_len;
1096 
1097         /* hmac key */
1098         map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
1099                                DMA_TO_DEVICE);
1100 
1101         sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ?: 1,
1102                               (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1103                                                            : DMA_TO_DEVICE);
1104         /* hmac data */
1105         desc->ptr[1].len = cpu_to_be16(areq->assoclen);
1106         if (sg_count > 1 &&
1107             (ret = sg_to_link_tbl_offset(areq->src, sg_count, 0,
1108                                          areq->assoclen,
1109                                          &edesc->link_tbl[tbl_off])) > 1) {
1110                 to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off *
1111                                sizeof(struct talitos_ptr), 0);
1112                 desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP;
1113 
1114                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1115                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1116 
1117                 tbl_off += ret;
1118         } else {
1119                 to_talitos_ptr(&desc->ptr[1], sg_dma_address(areq->src), 0);
1120                 desc->ptr[1].j_extent = 0;
1121         }
1122 
1123         /* cipher iv */
1124         to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, 0);
1125         desc->ptr[2].len = cpu_to_be16(ivsize);
1126         desc->ptr[2].j_extent = 0;
1127 
1128         /* cipher key */
1129         map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
1130                                (char *)&ctx->key + ctx->authkeylen,
1131                                DMA_TO_DEVICE);
1132 
1133         /*
1134          * cipher in
1135          * map and adjust cipher len to aead request cryptlen.
1136          * extent is bytes of HMAC postpended to ciphertext,
1137          * typically 12 for ipsec
1138          */
1139         desc->ptr[4].len = cpu_to_be16(cryptlen);
1140         desc->ptr[4].j_extent = authsize;
1141 
1142         sg_link_tbl_len = cryptlen;
1143         if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
1144                 sg_link_tbl_len += authsize;
1145 
1146         if (sg_count == 1) {
1147                 to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src) +
1148                                areq->assoclen, 0);
1149         } else if ((ret = sg_to_link_tbl_offset(areq->src, sg_count,
1150                                                 areq->assoclen, sg_link_tbl_len,
1151                                                 &edesc->link_tbl[tbl_off])) >
1152                    1) {
1153                 desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
1154                 to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl +
1155                                               tbl_off *
1156                                               sizeof(struct talitos_ptr), 0);
1157                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1158                                            edesc->dma_len,
1159                                            DMA_BIDIRECTIONAL);
1160                 tbl_off += ret;
1161         } else {
1162                 copy_talitos_ptr(&desc->ptr[4], &edesc->link_tbl[tbl_off], 0);
1163         }
1164 
1165         /* cipher out */
1166         desc->ptr[5].len = cpu_to_be16(cryptlen);
1167         desc->ptr[5].j_extent = authsize;
1168 
1169         if (areq->src != areq->dst)
1170                 sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1,
1171                                       DMA_FROM_DEVICE);
1172 
1173         edesc->icv_ool = false;
1174 
1175         if (sg_count == 1) {
1176                 to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst) +
1177                                areq->assoclen, 0);
1178         } else if ((sg_count =
1179                         sg_to_link_tbl_offset(areq->dst, sg_count,
1180                                               areq->assoclen, cryptlen,
1181                                               &edesc->link_tbl[tbl_off])) > 1) {
1182                 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1183 
1184                 to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl +
1185                                tbl_off * sizeof(struct talitos_ptr), 0);
1186 
1187                 /* Add an entry to the link table for ICV data */
1188                 tbl_ptr += sg_count - 1;
1189                 tbl_ptr->j_extent = 0;
1190                 tbl_ptr++;
1191                 tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1192                 tbl_ptr->len = cpu_to_be16(authsize);
1193 
1194                 /* icv data follows link tables */
1195                 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl +
1196                                         (edesc->src_nents + edesc->dst_nents +
1197                                          2) * sizeof(struct talitos_ptr) +
1198                                         authsize, 0);
1199                 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1200                 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1201                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1202 
1203                 edesc->icv_ool = true;
1204         } else {
1205                 copy_talitos_ptr(&desc->ptr[5], &edesc->link_tbl[tbl_off], 0);
1206         }
1207 
1208         /* iv out */
1209         map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1210                                DMA_FROM_DEVICE);
1211 
1212         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1213         if (ret != -EINPROGRESS) {
1214                 ipsec_esp_unmap(dev, edesc, areq);
1215                 kfree(edesc);
1216         }
1217         return ret;
1218 }
1219 
1220 /*
1221  * allocate and map the extended descriptor
1222  */
1223 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1224                                                  struct scatterlist *src,
1225                                                  struct scatterlist *dst,
1226                                                  u8 *iv,
1227                                                  unsigned int assoclen,
1228                                                  unsigned int cryptlen,
1229                                                  unsigned int authsize,
1230                                                  unsigned int ivsize,
1231                                                  int icv_stashing,
1232                                                  u32 cryptoflags,
1233                                                  bool encrypt)
1234 {
1235         struct talitos_edesc *edesc;
1236         int src_nents, dst_nents, alloc_len, dma_len;
1237         dma_addr_t iv_dma = 0;
1238         gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1239                       GFP_ATOMIC;
1240         struct talitos_private *priv = dev_get_drvdata(dev);
1241         bool is_sec1 = has_ftr_sec1(priv);
1242         int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
1243         void *err;
1244 
1245         if (cryptlen + authsize > max_len) {
1246                 dev_err(dev, "length exceeds h/w max limit\n");
1247                 return ERR_PTR(-EINVAL);
1248         }
1249 
1250         if (ivsize)
1251                 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1252 
1253         if (!dst || dst == src) {
1254                 src_nents = sg_nents_for_len(src,
1255                                              assoclen + cryptlen + authsize);
1256                 if (src_nents < 0) {
1257                         dev_err(dev, "Invalid number of src SG.\n");
1258                         err = ERR_PTR(-EINVAL);
1259                         goto error_sg;
1260                 }
1261                 src_nents = (src_nents == 1) ? 0 : src_nents;
1262                 dst_nents = dst ? src_nents : 0;
1263         } else { /* dst && dst != src*/
1264                 src_nents = sg_nents_for_len(src, assoclen + cryptlen +
1265                                                  (encrypt ? 0 : authsize));
1266                 if (src_nents < 0) {
1267                         dev_err(dev, "Invalid number of src SG.\n");
1268                         err = ERR_PTR(-EINVAL);
1269                         goto error_sg;
1270                 }
1271                 src_nents = (src_nents == 1) ? 0 : src_nents;
1272                 dst_nents = sg_nents_for_len(dst, assoclen + cryptlen +
1273                                                  (encrypt ? authsize : 0));
1274                 if (dst_nents < 0) {
1275                         dev_err(dev, "Invalid number of dst SG.\n");
1276                         err = ERR_PTR(-EINVAL);
1277                         goto error_sg;
1278                 }
1279                 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1280         }
1281 
1282         /*
1283          * allocate space for base edesc plus the link tables,
1284          * allowing for two separate entries for AD and generated ICV (+ 2),
1285          * and space for two sets of ICVs (stashed and generated)
1286          */
1287         alloc_len = sizeof(struct talitos_edesc);
1288         if (src_nents || dst_nents) {
1289                 if (is_sec1)
1290                         dma_len = (src_nents ? cryptlen : 0) +
1291                                   (dst_nents ? cryptlen : 0);
1292                 else
1293                         dma_len = (src_nents + dst_nents + 2) *
1294                                   sizeof(struct talitos_ptr) + authsize * 2;
1295                 alloc_len += dma_len;
1296         } else {
1297                 dma_len = 0;
1298                 alloc_len += icv_stashing ? authsize : 0;
1299         }
1300 
1301         edesc = kmalloc(alloc_len, GFP_DMA | flags);
1302         if (!edesc) {
1303                 dev_err(dev, "could not allocate edescriptor\n");
1304                 err = ERR_PTR(-ENOMEM);
1305                 goto error_sg;
1306         }
1307 
1308         edesc->src_nents = src_nents;
1309         edesc->dst_nents = dst_nents;
1310         edesc->iv_dma = iv_dma;
1311         edesc->dma_len = dma_len;
1312         if (dma_len)
1313                 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1314                                                      edesc->dma_len,
1315                                                      DMA_BIDIRECTIONAL);
1316 
1317         return edesc;
1318 error_sg:
1319         if (iv_dma)
1320                 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1321         return err;
1322 }
1323 
1324 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1325                                               int icv_stashing, bool encrypt)
1326 {
1327         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1328         unsigned int authsize = crypto_aead_authsize(authenc);
1329         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1330         unsigned int ivsize = crypto_aead_ivsize(authenc);
1331 
1332         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1333                                    iv, areq->assoclen, areq->cryptlen,
1334                                    authsize, ivsize, icv_stashing,
1335                                    areq->base.flags, encrypt);
1336 }
1337 
1338 static int aead_encrypt(struct aead_request *req)
1339 {
1340         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1341         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1342         struct talitos_edesc *edesc;
1343 
1344         /* allocate extended descriptor */
1345         edesc = aead_edesc_alloc(req, req->iv, 0, true);
1346         if (IS_ERR(edesc))
1347                 return PTR_ERR(edesc);
1348 
1349         /* set encrypt */
1350         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1351 
1352         return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
1353 }
1354 
1355 static int aead_decrypt(struct aead_request *req)
1356 {
1357         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1358         unsigned int authsize = crypto_aead_authsize(authenc);
1359         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1360         struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1361         struct talitos_edesc *edesc;
1362         struct scatterlist *sg;
1363         void *icvdata;
1364 
1365         req->cryptlen -= authsize;
1366 
1367         /* allocate extended descriptor */
1368         edesc = aead_edesc_alloc(req, req->iv, 1, false);
1369         if (IS_ERR(edesc))
1370                 return PTR_ERR(edesc);
1371 
1372         if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1373             ((!edesc->src_nents && !edesc->dst_nents) ||
1374              priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1375 
1376                 /* decrypt and check the ICV */
1377                 edesc->desc.hdr = ctx->desc_hdr_template |
1378                                   DESC_HDR_DIR_INBOUND |
1379                                   DESC_HDR_MODE1_MDEU_CICV;
1380 
1381                 /* reset integrity check result bits */
1382                 edesc->desc.hdr_lo = 0;
1383 
1384                 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
1385         }
1386 
1387         /* Have to check the ICV with software */
1388         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1389 
1390         /* stash incoming ICV for later cmp with ICV generated by the h/w */
1391         if (edesc->dma_len)
1392                 icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1393                                                    edesc->dst_nents + 2];
1394         else
1395                 icvdata = &edesc->link_tbl[0];
1396 
1397         sg = sg_last(req->src, edesc->src_nents ? : 1);
1398 
1399         memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
1400 
1401         return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
1402 }
1403 
1404 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1405                              const u8 *key, unsigned int keylen)
1406 {
1407         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1408 
1409         memcpy(&ctx->key, key, keylen);
1410         ctx->keylen = keylen;
1411 
1412         return 0;
1413 }
1414 
1415 static void unmap_sg_talitos_ptr(struct device *dev, struct scatterlist *src,
1416                                  struct scatterlist *dst, unsigned int len,
1417                                  struct talitos_edesc *edesc)
1418 {
1419         struct talitos_private *priv = dev_get_drvdata(dev);
1420         bool is_sec1 = has_ftr_sec1(priv);
1421 
1422         if (is_sec1) {
1423                 if (!edesc->src_nents) {
1424                         dma_unmap_sg(dev, src, 1,
1425                                      dst != src ? DMA_TO_DEVICE
1426                                                 : DMA_BIDIRECTIONAL);
1427                 }
1428                 if (dst && edesc->dst_nents) {
1429                         dma_sync_single_for_device(dev,
1430                                                    edesc->dma_link_tbl + len,
1431                                                    len, DMA_FROM_DEVICE);
1432                         sg_copy_from_buffer(dst, edesc->dst_nents ? : 1,
1433                                             edesc->buf + len, len);
1434                 } else if (dst && dst != src) {
1435                         dma_unmap_sg(dev, dst, 1, DMA_FROM_DEVICE);
1436                 }
1437         } else {
1438                 talitos_sg_unmap(dev, edesc, src, dst);
1439         }
1440 }
1441 
1442 static void common_nonsnoop_unmap(struct device *dev,
1443                                   struct talitos_edesc *edesc,
1444                                   struct ablkcipher_request *areq)
1445 {
1446         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1447 
1448         unmap_sg_talitos_ptr(dev, areq->src, areq->dst, areq->nbytes, edesc);
1449         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1450         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1451 
1452         if (edesc->dma_len)
1453                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1454                                  DMA_BIDIRECTIONAL);
1455 }
1456 
1457 static void ablkcipher_done(struct device *dev,
1458                             struct talitos_desc *desc, void *context,
1459                             int err)
1460 {
1461         struct ablkcipher_request *areq = context;
1462         struct talitos_edesc *edesc;
1463 
1464         edesc = container_of(desc, struct talitos_edesc, desc);
1465 
1466         common_nonsnoop_unmap(dev, edesc, areq);
1467 
1468         kfree(edesc);
1469 
1470         areq->base.complete(&areq->base, err);
1471 }
1472 
1473 int map_sg_in_talitos_ptr(struct device *dev, struct scatterlist *src,
1474                           unsigned int len, struct talitos_edesc *edesc,
1475                           enum dma_data_direction dir, struct talitos_ptr *ptr)
1476 {
1477         int sg_count;
1478         struct talitos_private *priv = dev_get_drvdata(dev);
1479         bool is_sec1 = has_ftr_sec1(priv);
1480 
1481         to_talitos_ptr_len(ptr, len, is_sec1);
1482 
1483         if (is_sec1) {
1484                 sg_count = edesc->src_nents ? : 1;
1485 
1486                 if (sg_count == 1) {
1487                         dma_map_sg(dev, src, 1, dir);
1488                         to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
1489                 } else {
1490                         sg_copy_to_buffer(src, sg_count, edesc->buf, len);
1491                         to_talitos_ptr(ptr, edesc->dma_link_tbl, is_sec1);
1492                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1493                                                    len, DMA_TO_DEVICE);
1494                 }
1495         } else {
1496                 to_talitos_ptr_extent_clear(ptr, is_sec1);
1497 
1498                 sg_count = dma_map_sg(dev, src, edesc->src_nents ? : 1, dir);
1499 
1500                 if (sg_count == 1) {
1501                         to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
1502                 } else {
1503                         sg_count = sg_to_link_tbl(src, sg_count, len,
1504                                                   &edesc->link_tbl[0]);
1505                         if (sg_count > 1) {
1506                                 to_talitos_ptr(ptr, edesc->dma_link_tbl, 0);
1507                                 ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
1508                                 dma_sync_single_for_device(dev,
1509                                                            edesc->dma_link_tbl,
1510                                                            edesc->dma_len,
1511                                                            DMA_BIDIRECTIONAL);
1512                         } else {
1513                                 /* Only one segment now, so no link tbl needed*/
1514                                 to_talitos_ptr(ptr, sg_dma_address(src),
1515                                                is_sec1);
1516                         }
1517                 }
1518         }
1519         return sg_count;
1520 }
1521 
1522 void map_sg_out_talitos_ptr(struct device *dev, struct scatterlist *dst,
1523                             unsigned int len, struct talitos_edesc *edesc,
1524                             enum dma_data_direction dir,
1525                             struct talitos_ptr *ptr, int sg_count)
1526 {
1527         struct talitos_private *priv = dev_get_drvdata(dev);
1528         bool is_sec1 = has_ftr_sec1(priv);
1529 
1530         if (dir != DMA_NONE)
1531                 sg_count = dma_map_sg(dev, dst, edesc->dst_nents ? : 1, dir);
1532 
1533         to_talitos_ptr_len(ptr, len, is_sec1);
1534 
1535         if (is_sec1) {
1536                 if (sg_count == 1) {
1537                         if (dir != DMA_NONE)
1538                                 dma_map_sg(dev, dst, 1, dir);
1539                         to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1);
1540                 } else {
1541                         to_talitos_ptr(ptr, edesc->dma_link_tbl + len, is_sec1);
1542                         dma_sync_single_for_device(dev,
1543                                                    edesc->dma_link_tbl + len,
1544                                                    len, DMA_FROM_DEVICE);
1545                 }
1546         } else {
1547                 to_talitos_ptr_extent_clear(ptr, is_sec1);
1548 
1549                 if (sg_count == 1) {
1550                         to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1);
1551                 } else {
1552                         struct talitos_ptr *link_tbl_ptr =
1553                                 &edesc->link_tbl[edesc->src_nents + 1];
1554 
1555                         to_talitos_ptr(ptr, edesc->dma_link_tbl +
1556                                             (edesc->src_nents + 1) *
1557                                              sizeof(struct talitos_ptr), 0);
1558                         ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
1559                         sg_to_link_tbl(dst, sg_count, len, link_tbl_ptr);
1560                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1561                                                    edesc->dma_len,
1562                                                    DMA_BIDIRECTIONAL);
1563                 }
1564         }
1565 }
1566 
1567 static int common_nonsnoop(struct talitos_edesc *edesc,
1568                            struct ablkcipher_request *areq,
1569                            void (*callback) (struct device *dev,
1570                                              struct talitos_desc *desc,
1571                                              void *context, int error))
1572 {
1573         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1574         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1575         struct device *dev = ctx->dev;
1576         struct talitos_desc *desc = &edesc->desc;
1577         unsigned int cryptlen = areq->nbytes;
1578         unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1579         int sg_count, ret;
1580         struct talitos_private *priv = dev_get_drvdata(dev);
1581         bool is_sec1 = has_ftr_sec1(priv);
1582 
1583         /* first DWORD empty */
1584         desc->ptr[0] = zero_entry;
1585 
1586         /* cipher iv */
1587         to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1);
1588         to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1);
1589         to_talitos_ptr_extent_clear(&desc->ptr[1], is_sec1);
1590 
1591         /* cipher key */
1592         map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1593                                (char *)&ctx->key, DMA_TO_DEVICE);
1594 
1595         /*
1596          * cipher in
1597          */
1598         sg_count = map_sg_in_talitos_ptr(dev, areq->src, cryptlen, edesc,
1599                                          (areq->src == areq->dst) ?
1600                                           DMA_BIDIRECTIONAL : DMA_TO_DEVICE,
1601                                           &desc->ptr[3]);
1602 
1603         /* cipher out */
1604         map_sg_out_talitos_ptr(dev, areq->dst, cryptlen, edesc,
1605                                (areq->src == areq->dst) ? DMA_NONE
1606                                                         : DMA_FROM_DEVICE,
1607                                &desc->ptr[4], sg_count);
1608 
1609         /* iv out */
1610         map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
1611                                DMA_FROM_DEVICE);
1612 
1613         /* last DWORD empty */
1614         desc->ptr[6] = zero_entry;
1615 
1616         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1617         if (ret != -EINPROGRESS) {
1618                 common_nonsnoop_unmap(dev, edesc, areq);
1619                 kfree(edesc);
1620         }
1621         return ret;
1622 }
1623 
1624 static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1625                                                     areq, bool encrypt)
1626 {
1627         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1628         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1629         unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1630 
1631         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1632                                    areq->info, 0, areq->nbytes, 0, ivsize, 0,
1633                                    areq->base.flags, encrypt);
1634 }
1635 
1636 static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1637 {
1638         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1639         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1640         struct talitos_edesc *edesc;
1641 
1642         /* allocate extended descriptor */
1643         edesc = ablkcipher_edesc_alloc(areq, true);
1644         if (IS_ERR(edesc))
1645                 return PTR_ERR(edesc);
1646 
1647         /* set encrypt */
1648         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1649 
1650         return common_nonsnoop(edesc, areq, ablkcipher_done);
1651 }
1652 
1653 static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1654 {
1655         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1656         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1657         struct talitos_edesc *edesc;
1658 
1659         /* allocate extended descriptor */
1660         edesc = ablkcipher_edesc_alloc(areq, false);
1661         if (IS_ERR(edesc))
1662                 return PTR_ERR(edesc);
1663 
1664         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1665 
1666         return common_nonsnoop(edesc, areq, ablkcipher_done);
1667 }
1668 
1669 static void common_nonsnoop_hash_unmap(struct device *dev,
1670                                        struct talitos_edesc *edesc,
1671                                        struct ahash_request *areq)
1672 {
1673         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1674         struct talitos_private *priv = dev_get_drvdata(dev);
1675         bool is_sec1 = has_ftr_sec1(priv);
1676 
1677         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1678 
1679         unmap_sg_talitos_ptr(dev, req_ctx->psrc, NULL, 0, edesc);
1680 
1681         /* When using hashctx-in, must unmap it. */
1682         if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
1683                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1684                                          DMA_TO_DEVICE);
1685 
1686         if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1))
1687                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1688                                          DMA_TO_DEVICE);
1689 
1690         if (edesc->dma_len)
1691                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1692                                  DMA_BIDIRECTIONAL);
1693 
1694 }
1695 
1696 static void ahash_done(struct device *dev,
1697                        struct talitos_desc *desc, void *context,
1698                        int err)
1699 {
1700         struct ahash_request *areq = context;
1701         struct talitos_edesc *edesc =
1702                  container_of(desc, struct talitos_edesc, desc);
1703         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1704 
1705         if (!req_ctx->last && req_ctx->to_hash_later) {
1706                 /* Position any partial block for next update/final/finup */
1707                 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
1708                 req_ctx->nbuf = req_ctx->to_hash_later;
1709         }
1710         common_nonsnoop_hash_unmap(dev, edesc, areq);
1711 
1712         kfree(edesc);
1713 
1714         areq->base.complete(&areq->base, err);
1715 }
1716 
1717 /*
1718  * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1719  * ourself and submit a padded block
1720  */
1721 void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1722                                struct talitos_edesc *edesc,
1723                                struct talitos_ptr *ptr)
1724 {
1725         static u8 padded_hash[64] = {
1726                 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1727                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1728                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1729                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1730         };
1731 
1732         pr_err_once("Bug in SEC1, padding ourself\n");
1733         edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1734         map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1735                                (char *)padded_hash, DMA_TO_DEVICE);
1736 }
1737 
1738 static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1739                                 struct ahash_request *areq, unsigned int length,
1740                                 void (*callback) (struct device *dev,
1741                                                   struct talitos_desc *desc,
1742                                                   void *context, int error))
1743 {
1744         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1745         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1746         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1747         struct device *dev = ctx->dev;
1748         struct talitos_desc *desc = &edesc->desc;
1749         int ret;
1750         struct talitos_private *priv = dev_get_drvdata(dev);
1751         bool is_sec1 = has_ftr_sec1(priv);
1752 
1753         /* first DWORD empty */
1754         desc->ptr[0] = zero_entry;
1755 
1756         /* hash context in */
1757         if (!req_ctx->first || req_ctx->swinit) {
1758                 map_single_talitos_ptr(dev, &desc->ptr[1],
1759                                        req_ctx->hw_context_size,
1760                                        (char *)req_ctx->hw_context,
1761                                        DMA_TO_DEVICE);
1762                 req_ctx->swinit = 0;
1763         } else {
1764                 desc->ptr[1] = zero_entry;
1765                 /* Indicate next op is not the first. */
1766                 req_ctx->first = 0;
1767         }
1768 
1769         /* HMAC key */
1770         if (ctx->keylen)
1771                 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1772                                        (char *)&ctx->key, DMA_TO_DEVICE);
1773         else
1774                 desc->ptr[2] = zero_entry;
1775 
1776         /*
1777          * data in
1778          */
1779         map_sg_in_talitos_ptr(dev, req_ctx->psrc, length, edesc,
1780                               DMA_TO_DEVICE, &desc->ptr[3]);
1781 
1782         /* fifth DWORD empty */
1783         desc->ptr[4] = zero_entry;
1784 
1785         /* hash/HMAC out -or- hash context out */
1786         if (req_ctx->last)
1787                 map_single_talitos_ptr(dev, &desc->ptr[5],
1788                                        crypto_ahash_digestsize(tfm),
1789                                        areq->result, DMA_FROM_DEVICE);
1790         else
1791                 map_single_talitos_ptr(dev, &desc->ptr[5],
1792                                        req_ctx->hw_context_size,
1793                                        req_ctx->hw_context, DMA_FROM_DEVICE);
1794 
1795         /* last DWORD empty */
1796         desc->ptr[6] = zero_entry;
1797 
1798         if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1799                 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1800 
1801         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1802         if (ret != -EINPROGRESS) {
1803                 common_nonsnoop_hash_unmap(dev, edesc, areq);
1804                 kfree(edesc);
1805         }
1806         return ret;
1807 }
1808 
1809 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1810                                                unsigned int nbytes)
1811 {
1812         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1813         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1814         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1815 
1816         return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
1817                                    nbytes, 0, 0, 0, areq->base.flags, false);
1818 }
1819 
1820 static int ahash_init(struct ahash_request *areq)
1821 {
1822         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1823         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1824 
1825         /* Initialize the context */
1826         req_ctx->nbuf = 0;
1827         req_ctx->first = 1; /* first indicates h/w must init its context */
1828         req_ctx->swinit = 0; /* assume h/w init of context */
1829         req_ctx->hw_context_size =
1830                 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1831                         ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1832                         : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1833 
1834         return 0;
1835 }
1836 
1837 /*
1838  * on h/w without explicit sha224 support, we initialize h/w context
1839  * manually with sha224 constants, and tell it to run sha256.
1840  */
1841 static int ahash_init_sha224_swinit(struct ahash_request *areq)
1842 {
1843         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1844 
1845         ahash_init(areq);
1846         req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1847 
1848         req_ctx->hw_context[0] = SHA224_H0;
1849         req_ctx->hw_context[1] = SHA224_H1;
1850         req_ctx->hw_context[2] = SHA224_H2;
1851         req_ctx->hw_context[3] = SHA224_H3;
1852         req_ctx->hw_context[4] = SHA224_H4;
1853         req_ctx->hw_context[5] = SHA224_H5;
1854         req_ctx->hw_context[6] = SHA224_H6;
1855         req_ctx->hw_context[7] = SHA224_H7;
1856 
1857         /* init 64-bit count */
1858         req_ctx->hw_context[8] = 0;
1859         req_ctx->hw_context[9] = 0;
1860 
1861         return 0;
1862 }
1863 
1864 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1865 {
1866         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1867         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1868         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1869         struct talitos_edesc *edesc;
1870         unsigned int blocksize =
1871                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1872         unsigned int nbytes_to_hash;
1873         unsigned int to_hash_later;
1874         unsigned int nsg;
1875         int nents;
1876 
1877         if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1878                 /* Buffer up to one whole block */
1879                 nents = sg_nents_for_len(areq->src, nbytes);
1880                 if (nents < 0) {
1881                         dev_err(ctx->dev, "Invalid number of src SG.\n");
1882                         return nents;
1883                 }
1884                 sg_copy_to_buffer(areq->src, nents,
1885                                   req_ctx->buf + req_ctx->nbuf, nbytes);
1886                 req_ctx->nbuf += nbytes;
1887                 return 0;
1888         }
1889 
1890         /* At least (blocksize + 1) bytes are available to hash */
1891         nbytes_to_hash = nbytes + req_ctx->nbuf;
1892         to_hash_later = nbytes_to_hash & (blocksize - 1);
1893 
1894         if (req_ctx->last)
1895                 to_hash_later = 0;
1896         else if (to_hash_later)
1897                 /* There is a partial block. Hash the full block(s) now */
1898                 nbytes_to_hash -= to_hash_later;
1899         else {
1900                 /* Keep one block buffered */
1901                 nbytes_to_hash -= blocksize;
1902                 to_hash_later = blocksize;
1903         }
1904 
1905         /* Chain in any previously buffered data */
1906         if (req_ctx->nbuf) {
1907                 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1908                 sg_init_table(req_ctx->bufsl, nsg);
1909                 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1910                 if (nsg > 1)
1911                         sg_chain(req_ctx->bufsl, 2, areq->src);
1912                 req_ctx->psrc = req_ctx->bufsl;
1913         } else
1914                 req_ctx->psrc = areq->src;
1915 
1916         if (to_hash_later) {
1917                 nents = sg_nents_for_len(areq->src, nbytes);
1918                 if (nents < 0) {
1919                         dev_err(ctx->dev, "Invalid number of src SG.\n");
1920                         return nents;
1921                 }
1922                 sg_pcopy_to_buffer(areq->src, nents,
1923                                       req_ctx->bufnext,
1924                                       to_hash_later,
1925                                       nbytes - to_hash_later);
1926         }
1927         req_ctx->to_hash_later = to_hash_later;
1928 
1929         /* Allocate extended descriptor */
1930         edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1931         if (IS_ERR(edesc))
1932                 return PTR_ERR(edesc);
1933 
1934         edesc->desc.hdr = ctx->desc_hdr_template;
1935 
1936         /* On last one, request SEC to pad; otherwise continue */
1937         if (req_ctx->last)
1938                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1939         else
1940                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1941 
1942         /* request SEC to INIT hash. */
1943         if (req_ctx->first && !req_ctx->swinit)
1944                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1945 
1946         /* When the tfm context has a keylen, it's an HMAC.
1947          * A first or last (ie. not middle) descriptor must request HMAC.
1948          */
1949         if (ctx->keylen && (req_ctx->first || req_ctx->last))
1950                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1951 
1952         return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1953                                     ahash_done);
1954 }
1955 
1956 static int ahash_update(struct ahash_request *areq)
1957 {
1958         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1959 
1960         req_ctx->last = 0;
1961 
1962         return ahash_process_req(areq, areq->nbytes);
1963 }
1964 
1965 static int ahash_final(struct ahash_request *areq)
1966 {
1967         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1968 
1969         req_ctx->last = 1;
1970 
1971         return ahash_process_req(areq, 0);
1972 }
1973 
1974 static int ahash_finup(struct ahash_request *areq)
1975 {
1976         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1977 
1978         req_ctx->last = 1;
1979 
1980         return ahash_process_req(areq, areq->nbytes);
1981 }
1982 
1983 static int ahash_digest(struct ahash_request *areq)
1984 {
1985         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1986         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
1987 
1988         ahash->init(areq);
1989         req_ctx->last = 1;
1990 
1991         return ahash_process_req(areq, areq->nbytes);
1992 }
1993 
1994 static int ahash_export(struct ahash_request *areq, void *out)
1995 {
1996         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1997         struct talitos_export_state *export = out;
1998 
1999         memcpy(export->hw_context, req_ctx->hw_context,
2000                req_ctx->hw_context_size);
2001         memcpy(export->buf, req_ctx->buf, req_ctx->nbuf);
2002         export->swinit = req_ctx->swinit;
2003         export->first = req_ctx->first;
2004         export->last = req_ctx->last;
2005         export->to_hash_later = req_ctx->to_hash_later;
2006         export->nbuf = req_ctx->nbuf;
2007 
2008         return 0;
2009 }
2010 
2011 static int ahash_import(struct ahash_request *areq, const void *in)
2012 {
2013         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2014         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2015         const struct talitos_export_state *export = in;
2016 
2017         memset(req_ctx, 0, sizeof(*req_ctx));
2018         req_ctx->hw_context_size =
2019                 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
2020                         ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2021                         : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2022         memcpy(req_ctx->hw_context, export->hw_context,
2023                req_ctx->hw_context_size);
2024         memcpy(req_ctx->buf, export->buf, export->nbuf);
2025         req_ctx->swinit = export->swinit;
2026         req_ctx->first = export->first;
2027         req_ctx->last = export->last;
2028         req_ctx->to_hash_later = export->to_hash_later;
2029         req_ctx->nbuf = export->nbuf;
2030 
2031         return 0;
2032 }
2033 
2034 struct keyhash_result {
2035         struct completion completion;
2036         int err;
2037 };
2038 
2039 static void keyhash_complete(struct crypto_async_request *req, int err)
2040 {
2041         struct keyhash_result *res = req->data;
2042 
2043         if (err == -EINPROGRESS)
2044                 return;
2045 
2046         res->err = err;
2047         complete(&res->completion);
2048 }
2049 
2050 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2051                    u8 *hash)
2052 {
2053         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2054 
2055         struct scatterlist sg[1];
2056         struct ahash_request *req;
2057         struct keyhash_result hresult;
2058         int ret;
2059 
2060         init_completion(&hresult.completion);
2061 
2062         req = ahash_request_alloc(tfm, GFP_KERNEL);
2063         if (!req)
2064                 return -ENOMEM;
2065 
2066         /* Keep tfm keylen == 0 during hash of the long key */
2067         ctx->keylen = 0;
2068         ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2069                                    keyhash_complete, &hresult);
2070 
2071         sg_init_one(&sg[0], key, keylen);
2072 
2073         ahash_request_set_crypt(req, sg, hash, keylen);
2074         ret = crypto_ahash_digest(req);
2075         switch (ret) {
2076         case 0:
2077                 break;
2078         case -EINPROGRESS:
2079         case -EBUSY:
2080                 ret = wait_for_completion_interruptible(
2081                         &hresult.completion);
2082                 if (!ret)
2083                         ret = hresult.err;
2084                 break;
2085         default:
2086                 break;
2087         }
2088         ahash_request_free(req);
2089 
2090         return ret;
2091 }
2092 
2093 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2094                         unsigned int keylen)
2095 {
2096         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2097         unsigned int blocksize =
2098                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2099         unsigned int digestsize = crypto_ahash_digestsize(tfm);
2100         unsigned int keysize = keylen;
2101         u8 hash[SHA512_DIGEST_SIZE];
2102         int ret;
2103 
2104         if (keylen <= blocksize)
2105                 memcpy(ctx->key, key, keysize);
2106         else {
2107                 /* Must get the hash of the long key */
2108                 ret = keyhash(tfm, key, keylen, hash);
2109 
2110                 if (ret) {
2111                         crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2112                         return -EINVAL;
2113                 }
2114 
2115                 keysize = digestsize;
2116                 memcpy(ctx->key, hash, digestsize);
2117         }
2118 
2119         ctx->keylen = keysize;
2120 
2121         return 0;
2122 }
2123 
2124 
2125 struct talitos_alg_template {
2126         u32 type;
2127         union {
2128                 struct crypto_alg crypto;
2129                 struct ahash_alg hash;
2130                 struct aead_alg aead;
2131         } alg;
2132         __be32 desc_hdr_template;
2133 };
2134 
2135 static struct talitos_alg_template driver_algs[] = {
2136         /* AEAD algorithms.  These use a single-pass ipsec_esp descriptor */
2137         {       .type = CRYPTO_ALG_TYPE_AEAD,
2138                 .alg.aead = {
2139                         .base = {
2140                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2141                                 .cra_driver_name = "authenc-hmac-sha1-"
2142                                                    "cbc-aes-talitos",
2143                                 .cra_blocksize = AES_BLOCK_SIZE,
2144                                 .cra_flags = CRYPTO_ALG_ASYNC,
2145                         },
2146                         .ivsize = AES_BLOCK_SIZE,
2147                         .maxauthsize = SHA1_DIGEST_SIZE,
2148                 },
2149                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2150                                      DESC_HDR_SEL0_AESU |
2151                                      DESC_HDR_MODE0_AESU_CBC |
2152                                      DESC_HDR_SEL1_MDEUA |
2153                                      DESC_HDR_MODE1_MDEU_INIT |
2154                                      DESC_HDR_MODE1_MDEU_PAD |
2155                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2156         },
2157         {       .type = CRYPTO_ALG_TYPE_AEAD,
2158                 .alg.aead = {
2159                         .base = {
2160                                 .cra_name = "authenc(hmac(sha1),"
2161                                             "cbc(des3_ede))",
2162                                 .cra_driver_name = "authenc-hmac-sha1-"
2163                                                    "cbc-3des-talitos",
2164                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2165                                 .cra_flags = CRYPTO_ALG_ASYNC,
2166                         },
2167                         .ivsize = DES3_EDE_BLOCK_SIZE,
2168                         .maxauthsize = SHA1_DIGEST_SIZE,
2169                 },
2170                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2171                                      DESC_HDR_SEL0_DEU |
2172                                      DESC_HDR_MODE0_DEU_CBC |
2173                                      DESC_HDR_MODE0_DEU_3DES |
2174                                      DESC_HDR_SEL1_MDEUA |
2175                                      DESC_HDR_MODE1_MDEU_INIT |
2176                                      DESC_HDR_MODE1_MDEU_PAD |
2177                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2178         },
2179         {       .type = CRYPTO_ALG_TYPE_AEAD,
2180                 .alg.aead = {
2181                         .base = {
2182                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2183                                 .cra_driver_name = "authenc-hmac-sha224-"
2184                                                    "cbc-aes-talitos",
2185                                 .cra_blocksize = AES_BLOCK_SIZE,
2186                                 .cra_flags = CRYPTO_ALG_ASYNC,
2187                         },
2188                         .ivsize = AES_BLOCK_SIZE,
2189                         .maxauthsize = SHA224_DIGEST_SIZE,
2190                 },
2191                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2192                                      DESC_HDR_SEL0_AESU |
2193                                      DESC_HDR_MODE0_AESU_CBC |
2194                                      DESC_HDR_SEL1_MDEUA |
2195                                      DESC_HDR_MODE1_MDEU_INIT |
2196                                      DESC_HDR_MODE1_MDEU_PAD |
2197                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2198         },
2199         {       .type = CRYPTO_ALG_TYPE_AEAD,
2200                 .alg.aead = {
2201                         .base = {
2202                                 .cra_name = "authenc(hmac(sha224),"
2203                                             "cbc(des3_ede))",
2204                                 .cra_driver_name = "authenc-hmac-sha224-"
2205                                                    "cbc-3des-talitos",
2206                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2207                                 .cra_flags = CRYPTO_ALG_ASYNC,
2208                         },
2209                         .ivsize = DES3_EDE_BLOCK_SIZE,
2210                         .maxauthsize = SHA224_DIGEST_SIZE,
2211                 },
2212                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2213                                      DESC_HDR_SEL0_DEU |
2214                                      DESC_HDR_MODE0_DEU_CBC |
2215                                      DESC_HDR_MODE0_DEU_3DES |
2216                                      DESC_HDR_SEL1_MDEUA |
2217                                      DESC_HDR_MODE1_MDEU_INIT |
2218                                      DESC_HDR_MODE1_MDEU_PAD |
2219                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2220         },
2221         {       .type = CRYPTO_ALG_TYPE_AEAD,
2222                 .alg.aead = {
2223                         .base = {
2224                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2225                                 .cra_driver_name = "authenc-hmac-sha256-"
2226                                                    "cbc-aes-talitos",
2227                                 .cra_blocksize = AES_BLOCK_SIZE,
2228                                 .cra_flags = CRYPTO_ALG_ASYNC,
2229                         },
2230                         .ivsize = AES_BLOCK_SIZE,
2231                         .maxauthsize = SHA256_DIGEST_SIZE,
2232                 },
2233                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2234                                      DESC_HDR_SEL0_AESU |
2235                                      DESC_HDR_MODE0_AESU_CBC |
2236                                      DESC_HDR_SEL1_MDEUA |
2237                                      DESC_HDR_MODE1_MDEU_INIT |
2238                                      DESC_HDR_MODE1_MDEU_PAD |
2239                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2240         },
2241         {       .type = CRYPTO_ALG_TYPE_AEAD,
2242                 .alg.aead = {
2243                         .base = {
2244                                 .cra_name = "authenc(hmac(sha256),"
2245                                             "cbc(des3_ede))",
2246                                 .cra_driver_name = "authenc-hmac-sha256-"
2247                                                    "cbc-3des-talitos",
2248                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2249                                 .cra_flags = CRYPTO_ALG_ASYNC,
2250                         },
2251                         .ivsize = DES3_EDE_BLOCK_SIZE,
2252                         .maxauthsize = SHA256_DIGEST_SIZE,
2253                 },
2254                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2255                                      DESC_HDR_SEL0_DEU |
2256                                      DESC_HDR_MODE0_DEU_CBC |
2257                                      DESC_HDR_MODE0_DEU_3DES |
2258                                      DESC_HDR_SEL1_MDEUA |
2259                                      DESC_HDR_MODE1_MDEU_INIT |
2260                                      DESC_HDR_MODE1_MDEU_PAD |
2261                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2262         },
2263         {       .type = CRYPTO_ALG_TYPE_AEAD,
2264                 .alg.aead = {
2265                         .base = {
2266                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2267                                 .cra_driver_name = "authenc-hmac-sha384-"
2268                                                    "cbc-aes-talitos",
2269                                 .cra_blocksize = AES_BLOCK_SIZE,
2270                                 .cra_flags = CRYPTO_ALG_ASYNC,
2271                         },
2272                         .ivsize = AES_BLOCK_SIZE,
2273                         .maxauthsize = SHA384_DIGEST_SIZE,
2274                 },
2275                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2276                                      DESC_HDR_SEL0_AESU |
2277                                      DESC_HDR_MODE0_AESU_CBC |
2278                                      DESC_HDR_SEL1_MDEUB |
2279                                      DESC_HDR_MODE1_MDEU_INIT |
2280                                      DESC_HDR_MODE1_MDEU_PAD |
2281                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2282         },
2283         {       .type = CRYPTO_ALG_TYPE_AEAD,
2284                 .alg.aead = {
2285                         .base = {
2286                                 .cra_name = "authenc(hmac(sha384),"
2287                                             "cbc(des3_ede))",
2288                                 .cra_driver_name = "authenc-hmac-sha384-"
2289                                                    "cbc-3des-talitos",
2290                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2291                                 .cra_flags = CRYPTO_ALG_ASYNC,
2292                         },
2293                         .ivsize = DES3_EDE_BLOCK_SIZE,
2294                         .maxauthsize = SHA384_DIGEST_SIZE,
2295                 },
2296                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2297                                      DESC_HDR_SEL0_DEU |
2298                                      DESC_HDR_MODE0_DEU_CBC |
2299                                      DESC_HDR_MODE0_DEU_3DES |
2300                                      DESC_HDR_SEL1_MDEUB |
2301                                      DESC_HDR_MODE1_MDEU_INIT |
2302                                      DESC_HDR_MODE1_MDEU_PAD |
2303                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2304         },
2305         {       .type = CRYPTO_ALG_TYPE_AEAD,
2306                 .alg.aead = {
2307                         .base = {
2308                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2309                                 .cra_driver_name = "authenc-hmac-sha512-"
2310                                                    "cbc-aes-talitos",
2311                                 .cra_blocksize = AES_BLOCK_SIZE,
2312                                 .cra_flags = CRYPTO_ALG_ASYNC,
2313                         },
2314                         .ivsize = AES_BLOCK_SIZE,
2315                         .maxauthsize = SHA512_DIGEST_SIZE,
2316                 },
2317                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2318                                      DESC_HDR_SEL0_AESU |
2319                                      DESC_HDR_MODE0_AESU_CBC |
2320                                      DESC_HDR_SEL1_MDEUB |
2321                                      DESC_HDR_MODE1_MDEU_INIT |
2322                                      DESC_HDR_MODE1_MDEU_PAD |
2323                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2324         },
2325         {       .type = CRYPTO_ALG_TYPE_AEAD,
2326                 .alg.aead = {
2327                         .base = {
2328                                 .cra_name = "authenc(hmac(sha512),"
2329                                             "cbc(des3_ede))",
2330                                 .cra_driver_name = "authenc-hmac-sha512-"
2331                                                    "cbc-3des-talitos",
2332                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2333                                 .cra_flags = CRYPTO_ALG_ASYNC,
2334                         },
2335                         .ivsize = DES3_EDE_BLOCK_SIZE,
2336                         .maxauthsize = SHA512_DIGEST_SIZE,
2337                 },
2338                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2339                                      DESC_HDR_SEL0_DEU |
2340                                      DESC_HDR_MODE0_DEU_CBC |
2341                                      DESC_HDR_MODE0_DEU_3DES |
2342                                      DESC_HDR_SEL1_MDEUB |
2343                                      DESC_HDR_MODE1_MDEU_INIT |
2344                                      DESC_HDR_MODE1_MDEU_PAD |
2345                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2346         },
2347         {       .type = CRYPTO_ALG_TYPE_AEAD,
2348                 .alg.aead = {
2349                         .base = {
2350                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
2351                                 .cra_driver_name = "authenc-hmac-md5-"
2352                                                    "cbc-aes-talitos",
2353                                 .cra_blocksize = AES_BLOCK_SIZE,
2354                                 .cra_flags = CRYPTO_ALG_ASYNC,
2355                         },
2356                         .ivsize = AES_BLOCK_SIZE,
2357                         .maxauthsize = MD5_DIGEST_SIZE,
2358                 },
2359                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2360                                      DESC_HDR_SEL0_AESU |
2361                                      DESC_HDR_MODE0_AESU_CBC |
2362                                      DESC_HDR_SEL1_MDEUA |
2363                                      DESC_HDR_MODE1_MDEU_INIT |
2364                                      DESC_HDR_MODE1_MDEU_PAD |
2365                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2366         },
2367         {       .type = CRYPTO_ALG_TYPE_AEAD,
2368                 .alg.aead = {
2369                         .base = {
2370                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2371                                 .cra_driver_name = "authenc-hmac-md5-"
2372                                                    "cbc-3des-talitos",
2373                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2374                                 .cra_flags = CRYPTO_ALG_ASYNC,
2375                         },
2376                         .ivsize = DES3_EDE_BLOCK_SIZE,
2377                         .maxauthsize = MD5_DIGEST_SIZE,
2378                 },
2379                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2380                                      DESC_HDR_SEL0_DEU |
2381                                      DESC_HDR_MODE0_DEU_CBC |
2382                                      DESC_HDR_MODE0_DEU_3DES |
2383                                      DESC_HDR_SEL1_MDEUA |
2384                                      DESC_HDR_MODE1_MDEU_INIT |
2385                                      DESC_HDR_MODE1_MDEU_PAD |
2386                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2387         },
2388         /* ABLKCIPHER algorithms. */
2389         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2390                 .alg.crypto = {
2391                         .cra_name = "ecb(aes)",
2392                         .cra_driver_name = "ecb-aes-talitos",
2393                         .cra_blocksize = AES_BLOCK_SIZE,
2394                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2395                                      CRYPTO_ALG_ASYNC,
2396                         .cra_ablkcipher = {
2397                                 .min_keysize = AES_MIN_KEY_SIZE,
2398                                 .max_keysize = AES_MAX_KEY_SIZE,
2399                                 .ivsize = AES_BLOCK_SIZE,
2400                         }
2401                 },
2402                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2403                                      DESC_HDR_SEL0_AESU,
2404         },
2405         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2406                 .alg.crypto = {
2407                         .cra_name = "cbc(aes)",
2408                         .cra_driver_name = "cbc-aes-talitos",
2409                         .cra_blocksize = AES_BLOCK_SIZE,
2410                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2411                                      CRYPTO_ALG_ASYNC,
2412                         .cra_ablkcipher = {
2413                                 .min_keysize = AES_MIN_KEY_SIZE,
2414                                 .max_keysize = AES_MAX_KEY_SIZE,
2415                                 .ivsize = AES_BLOCK_SIZE,
2416                         }
2417                 },
2418                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2419                                      DESC_HDR_SEL0_AESU |
2420                                      DESC_HDR_MODE0_AESU_CBC,
2421         },
2422         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2423                 .alg.crypto = {
2424                         .cra_name = "ctr(aes)",
2425                         .cra_driver_name = "ctr-aes-talitos",
2426                         .cra_blocksize = AES_BLOCK_SIZE,
2427                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2428                                      CRYPTO_ALG_ASYNC,
2429                         .cra_ablkcipher = {
2430                                 .min_keysize = AES_MIN_KEY_SIZE,
2431                                 .max_keysize = AES_MAX_KEY_SIZE,
2432                                 .ivsize = AES_BLOCK_SIZE,
2433                         }
2434                 },
2435                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2436                                      DESC_HDR_SEL0_AESU |
2437                                      DESC_HDR_MODE0_AESU_CTR,
2438         },
2439         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2440                 .alg.crypto = {
2441                         .cra_name = "ecb(des)",
2442                         .cra_driver_name = "ecb-des-talitos",
2443                         .cra_blocksize = DES_BLOCK_SIZE,
2444                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2445                                      CRYPTO_ALG_ASYNC,
2446                         .cra_ablkcipher = {
2447                                 .min_keysize = DES_KEY_SIZE,
2448                                 .max_keysize = DES_KEY_SIZE,
2449                                 .ivsize = DES_BLOCK_SIZE,
2450                         }
2451                 },
2452                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2453                                      DESC_HDR_SEL0_DEU,
2454         },
2455         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2456                 .alg.crypto = {
2457                         .cra_name = "cbc(des)",
2458                         .cra_driver_name = "cbc-des-talitos",
2459                         .cra_blocksize = DES_BLOCK_SIZE,
2460                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2461                                      CRYPTO_ALG_ASYNC,
2462                         .cra_ablkcipher = {
2463                                 .min_keysize = DES_KEY_SIZE,
2464                                 .max_keysize = DES_KEY_SIZE,
2465                                 .ivsize = DES_BLOCK_SIZE,
2466                         }
2467                 },
2468                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2469                                      DESC_HDR_SEL0_DEU |
2470                                      DESC_HDR_MODE0_DEU_CBC,
2471         },
2472         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2473                 .alg.crypto = {
2474                         .cra_name = "ecb(des3_ede)",
2475                         .cra_driver_name = "ecb-3des-talitos",
2476                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2477                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2478                                      CRYPTO_ALG_ASYNC,
2479                         .cra_ablkcipher = {
2480                                 .min_keysize = DES3_EDE_KEY_SIZE,
2481                                 .max_keysize = DES3_EDE_KEY_SIZE,
2482                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2483                         }
2484                 },
2485                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2486                                      DESC_HDR_SEL0_DEU |
2487                                      DESC_HDR_MODE0_DEU_3DES,
2488         },
2489         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2490                 .alg.crypto = {
2491                         .cra_name = "cbc(des3_ede)",
2492                         .cra_driver_name = "cbc-3des-talitos",
2493                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2494                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2495                                      CRYPTO_ALG_ASYNC,
2496                         .cra_ablkcipher = {
2497                                 .min_keysize = DES3_EDE_KEY_SIZE,
2498                                 .max_keysize = DES3_EDE_KEY_SIZE,
2499                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2500                         }
2501                 },
2502                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2503                                      DESC_HDR_SEL0_DEU |
2504                                      DESC_HDR_MODE0_DEU_CBC |
2505                                      DESC_HDR_MODE0_DEU_3DES,
2506         },
2507         /* AHASH algorithms. */
2508         {       .type = CRYPTO_ALG_TYPE_AHASH,
2509                 .alg.hash = {
2510                         .halg.digestsize = MD5_DIGEST_SIZE,
2511                         .halg.statesize = sizeof(struct talitos_export_state),
2512                         .halg.base = {
2513                                 .cra_name = "md5",
2514                                 .cra_driver_name = "md5-talitos",
2515                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2516                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2517                                              CRYPTO_ALG_ASYNC,
2518                         }
2519                 },
2520                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2521                                      DESC_HDR_SEL0_MDEUA |
2522                                      DESC_HDR_MODE0_MDEU_MD5,
2523         },
2524         {       .type = CRYPTO_ALG_TYPE_AHASH,
2525                 .alg.hash = {
2526                         .halg.digestsize = SHA1_DIGEST_SIZE,
2527                         .halg.statesize = sizeof(struct talitos_export_state),
2528                         .halg.base = {
2529                                 .cra_name = "sha1",
2530                                 .cra_driver_name = "sha1-talitos",
2531                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2532                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2533                                              CRYPTO_ALG_ASYNC,
2534                         }
2535                 },
2536                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2537                                      DESC_HDR_SEL0_MDEUA |
2538                                      DESC_HDR_MODE0_MDEU_SHA1,
2539         },
2540         {       .type = CRYPTO_ALG_TYPE_AHASH,
2541                 .alg.hash = {
2542                         .halg.digestsize = SHA224_DIGEST_SIZE,
2543                         .halg.statesize = sizeof(struct talitos_export_state),
2544                         .halg.base = {
2545                                 .cra_name = "sha224",
2546                                 .cra_driver_name = "sha224-talitos",
2547                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2548                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2549                                              CRYPTO_ALG_ASYNC,
2550                         }
2551                 },
2552                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2553                                      DESC_HDR_SEL0_MDEUA |
2554                                      DESC_HDR_MODE0_MDEU_SHA224,
2555         },
2556         {       .type = CRYPTO_ALG_TYPE_AHASH,
2557                 .alg.hash = {
2558                         .halg.digestsize = SHA256_DIGEST_SIZE,
2559                         .halg.statesize = sizeof(struct talitos_export_state),
2560                         .halg.base = {
2561                                 .cra_name = "sha256",
2562                                 .cra_driver_name = "sha256-talitos",
2563                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2564                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2565                                              CRYPTO_ALG_ASYNC,
2566                         }
2567                 },
2568                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2569                                      DESC_HDR_SEL0_MDEUA |
2570                                      DESC_HDR_MODE0_MDEU_SHA256,
2571         },
2572         {       .type = CRYPTO_ALG_TYPE_AHASH,
2573                 .alg.hash = {
2574                         .halg.digestsize = SHA384_DIGEST_SIZE,
2575                         .halg.statesize = sizeof(struct talitos_export_state),
2576                         .halg.base = {
2577                                 .cra_name = "sha384",
2578                                 .cra_driver_name = "sha384-talitos",
2579                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2580                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2581                                              CRYPTO_ALG_ASYNC,
2582                         }
2583                 },
2584                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2585                                      DESC_HDR_SEL0_MDEUB |
2586                                      DESC_HDR_MODE0_MDEUB_SHA384,
2587         },
2588         {       .type = CRYPTO_ALG_TYPE_AHASH,
2589                 .alg.hash = {
2590                         .halg.digestsize = SHA512_DIGEST_SIZE,
2591                         .halg.statesize = sizeof(struct talitos_export_state),
2592                         .halg.base = {
2593                                 .cra_name = "sha512",
2594                                 .cra_driver_name = "sha512-talitos",
2595                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2596                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2597                                              CRYPTO_ALG_ASYNC,
2598                         }
2599                 },
2600                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2601                                      DESC_HDR_SEL0_MDEUB |
2602                                      DESC_HDR_MODE0_MDEUB_SHA512,
2603         },
2604         {       .type = CRYPTO_ALG_TYPE_AHASH,
2605                 .alg.hash = {
2606                         .halg.digestsize = MD5_DIGEST_SIZE,
2607                         .halg.statesize = sizeof(struct talitos_export_state),
2608                         .halg.base = {
2609                                 .cra_name = "hmac(md5)",
2610                                 .cra_driver_name = "hmac-md5-talitos",
2611                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2612                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2613                                              CRYPTO_ALG_ASYNC,
2614                         }
2615                 },
2616                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2617                                      DESC_HDR_SEL0_MDEUA |
2618                                      DESC_HDR_MODE0_MDEU_MD5,
2619         },
2620         {       .type = CRYPTO_ALG_TYPE_AHASH,
2621                 .alg.hash = {
2622                         .halg.digestsize = SHA1_DIGEST_SIZE,
2623                         .halg.statesize = sizeof(struct talitos_export_state),
2624                         .halg.base = {
2625                                 .cra_name = "hmac(sha1)",
2626                                 .cra_driver_name = "hmac-sha1-talitos",
2627                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2628                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2629                                              CRYPTO_ALG_ASYNC,
2630                         }
2631                 },
2632                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2633                                      DESC_HDR_SEL0_MDEUA |
2634                                      DESC_HDR_MODE0_MDEU_SHA1,
2635         },
2636         {       .type = CRYPTO_ALG_TYPE_AHASH,
2637                 .alg.hash = {
2638                         .halg.digestsize = SHA224_DIGEST_SIZE,
2639                         .halg.statesize = sizeof(struct talitos_export_state),
2640                         .halg.base = {
2641                                 .cra_name = "hmac(sha224)",
2642                                 .cra_driver_name = "hmac-sha224-talitos",
2643                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2644                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2645                                              CRYPTO_ALG_ASYNC,
2646                         }
2647                 },
2648                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2649                                      DESC_HDR_SEL0_MDEUA |
2650                                      DESC_HDR_MODE0_MDEU_SHA224,
2651         },
2652         {       .type = CRYPTO_ALG_TYPE_AHASH,
2653                 .alg.hash = {
2654                         .halg.digestsize = SHA256_DIGEST_SIZE,
2655                         .halg.statesize = sizeof(struct talitos_export_state),
2656                         .halg.base = {
2657                                 .cra_name = "hmac(sha256)",
2658                                 .cra_driver_name = "hmac-sha256-talitos",
2659                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2660                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2661                                              CRYPTO_ALG_ASYNC,
2662                         }
2663                 },
2664                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2665                                      DESC_HDR_SEL0_MDEUA |
2666                                      DESC_HDR_MODE0_MDEU_SHA256,
2667         },
2668         {       .type = CRYPTO_ALG_TYPE_AHASH,
2669                 .alg.hash = {
2670                         .halg.digestsize = SHA384_DIGEST_SIZE,
2671                         .halg.statesize = sizeof(struct talitos_export_state),
2672                         .halg.base = {
2673                                 .cra_name = "hmac(sha384)",
2674                                 .cra_driver_name = "hmac-sha384-talitos",
2675                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2676                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2677                                              CRYPTO_ALG_ASYNC,
2678                         }
2679                 },
2680                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2681                                      DESC_HDR_SEL0_MDEUB |
2682                                      DESC_HDR_MODE0_MDEUB_SHA384,
2683         },
2684         {       .type = CRYPTO_ALG_TYPE_AHASH,
2685                 .alg.hash = {
2686                         .halg.digestsize = SHA512_DIGEST_SIZE,
2687                         .halg.statesize = sizeof(struct talitos_export_state),
2688                         .halg.base = {
2689                                 .cra_name = "hmac(sha512)",
2690                                 .cra_driver_name = "hmac-sha512-talitos",
2691                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2692                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2693                                              CRYPTO_ALG_ASYNC,
2694                         }
2695                 },
2696                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2697                                      DESC_HDR_SEL0_MDEUB |
2698                                      DESC_HDR_MODE0_MDEUB_SHA512,
2699         }
2700 };
2701 
2702 struct talitos_crypto_alg {
2703         struct list_head entry;
2704         struct device *dev;
2705         struct talitos_alg_template algt;
2706 };
2707 
2708 static int talitos_init_common(struct talitos_ctx *ctx,
2709                                struct talitos_crypto_alg *talitos_alg)
2710 {
2711         struct talitos_private *priv;
2712 
2713         /* update context with ptr to dev */
2714         ctx->dev = talitos_alg->dev;
2715 
2716         /* assign SEC channel to tfm in round-robin fashion */
2717         priv = dev_get_drvdata(ctx->dev);
2718         ctx->ch = atomic_inc_return(&priv->last_chan) &
2719                   (priv->num_channels - 1);
2720 
2721         /* copy descriptor header template value */
2722         ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
2723 
2724         /* select done notification */
2725         ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2726 
2727         return 0;
2728 }
2729 
2730 static int talitos_cra_init(struct crypto_tfm *tfm)
2731 {
2732         struct crypto_alg *alg = tfm->__crt_alg;
2733         struct talitos_crypto_alg *talitos_alg;
2734         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2735 
2736         if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2737                 talitos_alg = container_of(__crypto_ahash_alg(alg),
2738                                            struct talitos_crypto_alg,
2739                                            algt.alg.hash);
2740         else
2741                 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2742                                            algt.alg.crypto);
2743 
2744         return talitos_init_common(ctx, talitos_alg);
2745 }
2746 
2747 static int talitos_cra_init_aead(struct crypto_aead *tfm)
2748 {
2749         struct aead_alg *alg = crypto_aead_alg(tfm);
2750         struct talitos_crypto_alg *talitos_alg;
2751         struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
2752 
2753         talitos_alg = container_of(alg, struct talitos_crypto_alg,
2754                                    algt.alg.aead);
2755 
2756         return talitos_init_common(ctx, talitos_alg);
2757 }
2758 
2759 static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2760 {
2761         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2762 
2763         talitos_cra_init(tfm);
2764 
2765         ctx->keylen = 0;
2766         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2767                                  sizeof(struct talitos_ahash_req_ctx));
2768 
2769         return 0;
2770 }
2771 
2772 /*
2773  * given the alg's descriptor header template, determine whether descriptor
2774  * type and primary/secondary execution units required match the hw
2775  * capabilities description provided in the device tree node.
2776  */
2777 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2778 {
2779         struct talitos_private *priv = dev_get_drvdata(dev);
2780         int ret;
2781 
2782         ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2783               (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2784 
2785         if (SECONDARY_EU(desc_hdr_template))
2786                 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2787                               & priv->exec_units);
2788 
2789         return ret;
2790 }
2791 
2792 static int talitos_remove(struct platform_device *ofdev)
2793 {
2794         struct device *dev = &ofdev->dev;
2795         struct talitos_private *priv = dev_get_drvdata(dev);
2796         struct talitos_crypto_alg *t_alg, *n;
2797         int i;
2798 
2799         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
2800                 switch (t_alg->algt.type) {
2801                 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2802                         break;
2803                 case CRYPTO_ALG_TYPE_AEAD:
2804                         crypto_unregister_aead(&t_alg->algt.alg.aead);
2805                 case CRYPTO_ALG_TYPE_AHASH:
2806                         crypto_unregister_ahash(&t_alg->algt.alg.hash);
2807                         break;
2808                 }
2809                 list_del(&t_alg->entry);
2810                 kfree(t_alg);
2811         }
2812 
2813         if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2814                 talitos_unregister_rng(dev);
2815 
2816         for (i = 0; priv->chan && i < priv->num_channels; i++)
2817                 kfree(priv->chan[i].fifo);
2818 
2819         kfree(priv->chan);
2820 
2821         for (i = 0; i < 2; i++)
2822                 if (priv->irq[i]) {
2823                         free_irq(priv->irq[i], dev);
2824                         irq_dispose_mapping(priv->irq[i]);
2825                 }
2826 
2827         tasklet_kill(&priv->done_task[0]);
2828         if (priv->irq[1])
2829                 tasklet_kill(&priv->done_task[1]);
2830 
2831         iounmap(priv->reg);
2832 
2833         kfree(priv);
2834 
2835         return 0;
2836 }
2837 
2838 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
2839                                                     struct talitos_alg_template
2840                                                            *template)
2841 {
2842         struct talitos_private *priv = dev_get_drvdata(dev);
2843         struct talitos_crypto_alg *t_alg;
2844         struct crypto_alg *alg;
2845 
2846         t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
2847         if (!t_alg)
2848                 return ERR_PTR(-ENOMEM);
2849 
2850         t_alg->algt = *template;
2851 
2852         switch (t_alg->algt.type) {
2853         case CRYPTO_ALG_TYPE_ABLKCIPHER:
2854                 alg = &t_alg->algt.alg.crypto;
2855                 alg->cra_init = talitos_cra_init;
2856                 alg->cra_type = &crypto_ablkcipher_type;
2857                 alg->cra_ablkcipher.setkey = ablkcipher_setkey;
2858                 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
2859                 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
2860                 alg->cra_ablkcipher.geniv = "eseqiv";
2861                 break;
2862         case CRYPTO_ALG_TYPE_AEAD:
2863                 alg = &t_alg->algt.alg.aead.base;
2864                 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
2865                 t_alg->algt.alg.aead.setkey = aead_setkey;
2866                 t_alg->algt.alg.aead.encrypt = aead_encrypt;
2867                 t_alg->algt.alg.aead.decrypt = aead_decrypt;
2868                 break;
2869         case CRYPTO_ALG_TYPE_AHASH:
2870                 alg = &t_alg->algt.alg.hash.halg.base;
2871                 alg->cra_init = talitos_cra_init_ahash;
2872                 alg->cra_type = &crypto_ahash_type;
2873                 t_alg->algt.alg.hash.init = ahash_init;
2874                 t_alg->algt.alg.hash.update = ahash_update;
2875                 t_alg->algt.alg.hash.final = ahash_final;
2876                 t_alg->algt.alg.hash.finup = ahash_finup;
2877                 t_alg->algt.alg.hash.digest = ahash_digest;
2878                 t_alg->algt.alg.hash.setkey = ahash_setkey;
2879                 t_alg->algt.alg.hash.import = ahash_import;
2880                 t_alg->algt.alg.hash.export = ahash_export;
2881 
2882                 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
2883                     !strncmp(alg->cra_name, "hmac", 4)) {
2884                         kfree(t_alg);
2885                         return ERR_PTR(-ENOTSUPP);
2886                 }
2887                 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
2888                     (!strcmp(alg->cra_name, "sha224") ||
2889                      !strcmp(alg->cra_name, "hmac(sha224)"))) {
2890                         t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
2891                         t_alg->algt.desc_hdr_template =
2892                                         DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2893                                         DESC_HDR_SEL0_MDEUA |
2894                                         DESC_HDR_MODE0_MDEU_SHA256;
2895                 }
2896                 break;
2897         default:
2898                 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
2899                 kfree(t_alg);
2900                 return ERR_PTR(-EINVAL);
2901         }
2902 
2903         alg->cra_module = THIS_MODULE;
2904         alg->cra_priority = TALITOS_CRA_PRIORITY;
2905         alg->cra_alignmask = 0;
2906         alg->cra_ctxsize = sizeof(struct talitos_ctx);
2907         alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
2908 
2909         t_alg->dev = dev;
2910 
2911         return t_alg;
2912 }
2913 
2914 static int talitos_probe_irq(struct platform_device *ofdev)
2915 {
2916         struct device *dev = &ofdev->dev;
2917         struct device_node *np = ofdev->dev.of_node;
2918         struct talitos_private *priv = dev_get_drvdata(dev);
2919         int err;
2920         bool is_sec1 = has_ftr_sec1(priv);
2921 
2922         priv->irq[0] = irq_of_parse_and_map(np, 0);
2923         if (!priv->irq[0]) {
2924                 dev_err(dev, "failed to map irq\n");
2925                 return -EINVAL;
2926         }
2927         if (is_sec1) {
2928                 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
2929                                   dev_driver_string(dev), dev);
2930                 goto primary_out;
2931         }
2932 
2933         priv->irq[1] = irq_of_parse_and_map(np, 1);
2934 
2935         /* get the primary irq line */
2936         if (!priv->irq[1]) {
2937                 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
2938                                   dev_driver_string(dev), dev);
2939                 goto primary_out;
2940         }
2941 
2942         err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
2943                           dev_driver_string(dev), dev);
2944         if (err)
2945                 goto primary_out;
2946 
2947         /* get the secondary irq line */
2948         err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
2949                           dev_driver_string(dev), dev);
2950         if (err) {
2951                 dev_err(dev, "failed to request secondary irq\n");
2952                 irq_dispose_mapping(priv->irq[1]);
2953                 priv->irq[1] = 0;
2954         }
2955 
2956         return err;
2957 
2958 primary_out:
2959         if (err) {
2960                 dev_err(dev, "failed to request primary irq\n");
2961                 irq_dispose_mapping(priv->irq[0]);
2962                 priv->irq[0] = 0;
2963         }
2964 
2965         return err;
2966 }
2967 
2968 static int talitos_probe(struct platform_device *ofdev)
2969 {
2970         struct device *dev = &ofdev->dev;
2971         struct device_node *np = ofdev->dev.of_node;
2972         struct talitos_private *priv;
2973         const unsigned int *prop;
2974         int i, err;
2975         int stride;
2976 
2977         priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
2978         if (!priv)
2979                 return -ENOMEM;
2980 
2981         INIT_LIST_HEAD(&priv->alg_list);
2982 
2983         dev_set_drvdata(dev, priv);
2984 
2985         priv->ofdev = ofdev;
2986 
2987         spin_lock_init(&priv->reg_lock);
2988 
2989         priv->reg = of_iomap(np, 0);
2990         if (!priv->reg) {
2991                 dev_err(dev, "failed to of_iomap\n");
2992                 err = -ENOMEM;
2993                 goto err_out;
2994         }
2995 
2996         /* get SEC version capabilities from device tree */
2997         prop = of_get_property(np, "fsl,num-channels", NULL);
2998         if (prop)
2999                 priv->num_channels = *prop;
3000 
3001         prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
3002         if (prop)
3003                 priv->chfifo_len = *prop;
3004 
3005         prop = of_get_property(np, "fsl,exec-units-mask", NULL);
3006         if (prop)
3007                 priv->exec_units = *prop;
3008 
3009         prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
3010         if (prop)
3011                 priv->desc_types = *prop;
3012 
3013         if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
3014             !priv->exec_units || !priv->desc_types) {
3015                 dev_err(dev, "invalid property data in device tree node\n");
3016                 err = -EINVAL;
3017                 goto err_out;
3018         }
3019 
3020         if (of_device_is_compatible(np, "fsl,sec3.0"))
3021                 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3022 
3023         if (of_device_is_compatible(np, "fsl,sec2.1"))
3024                 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
3025                                   TALITOS_FTR_SHA224_HWINIT |
3026                                   TALITOS_FTR_HMAC_OK;
3027 
3028         if (of_device_is_compatible(np, "fsl,sec1.0"))
3029                 priv->features |= TALITOS_FTR_SEC1;
3030 
3031         if (of_device_is_compatible(np, "fsl,sec1.2")) {
3032                 priv->reg_deu = priv->reg + TALITOS12_DEU;
3033                 priv->reg_aesu = priv->reg + TALITOS12_AESU;
3034                 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3035                 stride = TALITOS1_CH_STRIDE;
3036         } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
3037                 priv->reg_deu = priv->reg + TALITOS10_DEU;
3038                 priv->reg_aesu = priv->reg + TALITOS10_AESU;
3039                 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3040                 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3041                 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3042                 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3043                 stride = TALITOS1_CH_STRIDE;
3044         } else {
3045                 priv->reg_deu = priv->reg + TALITOS2_DEU;
3046                 priv->reg_aesu = priv->reg + TALITOS2_AESU;
3047                 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3048                 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3049                 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3050                 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3051                 priv->reg_keu = priv->reg + TALITOS2_KEU;
3052                 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3053                 stride = TALITOS2_CH_STRIDE;
3054         }
3055 
3056         err = talitos_probe_irq(ofdev);
3057         if (err)
3058                 goto err_out;
3059 
3060         if (of_device_is_compatible(np, "fsl,sec1.0")) {
3061                 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
3062                              (unsigned long)dev);
3063         } else {
3064                 if (!priv->irq[1]) {
3065                         tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3066                                      (unsigned long)dev);
3067                 } else {
3068                         tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3069                                      (unsigned long)dev);
3070                         tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3071                                      (unsigned long)dev);
3072                 }
3073         }
3074 
3075         priv->chan = kzalloc(sizeof(struct talitos_channel) *
3076                              priv->num_channels, GFP_KERNEL);
3077         if (!priv->chan) {
3078                 dev_err(dev, "failed to allocate channel management space\n");
3079                 err = -ENOMEM;
3080                 goto err_out;
3081         }
3082 
3083         priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3084 
3085         for (i = 0; i < priv->num_channels; i++) {
3086                 priv->chan[i].reg = priv->reg + stride * (i + 1);
3087                 if (!priv->irq[1] || !(i & 1))
3088                         priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
3089 
3090                 spin_lock_init(&priv->chan[i].head_lock);
3091                 spin_lock_init(&priv->chan[i].tail_lock);
3092 
3093                 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
3094                                              priv->fifo_len, GFP_KERNEL);
3095                 if (!priv->chan[i].fifo) {
3096                         dev_err(dev, "failed to allocate request fifo %d\n", i);
3097                         err = -ENOMEM;
3098                         goto err_out;
3099                 }
3100 
3101                 atomic_set(&priv->chan[i].submit_count,
3102                            -(priv->chfifo_len - 1));
3103         }
3104 
3105         dma_set_mask(dev, DMA_BIT_MASK(36));
3106 
3107         /* reset and initialize the h/w */
3108         err = init_device(dev);
3109         if (err) {
3110                 dev_err(dev, "failed to initialize device\n");
3111                 goto err_out;
3112         }
3113 
3114         /* register the RNG, if available */
3115         if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3116                 err = talitos_register_rng(dev);
3117                 if (err) {
3118                         dev_err(dev, "failed to register hwrng: %d\n", err);
3119                         goto err_out;
3120                 } else
3121                         dev_info(dev, "hwrng\n");
3122         }
3123 
3124         /* register crypto algorithms the device supports */
3125         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3126                 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3127                         struct talitos_crypto_alg *t_alg;
3128                         struct crypto_alg *alg = NULL;
3129 
3130                         t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3131                         if (IS_ERR(t_alg)) {
3132                                 err = PTR_ERR(t_alg);
3133                                 if (err == -ENOTSUPP)
3134                                         continue;
3135                                 goto err_out;
3136                         }
3137 
3138                         switch (t_alg->algt.type) {
3139                         case CRYPTO_ALG_TYPE_ABLKCIPHER:
3140                                 err = crypto_register_alg(
3141                                                 &t_alg->algt.alg.crypto);
3142                                 alg = &t_alg->algt.alg.crypto;
3143                                 break;
3144 
3145                         case CRYPTO_ALG_TYPE_AEAD:
3146                                 err = crypto_register_aead(
3147                                         &t_alg->algt.alg.aead);
3148                                 alg = &t_alg->algt.alg.aead.base;
3149                                 break;
3150 
3151                         case CRYPTO_ALG_TYPE_AHASH:
3152                                 err = crypto_register_ahash(
3153                                                 &t_alg->algt.alg.hash);
3154                                 alg = &t_alg->algt.alg.hash.halg.base;
3155                                 break;
3156                         }
3157                         if (err) {
3158                                 dev_err(dev, "%s alg registration failed\n",
3159                                         alg->cra_driver_name);
3160                                 kfree(t_alg);
3161                         } else
3162                                 list_add_tail(&t_alg->entry, &priv->alg_list);
3163                 }
3164         }
3165         if (!list_empty(&priv->alg_list))
3166                 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3167                          (char *)of_get_property(np, "compatible", NULL));
3168 
3169         return 0;
3170 
3171 err_out:
3172         talitos_remove(ofdev);
3173 
3174         return err;
3175 }
3176 
3177 static const struct of_device_id talitos_match[] = {
3178 #ifdef CONFIG_CRYPTO_DEV_TALITOS1
3179         {
3180                 .compatible = "fsl,sec1.0",
3181         },
3182 #endif
3183 #ifdef CONFIG_CRYPTO_DEV_TALITOS2
3184         {
3185                 .compatible = "fsl,sec2.0",
3186         },
3187 #endif
3188         {},
3189 };
3190 MODULE_DEVICE_TABLE(of, talitos_match);
3191 
3192 static struct platform_driver talitos_driver = {
3193         .driver = {
3194                 .name = "talitos",
3195                 .of_match_table = talitos_match,
3196         },
3197         .probe = talitos_probe,
3198         .remove = talitos_remove,
3199 };
3200 
3201 module_platform_driver(talitos_driver);
3202 
3203 MODULE_LICENSE("GPL");
3204 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3205 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");
3206 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us