Version:  2.0.40 2.2.26 2.4.37 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6

Linux/drivers/crypto/talitos.c

  1 /*
  2  * talitos - Freescale Integrated Security Engine (SEC) device driver
  3  *
  4  * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
  5  *
  6  * Scatterlist Crypto API glue code copied from files with the following:
  7  * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
  8  *
  9  * Crypto algorithm registration code copied from hifn driver:
 10  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
 11  * All rights reserved.
 12  *
 13  * This program is free software; you can redistribute it and/or modify
 14  * it under the terms of the GNU General Public License as published by
 15  * the Free Software Foundation; either version 2 of the License, or
 16  * (at your option) any later version.
 17  *
 18  * This program is distributed in the hope that it will be useful,
 19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 21  * GNU General Public License for more details.
 22  *
 23  * You should have received a copy of the GNU General Public License
 24  * along with this program; if not, write to the Free Software
 25  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 26  */
 27 
 28 #include <linux/kernel.h>
 29 #include <linux/module.h>
 30 #include <linux/mod_devicetable.h>
 31 #include <linux/device.h>
 32 #include <linux/interrupt.h>
 33 #include <linux/crypto.h>
 34 #include <linux/hw_random.h>
 35 #include <linux/of_address.h>
 36 #include <linux/of_irq.h>
 37 #include <linux/of_platform.h>
 38 #include <linux/dma-mapping.h>
 39 #include <linux/io.h>
 40 #include <linux/spinlock.h>
 41 #include <linux/rtnetlink.h>
 42 #include <linux/slab.h>
 43 
 44 #include <crypto/algapi.h>
 45 #include <crypto/aes.h>
 46 #include <crypto/des.h>
 47 #include <crypto/sha.h>
 48 #include <crypto/md5.h>
 49 #include <crypto/internal/aead.h>
 50 #include <crypto/authenc.h>
 51 #include <crypto/skcipher.h>
 52 #include <crypto/hash.h>
 53 #include <crypto/internal/hash.h>
 54 #include <crypto/scatterwalk.h>
 55 
 56 #include "talitos.h"
 57 
 58 static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
 59                            bool is_sec1)
 60 {
 61         ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
 62         if (!is_sec1)
 63                 ptr->eptr = upper_32_bits(dma_addr);
 64 }
 65 
 66 static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
 67                              struct talitos_ptr *src_ptr, bool is_sec1)
 68 {
 69         dst_ptr->ptr = src_ptr->ptr;
 70         if (!is_sec1)
 71                 dst_ptr->eptr = src_ptr->eptr;
 72 }
 73 
 74 static void to_talitos_ptr_len(struct talitos_ptr *ptr, unsigned int len,
 75                                bool is_sec1)
 76 {
 77         if (is_sec1) {
 78                 ptr->res = 0;
 79                 ptr->len1 = cpu_to_be16(len);
 80         } else {
 81                 ptr->len = cpu_to_be16(len);
 82         }
 83 }
 84 
 85 static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
 86                                            bool is_sec1)
 87 {
 88         if (is_sec1)
 89                 return be16_to_cpu(ptr->len1);
 90         else
 91                 return be16_to_cpu(ptr->len);
 92 }
 93 
 94 static void to_talitos_ptr_extent_clear(struct talitos_ptr *ptr, bool is_sec1)
 95 {
 96         if (!is_sec1)
 97                 ptr->j_extent = 0;
 98 }
 99 
100 /*
101  * map virtual single (contiguous) pointer to h/w descriptor pointer
102  */
103 static void map_single_talitos_ptr(struct device *dev,
104                                    struct talitos_ptr *ptr,
105                                    unsigned int len, void *data,
106                                    enum dma_data_direction dir)
107 {
108         dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
109         struct talitos_private *priv = dev_get_drvdata(dev);
110         bool is_sec1 = has_ftr_sec1(priv);
111 
112         to_talitos_ptr_len(ptr, len, is_sec1);
113         to_talitos_ptr(ptr, dma_addr, is_sec1);
114         to_talitos_ptr_extent_clear(ptr, is_sec1);
115 }
116 
117 /*
118  * unmap bus single (contiguous) h/w descriptor pointer
119  */
120 static void unmap_single_talitos_ptr(struct device *dev,
121                                      struct talitos_ptr *ptr,
122                                      enum dma_data_direction dir)
123 {
124         struct talitos_private *priv = dev_get_drvdata(dev);
125         bool is_sec1 = has_ftr_sec1(priv);
126 
127         dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
128                          from_talitos_ptr_len(ptr, is_sec1), dir);
129 }
130 
131 static int reset_channel(struct device *dev, int ch)
132 {
133         struct talitos_private *priv = dev_get_drvdata(dev);
134         unsigned int timeout = TALITOS_TIMEOUT;
135         bool is_sec1 = has_ftr_sec1(priv);
136 
137         if (is_sec1) {
138                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
139                           TALITOS1_CCCR_LO_RESET);
140 
141                 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
142                         TALITOS1_CCCR_LO_RESET) && --timeout)
143                         cpu_relax();
144         } else {
145                 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
146                           TALITOS2_CCCR_RESET);
147 
148                 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
149                         TALITOS2_CCCR_RESET) && --timeout)
150                         cpu_relax();
151         }
152 
153         if (timeout == 0) {
154                 dev_err(dev, "failed to reset channel %d\n", ch);
155                 return -EIO;
156         }
157 
158         /* set 36-bit addressing, done writeback enable and done IRQ enable */
159         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
160                   TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
161 
162         /* and ICCR writeback, if available */
163         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
164                 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
165                           TALITOS_CCCR_LO_IWSE);
166 
167         return 0;
168 }
169 
170 static int reset_device(struct device *dev)
171 {
172         struct talitos_private *priv = dev_get_drvdata(dev);
173         unsigned int timeout = TALITOS_TIMEOUT;
174         bool is_sec1 = has_ftr_sec1(priv);
175         u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
176 
177         setbits32(priv->reg + TALITOS_MCR, mcr);
178 
179         while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
180                && --timeout)
181                 cpu_relax();
182 
183         if (priv->irq[1]) {
184                 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
185                 setbits32(priv->reg + TALITOS_MCR, mcr);
186         }
187 
188         if (timeout == 0) {
189                 dev_err(dev, "failed to reset device\n");
190                 return -EIO;
191         }
192 
193         return 0;
194 }
195 
196 /*
197  * Reset and initialize the device
198  */
199 static int init_device(struct device *dev)
200 {
201         struct talitos_private *priv = dev_get_drvdata(dev);
202         int ch, err;
203         bool is_sec1 = has_ftr_sec1(priv);
204 
205         /*
206          * Master reset
207          * errata documentation: warning: certain SEC interrupts
208          * are not fully cleared by writing the MCR:SWR bit,
209          * set bit twice to completely reset
210          */
211         err = reset_device(dev);
212         if (err)
213                 return err;
214 
215         err = reset_device(dev);
216         if (err)
217                 return err;
218 
219         /* reset channels */
220         for (ch = 0; ch < priv->num_channels; ch++) {
221                 err = reset_channel(dev, ch);
222                 if (err)
223                         return err;
224         }
225 
226         /* enable channel done and error interrupts */
227         if (is_sec1) {
228                 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
229                 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
230                 /* disable parity error check in DEU (erroneous? test vect.) */
231                 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
232         } else {
233                 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
234                 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
235         }
236 
237         /* disable integrity check error interrupts (use writeback instead) */
238         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
239                 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
240                           TALITOS_MDEUICR_LO_ICE);
241 
242         return 0;
243 }
244 
245 /**
246  * talitos_submit - submits a descriptor to the device for processing
247  * @dev:        the SEC device to be used
248  * @ch:         the SEC device channel to be used
249  * @desc:       the descriptor to be processed by the device
250  * @callback:   whom to call when processing is complete
251  * @context:    a handle for use by caller (optional)
252  *
253  * desc must contain valid dma-mapped (bus physical) address pointers.
254  * callback must check err and feedback in descriptor header
255  * for device processing status.
256  */
257 int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
258                    void (*callback)(struct device *dev,
259                                     struct talitos_desc *desc,
260                                     void *context, int error),
261                    void *context)
262 {
263         struct talitos_private *priv = dev_get_drvdata(dev);
264         struct talitos_request *request;
265         unsigned long flags;
266         int head;
267         bool is_sec1 = has_ftr_sec1(priv);
268 
269         spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
270 
271         if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
272                 /* h/w fifo is full */
273                 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
274                 return -EAGAIN;
275         }
276 
277         head = priv->chan[ch].head;
278         request = &priv->chan[ch].fifo[head];
279 
280         /* map descriptor and save caller data */
281         if (is_sec1) {
282                 desc->hdr1 = desc->hdr;
283                 desc->next_desc = 0;
284                 request->dma_desc = dma_map_single(dev, &desc->hdr1,
285                                                    TALITOS_DESC_SIZE,
286                                                    DMA_BIDIRECTIONAL);
287         } else {
288                 request->dma_desc = dma_map_single(dev, desc,
289                                                    TALITOS_DESC_SIZE,
290                                                    DMA_BIDIRECTIONAL);
291         }
292         request->callback = callback;
293         request->context = context;
294 
295         /* increment fifo head */
296         priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
297 
298         smp_wmb();
299         request->desc = desc;
300 
301         /* GO! */
302         wmb();
303         out_be32(priv->chan[ch].reg + TALITOS_FF,
304                  upper_32_bits(request->dma_desc));
305         out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
306                  lower_32_bits(request->dma_desc));
307 
308         spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
309 
310         return -EINPROGRESS;
311 }
312 EXPORT_SYMBOL(talitos_submit);
313 
314 /*
315  * process what was done, notify callback of error if not
316  */
317 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
318 {
319         struct talitos_private *priv = dev_get_drvdata(dev);
320         struct talitos_request *request, saved_req;
321         unsigned long flags;
322         int tail, status;
323         bool is_sec1 = has_ftr_sec1(priv);
324 
325         spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
326 
327         tail = priv->chan[ch].tail;
328         while (priv->chan[ch].fifo[tail].desc) {
329                 __be32 hdr;
330 
331                 request = &priv->chan[ch].fifo[tail];
332 
333                 /* descriptors with their done bits set don't get the error */
334                 rmb();
335                 hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr;
336 
337                 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
338                         status = 0;
339                 else
340                         if (!error)
341                                 break;
342                         else
343                                 status = error;
344 
345                 dma_unmap_single(dev, request->dma_desc,
346                                  TALITOS_DESC_SIZE,
347                                  DMA_BIDIRECTIONAL);
348 
349                 /* copy entries so we can call callback outside lock */
350                 saved_req.desc = request->desc;
351                 saved_req.callback = request->callback;
352                 saved_req.context = request->context;
353 
354                 /* release request entry in fifo */
355                 smp_wmb();
356                 request->desc = NULL;
357 
358                 /* increment fifo tail */
359                 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
360 
361                 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
362 
363                 atomic_dec(&priv->chan[ch].submit_count);
364 
365                 saved_req.callback(dev, saved_req.desc, saved_req.context,
366                                    status);
367                 /* channel may resume processing in single desc error case */
368                 if (error && !reset_ch && status == error)
369                         return;
370                 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
371                 tail = priv->chan[ch].tail;
372         }
373 
374         spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
375 }
376 
377 /*
378  * process completed requests for channels that have done status
379  */
380 #define DEF_TALITOS1_DONE(name, ch_done_mask)                           \
381 static void talitos1_done_##name(unsigned long data)                    \
382 {                                                                       \
383         struct device *dev = (struct device *)data;                     \
384         struct talitos_private *priv = dev_get_drvdata(dev);            \
385         unsigned long flags;                                            \
386                                                                         \
387         if (ch_done_mask & 0x10000000)                                  \
388                 flush_channel(dev, 0, 0, 0);                    \
389         if (priv->num_channels == 1)                                    \
390                 goto out;                                               \
391         if (ch_done_mask & 0x40000000)                                  \
392                 flush_channel(dev, 1, 0, 0);                    \
393         if (ch_done_mask & 0x00010000)                                  \
394                 flush_channel(dev, 2, 0, 0);                    \
395         if (ch_done_mask & 0x00040000)                                  \
396                 flush_channel(dev, 3, 0, 0);                    \
397                                                                         \
398 out:                                                                    \
399         /* At this point, all completed channels have been processed */ \
400         /* Unmask done interrupts for channels completed later on. */   \
401         spin_lock_irqsave(&priv->reg_lock, flags);                      \
402         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
403         clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);    \
404         spin_unlock_irqrestore(&priv->reg_lock, flags);                 \
405 }
406 
407 DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
408 
409 #define DEF_TALITOS2_DONE(name, ch_done_mask)                           \
410 static void talitos2_done_##name(unsigned long data)                    \
411 {                                                                       \
412         struct device *dev = (struct device *)data;                     \
413         struct talitos_private *priv = dev_get_drvdata(dev);            \
414         unsigned long flags;                                            \
415                                                                         \
416         if (ch_done_mask & 1)                                           \
417                 flush_channel(dev, 0, 0, 0);                            \
418         if (priv->num_channels == 1)                                    \
419                 goto out;                                               \
420         if (ch_done_mask & (1 << 2))                                    \
421                 flush_channel(dev, 1, 0, 0);                            \
422         if (ch_done_mask & (1 << 4))                                    \
423                 flush_channel(dev, 2, 0, 0);                            \
424         if (ch_done_mask & (1 << 6))                                    \
425                 flush_channel(dev, 3, 0, 0);                            \
426                                                                         \
427 out:                                                                    \
428         /* At this point, all completed channels have been processed */ \
429         /* Unmask done interrupts for channels completed later on. */   \
430         spin_lock_irqsave(&priv->reg_lock, flags);                      \
431         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);               \
432         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);    \
433         spin_unlock_irqrestore(&priv->reg_lock, flags);                 \
434 }
435 
436 DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
437 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
438 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
439 
440 /*
441  * locate current (offending) descriptor
442  */
443 static u32 current_desc_hdr(struct device *dev, int ch)
444 {
445         struct talitos_private *priv = dev_get_drvdata(dev);
446         int tail, iter;
447         dma_addr_t cur_desc;
448 
449         cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
450         cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
451 
452         if (!cur_desc) {
453                 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
454                 return 0;
455         }
456 
457         tail = priv->chan[ch].tail;
458 
459         iter = tail;
460         while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) {
461                 iter = (iter + 1) & (priv->fifo_len - 1);
462                 if (iter == tail) {
463                         dev_err(dev, "couldn't locate current descriptor\n");
464                         return 0;
465                 }
466         }
467 
468         return priv->chan[ch].fifo[iter].desc->hdr;
469 }
470 
471 /*
472  * user diagnostics; report root cause of error based on execution unit status
473  */
474 static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
475 {
476         struct talitos_private *priv = dev_get_drvdata(dev);
477         int i;
478 
479         if (!desc_hdr)
480                 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
481 
482         switch (desc_hdr & DESC_HDR_SEL0_MASK) {
483         case DESC_HDR_SEL0_AFEU:
484                 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
485                         in_be32(priv->reg_afeu + TALITOS_EUISR),
486                         in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
487                 break;
488         case DESC_HDR_SEL0_DEU:
489                 dev_err(dev, "DEUISR 0x%08x_%08x\n",
490                         in_be32(priv->reg_deu + TALITOS_EUISR),
491                         in_be32(priv->reg_deu + TALITOS_EUISR_LO));
492                 break;
493         case DESC_HDR_SEL0_MDEUA:
494         case DESC_HDR_SEL0_MDEUB:
495                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
496                         in_be32(priv->reg_mdeu + TALITOS_EUISR),
497                         in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
498                 break;
499         case DESC_HDR_SEL0_RNG:
500                 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
501                         in_be32(priv->reg_rngu + TALITOS_ISR),
502                         in_be32(priv->reg_rngu + TALITOS_ISR_LO));
503                 break;
504         case DESC_HDR_SEL0_PKEU:
505                 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
506                         in_be32(priv->reg_pkeu + TALITOS_EUISR),
507                         in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
508                 break;
509         case DESC_HDR_SEL0_AESU:
510                 dev_err(dev, "AESUISR 0x%08x_%08x\n",
511                         in_be32(priv->reg_aesu + TALITOS_EUISR),
512                         in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
513                 break;
514         case DESC_HDR_SEL0_CRCU:
515                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
516                         in_be32(priv->reg_crcu + TALITOS_EUISR),
517                         in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
518                 break;
519         case DESC_HDR_SEL0_KEU:
520                 dev_err(dev, "KEUISR 0x%08x_%08x\n",
521                         in_be32(priv->reg_pkeu + TALITOS_EUISR),
522                         in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
523                 break;
524         }
525 
526         switch (desc_hdr & DESC_HDR_SEL1_MASK) {
527         case DESC_HDR_SEL1_MDEUA:
528         case DESC_HDR_SEL1_MDEUB:
529                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
530                         in_be32(priv->reg_mdeu + TALITOS_EUISR),
531                         in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
532                 break;
533         case DESC_HDR_SEL1_CRCU:
534                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
535                         in_be32(priv->reg_crcu + TALITOS_EUISR),
536                         in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
537                 break;
538         }
539 
540         for (i = 0; i < 8; i++)
541                 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
542                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
543                         in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
544 }
545 
546 /*
547  * recover from error interrupts
548  */
549 static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
550 {
551         struct talitos_private *priv = dev_get_drvdata(dev);
552         unsigned int timeout = TALITOS_TIMEOUT;
553         int ch, error, reset_dev = 0;
554         u32 v_lo;
555         bool is_sec1 = has_ftr_sec1(priv);
556         int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
557 
558         for (ch = 0; ch < priv->num_channels; ch++) {
559                 /* skip channels without errors */
560                 if (is_sec1) {
561                         /* bits 29, 31, 17, 19 */
562                         if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
563                                 continue;
564                 } else {
565                         if (!(isr & (1 << (ch * 2 + 1))))
566                                 continue;
567                 }
568 
569                 error = -EINVAL;
570 
571                 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
572 
573                 if (v_lo & TALITOS_CCPSR_LO_DOF) {
574                         dev_err(dev, "double fetch fifo overflow error\n");
575                         error = -EAGAIN;
576                         reset_ch = 1;
577                 }
578                 if (v_lo & TALITOS_CCPSR_LO_SOF) {
579                         /* h/w dropped descriptor */
580                         dev_err(dev, "single fetch fifo overflow error\n");
581                         error = -EAGAIN;
582                 }
583                 if (v_lo & TALITOS_CCPSR_LO_MDTE)
584                         dev_err(dev, "master data transfer error\n");
585                 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
586                         dev_err(dev, is_sec1 ? "pointeur not complete error\n"
587                                              : "s/g data length zero error\n");
588                 if (v_lo & TALITOS_CCPSR_LO_FPZ)
589                         dev_err(dev, is_sec1 ? "parity error\n"
590                                              : "fetch pointer zero error\n");
591                 if (v_lo & TALITOS_CCPSR_LO_IDH)
592                         dev_err(dev, "illegal descriptor header error\n");
593                 if (v_lo & TALITOS_CCPSR_LO_IEU)
594                         dev_err(dev, is_sec1 ? "static assignment error\n"
595                                              : "invalid exec unit error\n");
596                 if (v_lo & TALITOS_CCPSR_LO_EU)
597                         report_eu_error(dev, ch, current_desc_hdr(dev, ch));
598                 if (!is_sec1) {
599                         if (v_lo & TALITOS_CCPSR_LO_GB)
600                                 dev_err(dev, "gather boundary error\n");
601                         if (v_lo & TALITOS_CCPSR_LO_GRL)
602                                 dev_err(dev, "gather return/length error\n");
603                         if (v_lo & TALITOS_CCPSR_LO_SB)
604                                 dev_err(dev, "scatter boundary error\n");
605                         if (v_lo & TALITOS_CCPSR_LO_SRL)
606                                 dev_err(dev, "scatter return/length error\n");
607                 }
608 
609                 flush_channel(dev, ch, error, reset_ch);
610 
611                 if (reset_ch) {
612                         reset_channel(dev, ch);
613                 } else {
614                         setbits32(priv->chan[ch].reg + TALITOS_CCCR,
615                                   TALITOS2_CCCR_CONT);
616                         setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
617                         while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
618                                TALITOS2_CCCR_CONT) && --timeout)
619                                 cpu_relax();
620                         if (timeout == 0) {
621                                 dev_err(dev, "failed to restart channel %d\n",
622                                         ch);
623                                 reset_dev = 1;
624                         }
625                 }
626         }
627         if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
628             (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
629                 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
630                         dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
631                                 isr, isr_lo);
632                 else
633                         dev_err(dev, "done overflow, internal time out, or "
634                                 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
635 
636                 /* purge request queues */
637                 for (ch = 0; ch < priv->num_channels; ch++)
638                         flush_channel(dev, ch, -EIO, 1);
639 
640                 /* reset and reinitialize the device */
641                 init_device(dev);
642         }
643 }
644 
645 #define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
646 static irqreturn_t talitos1_interrupt_##name(int irq, void *data)              \
647 {                                                                              \
648         struct device *dev = data;                                             \
649         struct talitos_private *priv = dev_get_drvdata(dev);                   \
650         u32 isr, isr_lo;                                                       \
651         unsigned long flags;                                                   \
652                                                                                \
653         spin_lock_irqsave(&priv->reg_lock, flags);                             \
654         isr = in_be32(priv->reg + TALITOS_ISR);                                \
655         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
656         /* Acknowledge interrupt */                                            \
657         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
658         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
659                                                                                \
660         if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) {    \
661                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
662                 talitos_error(dev, isr & ch_err_mask, isr_lo);                 \
663         }                                                                      \
664         else {                                                                 \
665                 if (likely(isr & ch_done_mask)) {                              \
666                         /* mask further done interrupts. */                    \
667                         setbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
668                         /* done_task will unmask done interrupts at exit */    \
669                         tasklet_schedule(&priv->done_task[tlet]);              \
670                 }                                                              \
671                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
672         }                                                                      \
673                                                                                \
674         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
675                                                                 IRQ_NONE;      \
676 }
677 
678 DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
679 
680 #define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet)          \
681 static irqreturn_t talitos2_interrupt_##name(int irq, void *data)              \
682 {                                                                              \
683         struct device *dev = data;                                             \
684         struct talitos_private *priv = dev_get_drvdata(dev);                   \
685         u32 isr, isr_lo;                                                       \
686         unsigned long flags;                                                   \
687                                                                                \
688         spin_lock_irqsave(&priv->reg_lock, flags);                             \
689         isr = in_be32(priv->reg + TALITOS_ISR);                                \
690         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);                          \
691         /* Acknowledge interrupt */                                            \
692         out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
693         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);                          \
694                                                                                \
695         if (unlikely(isr & ch_err_mask || isr_lo)) {                           \
696                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
697                 talitos_error(dev, isr & ch_err_mask, isr_lo);                 \
698         }                                                                      \
699         else {                                                                 \
700                 if (likely(isr & ch_done_mask)) {                              \
701                         /* mask further done interrupts. */                    \
702                         clrbits32(priv->reg + TALITOS_IMR, ch_done_mask);      \
703                         /* done_task will unmask done interrupts at exit */    \
704                         tasklet_schedule(&priv->done_task[tlet]);              \
705                 }                                                              \
706                 spin_unlock_irqrestore(&priv->reg_lock, flags);                \
707         }                                                                      \
708                                                                                \
709         return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED :  \
710                                                                 IRQ_NONE;      \
711 }
712 
713 DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
714 DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
715                        0)
716 DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
717                        1)
718 
719 /*
720  * hwrng
721  */
722 static int talitos_rng_data_present(struct hwrng *rng, int wait)
723 {
724         struct device *dev = (struct device *)rng->priv;
725         struct talitos_private *priv = dev_get_drvdata(dev);
726         u32 ofl;
727         int i;
728 
729         for (i = 0; i < 20; i++) {
730                 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
731                       TALITOS_RNGUSR_LO_OFL;
732                 if (ofl || !wait)
733                         break;
734                 udelay(10);
735         }
736 
737         return !!ofl;
738 }
739 
740 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
741 {
742         struct device *dev = (struct device *)rng->priv;
743         struct talitos_private *priv = dev_get_drvdata(dev);
744 
745         /* rng fifo requires 64-bit accesses */
746         *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
747         *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
748 
749         return sizeof(u32);
750 }
751 
752 static int talitos_rng_init(struct hwrng *rng)
753 {
754         struct device *dev = (struct device *)rng->priv;
755         struct talitos_private *priv = dev_get_drvdata(dev);
756         unsigned int timeout = TALITOS_TIMEOUT;
757 
758         setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
759         while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
760                  & TALITOS_RNGUSR_LO_RD)
761                && --timeout)
762                 cpu_relax();
763         if (timeout == 0) {
764                 dev_err(dev, "failed to reset rng hw\n");
765                 return -ENODEV;
766         }
767 
768         /* start generating */
769         setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
770 
771         return 0;
772 }
773 
774 static int talitos_register_rng(struct device *dev)
775 {
776         struct talitos_private *priv = dev_get_drvdata(dev);
777         int err;
778 
779         priv->rng.name          = dev_driver_string(dev),
780         priv->rng.init          = talitos_rng_init,
781         priv->rng.data_present  = talitos_rng_data_present,
782         priv->rng.data_read     = talitos_rng_data_read,
783         priv->rng.priv          = (unsigned long)dev;
784 
785         err = hwrng_register(&priv->rng);
786         if (!err)
787                 priv->rng_registered = true;
788 
789         return err;
790 }
791 
792 static void talitos_unregister_rng(struct device *dev)
793 {
794         struct talitos_private *priv = dev_get_drvdata(dev);
795 
796         if (!priv->rng_registered)
797                 return;
798 
799         hwrng_unregister(&priv->rng);
800         priv->rng_registered = false;
801 }
802 
803 /*
804  * crypto alg
805  */
806 #define TALITOS_CRA_PRIORITY            3000
807 #define TALITOS_MAX_KEY_SIZE            96
808 #define TALITOS_MAX_IV_LENGTH           16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
809 
810 struct talitos_ctx {
811         struct device *dev;
812         int ch;
813         __be32 desc_hdr_template;
814         u8 key[TALITOS_MAX_KEY_SIZE];
815         u8 iv[TALITOS_MAX_IV_LENGTH];
816         unsigned int keylen;
817         unsigned int enckeylen;
818         unsigned int authkeylen;
819 };
820 
821 #define HASH_MAX_BLOCK_SIZE             SHA512_BLOCK_SIZE
822 #define TALITOS_MDEU_MAX_CONTEXT_SIZE   TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
823 
824 struct talitos_ahash_req_ctx {
825         u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
826         unsigned int hw_context_size;
827         u8 buf[HASH_MAX_BLOCK_SIZE];
828         u8 bufnext[HASH_MAX_BLOCK_SIZE];
829         unsigned int swinit;
830         unsigned int first;
831         unsigned int last;
832         unsigned int to_hash_later;
833         unsigned int nbuf;
834         struct scatterlist bufsl[2];
835         struct scatterlist *psrc;
836 };
837 
838 static int aead_setkey(struct crypto_aead *authenc,
839                        const u8 *key, unsigned int keylen)
840 {
841         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
842         struct crypto_authenc_keys keys;
843 
844         if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
845                 goto badkey;
846 
847         if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
848                 goto badkey;
849 
850         memcpy(ctx->key, keys.authkey, keys.authkeylen);
851         memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
852 
853         ctx->keylen = keys.authkeylen + keys.enckeylen;
854         ctx->enckeylen = keys.enckeylen;
855         ctx->authkeylen = keys.authkeylen;
856 
857         return 0;
858 
859 badkey:
860         crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
861         return -EINVAL;
862 }
863 
864 /*
865  * talitos_edesc - s/w-extended descriptor
866  * @src_nents: number of segments in input scatterlist
867  * @dst_nents: number of segments in output scatterlist
868  * @icv_ool: whether ICV is out-of-line
869  * @iv_dma: dma address of iv for checking continuity and link table
870  * @dma_len: length of dma mapped link_tbl space
871  * @dma_link_tbl: bus physical address of link_tbl/buf
872  * @desc: h/w descriptor
873  * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
874  * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
875  *
876  * if decrypting (with authcheck), or either one of src_nents or dst_nents
877  * is greater than 1, an integrity check value is concatenated to the end
878  * of link_tbl data
879  */
880 struct talitos_edesc {
881         int src_nents;
882         int dst_nents;
883         bool icv_ool;
884         dma_addr_t iv_dma;
885         int dma_len;
886         dma_addr_t dma_link_tbl;
887         struct talitos_desc desc;
888         union {
889                 struct talitos_ptr link_tbl[0];
890                 u8 buf[0];
891         };
892 };
893 
894 static void talitos_sg_unmap(struct device *dev,
895                              struct talitos_edesc *edesc,
896                              struct scatterlist *src,
897                              struct scatterlist *dst)
898 {
899         unsigned int src_nents = edesc->src_nents ? : 1;
900         unsigned int dst_nents = edesc->dst_nents ? : 1;
901 
902         if (src != dst) {
903                 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
904 
905                 if (dst) {
906                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
907                 }
908         } else
909                 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
910 }
911 
912 static void ipsec_esp_unmap(struct device *dev,
913                             struct talitos_edesc *edesc,
914                             struct aead_request *areq)
915 {
916         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
917         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
918         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
919         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
920 
921         talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
922 
923         if (edesc->dma_len)
924                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
925                                  DMA_BIDIRECTIONAL);
926 }
927 
928 /*
929  * ipsec_esp descriptor callbacks
930  */
931 static void ipsec_esp_encrypt_done(struct device *dev,
932                                    struct talitos_desc *desc, void *context,
933                                    int err)
934 {
935         struct aead_request *areq = context;
936         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
937         unsigned int authsize = crypto_aead_authsize(authenc);
938         struct talitos_edesc *edesc;
939         struct scatterlist *sg;
940         void *icvdata;
941 
942         edesc = container_of(desc, struct talitos_edesc, desc);
943 
944         ipsec_esp_unmap(dev, edesc, areq);
945 
946         /* copy the generated ICV to dst */
947         if (edesc->icv_ool) {
948                 icvdata = &edesc->link_tbl[edesc->src_nents +
949                                            edesc->dst_nents + 2];
950                 sg = sg_last(areq->dst, edesc->dst_nents);
951                 memcpy((char *)sg_virt(sg) + sg->length - authsize,
952                        icvdata, authsize);
953         }
954 
955         kfree(edesc);
956 
957         aead_request_complete(areq, err);
958 }
959 
960 static void ipsec_esp_decrypt_swauth_done(struct device *dev,
961                                           struct talitos_desc *desc,
962                                           void *context, int err)
963 {
964         struct aead_request *req = context;
965         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
966         unsigned int authsize = crypto_aead_authsize(authenc);
967         struct talitos_edesc *edesc;
968         struct scatterlist *sg;
969         char *oicv, *icv;
970 
971         edesc = container_of(desc, struct talitos_edesc, desc);
972 
973         ipsec_esp_unmap(dev, edesc, req);
974 
975         if (!err) {
976                 /* auth check */
977                 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
978                 icv = (char *)sg_virt(sg) + sg->length - authsize;
979 
980                 if (edesc->dma_len) {
981                         oicv = (char *)&edesc->link_tbl[edesc->src_nents +
982                                                         edesc->dst_nents + 2];
983                         if (edesc->icv_ool)
984                                 icv = oicv + authsize;
985                 } else
986                         oicv = (char *)&edesc->link_tbl[0];
987 
988                 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
989         }
990 
991         kfree(edesc);
992 
993         aead_request_complete(req, err);
994 }
995 
996 static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
997                                           struct talitos_desc *desc,
998                                           void *context, int err)
999 {
1000         struct aead_request *req = context;
1001         struct talitos_edesc *edesc;
1002 
1003         edesc = container_of(desc, struct talitos_edesc, desc);
1004 
1005         ipsec_esp_unmap(dev, edesc, req);
1006 
1007         /* check ICV auth status */
1008         if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1009                      DESC_HDR_LO_ICCR1_PASS))
1010                 err = -EBADMSG;
1011 
1012         kfree(edesc);
1013 
1014         aead_request_complete(req, err);
1015 }
1016 
1017 /*
1018  * convert scatterlist to SEC h/w link table format
1019  * stop at cryptlen bytes
1020  */
1021 static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1022                                  unsigned int offset, int cryptlen,
1023                                  struct talitos_ptr *link_tbl_ptr)
1024 {
1025         int n_sg = sg_count;
1026         int count = 0;
1027 
1028         while (cryptlen && sg && n_sg--) {
1029                 unsigned int len = sg_dma_len(sg);
1030 
1031                 if (offset >= len) {
1032                         offset -= len;
1033                         goto next;
1034                 }
1035 
1036                 len -= offset;
1037 
1038                 if (len > cryptlen)
1039                         len = cryptlen;
1040 
1041                 to_talitos_ptr(link_tbl_ptr + count,
1042                                sg_dma_address(sg) + offset, 0);
1043                 link_tbl_ptr[count].len = cpu_to_be16(len);
1044                 link_tbl_ptr[count].j_extent = 0;
1045                 count++;
1046                 cryptlen -= len;
1047                 offset = 0;
1048 
1049 next:
1050                 sg = sg_next(sg);
1051         }
1052 
1053         /* tag end of link table */
1054         if (count > 0)
1055                 link_tbl_ptr[count - 1].j_extent = DESC_PTR_LNKTBL_RETURN;
1056 
1057         return count;
1058 }
1059 
1060 static inline int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
1061                                  int cryptlen,
1062                                  struct talitos_ptr *link_tbl_ptr)
1063 {
1064         return sg_to_link_tbl_offset(sg, sg_count, 0, cryptlen,
1065                                      link_tbl_ptr);
1066 }
1067 
1068 /*
1069  * fill in and submit ipsec_esp descriptor
1070  */
1071 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1072                      void (*callback)(struct device *dev,
1073                                       struct talitos_desc *desc,
1074                                       void *context, int error))
1075 {
1076         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1077         unsigned int authsize = crypto_aead_authsize(aead);
1078         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1079         struct device *dev = ctx->dev;
1080         struct talitos_desc *desc = &edesc->desc;
1081         unsigned int cryptlen = areq->cryptlen;
1082         unsigned int ivsize = crypto_aead_ivsize(aead);
1083         int tbl_off = 0;
1084         int sg_count, ret;
1085         int sg_link_tbl_len;
1086 
1087         /* hmac key */
1088         map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
1089                                DMA_TO_DEVICE);
1090 
1091         sg_count = dma_map_sg(dev, areq->src, edesc->src_nents ?: 1,
1092                               (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1093                                                            : DMA_TO_DEVICE);
1094         /* hmac data */
1095         desc->ptr[1].len = cpu_to_be16(areq->assoclen);
1096         if (sg_count > 1 &&
1097             (ret = sg_to_link_tbl_offset(areq->src, sg_count, 0,
1098                                          areq->assoclen,
1099                                          &edesc->link_tbl[tbl_off])) > 1) {
1100                 to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off *
1101                                sizeof(struct talitos_ptr), 0);
1102                 desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP;
1103 
1104                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1105                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1106 
1107                 tbl_off += ret;
1108         } else {
1109                 to_talitos_ptr(&desc->ptr[1], sg_dma_address(areq->src), 0);
1110                 desc->ptr[1].j_extent = 0;
1111         }
1112 
1113         /* cipher iv */
1114         to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, 0);
1115         desc->ptr[2].len = cpu_to_be16(ivsize);
1116         desc->ptr[2].j_extent = 0;
1117 
1118         /* cipher key */
1119         map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
1120                                (char *)&ctx->key + ctx->authkeylen,
1121                                DMA_TO_DEVICE);
1122 
1123         /*
1124          * cipher in
1125          * map and adjust cipher len to aead request cryptlen.
1126          * extent is bytes of HMAC postpended to ciphertext,
1127          * typically 12 for ipsec
1128          */
1129         desc->ptr[4].len = cpu_to_be16(cryptlen);
1130         desc->ptr[4].j_extent = authsize;
1131 
1132         sg_link_tbl_len = cryptlen;
1133         if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
1134                 sg_link_tbl_len += authsize;
1135 
1136         if (sg_count == 1) {
1137                 to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src) +
1138                                areq->assoclen, 0);
1139         } else if ((ret = sg_to_link_tbl_offset(areq->src, sg_count,
1140                                                 areq->assoclen, sg_link_tbl_len,
1141                                                 &edesc->link_tbl[tbl_off])) >
1142                    1) {
1143                 desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
1144                 to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl +
1145                                               tbl_off *
1146                                               sizeof(struct talitos_ptr), 0);
1147                 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1148                                            edesc->dma_len,
1149                                            DMA_BIDIRECTIONAL);
1150                 tbl_off += ret;
1151         } else {
1152                 copy_talitos_ptr(&desc->ptr[4], &edesc->link_tbl[tbl_off], 0);
1153         }
1154 
1155         /* cipher out */
1156         desc->ptr[5].len = cpu_to_be16(cryptlen);
1157         desc->ptr[5].j_extent = authsize;
1158 
1159         if (areq->src != areq->dst)
1160                 sg_count = dma_map_sg(dev, areq->dst, edesc->dst_nents ? : 1,
1161                                       DMA_FROM_DEVICE);
1162 
1163         edesc->icv_ool = false;
1164 
1165         if (sg_count == 1) {
1166                 to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst) +
1167                                areq->assoclen, 0);
1168         } else if ((sg_count =
1169                         sg_to_link_tbl_offset(areq->dst, sg_count,
1170                                               areq->assoclen, cryptlen,
1171                                               &edesc->link_tbl[tbl_off])) > 1) {
1172                 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1173 
1174                 to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl +
1175                                tbl_off * sizeof(struct talitos_ptr), 0);
1176 
1177                 /* Add an entry to the link table for ICV data */
1178                 tbl_ptr += sg_count - 1;
1179                 tbl_ptr->j_extent = 0;
1180                 tbl_ptr++;
1181                 tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1182                 tbl_ptr->len = cpu_to_be16(authsize);
1183 
1184                 /* icv data follows link tables */
1185                 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl +
1186                                         (edesc->src_nents + edesc->dst_nents +
1187                                          2) * sizeof(struct talitos_ptr) +
1188                                         authsize, 0);
1189                 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1190                 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1191                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1192 
1193                 edesc->icv_ool = true;
1194         } else {
1195                 copy_talitos_ptr(&desc->ptr[5], &edesc->link_tbl[tbl_off], 0);
1196         }
1197 
1198         /* iv out */
1199         map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1200                                DMA_FROM_DEVICE);
1201 
1202         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1203         if (ret != -EINPROGRESS) {
1204                 ipsec_esp_unmap(dev, edesc, areq);
1205                 kfree(edesc);
1206         }
1207         return ret;
1208 }
1209 
1210 /*
1211  * allocate and map the extended descriptor
1212  */
1213 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1214                                                  struct scatterlist *src,
1215                                                  struct scatterlist *dst,
1216                                                  u8 *iv,
1217                                                  unsigned int assoclen,
1218                                                  unsigned int cryptlen,
1219                                                  unsigned int authsize,
1220                                                  unsigned int ivsize,
1221                                                  int icv_stashing,
1222                                                  u32 cryptoflags,
1223                                                  bool encrypt)
1224 {
1225         struct talitos_edesc *edesc;
1226         int src_nents, dst_nents, alloc_len, dma_len;
1227         dma_addr_t iv_dma = 0;
1228         gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1229                       GFP_ATOMIC;
1230         struct talitos_private *priv = dev_get_drvdata(dev);
1231         bool is_sec1 = has_ftr_sec1(priv);
1232         int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
1233         void *err;
1234 
1235         if (cryptlen + authsize > max_len) {
1236                 dev_err(dev, "length exceeds h/w max limit\n");
1237                 return ERR_PTR(-EINVAL);
1238         }
1239 
1240         if (ivsize)
1241                 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1242 
1243         if (!dst || dst == src) {
1244                 src_nents = sg_nents_for_len(src,
1245                                              assoclen + cryptlen + authsize);
1246                 if (src_nents < 0) {
1247                         dev_err(dev, "Invalid number of src SG.\n");
1248                         err = ERR_PTR(-EINVAL);
1249                         goto error_sg;
1250                 }
1251                 src_nents = (src_nents == 1) ? 0 : src_nents;
1252                 dst_nents = dst ? src_nents : 0;
1253         } else { /* dst && dst != src*/
1254                 src_nents = sg_nents_for_len(src, assoclen + cryptlen +
1255                                                  (encrypt ? 0 : authsize));
1256                 if (src_nents < 0) {
1257                         dev_err(dev, "Invalid number of src SG.\n");
1258                         err = ERR_PTR(-EINVAL);
1259                         goto error_sg;
1260                 }
1261                 src_nents = (src_nents == 1) ? 0 : src_nents;
1262                 dst_nents = sg_nents_for_len(dst, assoclen + cryptlen +
1263                                                  (encrypt ? authsize : 0));
1264                 if (dst_nents < 0) {
1265                         dev_err(dev, "Invalid number of dst SG.\n");
1266                         err = ERR_PTR(-EINVAL);
1267                         goto error_sg;
1268                 }
1269                 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1270         }
1271 
1272         /*
1273          * allocate space for base edesc plus the link tables,
1274          * allowing for two separate entries for AD and generated ICV (+ 2),
1275          * and space for two sets of ICVs (stashed and generated)
1276          */
1277         alloc_len = sizeof(struct talitos_edesc);
1278         if (src_nents || dst_nents) {
1279                 if (is_sec1)
1280                         dma_len = (src_nents ? cryptlen : 0) +
1281                                   (dst_nents ? cryptlen : 0);
1282                 else
1283                         dma_len = (src_nents + dst_nents + 2) *
1284                                   sizeof(struct talitos_ptr) + authsize * 2;
1285                 alloc_len += dma_len;
1286         } else {
1287                 dma_len = 0;
1288                 alloc_len += icv_stashing ? authsize : 0;
1289         }
1290 
1291         edesc = kmalloc(alloc_len, GFP_DMA | flags);
1292         if (!edesc) {
1293                 dev_err(dev, "could not allocate edescriptor\n");
1294                 err = ERR_PTR(-ENOMEM);
1295                 goto error_sg;
1296         }
1297 
1298         edesc->src_nents = src_nents;
1299         edesc->dst_nents = dst_nents;
1300         edesc->iv_dma = iv_dma;
1301         edesc->dma_len = dma_len;
1302         if (dma_len)
1303                 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1304                                                      edesc->dma_len,
1305                                                      DMA_BIDIRECTIONAL);
1306 
1307         return edesc;
1308 error_sg:
1309         if (iv_dma)
1310                 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1311         return err;
1312 }
1313 
1314 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1315                                               int icv_stashing, bool encrypt)
1316 {
1317         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1318         unsigned int authsize = crypto_aead_authsize(authenc);
1319         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1320         unsigned int ivsize = crypto_aead_ivsize(authenc);
1321 
1322         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1323                                    iv, areq->assoclen, areq->cryptlen,
1324                                    authsize, ivsize, icv_stashing,
1325                                    areq->base.flags, encrypt);
1326 }
1327 
1328 static int aead_encrypt(struct aead_request *req)
1329 {
1330         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1331         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1332         struct talitos_edesc *edesc;
1333 
1334         /* allocate extended descriptor */
1335         edesc = aead_edesc_alloc(req, req->iv, 0, true);
1336         if (IS_ERR(edesc))
1337                 return PTR_ERR(edesc);
1338 
1339         /* set encrypt */
1340         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1341 
1342         return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
1343 }
1344 
1345 static int aead_decrypt(struct aead_request *req)
1346 {
1347         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1348         unsigned int authsize = crypto_aead_authsize(authenc);
1349         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1350         struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1351         struct talitos_edesc *edesc;
1352         struct scatterlist *sg;
1353         void *icvdata;
1354 
1355         req->cryptlen -= authsize;
1356 
1357         /* allocate extended descriptor */
1358         edesc = aead_edesc_alloc(req, req->iv, 1, false);
1359         if (IS_ERR(edesc))
1360                 return PTR_ERR(edesc);
1361 
1362         if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1363             ((!edesc->src_nents && !edesc->dst_nents) ||
1364              priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1365 
1366                 /* decrypt and check the ICV */
1367                 edesc->desc.hdr = ctx->desc_hdr_template |
1368                                   DESC_HDR_DIR_INBOUND |
1369                                   DESC_HDR_MODE1_MDEU_CICV;
1370 
1371                 /* reset integrity check result bits */
1372                 edesc->desc.hdr_lo = 0;
1373 
1374                 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
1375         }
1376 
1377         /* Have to check the ICV with software */
1378         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1379 
1380         /* stash incoming ICV for later cmp with ICV generated by the h/w */
1381         if (edesc->dma_len)
1382                 icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1383                                                    edesc->dst_nents + 2];
1384         else
1385                 icvdata = &edesc->link_tbl[0];
1386 
1387         sg = sg_last(req->src, edesc->src_nents ? : 1);
1388 
1389         memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
1390 
1391         return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
1392 }
1393 
1394 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1395                              const u8 *key, unsigned int keylen)
1396 {
1397         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1398 
1399         memcpy(&ctx->key, key, keylen);
1400         ctx->keylen = keylen;
1401 
1402         return 0;
1403 }
1404 
1405 static void unmap_sg_talitos_ptr(struct device *dev, struct scatterlist *src,
1406                                  struct scatterlist *dst, unsigned int len,
1407                                  struct talitos_edesc *edesc)
1408 {
1409         struct talitos_private *priv = dev_get_drvdata(dev);
1410         bool is_sec1 = has_ftr_sec1(priv);
1411 
1412         if (is_sec1) {
1413                 if (!edesc->src_nents) {
1414                         dma_unmap_sg(dev, src, 1,
1415                                      dst != src ? DMA_TO_DEVICE
1416                                                 : DMA_BIDIRECTIONAL);
1417                 }
1418                 if (dst && edesc->dst_nents) {
1419                         dma_sync_single_for_device(dev,
1420                                                    edesc->dma_link_tbl + len,
1421                                                    len, DMA_FROM_DEVICE);
1422                         sg_copy_from_buffer(dst, edesc->dst_nents ? : 1,
1423                                             edesc->buf + len, len);
1424                 } else if (dst && dst != src) {
1425                         dma_unmap_sg(dev, dst, 1, DMA_FROM_DEVICE);
1426                 }
1427         } else {
1428                 talitos_sg_unmap(dev, edesc, src, dst);
1429         }
1430 }
1431 
1432 static void common_nonsnoop_unmap(struct device *dev,
1433                                   struct talitos_edesc *edesc,
1434                                   struct ablkcipher_request *areq)
1435 {
1436         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1437 
1438         unmap_sg_talitos_ptr(dev, areq->src, areq->dst, areq->nbytes, edesc);
1439         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1440         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1441 
1442         if (edesc->dma_len)
1443                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1444                                  DMA_BIDIRECTIONAL);
1445 }
1446 
1447 static void ablkcipher_done(struct device *dev,
1448                             struct talitos_desc *desc, void *context,
1449                             int err)
1450 {
1451         struct ablkcipher_request *areq = context;
1452         struct talitos_edesc *edesc;
1453 
1454         edesc = container_of(desc, struct talitos_edesc, desc);
1455 
1456         common_nonsnoop_unmap(dev, edesc, areq);
1457 
1458         kfree(edesc);
1459 
1460         areq->base.complete(&areq->base, err);
1461 }
1462 
1463 int map_sg_in_talitos_ptr(struct device *dev, struct scatterlist *src,
1464                           unsigned int len, struct talitos_edesc *edesc,
1465                           enum dma_data_direction dir, struct talitos_ptr *ptr)
1466 {
1467         int sg_count;
1468         struct talitos_private *priv = dev_get_drvdata(dev);
1469         bool is_sec1 = has_ftr_sec1(priv);
1470 
1471         to_talitos_ptr_len(ptr, len, is_sec1);
1472 
1473         if (is_sec1) {
1474                 sg_count = edesc->src_nents ? : 1;
1475 
1476                 if (sg_count == 1) {
1477                         dma_map_sg(dev, src, 1, dir);
1478                         to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
1479                 } else {
1480                         sg_copy_to_buffer(src, sg_count, edesc->buf, len);
1481                         to_talitos_ptr(ptr, edesc->dma_link_tbl, is_sec1);
1482                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1483                                                    len, DMA_TO_DEVICE);
1484                 }
1485         } else {
1486                 to_talitos_ptr_extent_clear(ptr, is_sec1);
1487 
1488                 sg_count = dma_map_sg(dev, src, edesc->src_nents ? : 1, dir);
1489 
1490                 if (sg_count == 1) {
1491                         to_talitos_ptr(ptr, sg_dma_address(src), is_sec1);
1492                 } else {
1493                         sg_count = sg_to_link_tbl(src, sg_count, len,
1494                                                   &edesc->link_tbl[0]);
1495                         if (sg_count > 1) {
1496                                 to_talitos_ptr(ptr, edesc->dma_link_tbl, 0);
1497                                 ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
1498                                 dma_sync_single_for_device(dev,
1499                                                            edesc->dma_link_tbl,
1500                                                            edesc->dma_len,
1501                                                            DMA_BIDIRECTIONAL);
1502                         } else {
1503                                 /* Only one segment now, so no link tbl needed*/
1504                                 to_talitos_ptr(ptr, sg_dma_address(src),
1505                                                is_sec1);
1506                         }
1507                 }
1508         }
1509         return sg_count;
1510 }
1511 
1512 void map_sg_out_talitos_ptr(struct device *dev, struct scatterlist *dst,
1513                             unsigned int len, struct talitos_edesc *edesc,
1514                             enum dma_data_direction dir,
1515                             struct talitos_ptr *ptr, int sg_count)
1516 {
1517         struct talitos_private *priv = dev_get_drvdata(dev);
1518         bool is_sec1 = has_ftr_sec1(priv);
1519 
1520         if (dir != DMA_NONE)
1521                 sg_count = dma_map_sg(dev, dst, edesc->dst_nents ? : 1, dir);
1522 
1523         to_talitos_ptr_len(ptr, len, is_sec1);
1524 
1525         if (is_sec1) {
1526                 if (sg_count == 1) {
1527                         if (dir != DMA_NONE)
1528                                 dma_map_sg(dev, dst, 1, dir);
1529                         to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1);
1530                 } else {
1531                         to_talitos_ptr(ptr, edesc->dma_link_tbl + len, is_sec1);
1532                         dma_sync_single_for_device(dev,
1533                                                    edesc->dma_link_tbl + len,
1534                                                    len, DMA_FROM_DEVICE);
1535                 }
1536         } else {
1537                 to_talitos_ptr_extent_clear(ptr, is_sec1);
1538 
1539                 if (sg_count == 1) {
1540                         to_talitos_ptr(ptr, sg_dma_address(dst), is_sec1);
1541                 } else {
1542                         struct talitos_ptr *link_tbl_ptr =
1543                                 &edesc->link_tbl[edesc->src_nents + 1];
1544 
1545                         to_talitos_ptr(ptr, edesc->dma_link_tbl +
1546                                             (edesc->src_nents + 1) *
1547                                              sizeof(struct talitos_ptr), 0);
1548                         ptr->j_extent |= DESC_PTR_LNKTBL_JUMP;
1549                         sg_to_link_tbl(dst, sg_count, len, link_tbl_ptr);
1550                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1551                                                    edesc->dma_len,
1552                                                    DMA_BIDIRECTIONAL);
1553                 }
1554         }
1555 }
1556 
1557 static int common_nonsnoop(struct talitos_edesc *edesc,
1558                            struct ablkcipher_request *areq,
1559                            void (*callback) (struct device *dev,
1560                                              struct talitos_desc *desc,
1561                                              void *context, int error))
1562 {
1563         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1564         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1565         struct device *dev = ctx->dev;
1566         struct talitos_desc *desc = &edesc->desc;
1567         unsigned int cryptlen = areq->nbytes;
1568         unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1569         int sg_count, ret;
1570         struct talitos_private *priv = dev_get_drvdata(dev);
1571         bool is_sec1 = has_ftr_sec1(priv);
1572 
1573         /* first DWORD empty */
1574         desc->ptr[0] = zero_entry;
1575 
1576         /* cipher iv */
1577         to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, is_sec1);
1578         to_talitos_ptr_len(&desc->ptr[1], ivsize, is_sec1);
1579         to_talitos_ptr_extent_clear(&desc->ptr[1], is_sec1);
1580 
1581         /* cipher key */
1582         map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1583                                (char *)&ctx->key, DMA_TO_DEVICE);
1584 
1585         /*
1586          * cipher in
1587          */
1588         sg_count = map_sg_in_talitos_ptr(dev, areq->src, cryptlen, edesc,
1589                                          (areq->src == areq->dst) ?
1590                                           DMA_BIDIRECTIONAL : DMA_TO_DEVICE,
1591                                           &desc->ptr[3]);
1592 
1593         /* cipher out */
1594         map_sg_out_talitos_ptr(dev, areq->dst, cryptlen, edesc,
1595                                (areq->src == areq->dst) ? DMA_NONE
1596                                                         : DMA_FROM_DEVICE,
1597                                &desc->ptr[4], sg_count);
1598 
1599         /* iv out */
1600         map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
1601                                DMA_FROM_DEVICE);
1602 
1603         /* last DWORD empty */
1604         desc->ptr[6] = zero_entry;
1605 
1606         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1607         if (ret != -EINPROGRESS) {
1608                 common_nonsnoop_unmap(dev, edesc, areq);
1609                 kfree(edesc);
1610         }
1611         return ret;
1612 }
1613 
1614 static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1615                                                     areq, bool encrypt)
1616 {
1617         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1618         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1619         unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1620 
1621         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1622                                    areq->info, 0, areq->nbytes, 0, ivsize, 0,
1623                                    areq->base.flags, encrypt);
1624 }
1625 
1626 static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1627 {
1628         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1629         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1630         struct talitos_edesc *edesc;
1631 
1632         /* allocate extended descriptor */
1633         edesc = ablkcipher_edesc_alloc(areq, true);
1634         if (IS_ERR(edesc))
1635                 return PTR_ERR(edesc);
1636 
1637         /* set encrypt */
1638         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1639 
1640         return common_nonsnoop(edesc, areq, ablkcipher_done);
1641 }
1642 
1643 static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1644 {
1645         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1646         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1647         struct talitos_edesc *edesc;
1648 
1649         /* allocate extended descriptor */
1650         edesc = ablkcipher_edesc_alloc(areq, false);
1651         if (IS_ERR(edesc))
1652                 return PTR_ERR(edesc);
1653 
1654         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1655 
1656         return common_nonsnoop(edesc, areq, ablkcipher_done);
1657 }
1658 
1659 static void common_nonsnoop_hash_unmap(struct device *dev,
1660                                        struct talitos_edesc *edesc,
1661                                        struct ahash_request *areq)
1662 {
1663         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1664         struct talitos_private *priv = dev_get_drvdata(dev);
1665         bool is_sec1 = has_ftr_sec1(priv);
1666 
1667         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1668 
1669         unmap_sg_talitos_ptr(dev, req_ctx->psrc, NULL, 0, edesc);
1670 
1671         /* When using hashctx-in, must unmap it. */
1672         if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
1673                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1674                                          DMA_TO_DEVICE);
1675 
1676         if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1))
1677                 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1678                                          DMA_TO_DEVICE);
1679 
1680         if (edesc->dma_len)
1681                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1682                                  DMA_BIDIRECTIONAL);
1683 
1684 }
1685 
1686 static void ahash_done(struct device *dev,
1687                        struct talitos_desc *desc, void *context,
1688                        int err)
1689 {
1690         struct ahash_request *areq = context;
1691         struct talitos_edesc *edesc =
1692                  container_of(desc, struct talitos_edesc, desc);
1693         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1694 
1695         if (!req_ctx->last && req_ctx->to_hash_later) {
1696                 /* Position any partial block for next update/final/finup */
1697                 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
1698                 req_ctx->nbuf = req_ctx->to_hash_later;
1699         }
1700         common_nonsnoop_hash_unmap(dev, edesc, areq);
1701 
1702         kfree(edesc);
1703 
1704         areq->base.complete(&areq->base, err);
1705 }
1706 
1707 /*
1708  * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1709  * ourself and submit a padded block
1710  */
1711 void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1712                                struct talitos_edesc *edesc,
1713                                struct talitos_ptr *ptr)
1714 {
1715         static u8 padded_hash[64] = {
1716                 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1717                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1718                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1719                 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1720         };
1721 
1722         pr_err_once("Bug in SEC1, padding ourself\n");
1723         edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1724         map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1725                                (char *)padded_hash, DMA_TO_DEVICE);
1726 }
1727 
1728 static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1729                                 struct ahash_request *areq, unsigned int length,
1730                                 void (*callback) (struct device *dev,
1731                                                   struct talitos_desc *desc,
1732                                                   void *context, int error))
1733 {
1734         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1735         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1736         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1737         struct device *dev = ctx->dev;
1738         struct talitos_desc *desc = &edesc->desc;
1739         int ret;
1740         struct talitos_private *priv = dev_get_drvdata(dev);
1741         bool is_sec1 = has_ftr_sec1(priv);
1742 
1743         /* first DWORD empty */
1744         desc->ptr[0] = zero_entry;
1745 
1746         /* hash context in */
1747         if (!req_ctx->first || req_ctx->swinit) {
1748                 map_single_talitos_ptr(dev, &desc->ptr[1],
1749                                        req_ctx->hw_context_size,
1750                                        (char *)req_ctx->hw_context,
1751                                        DMA_TO_DEVICE);
1752                 req_ctx->swinit = 0;
1753         } else {
1754                 desc->ptr[1] = zero_entry;
1755                 /* Indicate next op is not the first. */
1756                 req_ctx->first = 0;
1757         }
1758 
1759         /* HMAC key */
1760         if (ctx->keylen)
1761                 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1762                                        (char *)&ctx->key, DMA_TO_DEVICE);
1763         else
1764                 desc->ptr[2] = zero_entry;
1765 
1766         /*
1767          * data in
1768          */
1769         map_sg_in_talitos_ptr(dev, req_ctx->psrc, length, edesc,
1770                               DMA_TO_DEVICE, &desc->ptr[3]);
1771 
1772         /* fifth DWORD empty */
1773         desc->ptr[4] = zero_entry;
1774 
1775         /* hash/HMAC out -or- hash context out */
1776         if (req_ctx->last)
1777                 map_single_talitos_ptr(dev, &desc->ptr[5],
1778                                        crypto_ahash_digestsize(tfm),
1779                                        areq->result, DMA_FROM_DEVICE);
1780         else
1781                 map_single_talitos_ptr(dev, &desc->ptr[5],
1782                                        req_ctx->hw_context_size,
1783                                        req_ctx->hw_context, DMA_FROM_DEVICE);
1784 
1785         /* last DWORD empty */
1786         desc->ptr[6] = zero_entry;
1787 
1788         if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1789                 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1790 
1791         ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1792         if (ret != -EINPROGRESS) {
1793                 common_nonsnoop_hash_unmap(dev, edesc, areq);
1794                 kfree(edesc);
1795         }
1796         return ret;
1797 }
1798 
1799 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1800                                                unsigned int nbytes)
1801 {
1802         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1803         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1804         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1805 
1806         return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
1807                                    nbytes, 0, 0, 0, areq->base.flags, false);
1808 }
1809 
1810 static int ahash_init(struct ahash_request *areq)
1811 {
1812         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1813         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1814 
1815         /* Initialize the context */
1816         req_ctx->nbuf = 0;
1817         req_ctx->first = 1; /* first indicates h/w must init its context */
1818         req_ctx->swinit = 0; /* assume h/w init of context */
1819         req_ctx->hw_context_size =
1820                 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1821                         ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1822                         : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1823 
1824         return 0;
1825 }
1826 
1827 /*
1828  * on h/w without explicit sha224 support, we initialize h/w context
1829  * manually with sha224 constants, and tell it to run sha256.
1830  */
1831 static int ahash_init_sha224_swinit(struct ahash_request *areq)
1832 {
1833         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1834 
1835         ahash_init(areq);
1836         req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1837 
1838         req_ctx->hw_context[0] = SHA224_H0;
1839         req_ctx->hw_context[1] = SHA224_H1;
1840         req_ctx->hw_context[2] = SHA224_H2;
1841         req_ctx->hw_context[3] = SHA224_H3;
1842         req_ctx->hw_context[4] = SHA224_H4;
1843         req_ctx->hw_context[5] = SHA224_H5;
1844         req_ctx->hw_context[6] = SHA224_H6;
1845         req_ctx->hw_context[7] = SHA224_H7;
1846 
1847         /* init 64-bit count */
1848         req_ctx->hw_context[8] = 0;
1849         req_ctx->hw_context[9] = 0;
1850 
1851         return 0;
1852 }
1853 
1854 static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1855 {
1856         struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1857         struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1858         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1859         struct talitos_edesc *edesc;
1860         unsigned int blocksize =
1861                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1862         unsigned int nbytes_to_hash;
1863         unsigned int to_hash_later;
1864         unsigned int nsg;
1865         int nents;
1866 
1867         if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1868                 /* Buffer up to one whole block */
1869                 nents = sg_nents_for_len(areq->src, nbytes);
1870                 if (nents < 0) {
1871                         dev_err(ctx->dev, "Invalid number of src SG.\n");
1872                         return nents;
1873                 }
1874                 sg_copy_to_buffer(areq->src, nents,
1875                                   req_ctx->buf + req_ctx->nbuf, nbytes);
1876                 req_ctx->nbuf += nbytes;
1877                 return 0;
1878         }
1879 
1880         /* At least (blocksize + 1) bytes are available to hash */
1881         nbytes_to_hash = nbytes + req_ctx->nbuf;
1882         to_hash_later = nbytes_to_hash & (blocksize - 1);
1883 
1884         if (req_ctx->last)
1885                 to_hash_later = 0;
1886         else if (to_hash_later)
1887                 /* There is a partial block. Hash the full block(s) now */
1888                 nbytes_to_hash -= to_hash_later;
1889         else {
1890                 /* Keep one block buffered */
1891                 nbytes_to_hash -= blocksize;
1892                 to_hash_later = blocksize;
1893         }
1894 
1895         /* Chain in any previously buffered data */
1896         if (req_ctx->nbuf) {
1897                 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1898                 sg_init_table(req_ctx->bufsl, nsg);
1899                 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1900                 if (nsg > 1)
1901                         sg_chain(req_ctx->bufsl, 2, areq->src);
1902                 req_ctx->psrc = req_ctx->bufsl;
1903         } else
1904                 req_ctx->psrc = areq->src;
1905 
1906         if (to_hash_later) {
1907                 nents = sg_nents_for_len(areq->src, nbytes);
1908                 if (nents < 0) {
1909                         dev_err(ctx->dev, "Invalid number of src SG.\n");
1910                         return nents;
1911                 }
1912                 sg_pcopy_to_buffer(areq->src, nents,
1913                                       req_ctx->bufnext,
1914                                       to_hash_later,
1915                                       nbytes - to_hash_later);
1916         }
1917         req_ctx->to_hash_later = to_hash_later;
1918 
1919         /* Allocate extended descriptor */
1920         edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1921         if (IS_ERR(edesc))
1922                 return PTR_ERR(edesc);
1923 
1924         edesc->desc.hdr = ctx->desc_hdr_template;
1925 
1926         /* On last one, request SEC to pad; otherwise continue */
1927         if (req_ctx->last)
1928                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1929         else
1930                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1931 
1932         /* request SEC to INIT hash. */
1933         if (req_ctx->first && !req_ctx->swinit)
1934                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1935 
1936         /* When the tfm context has a keylen, it's an HMAC.
1937          * A first or last (ie. not middle) descriptor must request HMAC.
1938          */
1939         if (ctx->keylen && (req_ctx->first || req_ctx->last))
1940                 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1941 
1942         return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1943                                     ahash_done);
1944 }
1945 
1946 static int ahash_update(struct ahash_request *areq)
1947 {
1948         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1949 
1950         req_ctx->last = 0;
1951 
1952         return ahash_process_req(areq, areq->nbytes);
1953 }
1954 
1955 static int ahash_final(struct ahash_request *areq)
1956 {
1957         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1958 
1959         req_ctx->last = 1;
1960 
1961         return ahash_process_req(areq, 0);
1962 }
1963 
1964 static int ahash_finup(struct ahash_request *areq)
1965 {
1966         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1967 
1968         req_ctx->last = 1;
1969 
1970         return ahash_process_req(areq, areq->nbytes);
1971 }
1972 
1973 static int ahash_digest(struct ahash_request *areq)
1974 {
1975         struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1976         struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
1977 
1978         ahash->init(areq);
1979         req_ctx->last = 1;
1980 
1981         return ahash_process_req(areq, areq->nbytes);
1982 }
1983 
1984 struct keyhash_result {
1985         struct completion completion;
1986         int err;
1987 };
1988 
1989 static void keyhash_complete(struct crypto_async_request *req, int err)
1990 {
1991         struct keyhash_result *res = req->data;
1992 
1993         if (err == -EINPROGRESS)
1994                 return;
1995 
1996         res->err = err;
1997         complete(&res->completion);
1998 }
1999 
2000 static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2001                    u8 *hash)
2002 {
2003         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2004 
2005         struct scatterlist sg[1];
2006         struct ahash_request *req;
2007         struct keyhash_result hresult;
2008         int ret;
2009 
2010         init_completion(&hresult.completion);
2011 
2012         req = ahash_request_alloc(tfm, GFP_KERNEL);
2013         if (!req)
2014                 return -ENOMEM;
2015 
2016         /* Keep tfm keylen == 0 during hash of the long key */
2017         ctx->keylen = 0;
2018         ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2019                                    keyhash_complete, &hresult);
2020 
2021         sg_init_one(&sg[0], key, keylen);
2022 
2023         ahash_request_set_crypt(req, sg, hash, keylen);
2024         ret = crypto_ahash_digest(req);
2025         switch (ret) {
2026         case 0:
2027                 break;
2028         case -EINPROGRESS:
2029         case -EBUSY:
2030                 ret = wait_for_completion_interruptible(
2031                         &hresult.completion);
2032                 if (!ret)
2033                         ret = hresult.err;
2034                 break;
2035         default:
2036                 break;
2037         }
2038         ahash_request_free(req);
2039 
2040         return ret;
2041 }
2042 
2043 static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2044                         unsigned int keylen)
2045 {
2046         struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2047         unsigned int blocksize =
2048                         crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2049         unsigned int digestsize = crypto_ahash_digestsize(tfm);
2050         unsigned int keysize = keylen;
2051         u8 hash[SHA512_DIGEST_SIZE];
2052         int ret;
2053 
2054         if (keylen <= blocksize)
2055                 memcpy(ctx->key, key, keysize);
2056         else {
2057                 /* Must get the hash of the long key */
2058                 ret = keyhash(tfm, key, keylen, hash);
2059 
2060                 if (ret) {
2061                         crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2062                         return -EINVAL;
2063                 }
2064 
2065                 keysize = digestsize;
2066                 memcpy(ctx->key, hash, digestsize);
2067         }
2068 
2069         ctx->keylen = keysize;
2070 
2071         return 0;
2072 }
2073 
2074 
2075 struct talitos_alg_template {
2076         u32 type;
2077         union {
2078                 struct crypto_alg crypto;
2079                 struct ahash_alg hash;
2080                 struct aead_alg aead;
2081         } alg;
2082         __be32 desc_hdr_template;
2083 };
2084 
2085 static struct talitos_alg_template driver_algs[] = {
2086         /* AEAD algorithms.  These use a single-pass ipsec_esp descriptor */
2087         {       .type = CRYPTO_ALG_TYPE_AEAD,
2088                 .alg.aead = {
2089                         .base = {
2090                                 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2091                                 .cra_driver_name = "authenc-hmac-sha1-"
2092                                                    "cbc-aes-talitos",
2093                                 .cra_blocksize = AES_BLOCK_SIZE,
2094                                 .cra_flags = CRYPTO_ALG_ASYNC,
2095                         },
2096                         .ivsize = AES_BLOCK_SIZE,
2097                         .maxauthsize = SHA1_DIGEST_SIZE,
2098                 },
2099                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2100                                      DESC_HDR_SEL0_AESU |
2101                                      DESC_HDR_MODE0_AESU_CBC |
2102                                      DESC_HDR_SEL1_MDEUA |
2103                                      DESC_HDR_MODE1_MDEU_INIT |
2104                                      DESC_HDR_MODE1_MDEU_PAD |
2105                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2106         },
2107         {       .type = CRYPTO_ALG_TYPE_AEAD,
2108                 .alg.aead = {
2109                         .base = {
2110                                 .cra_name = "authenc(hmac(sha1),"
2111                                             "cbc(des3_ede))",
2112                                 .cra_driver_name = "authenc-hmac-sha1-"
2113                                                    "cbc-3des-talitos",
2114                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2115                                 .cra_flags = CRYPTO_ALG_ASYNC,
2116                         },
2117                         .ivsize = DES3_EDE_BLOCK_SIZE,
2118                         .maxauthsize = SHA1_DIGEST_SIZE,
2119                 },
2120                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2121                                      DESC_HDR_SEL0_DEU |
2122                                      DESC_HDR_MODE0_DEU_CBC |
2123                                      DESC_HDR_MODE0_DEU_3DES |
2124                                      DESC_HDR_SEL1_MDEUA |
2125                                      DESC_HDR_MODE1_MDEU_INIT |
2126                                      DESC_HDR_MODE1_MDEU_PAD |
2127                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2128         },
2129         {       .type = CRYPTO_ALG_TYPE_AEAD,
2130                 .alg.aead = {
2131                         .base = {
2132                                 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2133                                 .cra_driver_name = "authenc-hmac-sha224-"
2134                                                    "cbc-aes-talitos",
2135                                 .cra_blocksize = AES_BLOCK_SIZE,
2136                                 .cra_flags = CRYPTO_ALG_ASYNC,
2137                         },
2138                         .ivsize = AES_BLOCK_SIZE,
2139                         .maxauthsize = SHA224_DIGEST_SIZE,
2140                 },
2141                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2142                                      DESC_HDR_SEL0_AESU |
2143                                      DESC_HDR_MODE0_AESU_CBC |
2144                                      DESC_HDR_SEL1_MDEUA |
2145                                      DESC_HDR_MODE1_MDEU_INIT |
2146                                      DESC_HDR_MODE1_MDEU_PAD |
2147                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2148         },
2149         {       .type = CRYPTO_ALG_TYPE_AEAD,
2150                 .alg.aead = {
2151                         .base = {
2152                                 .cra_name = "authenc(hmac(sha224),"
2153                                             "cbc(des3_ede))",
2154                                 .cra_driver_name = "authenc-hmac-sha224-"
2155                                                    "cbc-3des-talitos",
2156                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2157                                 .cra_flags = CRYPTO_ALG_ASYNC,
2158                         },
2159                         .ivsize = DES3_EDE_BLOCK_SIZE,
2160                         .maxauthsize = SHA224_DIGEST_SIZE,
2161                 },
2162                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2163                                      DESC_HDR_SEL0_DEU |
2164                                      DESC_HDR_MODE0_DEU_CBC |
2165                                      DESC_HDR_MODE0_DEU_3DES |
2166                                      DESC_HDR_SEL1_MDEUA |
2167                                      DESC_HDR_MODE1_MDEU_INIT |
2168                                      DESC_HDR_MODE1_MDEU_PAD |
2169                                      DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2170         },
2171         {       .type = CRYPTO_ALG_TYPE_AEAD,
2172                 .alg.aead = {
2173                         .base = {
2174                                 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2175                                 .cra_driver_name = "authenc-hmac-sha256-"
2176                                                    "cbc-aes-talitos",
2177                                 .cra_blocksize = AES_BLOCK_SIZE,
2178                                 .cra_flags = CRYPTO_ALG_ASYNC,
2179                         },
2180                         .ivsize = AES_BLOCK_SIZE,
2181                         .maxauthsize = SHA256_DIGEST_SIZE,
2182                 },
2183                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2184                                      DESC_HDR_SEL0_AESU |
2185                                      DESC_HDR_MODE0_AESU_CBC |
2186                                      DESC_HDR_SEL1_MDEUA |
2187                                      DESC_HDR_MODE1_MDEU_INIT |
2188                                      DESC_HDR_MODE1_MDEU_PAD |
2189                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2190         },
2191         {       .type = CRYPTO_ALG_TYPE_AEAD,
2192                 .alg.aead = {
2193                         .base = {
2194                                 .cra_name = "authenc(hmac(sha256),"
2195                                             "cbc(des3_ede))",
2196                                 .cra_driver_name = "authenc-hmac-sha256-"
2197                                                    "cbc-3des-talitos",
2198                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2199                                 .cra_flags = CRYPTO_ALG_ASYNC,
2200                         },
2201                         .ivsize = DES3_EDE_BLOCK_SIZE,
2202                         .maxauthsize = SHA256_DIGEST_SIZE,
2203                 },
2204                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2205                                      DESC_HDR_SEL0_DEU |
2206                                      DESC_HDR_MODE0_DEU_CBC |
2207                                      DESC_HDR_MODE0_DEU_3DES |
2208                                      DESC_HDR_SEL1_MDEUA |
2209                                      DESC_HDR_MODE1_MDEU_INIT |
2210                                      DESC_HDR_MODE1_MDEU_PAD |
2211                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2212         },
2213         {       .type = CRYPTO_ALG_TYPE_AEAD,
2214                 .alg.aead = {
2215                         .base = {
2216                                 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2217                                 .cra_driver_name = "authenc-hmac-sha384-"
2218                                                    "cbc-aes-talitos",
2219                                 .cra_blocksize = AES_BLOCK_SIZE,
2220                                 .cra_flags = CRYPTO_ALG_ASYNC,
2221                         },
2222                         .ivsize = AES_BLOCK_SIZE,
2223                         .maxauthsize = SHA384_DIGEST_SIZE,
2224                 },
2225                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2226                                      DESC_HDR_SEL0_AESU |
2227                                      DESC_HDR_MODE0_AESU_CBC |
2228                                      DESC_HDR_SEL1_MDEUB |
2229                                      DESC_HDR_MODE1_MDEU_INIT |
2230                                      DESC_HDR_MODE1_MDEU_PAD |
2231                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2232         },
2233         {       .type = CRYPTO_ALG_TYPE_AEAD,
2234                 .alg.aead = {
2235                         .base = {
2236                                 .cra_name = "authenc(hmac(sha384),"
2237                                             "cbc(des3_ede))",
2238                                 .cra_driver_name = "authenc-hmac-sha384-"
2239                                                    "cbc-3des-talitos",
2240                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2241                                 .cra_flags = CRYPTO_ALG_ASYNC,
2242                         },
2243                         .ivsize = DES3_EDE_BLOCK_SIZE,
2244                         .maxauthsize = SHA384_DIGEST_SIZE,
2245                 },
2246                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2247                                      DESC_HDR_SEL0_DEU |
2248                                      DESC_HDR_MODE0_DEU_CBC |
2249                                      DESC_HDR_MODE0_DEU_3DES |
2250                                      DESC_HDR_SEL1_MDEUB |
2251                                      DESC_HDR_MODE1_MDEU_INIT |
2252                                      DESC_HDR_MODE1_MDEU_PAD |
2253                                      DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2254         },
2255         {       .type = CRYPTO_ALG_TYPE_AEAD,
2256                 .alg.aead = {
2257                         .base = {
2258                                 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2259                                 .cra_driver_name = "authenc-hmac-sha512-"
2260                                                    "cbc-aes-talitos",
2261                                 .cra_blocksize = AES_BLOCK_SIZE,
2262                                 .cra_flags = CRYPTO_ALG_ASYNC,
2263                         },
2264                         .ivsize = AES_BLOCK_SIZE,
2265                         .maxauthsize = SHA512_DIGEST_SIZE,
2266                 },
2267                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2268                                      DESC_HDR_SEL0_AESU |
2269                                      DESC_HDR_MODE0_AESU_CBC |
2270                                      DESC_HDR_SEL1_MDEUB |
2271                                      DESC_HDR_MODE1_MDEU_INIT |
2272                                      DESC_HDR_MODE1_MDEU_PAD |
2273                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2274         },
2275         {       .type = CRYPTO_ALG_TYPE_AEAD,
2276                 .alg.aead = {
2277                         .base = {
2278                                 .cra_name = "authenc(hmac(sha512),"
2279                                             "cbc(des3_ede))",
2280                                 .cra_driver_name = "authenc-hmac-sha512-"
2281                                                    "cbc-3des-talitos",
2282                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2283                                 .cra_flags = CRYPTO_ALG_ASYNC,
2284                         },
2285                         .ivsize = DES3_EDE_BLOCK_SIZE,
2286                         .maxauthsize = SHA512_DIGEST_SIZE,
2287                 },
2288                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2289                                      DESC_HDR_SEL0_DEU |
2290                                      DESC_HDR_MODE0_DEU_CBC |
2291                                      DESC_HDR_MODE0_DEU_3DES |
2292                                      DESC_HDR_SEL1_MDEUB |
2293                                      DESC_HDR_MODE1_MDEU_INIT |
2294                                      DESC_HDR_MODE1_MDEU_PAD |
2295                                      DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2296         },
2297         {       .type = CRYPTO_ALG_TYPE_AEAD,
2298                 .alg.aead = {
2299                         .base = {
2300                                 .cra_name = "authenc(hmac(md5),cbc(aes))",
2301                                 .cra_driver_name = "authenc-hmac-md5-"
2302                                                    "cbc-aes-talitos",
2303                                 .cra_blocksize = AES_BLOCK_SIZE,
2304                                 .cra_flags = CRYPTO_ALG_ASYNC,
2305                         },
2306                         .ivsize = AES_BLOCK_SIZE,
2307                         .maxauthsize = MD5_DIGEST_SIZE,
2308                 },
2309                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2310                                      DESC_HDR_SEL0_AESU |
2311                                      DESC_HDR_MODE0_AESU_CBC |
2312                                      DESC_HDR_SEL1_MDEUA |
2313                                      DESC_HDR_MODE1_MDEU_INIT |
2314                                      DESC_HDR_MODE1_MDEU_PAD |
2315                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2316         },
2317         {       .type = CRYPTO_ALG_TYPE_AEAD,
2318                 .alg.aead = {
2319                         .base = {
2320                                 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2321                                 .cra_driver_name = "authenc-hmac-md5-"
2322                                                    "cbc-3des-talitos",
2323                                 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2324                                 .cra_flags = CRYPTO_ALG_ASYNC,
2325                         },
2326                         .ivsize = DES3_EDE_BLOCK_SIZE,
2327                         .maxauthsize = MD5_DIGEST_SIZE,
2328                 },
2329                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2330                                      DESC_HDR_SEL0_DEU |
2331                                      DESC_HDR_MODE0_DEU_CBC |
2332                                      DESC_HDR_MODE0_DEU_3DES |
2333                                      DESC_HDR_SEL1_MDEUA |
2334                                      DESC_HDR_MODE1_MDEU_INIT |
2335                                      DESC_HDR_MODE1_MDEU_PAD |
2336                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
2337         },
2338         /* ABLKCIPHER algorithms. */
2339         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2340                 .alg.crypto = {
2341                         .cra_name = "ecb(aes)",
2342                         .cra_driver_name = "ecb-aes-talitos",
2343                         .cra_blocksize = AES_BLOCK_SIZE,
2344                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2345                                      CRYPTO_ALG_ASYNC,
2346                         .cra_ablkcipher = {
2347                                 .min_keysize = AES_MIN_KEY_SIZE,
2348                                 .max_keysize = AES_MAX_KEY_SIZE,
2349                                 .ivsize = AES_BLOCK_SIZE,
2350                         }
2351                 },
2352                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2353                                      DESC_HDR_SEL0_AESU,
2354         },
2355         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2356                 .alg.crypto = {
2357                         .cra_name = "cbc(aes)",
2358                         .cra_driver_name = "cbc-aes-talitos",
2359                         .cra_blocksize = AES_BLOCK_SIZE,
2360                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2361                                      CRYPTO_ALG_ASYNC,
2362                         .cra_ablkcipher = {
2363                                 .min_keysize = AES_MIN_KEY_SIZE,
2364                                 .max_keysize = AES_MAX_KEY_SIZE,
2365                                 .ivsize = AES_BLOCK_SIZE,
2366                         }
2367                 },
2368                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2369                                      DESC_HDR_SEL0_AESU |
2370                                      DESC_HDR_MODE0_AESU_CBC,
2371         },
2372         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2373                 .alg.crypto = {
2374                         .cra_name = "ctr(aes)",
2375                         .cra_driver_name = "ctr-aes-talitos",
2376                         .cra_blocksize = AES_BLOCK_SIZE,
2377                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2378                                      CRYPTO_ALG_ASYNC,
2379                         .cra_ablkcipher = {
2380                                 .min_keysize = AES_MIN_KEY_SIZE,
2381                                 .max_keysize = AES_MAX_KEY_SIZE,
2382                                 .ivsize = AES_BLOCK_SIZE,
2383                         }
2384                 },
2385                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2386                                      DESC_HDR_SEL0_AESU |
2387                                      DESC_HDR_MODE0_AESU_CTR,
2388         },
2389         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2390                 .alg.crypto = {
2391                         .cra_name = "ecb(des)",
2392                         .cra_driver_name = "ecb-des-talitos",
2393                         .cra_blocksize = DES_BLOCK_SIZE,
2394                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2395                                      CRYPTO_ALG_ASYNC,
2396                         .cra_ablkcipher = {
2397                                 .min_keysize = DES_KEY_SIZE,
2398                                 .max_keysize = DES_KEY_SIZE,
2399                                 .ivsize = DES_BLOCK_SIZE,
2400                         }
2401                 },
2402                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2403                                      DESC_HDR_SEL0_DEU,
2404         },
2405         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2406                 .alg.crypto = {
2407                         .cra_name = "cbc(des)",
2408                         .cra_driver_name = "cbc-des-talitos",
2409                         .cra_blocksize = DES_BLOCK_SIZE,
2410                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2411                                      CRYPTO_ALG_ASYNC,
2412                         .cra_ablkcipher = {
2413                                 .min_keysize = DES_KEY_SIZE,
2414                                 .max_keysize = DES_KEY_SIZE,
2415                                 .ivsize = DES_BLOCK_SIZE,
2416                         }
2417                 },
2418                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2419                                      DESC_HDR_SEL0_DEU |
2420                                      DESC_HDR_MODE0_DEU_CBC,
2421         },
2422         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2423                 .alg.crypto = {
2424                         .cra_name = "ecb(des3_ede)",
2425                         .cra_driver_name = "ecb-3des-talitos",
2426                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2427                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2428                                      CRYPTO_ALG_ASYNC,
2429                         .cra_ablkcipher = {
2430                                 .min_keysize = DES3_EDE_KEY_SIZE,
2431                                 .max_keysize = DES3_EDE_KEY_SIZE,
2432                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2433                         }
2434                 },
2435                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2436                                      DESC_HDR_SEL0_DEU |
2437                                      DESC_HDR_MODE0_DEU_3DES,
2438         },
2439         {       .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2440                 .alg.crypto = {
2441                         .cra_name = "cbc(des3_ede)",
2442                         .cra_driver_name = "cbc-3des-talitos",
2443                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2444                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2445                                      CRYPTO_ALG_ASYNC,
2446                         .cra_ablkcipher = {
2447                                 .min_keysize = DES3_EDE_KEY_SIZE,
2448                                 .max_keysize = DES3_EDE_KEY_SIZE,
2449                                 .ivsize = DES3_EDE_BLOCK_SIZE,
2450                         }
2451                 },
2452                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2453                                      DESC_HDR_SEL0_DEU |
2454                                      DESC_HDR_MODE0_DEU_CBC |
2455                                      DESC_HDR_MODE0_DEU_3DES,
2456         },
2457         /* AHASH algorithms. */
2458         {       .type = CRYPTO_ALG_TYPE_AHASH,
2459                 .alg.hash = {
2460                         .halg.digestsize = MD5_DIGEST_SIZE,
2461                         .halg.base = {
2462                                 .cra_name = "md5",
2463                                 .cra_driver_name = "md5-talitos",
2464                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2465                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2466                                              CRYPTO_ALG_ASYNC,
2467                         }
2468                 },
2469                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2470                                      DESC_HDR_SEL0_MDEUA |
2471                                      DESC_HDR_MODE0_MDEU_MD5,
2472         },
2473         {       .type = CRYPTO_ALG_TYPE_AHASH,
2474                 .alg.hash = {
2475                         .halg.digestsize = SHA1_DIGEST_SIZE,
2476                         .halg.base = {
2477                                 .cra_name = "sha1",
2478                                 .cra_driver_name = "sha1-talitos",
2479                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2480                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2481                                              CRYPTO_ALG_ASYNC,
2482                         }
2483                 },
2484                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2485                                      DESC_HDR_SEL0_MDEUA |
2486                                      DESC_HDR_MODE0_MDEU_SHA1,
2487         },
2488         {       .type = CRYPTO_ALG_TYPE_AHASH,
2489                 .alg.hash = {
2490                         .halg.digestsize = SHA224_DIGEST_SIZE,
2491                         .halg.base = {
2492                                 .cra_name = "sha224",
2493                                 .cra_driver_name = "sha224-talitos",
2494                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2495                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2496                                              CRYPTO_ALG_ASYNC,
2497                         }
2498                 },
2499                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2500                                      DESC_HDR_SEL0_MDEUA |
2501                                      DESC_HDR_MODE0_MDEU_SHA224,
2502         },
2503         {       .type = CRYPTO_ALG_TYPE_AHASH,
2504                 .alg.hash = {
2505                         .halg.digestsize = SHA256_DIGEST_SIZE,
2506                         .halg.base = {
2507                                 .cra_name = "sha256",
2508                                 .cra_driver_name = "sha256-talitos",
2509                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2510                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2511                                              CRYPTO_ALG_ASYNC,
2512                         }
2513                 },
2514                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2515                                      DESC_HDR_SEL0_MDEUA |
2516                                      DESC_HDR_MODE0_MDEU_SHA256,
2517         },
2518         {       .type = CRYPTO_ALG_TYPE_AHASH,
2519                 .alg.hash = {
2520                         .halg.digestsize = SHA384_DIGEST_SIZE,
2521                         .halg.base = {
2522                                 .cra_name = "sha384",
2523                                 .cra_driver_name = "sha384-talitos",
2524                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2525                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2526                                              CRYPTO_ALG_ASYNC,
2527                         }
2528                 },
2529                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2530                                      DESC_HDR_SEL0_MDEUB |
2531                                      DESC_HDR_MODE0_MDEUB_SHA384,
2532         },
2533         {       .type = CRYPTO_ALG_TYPE_AHASH,
2534                 .alg.hash = {
2535                         .halg.digestsize = SHA512_DIGEST_SIZE,
2536                         .halg.base = {
2537                                 .cra_name = "sha512",
2538                                 .cra_driver_name = "sha512-talitos",
2539                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2540                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2541                                              CRYPTO_ALG_ASYNC,
2542                         }
2543                 },
2544                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2545                                      DESC_HDR_SEL0_MDEUB |
2546                                      DESC_HDR_MODE0_MDEUB_SHA512,
2547         },
2548         {       .type = CRYPTO_ALG_TYPE_AHASH,
2549                 .alg.hash = {
2550                         .halg.digestsize = MD5_DIGEST_SIZE,
2551                         .halg.base = {
2552                                 .cra_name = "hmac(md5)",
2553                                 .cra_driver_name = "hmac-md5-talitos",
2554                                 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2555                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2556                                              CRYPTO_ALG_ASYNC,
2557                         }
2558                 },
2559                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2560                                      DESC_HDR_SEL0_MDEUA |
2561                                      DESC_HDR_MODE0_MDEU_MD5,
2562         },
2563         {       .type = CRYPTO_ALG_TYPE_AHASH,
2564                 .alg.hash = {
2565                         .halg.digestsize = SHA1_DIGEST_SIZE,
2566                         .halg.base = {
2567                                 .cra_name = "hmac(sha1)",
2568                                 .cra_driver_name = "hmac-sha1-talitos",
2569                                 .cra_blocksize = SHA1_BLOCK_SIZE,
2570                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2571                                              CRYPTO_ALG_ASYNC,
2572                         }
2573                 },
2574                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2575                                      DESC_HDR_SEL0_MDEUA |
2576                                      DESC_HDR_MODE0_MDEU_SHA1,
2577         },
2578         {       .type = CRYPTO_ALG_TYPE_AHASH,
2579                 .alg.hash = {
2580                         .halg.digestsize = SHA224_DIGEST_SIZE,
2581                         .halg.base = {
2582                                 .cra_name = "hmac(sha224)",
2583                                 .cra_driver_name = "hmac-sha224-talitos",
2584                                 .cra_blocksize = SHA224_BLOCK_SIZE,
2585                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2586                                              CRYPTO_ALG_ASYNC,
2587                         }
2588                 },
2589                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2590                                      DESC_HDR_SEL0_MDEUA |
2591                                      DESC_HDR_MODE0_MDEU_SHA224,
2592         },
2593         {       .type = CRYPTO_ALG_TYPE_AHASH,
2594                 .alg.hash = {
2595                         .halg.digestsize = SHA256_DIGEST_SIZE,
2596                         .halg.base = {
2597                                 .cra_name = "hmac(sha256)",
2598                                 .cra_driver_name = "hmac-sha256-talitos",
2599                                 .cra_blocksize = SHA256_BLOCK_SIZE,
2600                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2601                                              CRYPTO_ALG_ASYNC,
2602                         }
2603                 },
2604                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2605                                      DESC_HDR_SEL0_MDEUA |
2606                                      DESC_HDR_MODE0_MDEU_SHA256,
2607         },
2608         {       .type = CRYPTO_ALG_TYPE_AHASH,
2609                 .alg.hash = {
2610                         .halg.digestsize = SHA384_DIGEST_SIZE,
2611                         .halg.base = {
2612                                 .cra_name = "hmac(sha384)",
2613                                 .cra_driver_name = "hmac-sha384-talitos",
2614                                 .cra_blocksize = SHA384_BLOCK_SIZE,
2615                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2616                                              CRYPTO_ALG_ASYNC,
2617                         }
2618                 },
2619                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2620                                      DESC_HDR_SEL0_MDEUB |
2621                                      DESC_HDR_MODE0_MDEUB_SHA384,
2622         },
2623         {       .type = CRYPTO_ALG_TYPE_AHASH,
2624                 .alg.hash = {
2625                         .halg.digestsize = SHA512_DIGEST_SIZE,
2626                         .halg.base = {
2627                                 .cra_name = "hmac(sha512)",
2628                                 .cra_driver_name = "hmac-sha512-talitos",
2629                                 .cra_blocksize = SHA512_BLOCK_SIZE,
2630                                 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2631                                              CRYPTO_ALG_ASYNC,
2632                         }
2633                 },
2634                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2635                                      DESC_HDR_SEL0_MDEUB |
2636                                      DESC_HDR_MODE0_MDEUB_SHA512,
2637         }
2638 };
2639 
2640 struct talitos_crypto_alg {
2641         struct list_head entry;
2642         struct device *dev;
2643         struct talitos_alg_template algt;
2644 };
2645 
2646 static int talitos_init_common(struct talitos_ctx *ctx,
2647                                struct talitos_crypto_alg *talitos_alg)
2648 {
2649         struct talitos_private *priv;
2650 
2651         /* update context with ptr to dev */
2652         ctx->dev = talitos_alg->dev;
2653 
2654         /* assign SEC channel to tfm in round-robin fashion */
2655         priv = dev_get_drvdata(ctx->dev);
2656         ctx->ch = atomic_inc_return(&priv->last_chan) &
2657                   (priv->num_channels - 1);
2658 
2659         /* copy descriptor header template value */
2660         ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
2661 
2662         /* select done notification */
2663         ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2664 
2665         return 0;
2666 }
2667 
2668 static int talitos_cra_init(struct crypto_tfm *tfm)
2669 {
2670         struct crypto_alg *alg = tfm->__crt_alg;
2671         struct talitos_crypto_alg *talitos_alg;
2672         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2673 
2674         if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2675                 talitos_alg = container_of(__crypto_ahash_alg(alg),
2676                                            struct talitos_crypto_alg,
2677                                            algt.alg.hash);
2678         else
2679                 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2680                                            algt.alg.crypto);
2681 
2682         return talitos_init_common(ctx, talitos_alg);
2683 }
2684 
2685 static int talitos_cra_init_aead(struct crypto_aead *tfm)
2686 {
2687         struct aead_alg *alg = crypto_aead_alg(tfm);
2688         struct talitos_crypto_alg *talitos_alg;
2689         struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
2690 
2691         talitos_alg = container_of(alg, struct talitos_crypto_alg,
2692                                    algt.alg.aead);
2693 
2694         return talitos_init_common(ctx, talitos_alg);
2695 }
2696 
2697 static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2698 {
2699         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2700 
2701         talitos_cra_init(tfm);
2702 
2703         ctx->keylen = 0;
2704         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2705                                  sizeof(struct talitos_ahash_req_ctx));
2706 
2707         return 0;
2708 }
2709 
2710 /*
2711  * given the alg's descriptor header template, determine whether descriptor
2712  * type and primary/secondary execution units required match the hw
2713  * capabilities description provided in the device tree node.
2714  */
2715 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2716 {
2717         struct talitos_private *priv = dev_get_drvdata(dev);
2718         int ret;
2719 
2720         ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2721               (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2722 
2723         if (SECONDARY_EU(desc_hdr_template))
2724                 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2725                               & priv->exec_units);
2726 
2727         return ret;
2728 }
2729 
2730 static int talitos_remove(struct platform_device *ofdev)
2731 {
2732         struct device *dev = &ofdev->dev;
2733         struct talitos_private *priv = dev_get_drvdata(dev);
2734         struct talitos_crypto_alg *t_alg, *n;
2735         int i;
2736 
2737         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
2738                 switch (t_alg->algt.type) {
2739                 case CRYPTO_ALG_TYPE_ABLKCIPHER:
2740                         break;
2741                 case CRYPTO_ALG_TYPE_AEAD:
2742                         crypto_unregister_aead(&t_alg->algt.alg.aead);
2743                 case CRYPTO_ALG_TYPE_AHASH:
2744                         crypto_unregister_ahash(&t_alg->algt.alg.hash);
2745                         break;
2746                 }
2747                 list_del(&t_alg->entry);
2748                 kfree(t_alg);
2749         }
2750 
2751         if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2752                 talitos_unregister_rng(dev);
2753 
2754         for (i = 0; priv->chan && i < priv->num_channels; i++)
2755                 kfree(priv->chan[i].fifo);
2756 
2757         kfree(priv->chan);
2758 
2759         for (i = 0; i < 2; i++)
2760                 if (priv->irq[i]) {
2761                         free_irq(priv->irq[i], dev);
2762                         irq_dispose_mapping(priv->irq[i]);
2763                 }
2764 
2765         tasklet_kill(&priv->done_task[0]);
2766         if (priv->irq[1])
2767                 tasklet_kill(&priv->done_task[1]);
2768 
2769         iounmap(priv->reg);
2770 
2771         kfree(priv);
2772 
2773         return 0;
2774 }
2775 
2776 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
2777                                                     struct talitos_alg_template
2778                                                            *template)
2779 {
2780         struct talitos_private *priv = dev_get_drvdata(dev);
2781         struct talitos_crypto_alg *t_alg;
2782         struct crypto_alg *alg;
2783 
2784         t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
2785         if (!t_alg)
2786                 return ERR_PTR(-ENOMEM);
2787 
2788         t_alg->algt = *template;
2789 
2790         switch (t_alg->algt.type) {
2791         case CRYPTO_ALG_TYPE_ABLKCIPHER:
2792                 alg = &t_alg->algt.alg.crypto;
2793                 alg->cra_init = talitos_cra_init;
2794                 alg->cra_type = &crypto_ablkcipher_type;
2795                 alg->cra_ablkcipher.setkey = ablkcipher_setkey;
2796                 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
2797                 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
2798                 alg->cra_ablkcipher.geniv = "eseqiv";
2799                 break;
2800         case CRYPTO_ALG_TYPE_AEAD:
2801                 alg = &t_alg->algt.alg.aead.base;
2802                 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
2803                 t_alg->algt.alg.aead.setkey = aead_setkey;
2804                 t_alg->algt.alg.aead.encrypt = aead_encrypt;
2805                 t_alg->algt.alg.aead.decrypt = aead_decrypt;
2806                 break;
2807         case CRYPTO_ALG_TYPE_AHASH:
2808                 alg = &t_alg->algt.alg.hash.halg.base;
2809                 alg->cra_init = talitos_cra_init_ahash;
2810                 alg->cra_type = &crypto_ahash_type;
2811                 t_alg->algt.alg.hash.init = ahash_init;
2812                 t_alg->algt.alg.hash.update = ahash_update;
2813                 t_alg->algt.alg.hash.final = ahash_final;
2814                 t_alg->algt.alg.hash.finup = ahash_finup;
2815                 t_alg->algt.alg.hash.digest = ahash_digest;
2816                 t_alg->algt.alg.hash.setkey = ahash_setkey;
2817 
2818                 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
2819                     !strncmp(alg->cra_name, "hmac", 4)) {
2820                         kfree(t_alg);
2821                         return ERR_PTR(-ENOTSUPP);
2822                 }
2823                 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
2824                     (!strcmp(alg->cra_name, "sha224") ||
2825                      !strcmp(alg->cra_name, "hmac(sha224)"))) {
2826                         t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
2827                         t_alg->algt.desc_hdr_template =
2828                                         DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2829                                         DESC_HDR_SEL0_MDEUA |
2830                                         DESC_HDR_MODE0_MDEU_SHA256;
2831                 }
2832                 break;
2833         default:
2834                 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
2835                 kfree(t_alg);
2836                 return ERR_PTR(-EINVAL);
2837         }
2838 
2839         alg->cra_module = THIS_MODULE;
2840         alg->cra_priority = TALITOS_CRA_PRIORITY;
2841         alg->cra_alignmask = 0;
2842         alg->cra_ctxsize = sizeof(struct talitos_ctx);
2843         alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
2844 
2845         t_alg->dev = dev;
2846 
2847         return t_alg;
2848 }
2849 
2850 static int talitos_probe_irq(struct platform_device *ofdev)
2851 {
2852         struct device *dev = &ofdev->dev;
2853         struct device_node *np = ofdev->dev.of_node;
2854         struct talitos_private *priv = dev_get_drvdata(dev);
2855         int err;
2856         bool is_sec1 = has_ftr_sec1(priv);
2857 
2858         priv->irq[0] = irq_of_parse_and_map(np, 0);
2859         if (!priv->irq[0]) {
2860                 dev_err(dev, "failed to map irq\n");
2861                 return -EINVAL;
2862         }
2863         if (is_sec1) {
2864                 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
2865                                   dev_driver_string(dev), dev);
2866                 goto primary_out;
2867         }
2868 
2869         priv->irq[1] = irq_of_parse_and_map(np, 1);
2870 
2871         /* get the primary irq line */
2872         if (!priv->irq[1]) {
2873                 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
2874                                   dev_driver_string(dev), dev);
2875                 goto primary_out;
2876         }
2877 
2878         err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
2879                           dev_driver_string(dev), dev);
2880         if (err)
2881                 goto primary_out;
2882 
2883         /* get the secondary irq line */
2884         err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
2885                           dev_driver_string(dev), dev);
2886         if (err) {
2887                 dev_err(dev, "failed to request secondary irq\n");
2888                 irq_dispose_mapping(priv->irq[1]);
2889                 priv->irq[1] = 0;
2890         }
2891 
2892         return err;
2893 
2894 primary_out:
2895         if (err) {
2896                 dev_err(dev, "failed to request primary irq\n");
2897                 irq_dispose_mapping(priv->irq[0]);
2898                 priv->irq[0] = 0;
2899         }
2900 
2901         return err;
2902 }
2903 
2904 static int talitos_probe(struct platform_device *ofdev)
2905 {
2906         struct device *dev = &ofdev->dev;
2907         struct device_node *np = ofdev->dev.of_node;
2908         struct talitos_private *priv;
2909         const unsigned int *prop;
2910         int i, err;
2911         int stride;
2912 
2913         priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
2914         if (!priv)
2915                 return -ENOMEM;
2916 
2917         INIT_LIST_HEAD(&priv->alg_list);
2918 
2919         dev_set_drvdata(dev, priv);
2920 
2921         priv->ofdev = ofdev;
2922 
2923         spin_lock_init(&priv->reg_lock);
2924 
2925         priv->reg = of_iomap(np, 0);
2926         if (!priv->reg) {
2927                 dev_err(dev, "failed to of_iomap\n");
2928                 err = -ENOMEM;
2929                 goto err_out;
2930         }
2931 
2932         /* get SEC version capabilities from device tree */
2933         prop = of_get_property(np, "fsl,num-channels", NULL);
2934         if (prop)
2935                 priv->num_channels = *prop;
2936 
2937         prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
2938         if (prop)
2939                 priv->chfifo_len = *prop;
2940 
2941         prop = of_get_property(np, "fsl,exec-units-mask", NULL);
2942         if (prop)
2943                 priv->exec_units = *prop;
2944 
2945         prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
2946         if (prop)
2947                 priv->desc_types = *prop;
2948 
2949         if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
2950             !priv->exec_units || !priv->desc_types) {
2951                 dev_err(dev, "invalid property data in device tree node\n");
2952                 err = -EINVAL;
2953                 goto err_out;
2954         }
2955 
2956         if (of_device_is_compatible(np, "fsl,sec3.0"))
2957                 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
2958 
2959         if (of_device_is_compatible(np, "fsl,sec2.1"))
2960                 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
2961                                   TALITOS_FTR_SHA224_HWINIT |
2962                                   TALITOS_FTR_HMAC_OK;
2963 
2964         if (of_device_is_compatible(np, "fsl,sec1.0"))
2965                 priv->features |= TALITOS_FTR_SEC1;
2966 
2967         if (of_device_is_compatible(np, "fsl,sec1.2")) {
2968                 priv->reg_deu = priv->reg + TALITOS12_DEU;
2969                 priv->reg_aesu = priv->reg + TALITOS12_AESU;
2970                 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
2971                 stride = TALITOS1_CH_STRIDE;
2972         } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
2973                 priv->reg_deu = priv->reg + TALITOS10_DEU;
2974                 priv->reg_aesu = priv->reg + TALITOS10_AESU;
2975                 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
2976                 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
2977                 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
2978                 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
2979                 stride = TALITOS1_CH_STRIDE;
2980         } else {
2981                 priv->reg_deu = priv->reg + TALITOS2_DEU;
2982                 priv->reg_aesu = priv->reg + TALITOS2_AESU;
2983                 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
2984                 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
2985                 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
2986                 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
2987                 priv->reg_keu = priv->reg + TALITOS2_KEU;
2988                 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
2989                 stride = TALITOS2_CH_STRIDE;
2990         }
2991 
2992         err = talitos_probe_irq(ofdev);
2993         if (err)
2994                 goto err_out;
2995 
2996         if (of_device_is_compatible(np, "fsl,sec1.0")) {
2997                 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
2998                              (unsigned long)dev);
2999         } else {
3000                 if (!priv->irq[1]) {
3001                         tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3002                                      (unsigned long)dev);
3003                 } else {
3004                         tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3005                                      (unsigned long)dev);
3006                         tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3007                                      (unsigned long)dev);
3008                 }
3009         }
3010 
3011         priv->chan = kzalloc(sizeof(struct talitos_channel) *
3012                              priv->num_channels, GFP_KERNEL);
3013         if (!priv->chan) {
3014                 dev_err(dev, "failed to allocate channel management space\n");
3015                 err = -ENOMEM;
3016                 goto err_out;
3017         }
3018 
3019         priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3020 
3021         for (i = 0; i < priv->num_channels; i++) {
3022                 priv->chan[i].reg = priv->reg + stride * (i + 1);
3023                 if (!priv->irq[1] || !(i & 1))
3024                         priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
3025 
3026                 spin_lock_init(&priv->chan[i].head_lock);
3027                 spin_lock_init(&priv->chan[i].tail_lock);
3028 
3029                 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
3030                                              priv->fifo_len, GFP_KERNEL);
3031                 if (!priv->chan[i].fifo) {
3032                         dev_err(dev, "failed to allocate request fifo %d\n", i);
3033                         err = -ENOMEM;
3034                         goto err_out;
3035                 }
3036 
3037                 atomic_set(&priv->chan[i].submit_count,
3038                            -(priv->chfifo_len - 1));
3039         }
3040 
3041         dma_set_mask(dev, DMA_BIT_MASK(36));
3042 
3043         /* reset and initialize the h/w */
3044         err = init_device(dev);
3045         if (err) {
3046                 dev_err(dev, "failed to initialize device\n");
3047                 goto err_out;
3048         }
3049 
3050         /* register the RNG, if available */
3051         if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3052                 err = talitos_register_rng(dev);
3053                 if (err) {
3054                         dev_err(dev, "failed to register hwrng: %d\n", err);
3055                         goto err_out;
3056                 } else
3057                         dev_info(dev, "hwrng\n");
3058         }
3059 
3060         /* register crypto algorithms the device supports */
3061         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3062                 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3063                         struct talitos_crypto_alg *t_alg;
3064                         struct crypto_alg *alg = NULL;
3065 
3066                         t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3067                         if (IS_ERR(t_alg)) {
3068                                 err = PTR_ERR(t_alg);
3069                                 if (err == -ENOTSUPP)
3070                                         continue;
3071                                 goto err_out;
3072                         }
3073 
3074                         switch (t_alg->algt.type) {
3075                         case CRYPTO_ALG_TYPE_ABLKCIPHER:
3076                                 err = crypto_register_alg(
3077                                                 &t_alg->algt.alg.crypto);
3078                                 alg = &t_alg->algt.alg.crypto;
3079                                 break;
3080 
3081                         case CRYPTO_ALG_TYPE_AEAD:
3082                                 err = crypto_register_aead(
3083                                         &t_alg->algt.alg.aead);
3084                                 alg = &t_alg->algt.alg.aead.base;
3085                                 break;
3086 
3087                         case CRYPTO_ALG_TYPE_AHASH:
3088                                 err = crypto_register_ahash(
3089                                                 &t_alg->algt.alg.hash);
3090                                 alg = &t_alg->algt.alg.hash.halg.base;
3091                                 break;
3092                         }
3093                         if (err) {
3094                                 dev_err(dev, "%s alg registration failed\n",
3095                                         alg->cra_driver_name);
3096                                 kfree(t_alg);
3097                         } else
3098                                 list_add_tail(&t_alg->entry, &priv->alg_list);
3099                 }
3100         }
3101         if (!list_empty(&priv->alg_list))
3102                 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3103                          (char *)of_get_property(np, "compatible", NULL));
3104 
3105         return 0;
3106 
3107 err_out:
3108         talitos_remove(ofdev);
3109 
3110         return err;
3111 }
3112 
3113 static const struct of_device_id talitos_match[] = {
3114 #ifdef CONFIG_CRYPTO_DEV_TALITOS1
3115         {
3116                 .compatible = "fsl,sec1.0",
3117         },
3118 #endif
3119 #ifdef CONFIG_CRYPTO_DEV_TALITOS2
3120         {
3121                 .compatible = "fsl,sec2.0",
3122         },
3123 #endif
3124         {},
3125 };
3126 MODULE_DEVICE_TABLE(of, talitos_match);
3127 
3128 static struct platform_driver talitos_driver = {
3129         .driver = {
3130                 .name = "talitos",
3131                 .of_match_table = talitos_match,
3132         },
3133         .probe = talitos_probe,
3134         .remove = talitos_remove,
3135 };
3136 
3137 module_platform_driver(talitos_driver);
3138 
3139 MODULE_LICENSE("GPL");
3140 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3141 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");
3142 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us