Version:  2.0.40 2.2.26 2.4.37 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6

Linux/drivers/crypto/atmel-sha.c

  1 /*
  2  * Cryptographic API.
  3  *
  4  * Support for ATMEL SHA1/SHA256 HW acceleration.
  5  *
  6  * Copyright (c) 2012 Eukréa Electromatique - ATMEL
  7  * Author: Nicolas Royer <nicolas@eukrea.com>
  8  *
  9  * This program is free software; you can redistribute it and/or modify
 10  * it under the terms of the GNU General Public License version 2 as published
 11  * by the Free Software Foundation.
 12  *
 13  * Some ideas are from omap-sham.c drivers.
 14  */
 15 
 16 
 17 #include <linux/kernel.h>
 18 #include <linux/module.h>
 19 #include <linux/slab.h>
 20 #include <linux/err.h>
 21 #include <linux/clk.h>
 22 #include <linux/io.h>
 23 #include <linux/hw_random.h>
 24 #include <linux/platform_device.h>
 25 
 26 #include <linux/device.h>
 27 #include <linux/init.h>
 28 #include <linux/errno.h>
 29 #include <linux/interrupt.h>
 30 #include <linux/irq.h>
 31 #include <linux/scatterlist.h>
 32 #include <linux/dma-mapping.h>
 33 #include <linux/of_device.h>
 34 #include <linux/delay.h>
 35 #include <linux/crypto.h>
 36 #include <linux/cryptohash.h>
 37 #include <crypto/scatterwalk.h>
 38 #include <crypto/algapi.h>
 39 #include <crypto/sha.h>
 40 #include <crypto/hash.h>
 41 #include <crypto/internal/hash.h>
 42 #include <linux/platform_data/crypto-atmel.h>
 43 #include "atmel-sha-regs.h"
 44 
 45 /* SHA flags */
 46 #define SHA_FLAGS_BUSY                  BIT(0)
 47 #define SHA_FLAGS_FINAL                 BIT(1)
 48 #define SHA_FLAGS_DMA_ACTIVE    BIT(2)
 49 #define SHA_FLAGS_OUTPUT_READY  BIT(3)
 50 #define SHA_FLAGS_INIT                  BIT(4)
 51 #define SHA_FLAGS_CPU                   BIT(5)
 52 #define SHA_FLAGS_DMA_READY             BIT(6)
 53 
 54 #define SHA_FLAGS_FINUP         BIT(16)
 55 #define SHA_FLAGS_SG            BIT(17)
 56 #define SHA_FLAGS_ALGO_MASK     GENMASK(22, 18)
 57 #define SHA_FLAGS_SHA1          BIT(18)
 58 #define SHA_FLAGS_SHA224        BIT(19)
 59 #define SHA_FLAGS_SHA256        BIT(20)
 60 #define SHA_FLAGS_SHA384        BIT(21)
 61 #define SHA_FLAGS_SHA512        BIT(22)
 62 #define SHA_FLAGS_ERROR         BIT(23)
 63 #define SHA_FLAGS_PAD           BIT(24)
 64 #define SHA_FLAGS_RESTORE       BIT(25)
 65 
 66 #define SHA_OP_UPDATE   1
 67 #define SHA_OP_FINAL    2
 68 
 69 #define SHA_BUFFER_LEN          (PAGE_SIZE / 16)
 70 
 71 #define ATMEL_SHA_DMA_THRESHOLD         56
 72 
 73 struct atmel_sha_caps {
 74         bool    has_dma;
 75         bool    has_dualbuff;
 76         bool    has_sha224;
 77         bool    has_sha_384_512;
 78         bool    has_uihv;
 79 };
 80 
 81 struct atmel_sha_dev;
 82 
 83 /*
 84  * .statesize = sizeof(struct atmel_sha_reqctx) must be <= PAGE_SIZE / 8 as
 85  * tested by the ahash_prepare_alg() function.
 86  */
 87 struct atmel_sha_reqctx {
 88         struct atmel_sha_dev    *dd;
 89         unsigned long   flags;
 90         unsigned long   op;
 91 
 92         u8      digest[SHA512_DIGEST_SIZE] __aligned(sizeof(u32));
 93         u64     digcnt[2];
 94         size_t  bufcnt;
 95         size_t  buflen;
 96         dma_addr_t      dma_addr;
 97 
 98         /* walk state */
 99         struct scatterlist      *sg;
100         unsigned int    offset; /* offset in current sg */
101         unsigned int    total;  /* total request */
102 
103         size_t block_size;
104 
105         u8 buffer[SHA_BUFFER_LEN + SHA512_BLOCK_SIZE] __aligned(sizeof(u32));
106 };
107 
108 struct atmel_sha_ctx {
109         struct atmel_sha_dev    *dd;
110 
111         unsigned long           flags;
112 };
113 
114 #define ATMEL_SHA_QUEUE_LENGTH  50
115 
116 struct atmel_sha_dma {
117         struct dma_chan                 *chan;
118         struct dma_slave_config dma_conf;
119 };
120 
121 struct atmel_sha_dev {
122         struct list_head        list;
123         unsigned long           phys_base;
124         struct device           *dev;
125         struct clk                      *iclk;
126         int                                     irq;
127         void __iomem            *io_base;
128 
129         spinlock_t              lock;
130         int                     err;
131         struct tasklet_struct   done_task;
132         struct tasklet_struct   queue_task;
133 
134         unsigned long           flags;
135         struct crypto_queue     queue;
136         struct ahash_request    *req;
137 
138         struct atmel_sha_dma    dma_lch_in;
139 
140         struct atmel_sha_caps   caps;
141 
142         u32     hw_version;
143 };
144 
145 struct atmel_sha_drv {
146         struct list_head        dev_list;
147         spinlock_t              lock;
148 };
149 
150 static struct atmel_sha_drv atmel_sha = {
151         .dev_list = LIST_HEAD_INIT(atmel_sha.dev_list),
152         .lock = __SPIN_LOCK_UNLOCKED(atmel_sha.lock),
153 };
154 
155 static inline u32 atmel_sha_read(struct atmel_sha_dev *dd, u32 offset)
156 {
157         return readl_relaxed(dd->io_base + offset);
158 }
159 
160 static inline void atmel_sha_write(struct atmel_sha_dev *dd,
161                                         u32 offset, u32 value)
162 {
163         writel_relaxed(value, dd->io_base + offset);
164 }
165 
166 static size_t atmel_sha_append_sg(struct atmel_sha_reqctx *ctx)
167 {
168         size_t count;
169 
170         while ((ctx->bufcnt < ctx->buflen) && ctx->total) {
171                 count = min(ctx->sg->length - ctx->offset, ctx->total);
172                 count = min(count, ctx->buflen - ctx->bufcnt);
173 
174                 if (count <= 0) {
175                         /*
176                         * Check if count <= 0 because the buffer is full or
177                         * because the sg length is 0. In the latest case,
178                         * check if there is another sg in the list, a 0 length
179                         * sg doesn't necessarily mean the end of the sg list.
180                         */
181                         if ((ctx->sg->length == 0) && !sg_is_last(ctx->sg)) {
182                                 ctx->sg = sg_next(ctx->sg);
183                                 continue;
184                         } else {
185                                 break;
186                         }
187                 }
188 
189                 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg,
190                         ctx->offset, count, 0);
191 
192                 ctx->bufcnt += count;
193                 ctx->offset += count;
194                 ctx->total -= count;
195 
196                 if (ctx->offset == ctx->sg->length) {
197                         ctx->sg = sg_next(ctx->sg);
198                         if (ctx->sg)
199                                 ctx->offset = 0;
200                         else
201                                 ctx->total = 0;
202                 }
203         }
204 
205         return 0;
206 }
207 
208 /*
209  * The purpose of this padding is to ensure that the padded message is a
210  * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512).
211  * The bit "1" is appended at the end of the message followed by
212  * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or
213  * 128 bits block (SHA384/SHA512) equals to the message length in bits
214  * is appended.
215  *
216  * For SHA1/SHA224/SHA256, padlen is calculated as followed:
217  *  - if message length < 56 bytes then padlen = 56 - message length
218  *  - else padlen = 64 + 56 - message length
219  *
220  * For SHA384/SHA512, padlen is calculated as followed:
221  *  - if message length < 112 bytes then padlen = 112 - message length
222  *  - else padlen = 128 + 112 - message length
223  */
224 static void atmel_sha_fill_padding(struct atmel_sha_reqctx *ctx, int length)
225 {
226         unsigned int index, padlen;
227         u64 bits[2];
228         u64 size[2];
229 
230         size[0] = ctx->digcnt[0];
231         size[1] = ctx->digcnt[1];
232 
233         size[0] += ctx->bufcnt;
234         if (size[0] < ctx->bufcnt)
235                 size[1]++;
236 
237         size[0] += length;
238         if (size[0]  < length)
239                 size[1]++;
240 
241         bits[1] = cpu_to_be64(size[0] << 3);
242         bits[0] = cpu_to_be64(size[1] << 3 | size[0] >> 61);
243 
244         if (ctx->flags & (SHA_FLAGS_SHA384 | SHA_FLAGS_SHA512)) {
245                 index = ctx->bufcnt & 0x7f;
246                 padlen = (index < 112) ? (112 - index) : ((128+112) - index);
247                 *(ctx->buffer + ctx->bufcnt) = 0x80;
248                 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1);
249                 memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16);
250                 ctx->bufcnt += padlen + 16;
251                 ctx->flags |= SHA_FLAGS_PAD;
252         } else {
253                 index = ctx->bufcnt & 0x3f;
254                 padlen = (index < 56) ? (56 - index) : ((64+56) - index);
255                 *(ctx->buffer + ctx->bufcnt) = 0x80;
256                 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen-1);
257                 memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8);
258                 ctx->bufcnt += padlen + 8;
259                 ctx->flags |= SHA_FLAGS_PAD;
260         }
261 }
262 
263 static int atmel_sha_init(struct ahash_request *req)
264 {
265         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
266         struct atmel_sha_ctx *tctx = crypto_ahash_ctx(tfm);
267         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
268         struct atmel_sha_dev *dd = NULL;
269         struct atmel_sha_dev *tmp;
270 
271         spin_lock_bh(&atmel_sha.lock);
272         if (!tctx->dd) {
273                 list_for_each_entry(tmp, &atmel_sha.dev_list, list) {
274                         dd = tmp;
275                         break;
276                 }
277                 tctx->dd = dd;
278         } else {
279                 dd = tctx->dd;
280         }
281 
282         spin_unlock_bh(&atmel_sha.lock);
283 
284         ctx->dd = dd;
285 
286         ctx->flags = 0;
287 
288         dev_dbg(dd->dev, "init: digest size: %d\n",
289                 crypto_ahash_digestsize(tfm));
290 
291         switch (crypto_ahash_digestsize(tfm)) {
292         case SHA1_DIGEST_SIZE:
293                 ctx->flags |= SHA_FLAGS_SHA1;
294                 ctx->block_size = SHA1_BLOCK_SIZE;
295                 break;
296         case SHA224_DIGEST_SIZE:
297                 ctx->flags |= SHA_FLAGS_SHA224;
298                 ctx->block_size = SHA224_BLOCK_SIZE;
299                 break;
300         case SHA256_DIGEST_SIZE:
301                 ctx->flags |= SHA_FLAGS_SHA256;
302                 ctx->block_size = SHA256_BLOCK_SIZE;
303                 break;
304         case SHA384_DIGEST_SIZE:
305                 ctx->flags |= SHA_FLAGS_SHA384;
306                 ctx->block_size = SHA384_BLOCK_SIZE;
307                 break;
308         case SHA512_DIGEST_SIZE:
309                 ctx->flags |= SHA_FLAGS_SHA512;
310                 ctx->block_size = SHA512_BLOCK_SIZE;
311                 break;
312         default:
313                 return -EINVAL;
314                 break;
315         }
316 
317         ctx->bufcnt = 0;
318         ctx->digcnt[0] = 0;
319         ctx->digcnt[1] = 0;
320         ctx->buflen = SHA_BUFFER_LEN;
321 
322         return 0;
323 }
324 
325 static void atmel_sha_write_ctrl(struct atmel_sha_dev *dd, int dma)
326 {
327         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
328         u32 valmr = SHA_MR_MODE_AUTO;
329         unsigned int i, hashsize = 0;
330 
331         if (likely(dma)) {
332                 if (!dd->caps.has_dma)
333                         atmel_sha_write(dd, SHA_IER, SHA_INT_TXBUFE);
334                 valmr = SHA_MR_MODE_PDC;
335                 if (dd->caps.has_dualbuff)
336                         valmr |= SHA_MR_DUALBUFF;
337         } else {
338                 atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);
339         }
340 
341         switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {
342         case SHA_FLAGS_SHA1:
343                 valmr |= SHA_MR_ALGO_SHA1;
344                 hashsize = SHA1_DIGEST_SIZE;
345                 break;
346 
347         case SHA_FLAGS_SHA224:
348                 valmr |= SHA_MR_ALGO_SHA224;
349                 hashsize = SHA256_DIGEST_SIZE;
350                 break;
351 
352         case SHA_FLAGS_SHA256:
353                 valmr |= SHA_MR_ALGO_SHA256;
354                 hashsize = SHA256_DIGEST_SIZE;
355                 break;
356 
357         case SHA_FLAGS_SHA384:
358                 valmr |= SHA_MR_ALGO_SHA384;
359                 hashsize = SHA512_DIGEST_SIZE;
360                 break;
361 
362         case SHA_FLAGS_SHA512:
363                 valmr |= SHA_MR_ALGO_SHA512;
364                 hashsize = SHA512_DIGEST_SIZE;
365                 break;
366 
367         default:
368                 break;
369         }
370 
371         /* Setting CR_FIRST only for the first iteration */
372         if (!(ctx->digcnt[0] || ctx->digcnt[1])) {
373                 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);
374         } else if (dd->caps.has_uihv && (ctx->flags & SHA_FLAGS_RESTORE)) {
375                 const u32 *hash = (const u32 *)ctx->digest;
376 
377                 /*
378                  * Restore the hardware context: update the User Initialize
379                  * Hash Value (UIHV) with the value saved when the latest
380                  * 'update' operation completed on this very same crypto
381                  * request.
382                  */
383                 ctx->flags &= ~SHA_FLAGS_RESTORE;
384                 atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV);
385                 for (i = 0; i < hashsize / sizeof(u32); ++i)
386                         atmel_sha_write(dd, SHA_REG_DIN(i), hash[i]);
387                 atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST);
388                 valmr |= SHA_MR_UIHV;
389         }
390         /*
391          * WARNING: If the UIHV feature is not available, the hardware CANNOT
392          * process concurrent requests: the internal registers used to store
393          * the hash/digest are still set to the partial digest output values
394          * computed during the latest round.
395          */
396 
397         atmel_sha_write(dd, SHA_MR, valmr);
398 }
399 
400 static int atmel_sha_xmit_cpu(struct atmel_sha_dev *dd, const u8 *buf,
401                               size_t length, int final)
402 {
403         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
404         int count, len32;
405         const u32 *buffer = (const u32 *)buf;
406 
407         dev_dbg(dd->dev, "xmit_cpu: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n",
408                 ctx->digcnt[1], ctx->digcnt[0], length, final);
409 
410         atmel_sha_write_ctrl(dd, 0);
411 
412         /* should be non-zero before next lines to disable clocks later */
413         ctx->digcnt[0] += length;
414         if (ctx->digcnt[0] < length)
415                 ctx->digcnt[1]++;
416 
417         if (final)
418                 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */
419 
420         len32 = DIV_ROUND_UP(length, sizeof(u32));
421 
422         dd->flags |= SHA_FLAGS_CPU;
423 
424         for (count = 0; count < len32; count++)
425                 atmel_sha_write(dd, SHA_REG_DIN(count), buffer[count]);
426 
427         return -EINPROGRESS;
428 }
429 
430 static int atmel_sha_xmit_pdc(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
431                 size_t length1, dma_addr_t dma_addr2, size_t length2, int final)
432 {
433         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
434         int len32;
435 
436         dev_dbg(dd->dev, "xmit_pdc: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n",
437                 ctx->digcnt[1], ctx->digcnt[0], length1, final);
438 
439         len32 = DIV_ROUND_UP(length1, sizeof(u32));
440         atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTDIS);
441         atmel_sha_write(dd, SHA_TPR, dma_addr1);
442         atmel_sha_write(dd, SHA_TCR, len32);
443 
444         len32 = DIV_ROUND_UP(length2, sizeof(u32));
445         atmel_sha_write(dd, SHA_TNPR, dma_addr2);
446         atmel_sha_write(dd, SHA_TNCR, len32);
447 
448         atmel_sha_write_ctrl(dd, 1);
449 
450         /* should be non-zero before next lines to disable clocks later */
451         ctx->digcnt[0] += length1;
452         if (ctx->digcnt[0] < length1)
453                 ctx->digcnt[1]++;
454 
455         if (final)
456                 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */
457 
458         dd->flags |=  SHA_FLAGS_DMA_ACTIVE;
459 
460         /* Start DMA transfer */
461         atmel_sha_write(dd, SHA_PTCR, SHA_PTCR_TXTEN);
462 
463         return -EINPROGRESS;
464 }
465 
466 static void atmel_sha_dma_callback(void *data)
467 {
468         struct atmel_sha_dev *dd = data;
469 
470         /* dma_lch_in - completed - wait DATRDY */
471         atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY);
472 }
473 
474 static int atmel_sha_xmit_dma(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
475                 size_t length1, dma_addr_t dma_addr2, size_t length2, int final)
476 {
477         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
478         struct dma_async_tx_descriptor  *in_desc;
479         struct scatterlist sg[2];
480 
481         dev_dbg(dd->dev, "xmit_dma: digcnt: 0x%llx 0x%llx, length: %d, final: %d\n",
482                 ctx->digcnt[1], ctx->digcnt[0], length1, final);
483 
484         dd->dma_lch_in.dma_conf.src_maxburst = 16;
485         dd->dma_lch_in.dma_conf.dst_maxburst = 16;
486 
487         dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf);
488 
489         if (length2) {
490                 sg_init_table(sg, 2);
491                 sg_dma_address(&sg[0]) = dma_addr1;
492                 sg_dma_len(&sg[0]) = length1;
493                 sg_dma_address(&sg[1]) = dma_addr2;
494                 sg_dma_len(&sg[1]) = length2;
495                 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 2,
496                         DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
497         } else {
498                 sg_init_table(sg, 1);
499                 sg_dma_address(&sg[0]) = dma_addr1;
500                 sg_dma_len(&sg[0]) = length1;
501                 in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, sg, 1,
502                         DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
503         }
504         if (!in_desc)
505                 return -EINVAL;
506 
507         in_desc->callback = atmel_sha_dma_callback;
508         in_desc->callback_param = dd;
509 
510         atmel_sha_write_ctrl(dd, 1);
511 
512         /* should be non-zero before next lines to disable clocks later */
513         ctx->digcnt[0] += length1;
514         if (ctx->digcnt[0] < length1)
515                 ctx->digcnt[1]++;
516 
517         if (final)
518                 dd->flags |= SHA_FLAGS_FINAL; /* catch last interrupt */
519 
520         dd->flags |=  SHA_FLAGS_DMA_ACTIVE;
521 
522         /* Start DMA transfer */
523         dmaengine_submit(in_desc);
524         dma_async_issue_pending(dd->dma_lch_in.chan);
525 
526         return -EINPROGRESS;
527 }
528 
529 static int atmel_sha_xmit_start(struct atmel_sha_dev *dd, dma_addr_t dma_addr1,
530                 size_t length1, dma_addr_t dma_addr2, size_t length2, int final)
531 {
532         if (dd->caps.has_dma)
533                 return atmel_sha_xmit_dma(dd, dma_addr1, length1,
534                                 dma_addr2, length2, final);
535         else
536                 return atmel_sha_xmit_pdc(dd, dma_addr1, length1,
537                                 dma_addr2, length2, final);
538 }
539 
540 static int atmel_sha_update_cpu(struct atmel_sha_dev *dd)
541 {
542         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
543         int bufcnt;
544 
545         atmel_sha_append_sg(ctx);
546         atmel_sha_fill_padding(ctx, 0);
547         bufcnt = ctx->bufcnt;
548         ctx->bufcnt = 0;
549 
550         return atmel_sha_xmit_cpu(dd, ctx->buffer, bufcnt, 1);
551 }
552 
553 static int atmel_sha_xmit_dma_map(struct atmel_sha_dev *dd,
554                                         struct atmel_sha_reqctx *ctx,
555                                         size_t length, int final)
556 {
557         ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer,
558                                 ctx->buflen + ctx->block_size, DMA_TO_DEVICE);
559         if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
560                 dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen +
561                                 ctx->block_size);
562                 return -EINVAL;
563         }
564 
565         ctx->flags &= ~SHA_FLAGS_SG;
566 
567         /* next call does not fail... so no unmap in the case of error */
568         return atmel_sha_xmit_start(dd, ctx->dma_addr, length, 0, 0, final);
569 }
570 
571 static int atmel_sha_update_dma_slow(struct atmel_sha_dev *dd)
572 {
573         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
574         unsigned int final;
575         size_t count;
576 
577         atmel_sha_append_sg(ctx);
578 
579         final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;
580 
581         dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: 0x%llx 0x%llx, final: %d\n",
582                  ctx->bufcnt, ctx->digcnt[1], ctx->digcnt[0], final);
583 
584         if (final)
585                 atmel_sha_fill_padding(ctx, 0);
586 
587         if (final || (ctx->bufcnt == ctx->buflen)) {
588                 count = ctx->bufcnt;
589                 ctx->bufcnt = 0;
590                 return atmel_sha_xmit_dma_map(dd, ctx, count, final);
591         }
592 
593         return 0;
594 }
595 
596 static int atmel_sha_update_dma_start(struct atmel_sha_dev *dd)
597 {
598         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
599         unsigned int length, final, tail;
600         struct scatterlist *sg;
601         unsigned int count;
602 
603         if (!ctx->total)
604                 return 0;
605 
606         if (ctx->bufcnt || ctx->offset)
607                 return atmel_sha_update_dma_slow(dd);
608 
609         dev_dbg(dd->dev, "fast: digcnt: 0x%llx 0x%llx, bufcnt: %u, total: %u\n",
610                 ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt, ctx->total);
611 
612         sg = ctx->sg;
613 
614         if (!IS_ALIGNED(sg->offset, sizeof(u32)))
615                 return atmel_sha_update_dma_slow(dd);
616 
617         if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, ctx->block_size))
618                 /* size is not ctx->block_size aligned */
619                 return atmel_sha_update_dma_slow(dd);
620 
621         length = min(ctx->total, sg->length);
622 
623         if (sg_is_last(sg)) {
624                 if (!(ctx->flags & SHA_FLAGS_FINUP)) {
625                         /* not last sg must be ctx->block_size aligned */
626                         tail = length & (ctx->block_size - 1);
627                         length -= tail;
628                 }
629         }
630 
631         ctx->total -= length;
632         ctx->offset = length; /* offset where to start slow */
633 
634         final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;
635 
636         /* Add padding */
637         if (final) {
638                 tail = length & (ctx->block_size - 1);
639                 length -= tail;
640                 ctx->total += tail;
641                 ctx->offset = length; /* offset where to start slow */
642 
643                 sg = ctx->sg;
644                 atmel_sha_append_sg(ctx);
645 
646                 atmel_sha_fill_padding(ctx, length);
647 
648                 ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer,
649                         ctx->buflen + ctx->block_size, DMA_TO_DEVICE);
650                 if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
651                         dev_err(dd->dev, "dma %u bytes error\n",
652                                 ctx->buflen + ctx->block_size);
653                         return -EINVAL;
654                 }
655 
656                 if (length == 0) {
657                         ctx->flags &= ~SHA_FLAGS_SG;
658                         count = ctx->bufcnt;
659                         ctx->bufcnt = 0;
660                         return atmel_sha_xmit_start(dd, ctx->dma_addr, count, 0,
661                                         0, final);
662                 } else {
663                         ctx->sg = sg;
664                         if (!dma_map_sg(dd->dev, ctx->sg, 1,
665                                 DMA_TO_DEVICE)) {
666                                         dev_err(dd->dev, "dma_map_sg  error\n");
667                                         return -EINVAL;
668                         }
669 
670                         ctx->flags |= SHA_FLAGS_SG;
671 
672                         count = ctx->bufcnt;
673                         ctx->bufcnt = 0;
674                         return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg),
675                                         length, ctx->dma_addr, count, final);
676                 }
677         }
678 
679         if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
680                 dev_err(dd->dev, "dma_map_sg  error\n");
681                 return -EINVAL;
682         }
683 
684         ctx->flags |= SHA_FLAGS_SG;
685 
686         /* next call does not fail... so no unmap in the case of error */
687         return atmel_sha_xmit_start(dd, sg_dma_address(ctx->sg), length, 0,
688                                                                 0, final);
689 }
690 
691 static int atmel_sha_update_dma_stop(struct atmel_sha_dev *dd)
692 {
693         struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req);
694 
695         if (ctx->flags & SHA_FLAGS_SG) {
696                 dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
697                 if (ctx->sg->length == ctx->offset) {
698                         ctx->sg = sg_next(ctx->sg);
699                         if (ctx->sg)
700                                 ctx->offset = 0;
701                 }
702                 if (ctx->flags & SHA_FLAGS_PAD) {
703                         dma_unmap_single(dd->dev, ctx->dma_addr,
704                                 ctx->buflen + ctx->block_size, DMA_TO_DEVICE);
705                 }
706         } else {
707                 dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen +
708                                                 ctx->block_size, DMA_TO_DEVICE);
709         }
710 
711         return 0;
712 }
713 
714 static int atmel_sha_update_req(struct atmel_sha_dev *dd)
715 {
716         struct ahash_request *req = dd->req;
717         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
718         int err;
719 
720         dev_dbg(dd->dev, "update_req: total: %u, digcnt: 0x%llx 0x%llx\n",
721                 ctx->total, ctx->digcnt[1], ctx->digcnt[0]);
722 
723         if (ctx->flags & SHA_FLAGS_CPU)
724                 err = atmel_sha_update_cpu(dd);
725         else
726                 err = atmel_sha_update_dma_start(dd);
727 
728         /* wait for dma completion before can take more data */
729         dev_dbg(dd->dev, "update: err: %d, digcnt: 0x%llx 0%llx\n",
730                         err, ctx->digcnt[1], ctx->digcnt[0]);
731 
732         return err;
733 }
734 
735 static int atmel_sha_final_req(struct atmel_sha_dev *dd)
736 {
737         struct ahash_request *req = dd->req;
738         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
739         int err = 0;
740         int count;
741 
742         if (ctx->bufcnt >= ATMEL_SHA_DMA_THRESHOLD) {
743                 atmel_sha_fill_padding(ctx, 0);
744                 count = ctx->bufcnt;
745                 ctx->bufcnt = 0;
746                 err = atmel_sha_xmit_dma_map(dd, ctx, count, 1);
747         }
748         /* faster to handle last block with cpu */
749         else {
750                 atmel_sha_fill_padding(ctx, 0);
751                 count = ctx->bufcnt;
752                 ctx->bufcnt = 0;
753                 err = atmel_sha_xmit_cpu(dd, ctx->buffer, count, 1);
754         }
755 
756         dev_dbg(dd->dev, "final_req: err: %d\n", err);
757 
758         return err;
759 }
760 
761 static void atmel_sha_copy_hash(struct ahash_request *req)
762 {
763         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
764         u32 *hash = (u32 *)ctx->digest;
765         unsigned int i, hashsize;
766 
767         switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {
768         case SHA_FLAGS_SHA1:
769                 hashsize = SHA1_DIGEST_SIZE;
770                 break;
771 
772         case SHA_FLAGS_SHA224:
773         case SHA_FLAGS_SHA256:
774                 hashsize = SHA256_DIGEST_SIZE;
775                 break;
776 
777         case SHA_FLAGS_SHA384:
778         case SHA_FLAGS_SHA512:
779                 hashsize = SHA512_DIGEST_SIZE;
780                 break;
781 
782         default:
783                 /* Should not happen... */
784                 return;
785         }
786 
787         for (i = 0; i < hashsize / sizeof(u32); ++i)
788                 hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i));
789         ctx->flags |= SHA_FLAGS_RESTORE;
790 }
791 
792 static void atmel_sha_copy_ready_hash(struct ahash_request *req)
793 {
794         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
795 
796         if (!req->result)
797                 return;
798 
799         if (ctx->flags & SHA_FLAGS_SHA1)
800                 memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE);
801         else if (ctx->flags & SHA_FLAGS_SHA224)
802                 memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE);
803         else if (ctx->flags & SHA_FLAGS_SHA256)
804                 memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE);
805         else if (ctx->flags & SHA_FLAGS_SHA384)
806                 memcpy(req->result, ctx->digest, SHA384_DIGEST_SIZE);
807         else
808                 memcpy(req->result, ctx->digest, SHA512_DIGEST_SIZE);
809 }
810 
811 static int atmel_sha_finish(struct ahash_request *req)
812 {
813         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
814         struct atmel_sha_dev *dd = ctx->dd;
815 
816         if (ctx->digcnt[0] || ctx->digcnt[1])
817                 atmel_sha_copy_ready_hash(req);
818 
819         dev_dbg(dd->dev, "digcnt: 0x%llx 0x%llx, bufcnt: %d\n", ctx->digcnt[1],
820                 ctx->digcnt[0], ctx->bufcnt);
821 
822         return 0;
823 }
824 
825 static void atmel_sha_finish_req(struct ahash_request *req, int err)
826 {
827         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
828         struct atmel_sha_dev *dd = ctx->dd;
829 
830         if (!err) {
831                 atmel_sha_copy_hash(req);
832                 if (SHA_FLAGS_FINAL & dd->flags)
833                         err = atmel_sha_finish(req);
834         } else {
835                 ctx->flags |= SHA_FLAGS_ERROR;
836         }
837 
838         /* atomic operation is not needed here */
839         dd->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL | SHA_FLAGS_CPU |
840                         SHA_FLAGS_DMA_READY | SHA_FLAGS_OUTPUT_READY);
841 
842         clk_disable(dd->iclk);
843 
844         if (req->base.complete)
845                 req->base.complete(&req->base, err);
846 
847         /* handle new request */
848         tasklet_schedule(&dd->queue_task);
849 }
850 
851 static int atmel_sha_hw_init(struct atmel_sha_dev *dd)
852 {
853         int err;
854 
855         err = clk_enable(dd->iclk);
856         if (err)
857                 return err;
858 
859         if (!(SHA_FLAGS_INIT & dd->flags)) {
860                 atmel_sha_write(dd, SHA_CR, SHA_CR_SWRST);
861                 dd->flags |= SHA_FLAGS_INIT;
862                 dd->err = 0;
863         }
864 
865         return 0;
866 }
867 
868 static inline unsigned int atmel_sha_get_version(struct atmel_sha_dev *dd)
869 {
870         return atmel_sha_read(dd, SHA_HW_VERSION) & 0x00000fff;
871 }
872 
873 static void atmel_sha_hw_version_init(struct atmel_sha_dev *dd)
874 {
875         atmel_sha_hw_init(dd);
876 
877         dd->hw_version = atmel_sha_get_version(dd);
878 
879         dev_info(dd->dev,
880                         "version: 0x%x\n", dd->hw_version);
881 
882         clk_disable(dd->iclk);
883 }
884 
885 static int atmel_sha_handle_queue(struct atmel_sha_dev *dd,
886                                   struct ahash_request *req)
887 {
888         struct crypto_async_request *async_req, *backlog;
889         struct atmel_sha_reqctx *ctx;
890         unsigned long flags;
891         int err = 0, ret = 0;
892 
893         spin_lock_irqsave(&dd->lock, flags);
894         if (req)
895                 ret = ahash_enqueue_request(&dd->queue, req);
896 
897         if (SHA_FLAGS_BUSY & dd->flags) {
898                 spin_unlock_irqrestore(&dd->lock, flags);
899                 return ret;
900         }
901 
902         backlog = crypto_get_backlog(&dd->queue);
903         async_req = crypto_dequeue_request(&dd->queue);
904         if (async_req)
905                 dd->flags |= SHA_FLAGS_BUSY;
906 
907         spin_unlock_irqrestore(&dd->lock, flags);
908 
909         if (!async_req)
910                 return ret;
911 
912         if (backlog)
913                 backlog->complete(backlog, -EINPROGRESS);
914 
915         req = ahash_request_cast(async_req);
916         dd->req = req;
917         ctx = ahash_request_ctx(req);
918 
919         dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
920                                                 ctx->op, req->nbytes);
921 
922         err = atmel_sha_hw_init(dd);
923 
924         if (err)
925                 goto err1;
926 
927         if (ctx->op == SHA_OP_UPDATE) {
928                 err = atmel_sha_update_req(dd);
929                 if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP))
930                         /* no final() after finup() */
931                         err = atmel_sha_final_req(dd);
932         } else if (ctx->op == SHA_OP_FINAL) {
933                 err = atmel_sha_final_req(dd);
934         }
935 
936 err1:
937         if (err != -EINPROGRESS)
938                 /* done_task will not finish it, so do it here */
939                 atmel_sha_finish_req(req, err);
940 
941         dev_dbg(dd->dev, "exit, err: %d\n", err);
942 
943         return ret;
944 }
945 
946 static int atmel_sha_enqueue(struct ahash_request *req, unsigned int op)
947 {
948         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
949         struct atmel_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
950         struct atmel_sha_dev *dd = tctx->dd;
951 
952         ctx->op = op;
953 
954         return atmel_sha_handle_queue(dd, req);
955 }
956 
957 static int atmel_sha_update(struct ahash_request *req)
958 {
959         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
960 
961         if (!req->nbytes)
962                 return 0;
963 
964         ctx->total = req->nbytes;
965         ctx->sg = req->src;
966         ctx->offset = 0;
967 
968         if (ctx->flags & SHA_FLAGS_FINUP) {
969                 if (ctx->bufcnt + ctx->total < ATMEL_SHA_DMA_THRESHOLD)
970                         /* faster to use CPU for short transfers */
971                         ctx->flags |= SHA_FLAGS_CPU;
972         } else if (ctx->bufcnt + ctx->total < ctx->buflen) {
973                 atmel_sha_append_sg(ctx);
974                 return 0;
975         }
976         return atmel_sha_enqueue(req, SHA_OP_UPDATE);
977 }
978 
979 static int atmel_sha_final(struct ahash_request *req)
980 {
981         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
982 
983         ctx->flags |= SHA_FLAGS_FINUP;
984 
985         if (ctx->flags & SHA_FLAGS_ERROR)
986                 return 0; /* uncompleted hash is not needed */
987 
988         if (ctx->flags & SHA_FLAGS_PAD)
989                 /* copy ready hash (+ finalize hmac) */
990                 return atmel_sha_finish(req);
991 
992         return atmel_sha_enqueue(req, SHA_OP_FINAL);
993 }
994 
995 static int atmel_sha_finup(struct ahash_request *req)
996 {
997         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
998         int err1, err2;
999 
1000         ctx->flags |= SHA_FLAGS_FINUP;
1001 
1002         err1 = atmel_sha_update(req);
1003         if (err1 == -EINPROGRESS || err1 == -EBUSY)
1004                 return err1;
1005 
1006         /*
1007          * final() has to be always called to cleanup resources
1008          * even if udpate() failed, except EINPROGRESS
1009          */
1010         err2 = atmel_sha_final(req);
1011 
1012         return err1 ?: err2;
1013 }
1014 
1015 static int atmel_sha_digest(struct ahash_request *req)
1016 {
1017         return atmel_sha_init(req) ?: atmel_sha_finup(req);
1018 }
1019 
1020 
1021 static int atmel_sha_export(struct ahash_request *req, void *out)
1022 {
1023         const struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1024 
1025         memcpy(out, ctx, sizeof(*ctx));
1026         return 0;
1027 }
1028 
1029 static int atmel_sha_import(struct ahash_request *req, const void *in)
1030 {
1031         struct atmel_sha_reqctx *ctx = ahash_request_ctx(req);
1032 
1033         memcpy(ctx, in, sizeof(*ctx));
1034         return 0;
1035 }
1036 
1037 static int atmel_sha_cra_init(struct crypto_tfm *tfm)
1038 {
1039         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1040                                  sizeof(struct atmel_sha_reqctx));
1041 
1042         return 0;
1043 }
1044 
1045 static struct ahash_alg sha_1_256_algs[] = {
1046 {
1047         .init           = atmel_sha_init,
1048         .update         = atmel_sha_update,
1049         .final          = atmel_sha_final,
1050         .finup          = atmel_sha_finup,
1051         .digest         = atmel_sha_digest,
1052         .export         = atmel_sha_export,
1053         .import         = atmel_sha_import,
1054         .halg = {
1055                 .digestsize     = SHA1_DIGEST_SIZE,
1056                 .statesize      = sizeof(struct atmel_sha_reqctx),
1057                 .base   = {
1058                         .cra_name               = "sha1",
1059                         .cra_driver_name        = "atmel-sha1",
1060                         .cra_priority           = 100,
1061                         .cra_flags              = CRYPTO_ALG_ASYNC,
1062                         .cra_blocksize          = SHA1_BLOCK_SIZE,
1063                         .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
1064                         .cra_alignmask          = 0,
1065                         .cra_module             = THIS_MODULE,
1066                         .cra_init               = atmel_sha_cra_init,
1067                 }
1068         }
1069 },
1070 {
1071         .init           = atmel_sha_init,
1072         .update         = atmel_sha_update,
1073         .final          = atmel_sha_final,
1074         .finup          = atmel_sha_finup,
1075         .digest         = atmel_sha_digest,
1076         .export         = atmel_sha_export,
1077         .import         = atmel_sha_import,
1078         .halg = {
1079                 .digestsize     = SHA256_DIGEST_SIZE,
1080                 .statesize      = sizeof(struct atmel_sha_reqctx),
1081                 .base   = {
1082                         .cra_name               = "sha256",
1083                         .cra_driver_name        = "atmel-sha256",
1084                         .cra_priority           = 100,
1085                         .cra_flags              = CRYPTO_ALG_ASYNC,
1086                         .cra_blocksize          = SHA256_BLOCK_SIZE,
1087                         .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
1088                         .cra_alignmask          = 0,
1089                         .cra_module             = THIS_MODULE,
1090                         .cra_init               = atmel_sha_cra_init,
1091                 }
1092         }
1093 },
1094 };
1095 
1096 static struct ahash_alg sha_224_alg = {
1097         .init           = atmel_sha_init,
1098         .update         = atmel_sha_update,
1099         .final          = atmel_sha_final,
1100         .finup          = atmel_sha_finup,
1101         .digest         = atmel_sha_digest,
1102         .export         = atmel_sha_export,
1103         .import         = atmel_sha_import,
1104         .halg = {
1105                 .digestsize     = SHA224_DIGEST_SIZE,
1106                 .statesize      = sizeof(struct atmel_sha_reqctx),
1107                 .base   = {
1108                         .cra_name               = "sha224",
1109                         .cra_driver_name        = "atmel-sha224",
1110                         .cra_priority           = 100,
1111                         .cra_flags              = CRYPTO_ALG_ASYNC,
1112                         .cra_blocksize          = SHA224_BLOCK_SIZE,
1113                         .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
1114                         .cra_alignmask          = 0,
1115                         .cra_module             = THIS_MODULE,
1116                         .cra_init               = atmel_sha_cra_init,
1117                 }
1118         }
1119 };
1120 
1121 static struct ahash_alg sha_384_512_algs[] = {
1122 {
1123         .init           = atmel_sha_init,
1124         .update         = atmel_sha_update,
1125         .final          = atmel_sha_final,
1126         .finup          = atmel_sha_finup,
1127         .digest         = atmel_sha_digest,
1128         .export         = atmel_sha_export,
1129         .import         = atmel_sha_import,
1130         .halg = {
1131                 .digestsize     = SHA384_DIGEST_SIZE,
1132                 .statesize      = sizeof(struct atmel_sha_reqctx),
1133                 .base   = {
1134                         .cra_name               = "sha384",
1135                         .cra_driver_name        = "atmel-sha384",
1136                         .cra_priority           = 100,
1137                         .cra_flags              = CRYPTO_ALG_ASYNC,
1138                         .cra_blocksize          = SHA384_BLOCK_SIZE,
1139                         .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
1140                         .cra_alignmask          = 0x3,
1141                         .cra_module             = THIS_MODULE,
1142                         .cra_init               = atmel_sha_cra_init,
1143                 }
1144         }
1145 },
1146 {
1147         .init           = atmel_sha_init,
1148         .update         = atmel_sha_update,
1149         .final          = atmel_sha_final,
1150         .finup          = atmel_sha_finup,
1151         .digest         = atmel_sha_digest,
1152         .export         = atmel_sha_export,
1153         .import         = atmel_sha_import,
1154         .halg = {
1155                 .digestsize     = SHA512_DIGEST_SIZE,
1156                 .statesize      = sizeof(struct atmel_sha_reqctx),
1157                 .base   = {
1158                         .cra_name               = "sha512",
1159                         .cra_driver_name        = "atmel-sha512",
1160                         .cra_priority           = 100,
1161                         .cra_flags              = CRYPTO_ALG_ASYNC,
1162                         .cra_blocksize          = SHA512_BLOCK_SIZE,
1163                         .cra_ctxsize            = sizeof(struct atmel_sha_ctx),
1164                         .cra_alignmask          = 0x3,
1165                         .cra_module             = THIS_MODULE,
1166                         .cra_init               = atmel_sha_cra_init,
1167                 }
1168         }
1169 },
1170 };
1171 
1172 static void atmel_sha_queue_task(unsigned long data)
1173 {
1174         struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data;
1175 
1176         atmel_sha_handle_queue(dd, NULL);
1177 }
1178 
1179 static void atmel_sha_done_task(unsigned long data)
1180 {
1181         struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data;
1182         int err = 0;
1183 
1184         if (SHA_FLAGS_CPU & dd->flags) {
1185                 if (SHA_FLAGS_OUTPUT_READY & dd->flags) {
1186                         dd->flags &= ~SHA_FLAGS_OUTPUT_READY;
1187                         goto finish;
1188                 }
1189         } else if (SHA_FLAGS_DMA_READY & dd->flags) {
1190                 if (SHA_FLAGS_DMA_ACTIVE & dd->flags) {
1191                         dd->flags &= ~SHA_FLAGS_DMA_ACTIVE;
1192                         atmel_sha_update_dma_stop(dd);
1193                         if (dd->err) {
1194                                 err = dd->err;
1195                                 goto finish;
1196                         }
1197                 }
1198                 if (SHA_FLAGS_OUTPUT_READY & dd->flags) {
1199                         /* hash or semi-hash ready */
1200                         dd->flags &= ~(SHA_FLAGS_DMA_READY |
1201                                                 SHA_FLAGS_OUTPUT_READY);
1202                         err = atmel_sha_update_dma_start(dd);
1203                         if (err != -EINPROGRESS)
1204                                 goto finish;
1205                 }
1206         }
1207         return;
1208 
1209 finish:
1210         /* finish curent request */
1211         atmel_sha_finish_req(dd->req, err);
1212 }
1213 
1214 static irqreturn_t atmel_sha_irq(int irq, void *dev_id)
1215 {
1216         struct atmel_sha_dev *sha_dd = dev_id;
1217         u32 reg;
1218 
1219         reg = atmel_sha_read(sha_dd, SHA_ISR);
1220         if (reg & atmel_sha_read(sha_dd, SHA_IMR)) {
1221                 atmel_sha_write(sha_dd, SHA_IDR, reg);
1222                 if (SHA_FLAGS_BUSY & sha_dd->flags) {
1223                         sha_dd->flags |= SHA_FLAGS_OUTPUT_READY;
1224                         if (!(SHA_FLAGS_CPU & sha_dd->flags))
1225                                 sha_dd->flags |= SHA_FLAGS_DMA_READY;
1226                         tasklet_schedule(&sha_dd->done_task);
1227                 } else {
1228                         dev_warn(sha_dd->dev, "SHA interrupt when no active requests.\n");
1229                 }
1230                 return IRQ_HANDLED;
1231         }
1232 
1233         return IRQ_NONE;
1234 }
1235 
1236 static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd)
1237 {
1238         int i;
1239 
1240         for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++)
1241                 crypto_unregister_ahash(&sha_1_256_algs[i]);
1242 
1243         if (dd->caps.has_sha224)
1244                 crypto_unregister_ahash(&sha_224_alg);
1245 
1246         if (dd->caps.has_sha_384_512) {
1247                 for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++)
1248                         crypto_unregister_ahash(&sha_384_512_algs[i]);
1249         }
1250 }
1251 
1252 static int atmel_sha_register_algs(struct atmel_sha_dev *dd)
1253 {
1254         int err, i, j;
1255 
1256         for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) {
1257                 err = crypto_register_ahash(&sha_1_256_algs[i]);
1258                 if (err)
1259                         goto err_sha_1_256_algs;
1260         }
1261 
1262         if (dd->caps.has_sha224) {
1263                 err = crypto_register_ahash(&sha_224_alg);
1264                 if (err)
1265                         goto err_sha_224_algs;
1266         }
1267 
1268         if (dd->caps.has_sha_384_512) {
1269                 for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) {
1270                         err = crypto_register_ahash(&sha_384_512_algs[i]);
1271                         if (err)
1272                                 goto err_sha_384_512_algs;
1273                 }
1274         }
1275 
1276         return 0;
1277 
1278 err_sha_384_512_algs:
1279         for (j = 0; j < i; j++)
1280                 crypto_unregister_ahash(&sha_384_512_algs[j]);
1281         crypto_unregister_ahash(&sha_224_alg);
1282 err_sha_224_algs:
1283         i = ARRAY_SIZE(sha_1_256_algs);
1284 err_sha_1_256_algs:
1285         for (j = 0; j < i; j++)
1286                 crypto_unregister_ahash(&sha_1_256_algs[j]);
1287 
1288         return err;
1289 }
1290 
1291 static bool atmel_sha_filter(struct dma_chan *chan, void *slave)
1292 {
1293         struct at_dma_slave     *sl = slave;
1294 
1295         if (sl && sl->dma_dev == chan->device->dev) {
1296                 chan->private = sl;
1297                 return true;
1298         } else {
1299                 return false;
1300         }
1301 }
1302 
1303 static int atmel_sha_dma_init(struct atmel_sha_dev *dd,
1304                                 struct crypto_platform_data *pdata)
1305 {
1306         int err = -ENOMEM;
1307         dma_cap_mask_t mask_in;
1308 
1309         /* Try to grab DMA channel */
1310         dma_cap_zero(mask_in);
1311         dma_cap_set(DMA_SLAVE, mask_in);
1312 
1313         dd->dma_lch_in.chan = dma_request_slave_channel_compat(mask_in,
1314                         atmel_sha_filter, &pdata->dma_slave->rxdata, dd->dev, "tx");
1315         if (!dd->dma_lch_in.chan) {
1316                 dev_warn(dd->dev, "no DMA channel available\n");
1317                 return err;
1318         }
1319 
1320         dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV;
1321         dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base +
1322                 SHA_REG_DIN(0);
1323         dd->dma_lch_in.dma_conf.src_maxburst = 1;
1324         dd->dma_lch_in.dma_conf.src_addr_width =
1325                 DMA_SLAVE_BUSWIDTH_4_BYTES;
1326         dd->dma_lch_in.dma_conf.dst_maxburst = 1;
1327         dd->dma_lch_in.dma_conf.dst_addr_width =
1328                 DMA_SLAVE_BUSWIDTH_4_BYTES;
1329         dd->dma_lch_in.dma_conf.device_fc = false;
1330 
1331         return 0;
1332 }
1333 
1334 static void atmel_sha_dma_cleanup(struct atmel_sha_dev *dd)
1335 {
1336         dma_release_channel(dd->dma_lch_in.chan);
1337 }
1338 
1339 static void atmel_sha_get_cap(struct atmel_sha_dev *dd)
1340 {
1341 
1342         dd->caps.has_dma = 0;
1343         dd->caps.has_dualbuff = 0;
1344         dd->caps.has_sha224 = 0;
1345         dd->caps.has_sha_384_512 = 0;
1346         dd->caps.has_uihv = 0;
1347 
1348         /* keep only major version number */
1349         switch (dd->hw_version & 0xff0) {
1350         case 0x510:
1351                 dd->caps.has_dma = 1;
1352                 dd->caps.has_dualbuff = 1;
1353                 dd->caps.has_sha224 = 1;
1354                 dd->caps.has_sha_384_512 = 1;
1355                 dd->caps.has_uihv = 1;
1356                 break;
1357         case 0x420:
1358                 dd->caps.has_dma = 1;
1359                 dd->caps.has_dualbuff = 1;
1360                 dd->caps.has_sha224 = 1;
1361                 dd->caps.has_sha_384_512 = 1;
1362                 dd->caps.has_uihv = 1;
1363                 break;
1364         case 0x410:
1365                 dd->caps.has_dma = 1;
1366                 dd->caps.has_dualbuff = 1;
1367                 dd->caps.has_sha224 = 1;
1368                 dd->caps.has_sha_384_512 = 1;
1369                 break;
1370         case 0x400:
1371                 dd->caps.has_dma = 1;
1372                 dd->caps.has_dualbuff = 1;
1373                 dd->caps.has_sha224 = 1;
1374                 break;
1375         case 0x320:
1376                 break;
1377         default:
1378                 dev_warn(dd->dev,
1379                                 "Unmanaged sha version, set minimum capabilities\n");
1380                 break;
1381         }
1382 }
1383 
1384 #if defined(CONFIG_OF)
1385 static const struct of_device_id atmel_sha_dt_ids[] = {
1386         { .compatible = "atmel,at91sam9g46-sha" },
1387         { /* sentinel */ }
1388 };
1389 
1390 MODULE_DEVICE_TABLE(of, atmel_sha_dt_ids);
1391 
1392 static struct crypto_platform_data *atmel_sha_of_init(struct platform_device *pdev)
1393 {
1394         struct device_node *np = pdev->dev.of_node;
1395         struct crypto_platform_data *pdata;
1396 
1397         if (!np) {
1398                 dev_err(&pdev->dev, "device node not found\n");
1399                 return ERR_PTR(-EINVAL);
1400         }
1401 
1402         pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
1403         if (!pdata) {
1404                 dev_err(&pdev->dev, "could not allocate memory for pdata\n");
1405                 return ERR_PTR(-ENOMEM);
1406         }
1407 
1408         pdata->dma_slave = devm_kzalloc(&pdev->dev,
1409                                         sizeof(*(pdata->dma_slave)),
1410                                         GFP_KERNEL);
1411         if (!pdata->dma_slave) {
1412                 dev_err(&pdev->dev, "could not allocate memory for dma_slave\n");
1413                 return ERR_PTR(-ENOMEM);
1414         }
1415 
1416         return pdata;
1417 }
1418 #else /* CONFIG_OF */
1419 static inline struct crypto_platform_data *atmel_sha_of_init(struct platform_device *dev)
1420 {
1421         return ERR_PTR(-EINVAL);
1422 }
1423 #endif
1424 
1425 static int atmel_sha_probe(struct platform_device *pdev)
1426 {
1427         struct atmel_sha_dev *sha_dd;
1428         struct crypto_platform_data     *pdata;
1429         struct device *dev = &pdev->dev;
1430         struct resource *sha_res;
1431         int err;
1432 
1433         sha_dd = devm_kzalloc(&pdev->dev, sizeof(*sha_dd), GFP_KERNEL);
1434         if (sha_dd == NULL) {
1435                 dev_err(dev, "unable to alloc data struct.\n");
1436                 err = -ENOMEM;
1437                 goto sha_dd_err;
1438         }
1439 
1440         sha_dd->dev = dev;
1441 
1442         platform_set_drvdata(pdev, sha_dd);
1443 
1444         INIT_LIST_HEAD(&sha_dd->list);
1445         spin_lock_init(&sha_dd->lock);
1446 
1447         tasklet_init(&sha_dd->done_task, atmel_sha_done_task,
1448                                         (unsigned long)sha_dd);
1449         tasklet_init(&sha_dd->queue_task, atmel_sha_queue_task,
1450                                         (unsigned long)sha_dd);
1451 
1452         crypto_init_queue(&sha_dd->queue, ATMEL_SHA_QUEUE_LENGTH);
1453 
1454         sha_dd->irq = -1;
1455 
1456         /* Get the base address */
1457         sha_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1458         if (!sha_res) {
1459                 dev_err(dev, "no MEM resource info\n");
1460                 err = -ENODEV;
1461                 goto res_err;
1462         }
1463         sha_dd->phys_base = sha_res->start;
1464 
1465         /* Get the IRQ */
1466         sha_dd->irq = platform_get_irq(pdev,  0);
1467         if (sha_dd->irq < 0) {
1468                 dev_err(dev, "no IRQ resource info\n");
1469                 err = sha_dd->irq;
1470                 goto res_err;
1471         }
1472 
1473         err = devm_request_irq(&pdev->dev, sha_dd->irq, atmel_sha_irq,
1474                                IRQF_SHARED, "atmel-sha", sha_dd);
1475         if (err) {
1476                 dev_err(dev, "unable to request sha irq.\n");
1477                 goto res_err;
1478         }
1479 
1480         /* Initializing the clock */
1481         sha_dd->iclk = devm_clk_get(&pdev->dev, "sha_clk");
1482         if (IS_ERR(sha_dd->iclk)) {
1483                 dev_err(dev, "clock initialization failed.\n");
1484                 err = PTR_ERR(sha_dd->iclk);
1485                 goto res_err;
1486         }
1487 
1488         sha_dd->io_base = devm_ioremap_resource(&pdev->dev, sha_res);
1489         if (IS_ERR(sha_dd->io_base)) {
1490                 dev_err(dev, "can't ioremap\n");
1491                 err = PTR_ERR(sha_dd->io_base);
1492                 goto res_err;
1493         }
1494 
1495         err = clk_prepare(sha_dd->iclk);
1496         if (err)
1497                 goto res_err;
1498 
1499         atmel_sha_hw_version_init(sha_dd);
1500 
1501         atmel_sha_get_cap(sha_dd);
1502 
1503         if (sha_dd->caps.has_dma) {
1504                 pdata = pdev->dev.platform_data;
1505                 if (!pdata) {
1506                         pdata = atmel_sha_of_init(pdev);
1507                         if (IS_ERR(pdata)) {
1508                                 dev_err(&pdev->dev, "platform data not available\n");
1509                                 err = PTR_ERR(pdata);
1510                                 goto iclk_unprepare;
1511                         }
1512                 }
1513                 if (!pdata->dma_slave) {
1514                         err = -ENXIO;
1515                         goto iclk_unprepare;
1516                 }
1517                 err = atmel_sha_dma_init(sha_dd, pdata);
1518                 if (err)
1519                         goto err_sha_dma;
1520 
1521                 dev_info(dev, "using %s for DMA transfers\n",
1522                                 dma_chan_name(sha_dd->dma_lch_in.chan));
1523         }
1524 
1525         spin_lock(&atmel_sha.lock);
1526         list_add_tail(&sha_dd->list, &atmel_sha.dev_list);
1527         spin_unlock(&atmel_sha.lock);
1528 
1529         err = atmel_sha_register_algs(sha_dd);
1530         if (err)
1531                 goto err_algs;
1532 
1533         dev_info(dev, "Atmel SHA1/SHA256%s%s\n",
1534                         sha_dd->caps.has_sha224 ? "/SHA224" : "",
1535                         sha_dd->caps.has_sha_384_512 ? "/SHA384/SHA512" : "");
1536 
1537         return 0;
1538 
1539 err_algs:
1540         spin_lock(&atmel_sha.lock);
1541         list_del(&sha_dd->list);
1542         spin_unlock(&atmel_sha.lock);
1543         if (sha_dd->caps.has_dma)
1544                 atmel_sha_dma_cleanup(sha_dd);
1545 err_sha_dma:
1546 iclk_unprepare:
1547         clk_unprepare(sha_dd->iclk);
1548 res_err:
1549         tasklet_kill(&sha_dd->queue_task);
1550         tasklet_kill(&sha_dd->done_task);
1551 sha_dd_err:
1552         dev_err(dev, "initialization failed.\n");
1553 
1554         return err;
1555 }
1556 
1557 static int atmel_sha_remove(struct platform_device *pdev)
1558 {
1559         static struct atmel_sha_dev *sha_dd;
1560 
1561         sha_dd = platform_get_drvdata(pdev);
1562         if (!sha_dd)
1563                 return -ENODEV;
1564         spin_lock(&atmel_sha.lock);
1565         list_del(&sha_dd->list);
1566         spin_unlock(&atmel_sha.lock);
1567 
1568         atmel_sha_unregister_algs(sha_dd);
1569 
1570         tasklet_kill(&sha_dd->queue_task);
1571         tasklet_kill(&sha_dd->done_task);
1572 
1573         if (sha_dd->caps.has_dma)
1574                 atmel_sha_dma_cleanup(sha_dd);
1575 
1576         clk_unprepare(sha_dd->iclk);
1577 
1578         return 0;
1579 }
1580 
1581 static struct platform_driver atmel_sha_driver = {
1582         .probe          = atmel_sha_probe,
1583         .remove         = atmel_sha_remove,
1584         .driver         = {
1585                 .name   = "atmel_sha",
1586                 .of_match_table = of_match_ptr(atmel_sha_dt_ids),
1587         },
1588 };
1589 
1590 module_platform_driver(atmel_sha_driver);
1591 
1592 MODULE_DESCRIPTION("Atmel SHA (1/256/224/384/512) hw acceleration support.");
1593 MODULE_LICENSE("GPL v2");
1594 MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique");
1595 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us