Version:  2.0.40 2.2.26 2.4.37 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0

Linux/drivers/crypto/sahara.c

  1 /*
  2  * Cryptographic API.
  3  *
  4  * Support for SAHARA cryptographic accelerator.
  5  *
  6  * Copyright (c) 2014 Steffen Trumtrar <s.trumtrar@pengutronix.de>
  7  * Copyright (c) 2013 Vista Silicon S.L.
  8  * Author: Javier Martin <javier.martin@vista-silicon.com>
  9  *
 10  * This program is free software; you can redistribute it and/or modify
 11  * it under the terms of the GNU General Public License version 2 as published
 12  * by the Free Software Foundation.
 13  *
 14  * Based on omap-aes.c and tegra-aes.c
 15  */
 16 
 17 #include <crypto/algapi.h>
 18 #include <crypto/aes.h>
 19 #include <crypto/hash.h>
 20 #include <crypto/internal/hash.h>
 21 #include <crypto/scatterwalk.h>
 22 #include <crypto/sha.h>
 23 
 24 #include <linux/clk.h>
 25 #include <linux/crypto.h>
 26 #include <linux/interrupt.h>
 27 #include <linux/io.h>
 28 #include <linux/irq.h>
 29 #include <linux/kernel.h>
 30 #include <linux/kthread.h>
 31 #include <linux/module.h>
 32 #include <linux/mutex.h>
 33 #include <linux/of.h>
 34 #include <linux/of_device.h>
 35 #include <linux/platform_device.h>
 36 
 37 #define SHA_BUFFER_LEN          PAGE_SIZE
 38 #define SAHARA_MAX_SHA_BLOCK_SIZE       SHA256_BLOCK_SIZE
 39 
 40 #define SAHARA_NAME "sahara"
 41 #define SAHARA_VERSION_3        3
 42 #define SAHARA_VERSION_4        4
 43 #define SAHARA_TIMEOUT_MS       1000
 44 #define SAHARA_MAX_HW_DESC      2
 45 #define SAHARA_MAX_HW_LINK      20
 46 
 47 #define FLAGS_MODE_MASK         0x000f
 48 #define FLAGS_ENCRYPT           BIT(0)
 49 #define FLAGS_CBC               BIT(1)
 50 #define FLAGS_NEW_KEY           BIT(3)
 51 
 52 #define SAHARA_HDR_BASE                 0x00800000
 53 #define SAHARA_HDR_SKHA_ALG_AES 0
 54 #define SAHARA_HDR_SKHA_OP_ENC          (1 << 2)
 55 #define SAHARA_HDR_SKHA_MODE_ECB        (0 << 3)
 56 #define SAHARA_HDR_SKHA_MODE_CBC        (1 << 3)
 57 #define SAHARA_HDR_FORM_DATA            (5 << 16)
 58 #define SAHARA_HDR_FORM_KEY             (8 << 16)
 59 #define SAHARA_HDR_LLO                  (1 << 24)
 60 #define SAHARA_HDR_CHA_SKHA             (1 << 28)
 61 #define SAHARA_HDR_CHA_MDHA             (2 << 28)
 62 #define SAHARA_HDR_PARITY_BIT           (1 << 31)
 63 
 64 #define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000
 65 #define SAHARA_HDR_MDHA_SET_MODE_HASH   0x208D0000
 66 #define SAHARA_HDR_MDHA_HASH            0xA0850000
 67 #define SAHARA_HDR_MDHA_STORE_DIGEST    0x20820000
 68 #define SAHARA_HDR_MDHA_ALG_SHA1        0
 69 #define SAHARA_HDR_MDHA_ALG_MD5         1
 70 #define SAHARA_HDR_MDHA_ALG_SHA256      2
 71 #define SAHARA_HDR_MDHA_ALG_SHA224      3
 72 #define SAHARA_HDR_MDHA_PDATA           (1 << 2)
 73 #define SAHARA_HDR_MDHA_HMAC            (1 << 3)
 74 #define SAHARA_HDR_MDHA_INIT            (1 << 5)
 75 #define SAHARA_HDR_MDHA_IPAD            (1 << 6)
 76 #define SAHARA_HDR_MDHA_OPAD            (1 << 7)
 77 #define SAHARA_HDR_MDHA_SWAP            (1 << 8)
 78 #define SAHARA_HDR_MDHA_MAC_FULL        (1 << 9)
 79 #define SAHARA_HDR_MDHA_SSL             (1 << 10)
 80 
 81 /* SAHARA can only process one request at a time */
 82 #define SAHARA_QUEUE_LENGTH     1
 83 
 84 #define SAHARA_REG_VERSION      0x00
 85 #define SAHARA_REG_DAR          0x04
 86 #define SAHARA_REG_CONTROL      0x08
 87 #define         SAHARA_CONTROL_SET_THROTTLE(x)  (((x) & 0xff) << 24)
 88 #define         SAHARA_CONTROL_SET_MAXBURST(x)  (((x) & 0xff) << 16)
 89 #define         SAHARA_CONTROL_RNG_AUTORSD      (1 << 7)
 90 #define         SAHARA_CONTROL_ENABLE_INT       (1 << 4)
 91 #define SAHARA_REG_CMD          0x0C
 92 #define         SAHARA_CMD_RESET                (1 << 0)
 93 #define         SAHARA_CMD_CLEAR_INT            (1 << 8)
 94 #define         SAHARA_CMD_CLEAR_ERR            (1 << 9)
 95 #define         SAHARA_CMD_SINGLE_STEP          (1 << 10)
 96 #define         SAHARA_CMD_MODE_BATCH           (1 << 16)
 97 #define         SAHARA_CMD_MODE_DEBUG           (1 << 18)
 98 #define SAHARA_REG_STATUS       0x10
 99 #define         SAHARA_STATUS_GET_STATE(x)      ((x) & 0x7)
100 #define                 SAHARA_STATE_IDLE       0
101 #define                 SAHARA_STATE_BUSY       1
102 #define                 SAHARA_STATE_ERR        2
103 #define                 SAHARA_STATE_FAULT      3
104 #define                 SAHARA_STATE_COMPLETE   4
105 #define                 SAHARA_STATE_COMP_FLAG  (1 << 2)
106 #define         SAHARA_STATUS_DAR_FULL          (1 << 3)
107 #define         SAHARA_STATUS_ERROR             (1 << 4)
108 #define         SAHARA_STATUS_SECURE            (1 << 5)
109 #define         SAHARA_STATUS_FAIL              (1 << 6)
110 #define         SAHARA_STATUS_INIT              (1 << 7)
111 #define         SAHARA_STATUS_RNG_RESEED        (1 << 8)
112 #define         SAHARA_STATUS_ACTIVE_RNG        (1 << 9)
113 #define         SAHARA_STATUS_ACTIVE_MDHA       (1 << 10)
114 #define         SAHARA_STATUS_ACTIVE_SKHA       (1 << 11)
115 #define         SAHARA_STATUS_MODE_BATCH        (1 << 16)
116 #define         SAHARA_STATUS_MODE_DEDICATED    (1 << 17)
117 #define         SAHARA_STATUS_MODE_DEBUG        (1 << 18)
118 #define         SAHARA_STATUS_GET_ISTATE(x)     (((x) >> 24) & 0xff)
119 #define SAHARA_REG_ERRSTATUS    0x14
120 #define         SAHARA_ERRSTATUS_GET_SOURCE(x)  ((x) & 0xf)
121 #define                 SAHARA_ERRSOURCE_CHA    14
122 #define                 SAHARA_ERRSOURCE_DMA    15
123 #define         SAHARA_ERRSTATUS_DMA_DIR        (1 << 8)
124 #define         SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3)
125 #define         SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7)
126 #define         SAHARA_ERRSTATUS_GET_CHASRC(x)  (((x) >> 16) & 0xfff)
127 #define         SAHARA_ERRSTATUS_GET_CHAERR(x)  (((x) >> 28) & 0x3)
128 #define SAHARA_REG_FADDR        0x18
129 #define SAHARA_REG_CDAR         0x1C
130 #define SAHARA_REG_IDAR         0x20
131 
132 struct sahara_hw_desc {
133         u32             hdr;
134         u32             len1;
135         dma_addr_t      p1;
136         u32             len2;
137         dma_addr_t      p2;
138         dma_addr_t      next;
139 };
140 
141 struct sahara_hw_link {
142         u32             len;
143         dma_addr_t      p;
144         dma_addr_t      next;
145 };
146 
147 struct sahara_ctx {
148         unsigned long flags;
149 
150         /* AES-specific context */
151         int keylen;
152         u8 key[AES_KEYSIZE_128];
153         struct crypto_ablkcipher *fallback;
154 
155         /* SHA-specific context */
156         struct crypto_shash *shash_fallback;
157 };
158 
159 struct sahara_aes_reqctx {
160         unsigned long mode;
161 };
162 
163 /*
164  * struct sahara_sha_reqctx - private data per request
165  * @buf: holds data for requests smaller than block_size
166  * @rembuf: used to prepare one block_size-aligned request
167  * @context: hw-specific context for request. Digest is extracted from this
168  * @mode: specifies what type of hw-descriptor needs to be built
169  * @digest_size: length of digest for this request
170  * @context_size: length of hw-context for this request.
171  *                Always digest_size + 4
172  * @buf_cnt: number of bytes saved in buf
173  * @sg_in_idx: number of hw links
174  * @in_sg: scatterlist for input data
175  * @in_sg_chain: scatterlists for chained input data
176  * @in_sg_chained: specifies if chained scatterlists are used or not
177  * @total: total number of bytes for transfer
178  * @last: is this the last block
179  * @first: is this the first block
180  * @active: inside a transfer
181  */
182 struct sahara_sha_reqctx {
183         u8                      buf[SAHARA_MAX_SHA_BLOCK_SIZE];
184         u8                      rembuf[SAHARA_MAX_SHA_BLOCK_SIZE];
185         u8                      context[SHA256_DIGEST_SIZE + 4];
186         struct mutex            mutex;
187         unsigned int            mode;
188         unsigned int            digest_size;
189         unsigned int            context_size;
190         unsigned int            buf_cnt;
191         unsigned int            sg_in_idx;
192         struct scatterlist      *in_sg;
193         struct scatterlist      in_sg_chain[2];
194         bool                    in_sg_chained;
195         size_t                  total;
196         unsigned int            last;
197         unsigned int            first;
198         unsigned int            active;
199 };
200 
201 struct sahara_dev {
202         struct device           *device;
203         unsigned int            version;
204         void __iomem            *regs_base;
205         struct clk              *clk_ipg;
206         struct clk              *clk_ahb;
207         struct mutex            queue_mutex;
208         struct task_struct      *kthread;
209         struct completion       dma_completion;
210 
211         struct sahara_ctx       *ctx;
212         spinlock_t              lock;
213         struct crypto_queue     queue;
214         unsigned long           flags;
215 
216         struct sahara_hw_desc   *hw_desc[SAHARA_MAX_HW_DESC];
217         dma_addr_t              hw_phys_desc[SAHARA_MAX_HW_DESC];
218 
219         u8                      *key_base;
220         dma_addr_t              key_phys_base;
221 
222         u8                      *iv_base;
223         dma_addr_t              iv_phys_base;
224 
225         u8                      *context_base;
226         dma_addr_t              context_phys_base;
227 
228         struct sahara_hw_link   *hw_link[SAHARA_MAX_HW_LINK];
229         dma_addr_t              hw_phys_link[SAHARA_MAX_HW_LINK];
230 
231         size_t                  total;
232         struct scatterlist      *in_sg;
233         unsigned int            nb_in_sg;
234         struct scatterlist      *out_sg;
235         unsigned int            nb_out_sg;
236 
237         u32                     error;
238 };
239 
240 static struct sahara_dev *dev_ptr;
241 
242 static inline void sahara_write(struct sahara_dev *dev, u32 data, u32 reg)
243 {
244         writel(data, dev->regs_base + reg);
245 }
246 
247 static inline unsigned int sahara_read(struct sahara_dev *dev, u32 reg)
248 {
249         return readl(dev->regs_base + reg);
250 }
251 
252 static u32 sahara_aes_key_hdr(struct sahara_dev *dev)
253 {
254         u32 hdr = SAHARA_HDR_BASE | SAHARA_HDR_SKHA_ALG_AES |
255                         SAHARA_HDR_FORM_KEY | SAHARA_HDR_LLO |
256                         SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
257 
258         if (dev->flags & FLAGS_CBC) {
259                 hdr |= SAHARA_HDR_SKHA_MODE_CBC;
260                 hdr ^= SAHARA_HDR_PARITY_BIT;
261         }
262 
263         if (dev->flags & FLAGS_ENCRYPT) {
264                 hdr |= SAHARA_HDR_SKHA_OP_ENC;
265                 hdr ^= SAHARA_HDR_PARITY_BIT;
266         }
267 
268         return hdr;
269 }
270 
271 static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev)
272 {
273         return SAHARA_HDR_BASE | SAHARA_HDR_FORM_DATA |
274                         SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
275 }
276 
277 static int sahara_sg_length(struct scatterlist *sg,
278                             unsigned int total)
279 {
280         int sg_nb;
281         unsigned int len;
282         struct scatterlist *sg_list;
283 
284         sg_nb = 0;
285         sg_list = sg;
286 
287         while (total) {
288                 len = min(sg_list->length, total);
289 
290                 sg_nb++;
291                 total -= len;
292 
293                 sg_list = sg_next(sg_list);
294                 if (!sg_list)
295                         total = 0;
296         }
297 
298         return sg_nb;
299 }
300 
301 static char *sahara_err_src[16] = {
302         "No error",
303         "Header error",
304         "Descriptor length error",
305         "Descriptor length or pointer error",
306         "Link length error",
307         "Link pointer error",
308         "Input buffer error",
309         "Output buffer error",
310         "Output buffer starvation",
311         "Internal state fault",
312         "General descriptor problem",
313         "Reserved",
314         "Descriptor address error",
315         "Link address error",
316         "CHA error",
317         "DMA error"
318 };
319 
320 static char *sahara_err_dmasize[4] = {
321         "Byte transfer",
322         "Half-word transfer",
323         "Word transfer",
324         "Reserved"
325 };
326 
327 static char *sahara_err_dmasrc[8] = {
328         "No error",
329         "AHB bus error",
330         "Internal IP bus error",
331         "Parity error",
332         "DMA crosses 256 byte boundary",
333         "DMA is busy",
334         "Reserved",
335         "DMA HW error"
336 };
337 
338 static char *sahara_cha_errsrc[12] = {
339         "Input buffer non-empty",
340         "Illegal address",
341         "Illegal mode",
342         "Illegal data size",
343         "Illegal key size",
344         "Write during processing",
345         "CTX read during processing",
346         "HW error",
347         "Input buffer disabled/underflow",
348         "Output buffer disabled/overflow",
349         "DES key parity error",
350         "Reserved"
351 };
352 
353 static char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" };
354 
355 static void sahara_decode_error(struct sahara_dev *dev, unsigned int error)
356 {
357         u8 source = SAHARA_ERRSTATUS_GET_SOURCE(error);
358         u16 chasrc = ffs(SAHARA_ERRSTATUS_GET_CHASRC(error));
359 
360         dev_err(dev->device, "%s: Error Register = 0x%08x\n", __func__, error);
361 
362         dev_err(dev->device, "  - %s.\n", sahara_err_src[source]);
363 
364         if (source == SAHARA_ERRSOURCE_DMA) {
365                 if (error & SAHARA_ERRSTATUS_DMA_DIR)
366                         dev_err(dev->device, "          * DMA read.\n");
367                 else
368                         dev_err(dev->device, "          * DMA write.\n");
369 
370                 dev_err(dev->device, "          * %s.\n",
371                        sahara_err_dmasize[SAHARA_ERRSTATUS_GET_DMASZ(error)]);
372                 dev_err(dev->device, "          * %s.\n",
373                        sahara_err_dmasrc[SAHARA_ERRSTATUS_GET_DMASRC(error)]);
374         } else if (source == SAHARA_ERRSOURCE_CHA) {
375                 dev_err(dev->device, "          * %s.\n",
376                         sahara_cha_errsrc[chasrc]);
377                 dev_err(dev->device, "          * %s.\n",
378                        sahara_cha_err[SAHARA_ERRSTATUS_GET_CHAERR(error)]);
379         }
380         dev_err(dev->device, "\n");
381 }
382 
383 static char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" };
384 
385 static void sahara_decode_status(struct sahara_dev *dev, unsigned int status)
386 {
387         u8 state;
388 
389         if (!IS_ENABLED(DEBUG))
390                 return;
391 
392         state = SAHARA_STATUS_GET_STATE(status);
393 
394         dev_dbg(dev->device, "%s: Status Register = 0x%08x\n",
395                 __func__, status);
396 
397         dev_dbg(dev->device, "  - State = %d:\n", state);
398         if (state & SAHARA_STATE_COMP_FLAG)
399                 dev_dbg(dev->device, "          * Descriptor completed. IRQ pending.\n");
400 
401         dev_dbg(dev->device, "          * %s.\n",
402                sahara_state[state & ~SAHARA_STATE_COMP_FLAG]);
403 
404         if (status & SAHARA_STATUS_DAR_FULL)
405                 dev_dbg(dev->device, "  - DAR Full.\n");
406         if (status & SAHARA_STATUS_ERROR)
407                 dev_dbg(dev->device, "  - Error.\n");
408         if (status & SAHARA_STATUS_SECURE)
409                 dev_dbg(dev->device, "  - Secure.\n");
410         if (status & SAHARA_STATUS_FAIL)
411                 dev_dbg(dev->device, "  - Fail.\n");
412         if (status & SAHARA_STATUS_RNG_RESEED)
413                 dev_dbg(dev->device, "  - RNG Reseed Request.\n");
414         if (status & SAHARA_STATUS_ACTIVE_RNG)
415                 dev_dbg(dev->device, "  - RNG Active.\n");
416         if (status & SAHARA_STATUS_ACTIVE_MDHA)
417                 dev_dbg(dev->device, "  - MDHA Active.\n");
418         if (status & SAHARA_STATUS_ACTIVE_SKHA)
419                 dev_dbg(dev->device, "  - SKHA Active.\n");
420 
421         if (status & SAHARA_STATUS_MODE_BATCH)
422                 dev_dbg(dev->device, "  - Batch Mode.\n");
423         else if (status & SAHARA_STATUS_MODE_DEDICATED)
424                 dev_dbg(dev->device, "  - Decidated Mode.\n");
425         else if (status & SAHARA_STATUS_MODE_DEBUG)
426                 dev_dbg(dev->device, "  - Debug Mode.\n");
427 
428         dev_dbg(dev->device, "  - Internal state = 0x%02x\n",
429                SAHARA_STATUS_GET_ISTATE(status));
430 
431         dev_dbg(dev->device, "Current DAR: 0x%08x\n",
432                 sahara_read(dev, SAHARA_REG_CDAR));
433         dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n",
434                 sahara_read(dev, SAHARA_REG_IDAR));
435 }
436 
437 static void sahara_dump_descriptors(struct sahara_dev *dev)
438 {
439         int i;
440 
441         if (!IS_ENABLED(DEBUG))
442                 return;
443 
444         for (i = 0; i < SAHARA_MAX_HW_DESC; i++) {
445                 dev_dbg(dev->device, "Descriptor (%d) (0x%08x):\n",
446                         i, dev->hw_phys_desc[i]);
447                 dev_dbg(dev->device, "\thdr = 0x%08x\n", dev->hw_desc[i]->hdr);
448                 dev_dbg(dev->device, "\tlen1 = %u\n", dev->hw_desc[i]->len1);
449                 dev_dbg(dev->device, "\tp1 = 0x%08x\n", dev->hw_desc[i]->p1);
450                 dev_dbg(dev->device, "\tlen2 = %u\n", dev->hw_desc[i]->len2);
451                 dev_dbg(dev->device, "\tp2 = 0x%08x\n", dev->hw_desc[i]->p2);
452                 dev_dbg(dev->device, "\tnext = 0x%08x\n",
453                         dev->hw_desc[i]->next);
454         }
455         dev_dbg(dev->device, "\n");
456 }
457 
458 static void sahara_dump_links(struct sahara_dev *dev)
459 {
460         int i;
461 
462         if (!IS_ENABLED(DEBUG))
463                 return;
464 
465         for (i = 0; i < SAHARA_MAX_HW_LINK; i++) {
466                 dev_dbg(dev->device, "Link (%d) (0x%08x):\n",
467                         i, dev->hw_phys_link[i]);
468                 dev_dbg(dev->device, "\tlen = %u\n", dev->hw_link[i]->len);
469                 dev_dbg(dev->device, "\tp = 0x%08x\n", dev->hw_link[i]->p);
470                 dev_dbg(dev->device, "\tnext = 0x%08x\n",
471                         dev->hw_link[i]->next);
472         }
473         dev_dbg(dev->device, "\n");
474 }
475 
476 static int sahara_hw_descriptor_create(struct sahara_dev *dev)
477 {
478         struct sahara_ctx *ctx = dev->ctx;
479         struct scatterlist *sg;
480         int ret;
481         int i, j;
482 
483         /* Copy new key if necessary */
484         if (ctx->flags & FLAGS_NEW_KEY) {
485                 memcpy(dev->key_base, ctx->key, ctx->keylen);
486                 ctx->flags &= ~FLAGS_NEW_KEY;
487 
488                 if (dev->flags & FLAGS_CBC) {
489                         dev->hw_desc[0]->len1 = AES_BLOCK_SIZE;
490                         dev->hw_desc[0]->p1 = dev->iv_phys_base;
491                 } else {
492                         dev->hw_desc[0]->len1 = 0;
493                         dev->hw_desc[0]->p1 = 0;
494                 }
495                 dev->hw_desc[0]->len2 = ctx->keylen;
496                 dev->hw_desc[0]->p2 = dev->key_phys_base;
497                 dev->hw_desc[0]->next = dev->hw_phys_desc[1];
498         }
499         dev->hw_desc[0]->hdr = sahara_aes_key_hdr(dev);
500 
501         dev->nb_in_sg = sahara_sg_length(dev->in_sg, dev->total);
502         dev->nb_out_sg = sahara_sg_length(dev->out_sg, dev->total);
503         if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) {
504                 dev_err(dev->device, "not enough hw links (%d)\n",
505                         dev->nb_in_sg + dev->nb_out_sg);
506                 return -EINVAL;
507         }
508 
509         ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg,
510                          DMA_TO_DEVICE);
511         if (ret != dev->nb_in_sg) {
512                 dev_err(dev->device, "couldn't map in sg\n");
513                 goto unmap_in;
514         }
515         ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg,
516                          DMA_FROM_DEVICE);
517         if (ret != dev->nb_out_sg) {
518                 dev_err(dev->device, "couldn't map out sg\n");
519                 goto unmap_out;
520         }
521 
522         /* Create input links */
523         dev->hw_desc[1]->p1 = dev->hw_phys_link[0];
524         sg = dev->in_sg;
525         for (i = 0; i < dev->nb_in_sg; i++) {
526                 dev->hw_link[i]->len = sg->length;
527                 dev->hw_link[i]->p = sg->dma_address;
528                 if (i == (dev->nb_in_sg - 1)) {
529                         dev->hw_link[i]->next = 0;
530                 } else {
531                         dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
532                         sg = sg_next(sg);
533                 }
534         }
535 
536         /* Create output links */
537         dev->hw_desc[1]->p2 = dev->hw_phys_link[i];
538         sg = dev->out_sg;
539         for (j = i; j < dev->nb_out_sg + i; j++) {
540                 dev->hw_link[j]->len = sg->length;
541                 dev->hw_link[j]->p = sg->dma_address;
542                 if (j == (dev->nb_out_sg + i - 1)) {
543                         dev->hw_link[j]->next = 0;
544                 } else {
545                         dev->hw_link[j]->next = dev->hw_phys_link[j + 1];
546                         sg = sg_next(sg);
547                 }
548         }
549 
550         /* Fill remaining fields of hw_desc[1] */
551         dev->hw_desc[1]->hdr = sahara_aes_data_link_hdr(dev);
552         dev->hw_desc[1]->len1 = dev->total;
553         dev->hw_desc[1]->len2 = dev->total;
554         dev->hw_desc[1]->next = 0;
555 
556         sahara_dump_descriptors(dev);
557         sahara_dump_links(dev);
558 
559         sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
560 
561         return 0;
562 
563 unmap_out:
564         dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
565                 DMA_TO_DEVICE);
566 unmap_in:
567         dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
568                 DMA_FROM_DEVICE);
569 
570         return -EINVAL;
571 }
572 
573 static int sahara_aes_process(struct ablkcipher_request *req)
574 {
575         struct sahara_dev *dev = dev_ptr;
576         struct sahara_ctx *ctx;
577         struct sahara_aes_reqctx *rctx;
578         int ret;
579 
580         /* Request is ready to be dispatched by the device */
581         dev_dbg(dev->device,
582                 "dispatch request (nbytes=%d, src=%p, dst=%p)\n",
583                 req->nbytes, req->src, req->dst);
584 
585         /* assign new request to device */
586         dev->total = req->nbytes;
587         dev->in_sg = req->src;
588         dev->out_sg = req->dst;
589 
590         rctx = ablkcipher_request_ctx(req);
591         ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
592         rctx->mode &= FLAGS_MODE_MASK;
593         dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode;
594 
595         if ((dev->flags & FLAGS_CBC) && req->info)
596                 memcpy(dev->iv_base, req->info, AES_KEYSIZE_128);
597 
598         /* assign new context to device */
599         dev->ctx = ctx;
600 
601         reinit_completion(&dev->dma_completion);
602 
603         ret = sahara_hw_descriptor_create(dev);
604 
605         ret = wait_for_completion_timeout(&dev->dma_completion,
606                                 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
607         if (!ret) {
608                 dev_err(dev->device, "AES timeout\n");
609                 return -ETIMEDOUT;
610         }
611 
612         dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
613                 DMA_TO_DEVICE);
614         dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
615                 DMA_FROM_DEVICE);
616 
617         return 0;
618 }
619 
620 static int sahara_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
621                              unsigned int keylen)
622 {
623         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(tfm);
624         int ret;
625 
626         ctx->keylen = keylen;
627 
628         /* SAHARA only supports 128bit keys */
629         if (keylen == AES_KEYSIZE_128) {
630                 memcpy(ctx->key, key, keylen);
631                 ctx->flags |= FLAGS_NEW_KEY;
632                 return 0;
633         }
634 
635         if (keylen != AES_KEYSIZE_128 &&
636             keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256)
637                 return -EINVAL;
638 
639         /*
640          * The requested key size is not supported by HW, do a fallback.
641          */
642         ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
643         ctx->fallback->base.crt_flags |=
644                 (tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
645 
646         ret = crypto_ablkcipher_setkey(ctx->fallback, key, keylen);
647         if (ret) {
648                 struct crypto_tfm *tfm_aux = crypto_ablkcipher_tfm(tfm);
649 
650                 tfm_aux->crt_flags &= ~CRYPTO_TFM_RES_MASK;
651                 tfm_aux->crt_flags |=
652                         (ctx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK);
653         }
654         return ret;
655 }
656 
657 static int sahara_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
658 {
659         struct sahara_aes_reqctx *rctx = ablkcipher_request_ctx(req);
660         struct sahara_dev *dev = dev_ptr;
661         int err = 0;
662 
663         dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n",
664                 req->nbytes, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC));
665 
666         if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
667                 dev_err(dev->device,
668                         "request size is not exact amount of AES blocks\n");
669                 return -EINVAL;
670         }
671 
672         rctx->mode = mode;
673 
674         mutex_lock(&dev->queue_mutex);
675         err = ablkcipher_enqueue_request(&dev->queue, req);
676         mutex_unlock(&dev->queue_mutex);
677 
678         wake_up_process(dev->kthread);
679 
680         return err;
681 }
682 
683 static int sahara_aes_ecb_encrypt(struct ablkcipher_request *req)
684 {
685         struct crypto_tfm *tfm =
686                 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
687         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
688                 crypto_ablkcipher_reqtfm(req));
689         int err;
690 
691         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
692                 ablkcipher_request_set_tfm(req, ctx->fallback);
693                 err = crypto_ablkcipher_encrypt(req);
694                 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
695                 return err;
696         }
697 
698         return sahara_aes_crypt(req, FLAGS_ENCRYPT);
699 }
700 
701 static int sahara_aes_ecb_decrypt(struct ablkcipher_request *req)
702 {
703         struct crypto_tfm *tfm =
704                 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
705         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
706                 crypto_ablkcipher_reqtfm(req));
707         int err;
708 
709         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
710                 ablkcipher_request_set_tfm(req, ctx->fallback);
711                 err = crypto_ablkcipher_decrypt(req);
712                 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
713                 return err;
714         }
715 
716         return sahara_aes_crypt(req, 0);
717 }
718 
719 static int sahara_aes_cbc_encrypt(struct ablkcipher_request *req)
720 {
721         struct crypto_tfm *tfm =
722                 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
723         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
724                 crypto_ablkcipher_reqtfm(req));
725         int err;
726 
727         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
728                 ablkcipher_request_set_tfm(req, ctx->fallback);
729                 err = crypto_ablkcipher_encrypt(req);
730                 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
731                 return err;
732         }
733 
734         return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
735 }
736 
737 static int sahara_aes_cbc_decrypt(struct ablkcipher_request *req)
738 {
739         struct crypto_tfm *tfm =
740                 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
741         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
742                 crypto_ablkcipher_reqtfm(req));
743         int err;
744 
745         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
746                 ablkcipher_request_set_tfm(req, ctx->fallback);
747                 err = crypto_ablkcipher_decrypt(req);
748                 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
749                 return err;
750         }
751 
752         return sahara_aes_crypt(req, FLAGS_CBC);
753 }
754 
755 static int sahara_aes_cra_init(struct crypto_tfm *tfm)
756 {
757         const char *name = crypto_tfm_alg_name(tfm);
758         struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
759 
760         ctx->fallback = crypto_alloc_ablkcipher(name, 0,
761                                 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
762         if (IS_ERR(ctx->fallback)) {
763                 pr_err("Error allocating fallback algo %s\n", name);
764                 return PTR_ERR(ctx->fallback);
765         }
766 
767         tfm->crt_ablkcipher.reqsize = sizeof(struct sahara_aes_reqctx);
768 
769         return 0;
770 }
771 
772 static void sahara_aes_cra_exit(struct crypto_tfm *tfm)
773 {
774         struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
775 
776         if (ctx->fallback)
777                 crypto_free_ablkcipher(ctx->fallback);
778         ctx->fallback = NULL;
779 }
780 
781 static u32 sahara_sha_init_hdr(struct sahara_dev *dev,
782                               struct sahara_sha_reqctx *rctx)
783 {
784         u32 hdr = 0;
785 
786         hdr = rctx->mode;
787 
788         if (rctx->first) {
789                 hdr |= SAHARA_HDR_MDHA_SET_MODE_HASH;
790                 hdr |= SAHARA_HDR_MDHA_INIT;
791         } else {
792                 hdr |= SAHARA_HDR_MDHA_SET_MODE_MD_KEY;
793         }
794 
795         if (rctx->last)
796                 hdr |= SAHARA_HDR_MDHA_PDATA;
797 
798         if (hweight_long(hdr) % 2 == 0)
799                 hdr |= SAHARA_HDR_PARITY_BIT;
800 
801         return hdr;
802 }
803 
804 static int sahara_sha_hw_links_create(struct sahara_dev *dev,
805                                        struct sahara_sha_reqctx *rctx,
806                                        int start)
807 {
808         struct scatterlist *sg;
809         unsigned int i;
810         int ret;
811 
812         dev->in_sg = rctx->in_sg;
813 
814         dev->nb_in_sg = sahara_sg_length(dev->in_sg, rctx->total);
815         if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) {
816                 dev_err(dev->device, "not enough hw links (%d)\n",
817                         dev->nb_in_sg + dev->nb_out_sg);
818                 return -EINVAL;
819         }
820 
821         if (rctx->in_sg_chained) {
822                 i = start;
823                 sg = dev->in_sg;
824                 while (sg) {
825                         ret = dma_map_sg(dev->device, sg, 1,
826                                          DMA_TO_DEVICE);
827                         if (!ret)
828                                 return -EFAULT;
829 
830                         dev->hw_link[i]->len = sg->length;
831                         dev->hw_link[i]->p = sg->dma_address;
832                         dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
833                         sg = sg_next(sg);
834                         i += 1;
835                 }
836                 dev->hw_link[i-1]->next = 0;
837         } else {
838                 sg = dev->in_sg;
839                 ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg,
840                                  DMA_TO_DEVICE);
841                 if (!ret)
842                         return -EFAULT;
843 
844                 for (i = start; i < dev->nb_in_sg + start; i++) {
845                         dev->hw_link[i]->len = sg->length;
846                         dev->hw_link[i]->p = sg->dma_address;
847                         if (i == (dev->nb_in_sg + start - 1)) {
848                                 dev->hw_link[i]->next = 0;
849                         } else {
850                                 dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
851                                 sg = sg_next(sg);
852                         }
853                 }
854         }
855 
856         return i;
857 }
858 
859 static int sahara_sha_hw_data_descriptor_create(struct sahara_dev *dev,
860                                                 struct sahara_sha_reqctx *rctx,
861                                                 struct ahash_request *req,
862                                                 int index)
863 {
864         unsigned result_len;
865         int i = index;
866 
867         if (rctx->first)
868                 /* Create initial descriptor: #8*/
869                 dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
870         else
871                 /* Create hash descriptor: #10. Must follow #6. */
872                 dev->hw_desc[index]->hdr = SAHARA_HDR_MDHA_HASH;
873 
874         dev->hw_desc[index]->len1 = rctx->total;
875         if (dev->hw_desc[index]->len1 == 0) {
876                 /* if len1 is 0, p1 must be 0, too */
877                 dev->hw_desc[index]->p1 = 0;
878                 rctx->sg_in_idx = 0;
879         } else {
880                 /* Create input links */
881                 dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
882                 i = sahara_sha_hw_links_create(dev, rctx, index);
883 
884                 rctx->sg_in_idx = index;
885                 if (i < 0)
886                         return i;
887         }
888 
889         dev->hw_desc[index]->p2 = dev->hw_phys_link[i];
890 
891         /* Save the context for the next operation */
892         result_len = rctx->context_size;
893         dev->hw_link[i]->p = dev->context_phys_base;
894 
895         dev->hw_link[i]->len = result_len;
896         dev->hw_desc[index]->len2 = result_len;
897 
898         dev->hw_link[i]->next = 0;
899 
900         return 0;
901 }
902 
903 /*
904  * Load descriptor aka #6
905  *
906  * To load a previously saved context back to the MDHA unit
907  *
908  * p1: Saved Context
909  * p2: NULL
910  *
911  */
912 static int sahara_sha_hw_context_descriptor_create(struct sahara_dev *dev,
913                                                 struct sahara_sha_reqctx *rctx,
914                                                 struct ahash_request *req,
915                                                 int index)
916 {
917         dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
918 
919         dev->hw_desc[index]->len1 = rctx->context_size;
920         dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
921         dev->hw_desc[index]->len2 = 0;
922         dev->hw_desc[index]->p2 = 0;
923 
924         dev->hw_link[index]->len = rctx->context_size;
925         dev->hw_link[index]->p = dev->context_phys_base;
926         dev->hw_link[index]->next = 0;
927 
928         return 0;
929 }
930 
931 static int sahara_walk_and_recalc(struct scatterlist *sg, unsigned int nbytes)
932 {
933         if (!sg || !sg->length)
934                 return nbytes;
935 
936         while (nbytes && sg) {
937                 if (nbytes <= sg->length) {
938                         sg->length = nbytes;
939                         sg_mark_end(sg);
940                         break;
941                 }
942                 nbytes -= sg->length;
943                 sg = sg_next(sg);
944         }
945 
946         return nbytes;
947 }
948 
949 static int sahara_sha_prepare_request(struct ahash_request *req)
950 {
951         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
952         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
953         unsigned int hash_later;
954         unsigned int block_size;
955         unsigned int len;
956 
957         block_size = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
958 
959         /* append bytes from previous operation */
960         len = rctx->buf_cnt + req->nbytes;
961 
962         /* only the last transfer can be padded in hardware */
963         if (!rctx->last && (len < block_size)) {
964                 /* to few data, save for next operation */
965                 scatterwalk_map_and_copy(rctx->buf + rctx->buf_cnt, req->src,
966                                          0, req->nbytes, 0);
967                 rctx->buf_cnt += req->nbytes;
968 
969                 return 0;
970         }
971 
972         /* add data from previous operation first */
973         if (rctx->buf_cnt)
974                 memcpy(rctx->rembuf, rctx->buf, rctx->buf_cnt);
975 
976         /* data must always be a multiple of block_size */
977         hash_later = rctx->last ? 0 : len & (block_size - 1);
978         if (hash_later) {
979                 unsigned int offset = req->nbytes - hash_later;
980                 /* Save remaining bytes for later use */
981                 scatterwalk_map_and_copy(rctx->buf, req->src, offset,
982                                         hash_later, 0);
983         }
984 
985         /* nbytes should now be multiple of blocksize */
986         req->nbytes = req->nbytes - hash_later;
987 
988         sahara_walk_and_recalc(req->src, req->nbytes);
989 
990         /* have data from previous operation and current */
991         if (rctx->buf_cnt && req->nbytes) {
992                 sg_init_table(rctx->in_sg_chain, 2);
993                 sg_set_buf(rctx->in_sg_chain, rctx->rembuf, rctx->buf_cnt);
994 
995                 scatterwalk_sg_chain(rctx->in_sg_chain, 2, req->src);
996 
997                 rctx->total = req->nbytes + rctx->buf_cnt;
998                 rctx->in_sg = rctx->in_sg_chain;
999 
1000                 rctx->in_sg_chained = true;
1001                 req->src = rctx->in_sg_chain;
1002         /* only data from previous operation */
1003         } else if (rctx->buf_cnt) {
1004                 if (req->src)
1005                         rctx->in_sg = req->src;
1006                 else
1007                         rctx->in_sg = rctx->in_sg_chain;
1008                 /* buf was copied into rembuf above */
1009                 sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt);
1010                 rctx->total = rctx->buf_cnt;
1011                 rctx->in_sg_chained = false;
1012         /* no data from previous operation */
1013         } else {
1014                 rctx->in_sg = req->src;
1015                 rctx->total = req->nbytes;
1016                 req->src = rctx->in_sg;
1017                 rctx->in_sg_chained = false;
1018         }
1019 
1020         /* on next call, we only have the remaining data in the buffer */
1021         rctx->buf_cnt = hash_later;
1022 
1023         return -EINPROGRESS;
1024 }
1025 
1026 static void sahara_sha_unmap_sg(struct sahara_dev *dev,
1027                                 struct sahara_sha_reqctx *rctx)
1028 {
1029         struct scatterlist *sg;
1030 
1031         if (rctx->in_sg_chained) {
1032                 sg = dev->in_sg;
1033                 while (sg) {
1034                         dma_unmap_sg(dev->device, sg, 1, DMA_TO_DEVICE);
1035                         sg = sg_next(sg);
1036                 }
1037         } else {
1038                 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
1039                         DMA_TO_DEVICE);
1040         }
1041 }
1042 
1043 static int sahara_sha_process(struct ahash_request *req)
1044 {
1045         struct sahara_dev *dev = dev_ptr;
1046         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1047         int ret = -EINPROGRESS;
1048 
1049         ret = sahara_sha_prepare_request(req);
1050         if (!ret)
1051                 return ret;
1052 
1053         if (rctx->first) {
1054                 sahara_sha_hw_data_descriptor_create(dev, rctx, req, 0);
1055                 dev->hw_desc[0]->next = 0;
1056                 rctx->first = 0;
1057         } else {
1058                 memcpy(dev->context_base, rctx->context, rctx->context_size);
1059 
1060                 sahara_sha_hw_context_descriptor_create(dev, rctx, req, 0);
1061                 dev->hw_desc[0]->next = dev->hw_phys_desc[1];
1062                 sahara_sha_hw_data_descriptor_create(dev, rctx, req, 1);
1063                 dev->hw_desc[1]->next = 0;
1064         }
1065 
1066         sahara_dump_descriptors(dev);
1067         sahara_dump_links(dev);
1068 
1069         reinit_completion(&dev->dma_completion);
1070 
1071         sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
1072 
1073         ret = wait_for_completion_timeout(&dev->dma_completion,
1074                                 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
1075         if (!ret) {
1076                 dev_err(dev->device, "SHA timeout\n");
1077                 return -ETIMEDOUT;
1078         }
1079 
1080         if (rctx->sg_in_idx)
1081                 sahara_sha_unmap_sg(dev, rctx);
1082 
1083         memcpy(rctx->context, dev->context_base, rctx->context_size);
1084 
1085         if (req->result)
1086                 memcpy(req->result, rctx->context, rctx->digest_size);
1087 
1088         return 0;
1089 }
1090 
1091 static int sahara_queue_manage(void *data)
1092 {
1093         struct sahara_dev *dev = (struct sahara_dev *)data;
1094         struct crypto_async_request *async_req;
1095         int ret = 0;
1096 
1097         do {
1098                 __set_current_state(TASK_INTERRUPTIBLE);
1099 
1100                 mutex_lock(&dev->queue_mutex);
1101                 async_req = crypto_dequeue_request(&dev->queue);
1102                 mutex_unlock(&dev->queue_mutex);
1103 
1104                 if (async_req) {
1105                         if (crypto_tfm_alg_type(async_req->tfm) ==
1106                             CRYPTO_ALG_TYPE_AHASH) {
1107                                 struct ahash_request *req =
1108                                         ahash_request_cast(async_req);
1109 
1110                                 ret = sahara_sha_process(req);
1111                         } else {
1112                                 struct ablkcipher_request *req =
1113                                         ablkcipher_request_cast(async_req);
1114 
1115                                 ret = sahara_aes_process(req);
1116                         }
1117 
1118                         async_req->complete(async_req, ret);
1119 
1120                         continue;
1121                 }
1122 
1123                 schedule();
1124         } while (!kthread_should_stop());
1125 
1126         return 0;
1127 }
1128 
1129 static int sahara_sha_enqueue(struct ahash_request *req, int last)
1130 {
1131         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1132         struct sahara_dev *dev = dev_ptr;
1133         int ret;
1134 
1135         if (!req->nbytes && !last)
1136                 return 0;
1137 
1138         mutex_lock(&rctx->mutex);
1139         rctx->last = last;
1140 
1141         if (!rctx->active) {
1142                 rctx->active = 1;
1143                 rctx->first = 1;
1144         }
1145 
1146         mutex_lock(&dev->queue_mutex);
1147         ret = crypto_enqueue_request(&dev->queue, &req->base);
1148         mutex_unlock(&dev->queue_mutex);
1149 
1150         wake_up_process(dev->kthread);
1151         mutex_unlock(&rctx->mutex);
1152 
1153         return ret;
1154 }
1155 
1156 static int sahara_sha_init(struct ahash_request *req)
1157 {
1158         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1159         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1160 
1161         memset(rctx, 0, sizeof(*rctx));
1162 
1163         switch (crypto_ahash_digestsize(tfm)) {
1164         case SHA1_DIGEST_SIZE:
1165                 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA1;
1166                 rctx->digest_size = SHA1_DIGEST_SIZE;
1167                 break;
1168         case SHA256_DIGEST_SIZE:
1169                 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA256;
1170                 rctx->digest_size = SHA256_DIGEST_SIZE;
1171                 break;
1172         default:
1173                 return -EINVAL;
1174         }
1175 
1176         rctx->context_size = rctx->digest_size + 4;
1177         rctx->active = 0;
1178 
1179         mutex_init(&rctx->mutex);
1180 
1181         return 0;
1182 }
1183 
1184 static int sahara_sha_update(struct ahash_request *req)
1185 {
1186         return sahara_sha_enqueue(req, 0);
1187 }
1188 
1189 static int sahara_sha_final(struct ahash_request *req)
1190 {
1191         req->nbytes = 0;
1192         return sahara_sha_enqueue(req, 1);
1193 }
1194 
1195 static int sahara_sha_finup(struct ahash_request *req)
1196 {
1197         return sahara_sha_enqueue(req, 1);
1198 }
1199 
1200 static int sahara_sha_digest(struct ahash_request *req)
1201 {
1202         sahara_sha_init(req);
1203 
1204         return sahara_sha_finup(req);
1205 }
1206 
1207 static int sahara_sha_export(struct ahash_request *req, void *out)
1208 {
1209         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1210         struct sahara_ctx *ctx = crypto_ahash_ctx(ahash);
1211         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1212 
1213         memcpy(out, ctx, sizeof(struct sahara_ctx));
1214         memcpy(out + sizeof(struct sahara_sha_reqctx), rctx,
1215                sizeof(struct sahara_sha_reqctx));
1216 
1217         return 0;
1218 }
1219 
1220 static int sahara_sha_import(struct ahash_request *req, const void *in)
1221 {
1222         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1223         struct sahara_ctx *ctx = crypto_ahash_ctx(ahash);
1224         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1225 
1226         memcpy(ctx, in, sizeof(struct sahara_ctx));
1227         memcpy(rctx, in + sizeof(struct sahara_sha_reqctx),
1228                sizeof(struct sahara_sha_reqctx));
1229 
1230         return 0;
1231 }
1232 
1233 static int sahara_sha_cra_init(struct crypto_tfm *tfm)
1234 {
1235         const char *name = crypto_tfm_alg_name(tfm);
1236         struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
1237 
1238         ctx->shash_fallback = crypto_alloc_shash(name, 0,
1239                                         CRYPTO_ALG_NEED_FALLBACK);
1240         if (IS_ERR(ctx->shash_fallback)) {
1241                 pr_err("Error allocating fallback algo %s\n", name);
1242                 return PTR_ERR(ctx->shash_fallback);
1243         }
1244         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1245                                  sizeof(struct sahara_sha_reqctx) +
1246                                  SHA_BUFFER_LEN + SHA256_BLOCK_SIZE);
1247 
1248         return 0;
1249 }
1250 
1251 static void sahara_sha_cra_exit(struct crypto_tfm *tfm)
1252 {
1253         struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
1254 
1255         crypto_free_shash(ctx->shash_fallback);
1256         ctx->shash_fallback = NULL;
1257 }
1258 
1259 static struct crypto_alg aes_algs[] = {
1260 {
1261         .cra_name               = "ecb(aes)",
1262         .cra_driver_name        = "sahara-ecb-aes",
1263         .cra_priority           = 300,
1264         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
1265                         CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1266         .cra_blocksize          = AES_BLOCK_SIZE,
1267         .cra_ctxsize            = sizeof(struct sahara_ctx),
1268         .cra_alignmask          = 0x0,
1269         .cra_type               = &crypto_ablkcipher_type,
1270         .cra_module             = THIS_MODULE,
1271         .cra_init               = sahara_aes_cra_init,
1272         .cra_exit               = sahara_aes_cra_exit,
1273         .cra_u.ablkcipher = {
1274                 .min_keysize    = AES_MIN_KEY_SIZE ,
1275                 .max_keysize    = AES_MAX_KEY_SIZE,
1276                 .setkey         = sahara_aes_setkey,
1277                 .encrypt        = sahara_aes_ecb_encrypt,
1278                 .decrypt        = sahara_aes_ecb_decrypt,
1279         }
1280 }, {
1281         .cra_name               = "cbc(aes)",
1282         .cra_driver_name        = "sahara-cbc-aes",
1283         .cra_priority           = 300,
1284         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
1285                         CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1286         .cra_blocksize          = AES_BLOCK_SIZE,
1287         .cra_ctxsize            = sizeof(struct sahara_ctx),
1288         .cra_alignmask          = 0x0,
1289         .cra_type               = &crypto_ablkcipher_type,
1290         .cra_module             = THIS_MODULE,
1291         .cra_init               = sahara_aes_cra_init,
1292         .cra_exit               = sahara_aes_cra_exit,
1293         .cra_u.ablkcipher = {
1294                 .min_keysize    = AES_MIN_KEY_SIZE ,
1295                 .max_keysize    = AES_MAX_KEY_SIZE,
1296                 .ivsize         = AES_BLOCK_SIZE,
1297                 .setkey         = sahara_aes_setkey,
1298                 .encrypt        = sahara_aes_cbc_encrypt,
1299                 .decrypt        = sahara_aes_cbc_decrypt,
1300         }
1301 }
1302 };
1303 
1304 static struct ahash_alg sha_v3_algs[] = {
1305 {
1306         .init           = sahara_sha_init,
1307         .update         = sahara_sha_update,
1308         .final          = sahara_sha_final,
1309         .finup          = sahara_sha_finup,
1310         .digest         = sahara_sha_digest,
1311         .export         = sahara_sha_export,
1312         .import         = sahara_sha_import,
1313         .halg.digestsize        = SHA1_DIGEST_SIZE,
1314         .halg.base      = {
1315                 .cra_name               = "sha1",
1316                 .cra_driver_name        = "sahara-sha1",
1317                 .cra_priority           = 300,
1318                 .cra_flags              = CRYPTO_ALG_TYPE_AHASH |
1319                                                 CRYPTO_ALG_ASYNC |
1320                                                 CRYPTO_ALG_NEED_FALLBACK,
1321                 .cra_blocksize          = SHA1_BLOCK_SIZE,
1322                 .cra_ctxsize            = sizeof(struct sahara_ctx),
1323                 .cra_alignmask          = 0,
1324                 .cra_module             = THIS_MODULE,
1325                 .cra_init               = sahara_sha_cra_init,
1326                 .cra_exit               = sahara_sha_cra_exit,
1327         }
1328 },
1329 };
1330 
1331 static struct ahash_alg sha_v4_algs[] = {
1332 {
1333         .init           = sahara_sha_init,
1334         .update         = sahara_sha_update,
1335         .final          = sahara_sha_final,
1336         .finup          = sahara_sha_finup,
1337         .digest         = sahara_sha_digest,
1338         .export         = sahara_sha_export,
1339         .import         = sahara_sha_import,
1340         .halg.digestsize        = SHA256_DIGEST_SIZE,
1341         .halg.base      = {
1342                 .cra_name               = "sha256",
1343                 .cra_driver_name        = "sahara-sha256",
1344                 .cra_priority           = 300,
1345                 .cra_flags              = CRYPTO_ALG_TYPE_AHASH |
1346                                                 CRYPTO_ALG_ASYNC |
1347                                                 CRYPTO_ALG_NEED_FALLBACK,
1348                 .cra_blocksize          = SHA256_BLOCK_SIZE,
1349                 .cra_ctxsize            = sizeof(struct sahara_ctx),
1350                 .cra_alignmask          = 0,
1351                 .cra_module             = THIS_MODULE,
1352                 .cra_init               = sahara_sha_cra_init,
1353                 .cra_exit               = sahara_sha_cra_exit,
1354         }
1355 },
1356 };
1357 
1358 static irqreturn_t sahara_irq_handler(int irq, void *data)
1359 {
1360         struct sahara_dev *dev = (struct sahara_dev *)data;
1361         unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS);
1362         unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS);
1363 
1364         sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR,
1365                      SAHARA_REG_CMD);
1366 
1367         sahara_decode_status(dev, stat);
1368 
1369         if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) {
1370                 return IRQ_NONE;
1371         } else if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_COMPLETE) {
1372                 dev->error = 0;
1373         } else {
1374                 sahara_decode_error(dev, err);
1375                 dev->error = -EINVAL;
1376         }
1377 
1378         complete(&dev->dma_completion);
1379 
1380         return IRQ_HANDLED;
1381 }
1382 
1383 
1384 static int sahara_register_algs(struct sahara_dev *dev)
1385 {
1386         int err;
1387         unsigned int i, j, k, l;
1388 
1389         for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
1390                 INIT_LIST_HEAD(&aes_algs[i].cra_list);
1391                 err = crypto_register_alg(&aes_algs[i]);
1392                 if (err)
1393                         goto err_aes_algs;
1394         }
1395 
1396         for (k = 0; k < ARRAY_SIZE(sha_v3_algs); k++) {
1397                 err = crypto_register_ahash(&sha_v3_algs[k]);
1398                 if (err)
1399                         goto err_sha_v3_algs;
1400         }
1401 
1402         if (dev->version > SAHARA_VERSION_3)
1403                 for (l = 0; l < ARRAY_SIZE(sha_v4_algs); l++) {
1404                         err = crypto_register_ahash(&sha_v4_algs[l]);
1405                         if (err)
1406                                 goto err_sha_v4_algs;
1407                 }
1408 
1409         return 0;
1410 
1411 err_sha_v4_algs:
1412         for (j = 0; j < l; j++)
1413                 crypto_unregister_ahash(&sha_v4_algs[j]);
1414 
1415 err_sha_v3_algs:
1416         for (j = 0; j < k; j++)
1417                 crypto_unregister_ahash(&sha_v4_algs[j]);
1418 
1419 err_aes_algs:
1420         for (j = 0; j < i; j++)
1421                 crypto_unregister_alg(&aes_algs[j]);
1422 
1423         return err;
1424 }
1425 
1426 static void sahara_unregister_algs(struct sahara_dev *dev)
1427 {
1428         unsigned int i;
1429 
1430         for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
1431                 crypto_unregister_alg(&aes_algs[i]);
1432 
1433         for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++)
1434                 crypto_unregister_ahash(&sha_v3_algs[i]);
1435 
1436         if (dev->version > SAHARA_VERSION_3)
1437                 for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++)
1438                         crypto_unregister_ahash(&sha_v4_algs[i]);
1439 }
1440 
1441 static struct platform_device_id sahara_platform_ids[] = {
1442         { .name = "sahara-imx27" },
1443         { /* sentinel */ }
1444 };
1445 MODULE_DEVICE_TABLE(platform, sahara_platform_ids);
1446 
1447 static struct of_device_id sahara_dt_ids[] = {
1448         { .compatible = "fsl,imx53-sahara" },
1449         { .compatible = "fsl,imx27-sahara" },
1450         { /* sentinel */ }
1451 };
1452 MODULE_DEVICE_TABLE(of, sahara_dt_ids);
1453 
1454 static int sahara_probe(struct platform_device *pdev)
1455 {
1456         struct sahara_dev *dev;
1457         struct resource *res;
1458         u32 version;
1459         int irq;
1460         int err;
1461         int i;
1462 
1463         dev = devm_kzalloc(&pdev->dev, sizeof(struct sahara_dev), GFP_KERNEL);
1464         if (dev == NULL) {
1465                 dev_err(&pdev->dev, "unable to alloc data struct.\n");
1466                 return -ENOMEM;
1467         }
1468 
1469         dev->device = &pdev->dev;
1470         platform_set_drvdata(pdev, dev);
1471 
1472         /* Get the base address */
1473         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1474         dev->regs_base = devm_ioremap_resource(&pdev->dev, res);
1475         if (IS_ERR(dev->regs_base))
1476                 return PTR_ERR(dev->regs_base);
1477 
1478         /* Get the IRQ */
1479         irq = platform_get_irq(pdev,  0);
1480         if (irq < 0) {
1481                 dev_err(&pdev->dev, "failed to get irq resource\n");
1482                 return irq;
1483         }
1484 
1485         err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler,
1486                                0, dev_name(&pdev->dev), dev);
1487         if (err) {
1488                 dev_err(&pdev->dev, "failed to request irq\n");
1489                 return err;
1490         }
1491 
1492         /* clocks */
1493         dev->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
1494         if (IS_ERR(dev->clk_ipg)) {
1495                 dev_err(&pdev->dev, "Could not get ipg clock\n");
1496                 return PTR_ERR(dev->clk_ipg);
1497         }
1498 
1499         dev->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
1500         if (IS_ERR(dev->clk_ahb)) {
1501                 dev_err(&pdev->dev, "Could not get ahb clock\n");
1502                 return PTR_ERR(dev->clk_ahb);
1503         }
1504 
1505         /* Allocate HW descriptors */
1506         dev->hw_desc[0] = dma_alloc_coherent(&pdev->dev,
1507                         SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1508                         &dev->hw_phys_desc[0], GFP_KERNEL);
1509         if (!dev->hw_desc[0]) {
1510                 dev_err(&pdev->dev, "Could not allocate hw descriptors\n");
1511                 return -ENOMEM;
1512         }
1513         dev->hw_desc[1] = dev->hw_desc[0] + 1;
1514         dev->hw_phys_desc[1] = dev->hw_phys_desc[0] +
1515                                 sizeof(struct sahara_hw_desc);
1516 
1517         /* Allocate space for iv and key */
1518         dev->key_base = dma_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128,
1519                                 &dev->key_phys_base, GFP_KERNEL);
1520         if (!dev->key_base) {
1521                 dev_err(&pdev->dev, "Could not allocate memory for key\n");
1522                 err = -ENOMEM;
1523                 goto err_key;
1524         }
1525         dev->iv_base = dev->key_base + AES_KEYSIZE_128;
1526         dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128;
1527 
1528         /* Allocate space for context: largest digest + message length field */
1529         dev->context_base = dma_alloc_coherent(&pdev->dev,
1530                                         SHA256_DIGEST_SIZE + 4,
1531                                         &dev->context_phys_base, GFP_KERNEL);
1532         if (!dev->context_base) {
1533                 dev_err(&pdev->dev, "Could not allocate memory for MDHA context\n");
1534                 err = -ENOMEM;
1535                 goto err_key;
1536         }
1537 
1538         /* Allocate space for HW links */
1539         dev->hw_link[0] = dma_alloc_coherent(&pdev->dev,
1540                         SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1541                         &dev->hw_phys_link[0], GFP_KERNEL);
1542         if (!dev->hw_link[0]) {
1543                 dev_err(&pdev->dev, "Could not allocate hw links\n");
1544                 err = -ENOMEM;
1545                 goto err_link;
1546         }
1547         for (i = 1; i < SAHARA_MAX_HW_LINK; i++) {
1548                 dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] +
1549                                         sizeof(struct sahara_hw_link);
1550                 dev->hw_link[i] = dev->hw_link[i - 1] + 1;
1551         }
1552 
1553         crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH);
1554 
1555         spin_lock_init(&dev->lock);
1556         mutex_init(&dev->queue_mutex);
1557 
1558         dev_ptr = dev;
1559 
1560         dev->kthread = kthread_run(sahara_queue_manage, dev, "sahara_crypto");
1561         if (IS_ERR(dev->kthread)) {
1562                 err = PTR_ERR(dev->kthread);
1563                 goto err_link;
1564         }
1565 
1566         init_completion(&dev->dma_completion);
1567 
1568         clk_prepare_enable(dev->clk_ipg);
1569         clk_prepare_enable(dev->clk_ahb);
1570 
1571         version = sahara_read(dev, SAHARA_REG_VERSION);
1572         if (of_device_is_compatible(pdev->dev.of_node, "fsl,imx27-sahara")) {
1573                 if (version != SAHARA_VERSION_3)
1574                         err = -ENODEV;
1575         } else if (of_device_is_compatible(pdev->dev.of_node,
1576                         "fsl,imx53-sahara")) {
1577                 if (((version >> 8) & 0xff) != SAHARA_VERSION_4)
1578                         err = -ENODEV;
1579                 version = (version >> 8) & 0xff;
1580         }
1581         if (err == -ENODEV) {
1582                 dev_err(&pdev->dev, "SAHARA version %d not supported\n",
1583                                 version);
1584                 goto err_algs;
1585         }
1586 
1587         dev->version = version;
1588 
1589         sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH,
1590                      SAHARA_REG_CMD);
1591         sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) |
1592                         SAHARA_CONTROL_SET_MAXBURST(8) |
1593                         SAHARA_CONTROL_RNG_AUTORSD |
1594                         SAHARA_CONTROL_ENABLE_INT,
1595                         SAHARA_REG_CONTROL);
1596 
1597         err = sahara_register_algs(dev);
1598         if (err)
1599                 goto err_algs;
1600 
1601         dev_info(&pdev->dev, "SAHARA version %d initialized\n", version);
1602 
1603         return 0;
1604 
1605 err_algs:
1606         dma_free_coherent(&pdev->dev,
1607                           SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1608                           dev->hw_link[0], dev->hw_phys_link[0]);
1609         clk_disable_unprepare(dev->clk_ipg);
1610         clk_disable_unprepare(dev->clk_ahb);
1611         kthread_stop(dev->kthread);
1612         dev_ptr = NULL;
1613 err_link:
1614         dma_free_coherent(&pdev->dev,
1615                           2 * AES_KEYSIZE_128,
1616                           dev->key_base, dev->key_phys_base);
1617         dma_free_coherent(&pdev->dev,
1618                           SHA256_DIGEST_SIZE,
1619                           dev->context_base, dev->context_phys_base);
1620 err_key:
1621         dma_free_coherent(&pdev->dev,
1622                           SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1623                           dev->hw_desc[0], dev->hw_phys_desc[0]);
1624 
1625         return err;
1626 }
1627 
1628 static int sahara_remove(struct platform_device *pdev)
1629 {
1630         struct sahara_dev *dev = platform_get_drvdata(pdev);
1631 
1632         dma_free_coherent(&pdev->dev,
1633                           SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1634                           dev->hw_link[0], dev->hw_phys_link[0]);
1635         dma_free_coherent(&pdev->dev,
1636                           2 * AES_KEYSIZE_128,
1637                           dev->key_base, dev->key_phys_base);
1638         dma_free_coherent(&pdev->dev,
1639                           SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1640                           dev->hw_desc[0], dev->hw_phys_desc[0]);
1641 
1642         kthread_stop(dev->kthread);
1643 
1644         sahara_unregister_algs(dev);
1645 
1646         clk_disable_unprepare(dev->clk_ipg);
1647         clk_disable_unprepare(dev->clk_ahb);
1648 
1649         dev_ptr = NULL;
1650 
1651         return 0;
1652 }
1653 
1654 static struct platform_driver sahara_driver = {
1655         .probe          = sahara_probe,
1656         .remove         = sahara_remove,
1657         .driver         = {
1658                 .name   = SAHARA_NAME,
1659                 .of_match_table = sahara_dt_ids,
1660         },
1661         .id_table = sahara_platform_ids,
1662 };
1663 
1664 module_platform_driver(sahara_driver);
1665 
1666 MODULE_LICENSE("GPL");
1667 MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>");
1668 MODULE_AUTHOR("Steffen Trumtrar <s.trumtrar@pengutronix.de>");
1669 MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator");
1670 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us