Version:  2.0.40 2.2.26 2.4.37 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8

Linux/drivers/crypto/sahara.c

  1 /*
  2  * Cryptographic API.
  3  *
  4  * Support for SAHARA cryptographic accelerator.
  5  *
  6  * Copyright (c) 2014 Steffen Trumtrar <s.trumtrar@pengutronix.de>
  7  * Copyright (c) 2013 Vista Silicon S.L.
  8  * Author: Javier Martin <javier.martin@vista-silicon.com>
  9  *
 10  * This program is free software; you can redistribute it and/or modify
 11  * it under the terms of the GNU General Public License version 2 as published
 12  * by the Free Software Foundation.
 13  *
 14  * Based on omap-aes.c and tegra-aes.c
 15  */
 16 
 17 #include <crypto/aes.h>
 18 #include <crypto/internal/hash.h>
 19 #include <crypto/internal/skcipher.h>
 20 #include <crypto/scatterwalk.h>
 21 #include <crypto/sha.h>
 22 
 23 #include <linux/clk.h>
 24 #include <linux/crypto.h>
 25 #include <linux/interrupt.h>
 26 #include <linux/io.h>
 27 #include <linux/irq.h>
 28 #include <linux/kernel.h>
 29 #include <linux/kthread.h>
 30 #include <linux/module.h>
 31 #include <linux/mutex.h>
 32 #include <linux/of.h>
 33 #include <linux/of_device.h>
 34 #include <linux/platform_device.h>
 35 
 36 #define SHA_BUFFER_LEN          PAGE_SIZE
 37 #define SAHARA_MAX_SHA_BLOCK_SIZE       SHA256_BLOCK_SIZE
 38 
 39 #define SAHARA_NAME "sahara"
 40 #define SAHARA_VERSION_3        3
 41 #define SAHARA_VERSION_4        4
 42 #define SAHARA_TIMEOUT_MS       1000
 43 #define SAHARA_MAX_HW_DESC      2
 44 #define SAHARA_MAX_HW_LINK      20
 45 
 46 #define FLAGS_MODE_MASK         0x000f
 47 #define FLAGS_ENCRYPT           BIT(0)
 48 #define FLAGS_CBC               BIT(1)
 49 #define FLAGS_NEW_KEY           BIT(3)
 50 
 51 #define SAHARA_HDR_BASE                 0x00800000
 52 #define SAHARA_HDR_SKHA_ALG_AES 0
 53 #define SAHARA_HDR_SKHA_OP_ENC          (1 << 2)
 54 #define SAHARA_HDR_SKHA_MODE_ECB        (0 << 3)
 55 #define SAHARA_HDR_SKHA_MODE_CBC        (1 << 3)
 56 #define SAHARA_HDR_FORM_DATA            (5 << 16)
 57 #define SAHARA_HDR_FORM_KEY             (8 << 16)
 58 #define SAHARA_HDR_LLO                  (1 << 24)
 59 #define SAHARA_HDR_CHA_SKHA             (1 << 28)
 60 #define SAHARA_HDR_CHA_MDHA             (2 << 28)
 61 #define SAHARA_HDR_PARITY_BIT           (1 << 31)
 62 
 63 #define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000
 64 #define SAHARA_HDR_MDHA_SET_MODE_HASH   0x208D0000
 65 #define SAHARA_HDR_MDHA_HASH            0xA0850000
 66 #define SAHARA_HDR_MDHA_STORE_DIGEST    0x20820000
 67 #define SAHARA_HDR_MDHA_ALG_SHA1        0
 68 #define SAHARA_HDR_MDHA_ALG_MD5         1
 69 #define SAHARA_HDR_MDHA_ALG_SHA256      2
 70 #define SAHARA_HDR_MDHA_ALG_SHA224      3
 71 #define SAHARA_HDR_MDHA_PDATA           (1 << 2)
 72 #define SAHARA_HDR_MDHA_HMAC            (1 << 3)
 73 #define SAHARA_HDR_MDHA_INIT            (1 << 5)
 74 #define SAHARA_HDR_MDHA_IPAD            (1 << 6)
 75 #define SAHARA_HDR_MDHA_OPAD            (1 << 7)
 76 #define SAHARA_HDR_MDHA_SWAP            (1 << 8)
 77 #define SAHARA_HDR_MDHA_MAC_FULL        (1 << 9)
 78 #define SAHARA_HDR_MDHA_SSL             (1 << 10)
 79 
 80 /* SAHARA can only process one request at a time */
 81 #define SAHARA_QUEUE_LENGTH     1
 82 
 83 #define SAHARA_REG_VERSION      0x00
 84 #define SAHARA_REG_DAR          0x04
 85 #define SAHARA_REG_CONTROL      0x08
 86 #define         SAHARA_CONTROL_SET_THROTTLE(x)  (((x) & 0xff) << 24)
 87 #define         SAHARA_CONTROL_SET_MAXBURST(x)  (((x) & 0xff) << 16)
 88 #define         SAHARA_CONTROL_RNG_AUTORSD      (1 << 7)
 89 #define         SAHARA_CONTROL_ENABLE_INT       (1 << 4)
 90 #define SAHARA_REG_CMD          0x0C
 91 #define         SAHARA_CMD_RESET                (1 << 0)
 92 #define         SAHARA_CMD_CLEAR_INT            (1 << 8)
 93 #define         SAHARA_CMD_CLEAR_ERR            (1 << 9)
 94 #define         SAHARA_CMD_SINGLE_STEP          (1 << 10)
 95 #define         SAHARA_CMD_MODE_BATCH           (1 << 16)
 96 #define         SAHARA_CMD_MODE_DEBUG           (1 << 18)
 97 #define SAHARA_REG_STATUS       0x10
 98 #define         SAHARA_STATUS_GET_STATE(x)      ((x) & 0x7)
 99 #define                 SAHARA_STATE_IDLE       0
100 #define                 SAHARA_STATE_BUSY       1
101 #define                 SAHARA_STATE_ERR        2
102 #define                 SAHARA_STATE_FAULT      3
103 #define                 SAHARA_STATE_COMPLETE   4
104 #define                 SAHARA_STATE_COMP_FLAG  (1 << 2)
105 #define         SAHARA_STATUS_DAR_FULL          (1 << 3)
106 #define         SAHARA_STATUS_ERROR             (1 << 4)
107 #define         SAHARA_STATUS_SECURE            (1 << 5)
108 #define         SAHARA_STATUS_FAIL              (1 << 6)
109 #define         SAHARA_STATUS_INIT              (1 << 7)
110 #define         SAHARA_STATUS_RNG_RESEED        (1 << 8)
111 #define         SAHARA_STATUS_ACTIVE_RNG        (1 << 9)
112 #define         SAHARA_STATUS_ACTIVE_MDHA       (1 << 10)
113 #define         SAHARA_STATUS_ACTIVE_SKHA       (1 << 11)
114 #define         SAHARA_STATUS_MODE_BATCH        (1 << 16)
115 #define         SAHARA_STATUS_MODE_DEDICATED    (1 << 17)
116 #define         SAHARA_STATUS_MODE_DEBUG        (1 << 18)
117 #define         SAHARA_STATUS_GET_ISTATE(x)     (((x) >> 24) & 0xff)
118 #define SAHARA_REG_ERRSTATUS    0x14
119 #define         SAHARA_ERRSTATUS_GET_SOURCE(x)  ((x) & 0xf)
120 #define                 SAHARA_ERRSOURCE_CHA    14
121 #define                 SAHARA_ERRSOURCE_DMA    15
122 #define         SAHARA_ERRSTATUS_DMA_DIR        (1 << 8)
123 #define         SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3)
124 #define         SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7)
125 #define         SAHARA_ERRSTATUS_GET_CHASRC(x)  (((x) >> 16) & 0xfff)
126 #define         SAHARA_ERRSTATUS_GET_CHAERR(x)  (((x) >> 28) & 0x3)
127 #define SAHARA_REG_FADDR        0x18
128 #define SAHARA_REG_CDAR         0x1C
129 #define SAHARA_REG_IDAR         0x20
130 
131 struct sahara_hw_desc {
132         u32     hdr;
133         u32     len1;
134         u32     p1;
135         u32     len2;
136         u32     p2;
137         u32     next;
138 };
139 
140 struct sahara_hw_link {
141         u32     len;
142         u32     p;
143         u32     next;
144 };
145 
146 struct sahara_ctx {
147         unsigned long flags;
148 
149         /* AES-specific context */
150         int keylen;
151         u8 key[AES_KEYSIZE_128];
152         struct crypto_skcipher *fallback;
153 };
154 
155 struct sahara_aes_reqctx {
156         unsigned long mode;
157 };
158 
159 /*
160  * struct sahara_sha_reqctx - private data per request
161  * @buf: holds data for requests smaller than block_size
162  * @rembuf: used to prepare one block_size-aligned request
163  * @context: hw-specific context for request. Digest is extracted from this
164  * @mode: specifies what type of hw-descriptor needs to be built
165  * @digest_size: length of digest for this request
166  * @context_size: length of hw-context for this request.
167  *                Always digest_size + 4
168  * @buf_cnt: number of bytes saved in buf
169  * @sg_in_idx: number of hw links
170  * @in_sg: scatterlist for input data
171  * @in_sg_chain: scatterlists for chained input data
172  * @total: total number of bytes for transfer
173  * @last: is this the last block
174  * @first: is this the first block
175  * @active: inside a transfer
176  */
177 struct sahara_sha_reqctx {
178         u8                      buf[SAHARA_MAX_SHA_BLOCK_SIZE];
179         u8                      rembuf[SAHARA_MAX_SHA_BLOCK_SIZE];
180         u8                      context[SHA256_DIGEST_SIZE + 4];
181         unsigned int            mode;
182         unsigned int            digest_size;
183         unsigned int            context_size;
184         unsigned int            buf_cnt;
185         unsigned int            sg_in_idx;
186         struct scatterlist      *in_sg;
187         struct scatterlist      in_sg_chain[2];
188         size_t                  total;
189         unsigned int            last;
190         unsigned int            first;
191         unsigned int            active;
192 };
193 
194 struct sahara_dev {
195         struct device           *device;
196         unsigned int            version;
197         void __iomem            *regs_base;
198         struct clk              *clk_ipg;
199         struct clk              *clk_ahb;
200         struct mutex            queue_mutex;
201         struct task_struct      *kthread;
202         struct completion       dma_completion;
203 
204         struct sahara_ctx       *ctx;
205         spinlock_t              lock;
206         struct crypto_queue     queue;
207         unsigned long           flags;
208 
209         struct sahara_hw_desc   *hw_desc[SAHARA_MAX_HW_DESC];
210         dma_addr_t              hw_phys_desc[SAHARA_MAX_HW_DESC];
211 
212         u8                      *key_base;
213         dma_addr_t              key_phys_base;
214 
215         u8                      *iv_base;
216         dma_addr_t              iv_phys_base;
217 
218         u8                      *context_base;
219         dma_addr_t              context_phys_base;
220 
221         struct sahara_hw_link   *hw_link[SAHARA_MAX_HW_LINK];
222         dma_addr_t              hw_phys_link[SAHARA_MAX_HW_LINK];
223 
224         size_t                  total;
225         struct scatterlist      *in_sg;
226         int             nb_in_sg;
227         struct scatterlist      *out_sg;
228         int             nb_out_sg;
229 
230         u32                     error;
231 };
232 
233 static struct sahara_dev *dev_ptr;
234 
235 static inline void sahara_write(struct sahara_dev *dev, u32 data, u32 reg)
236 {
237         writel(data, dev->regs_base + reg);
238 }
239 
240 static inline unsigned int sahara_read(struct sahara_dev *dev, u32 reg)
241 {
242         return readl(dev->regs_base + reg);
243 }
244 
245 static u32 sahara_aes_key_hdr(struct sahara_dev *dev)
246 {
247         u32 hdr = SAHARA_HDR_BASE | SAHARA_HDR_SKHA_ALG_AES |
248                         SAHARA_HDR_FORM_KEY | SAHARA_HDR_LLO |
249                         SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
250 
251         if (dev->flags & FLAGS_CBC) {
252                 hdr |= SAHARA_HDR_SKHA_MODE_CBC;
253                 hdr ^= SAHARA_HDR_PARITY_BIT;
254         }
255 
256         if (dev->flags & FLAGS_ENCRYPT) {
257                 hdr |= SAHARA_HDR_SKHA_OP_ENC;
258                 hdr ^= SAHARA_HDR_PARITY_BIT;
259         }
260 
261         return hdr;
262 }
263 
264 static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev)
265 {
266         return SAHARA_HDR_BASE | SAHARA_HDR_FORM_DATA |
267                         SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
268 }
269 
270 static const char *sahara_err_src[16] = {
271         "No error",
272         "Header error",
273         "Descriptor length error",
274         "Descriptor length or pointer error",
275         "Link length error",
276         "Link pointer error",
277         "Input buffer error",
278         "Output buffer error",
279         "Output buffer starvation",
280         "Internal state fault",
281         "General descriptor problem",
282         "Reserved",
283         "Descriptor address error",
284         "Link address error",
285         "CHA error",
286         "DMA error"
287 };
288 
289 static const char *sahara_err_dmasize[4] = {
290         "Byte transfer",
291         "Half-word transfer",
292         "Word transfer",
293         "Reserved"
294 };
295 
296 static const char *sahara_err_dmasrc[8] = {
297         "No error",
298         "AHB bus error",
299         "Internal IP bus error",
300         "Parity error",
301         "DMA crosses 256 byte boundary",
302         "DMA is busy",
303         "Reserved",
304         "DMA HW error"
305 };
306 
307 static const char *sahara_cha_errsrc[12] = {
308         "Input buffer non-empty",
309         "Illegal address",
310         "Illegal mode",
311         "Illegal data size",
312         "Illegal key size",
313         "Write during processing",
314         "CTX read during processing",
315         "HW error",
316         "Input buffer disabled/underflow",
317         "Output buffer disabled/overflow",
318         "DES key parity error",
319         "Reserved"
320 };
321 
322 static const char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" };
323 
324 static void sahara_decode_error(struct sahara_dev *dev, unsigned int error)
325 {
326         u8 source = SAHARA_ERRSTATUS_GET_SOURCE(error);
327         u16 chasrc = ffs(SAHARA_ERRSTATUS_GET_CHASRC(error));
328 
329         dev_err(dev->device, "%s: Error Register = 0x%08x\n", __func__, error);
330 
331         dev_err(dev->device, "  - %s.\n", sahara_err_src[source]);
332 
333         if (source == SAHARA_ERRSOURCE_DMA) {
334                 if (error & SAHARA_ERRSTATUS_DMA_DIR)
335                         dev_err(dev->device, "          * DMA read.\n");
336                 else
337                         dev_err(dev->device, "          * DMA write.\n");
338 
339                 dev_err(dev->device, "          * %s.\n",
340                        sahara_err_dmasize[SAHARA_ERRSTATUS_GET_DMASZ(error)]);
341                 dev_err(dev->device, "          * %s.\n",
342                        sahara_err_dmasrc[SAHARA_ERRSTATUS_GET_DMASRC(error)]);
343         } else if (source == SAHARA_ERRSOURCE_CHA) {
344                 dev_err(dev->device, "          * %s.\n",
345                         sahara_cha_errsrc[chasrc]);
346                 dev_err(dev->device, "          * %s.\n",
347                        sahara_cha_err[SAHARA_ERRSTATUS_GET_CHAERR(error)]);
348         }
349         dev_err(dev->device, "\n");
350 }
351 
352 static const char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" };
353 
354 static void sahara_decode_status(struct sahara_dev *dev, unsigned int status)
355 {
356         u8 state;
357 
358         if (!IS_ENABLED(DEBUG))
359                 return;
360 
361         state = SAHARA_STATUS_GET_STATE(status);
362 
363         dev_dbg(dev->device, "%s: Status Register = 0x%08x\n",
364                 __func__, status);
365 
366         dev_dbg(dev->device, "  - State = %d:\n", state);
367         if (state & SAHARA_STATE_COMP_FLAG)
368                 dev_dbg(dev->device, "          * Descriptor completed. IRQ pending.\n");
369 
370         dev_dbg(dev->device, "          * %s.\n",
371                sahara_state[state & ~SAHARA_STATE_COMP_FLAG]);
372 
373         if (status & SAHARA_STATUS_DAR_FULL)
374                 dev_dbg(dev->device, "  - DAR Full.\n");
375         if (status & SAHARA_STATUS_ERROR)
376                 dev_dbg(dev->device, "  - Error.\n");
377         if (status & SAHARA_STATUS_SECURE)
378                 dev_dbg(dev->device, "  - Secure.\n");
379         if (status & SAHARA_STATUS_FAIL)
380                 dev_dbg(dev->device, "  - Fail.\n");
381         if (status & SAHARA_STATUS_RNG_RESEED)
382                 dev_dbg(dev->device, "  - RNG Reseed Request.\n");
383         if (status & SAHARA_STATUS_ACTIVE_RNG)
384                 dev_dbg(dev->device, "  - RNG Active.\n");
385         if (status & SAHARA_STATUS_ACTIVE_MDHA)
386                 dev_dbg(dev->device, "  - MDHA Active.\n");
387         if (status & SAHARA_STATUS_ACTIVE_SKHA)
388                 dev_dbg(dev->device, "  - SKHA Active.\n");
389 
390         if (status & SAHARA_STATUS_MODE_BATCH)
391                 dev_dbg(dev->device, "  - Batch Mode.\n");
392         else if (status & SAHARA_STATUS_MODE_DEDICATED)
393                 dev_dbg(dev->device, "  - Decidated Mode.\n");
394         else if (status & SAHARA_STATUS_MODE_DEBUG)
395                 dev_dbg(dev->device, "  - Debug Mode.\n");
396 
397         dev_dbg(dev->device, "  - Internal state = 0x%02x\n",
398                SAHARA_STATUS_GET_ISTATE(status));
399 
400         dev_dbg(dev->device, "Current DAR: 0x%08x\n",
401                 sahara_read(dev, SAHARA_REG_CDAR));
402         dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n",
403                 sahara_read(dev, SAHARA_REG_IDAR));
404 }
405 
406 static void sahara_dump_descriptors(struct sahara_dev *dev)
407 {
408         int i;
409 
410         if (!IS_ENABLED(DEBUG))
411                 return;
412 
413         for (i = 0; i < SAHARA_MAX_HW_DESC; i++) {
414                 dev_dbg(dev->device, "Descriptor (%d) (%pad):\n",
415                         i, &dev->hw_phys_desc[i]);
416                 dev_dbg(dev->device, "\thdr = 0x%08x\n", dev->hw_desc[i]->hdr);
417                 dev_dbg(dev->device, "\tlen1 = %u\n", dev->hw_desc[i]->len1);
418                 dev_dbg(dev->device, "\tp1 = 0x%08x\n", dev->hw_desc[i]->p1);
419                 dev_dbg(dev->device, "\tlen2 = %u\n", dev->hw_desc[i]->len2);
420                 dev_dbg(dev->device, "\tp2 = 0x%08x\n", dev->hw_desc[i]->p2);
421                 dev_dbg(dev->device, "\tnext = 0x%08x\n",
422                         dev->hw_desc[i]->next);
423         }
424         dev_dbg(dev->device, "\n");
425 }
426 
427 static void sahara_dump_links(struct sahara_dev *dev)
428 {
429         int i;
430 
431         if (!IS_ENABLED(DEBUG))
432                 return;
433 
434         for (i = 0; i < SAHARA_MAX_HW_LINK; i++) {
435                 dev_dbg(dev->device, "Link (%d) (%pad):\n",
436                         i, &dev->hw_phys_link[i]);
437                 dev_dbg(dev->device, "\tlen = %u\n", dev->hw_link[i]->len);
438                 dev_dbg(dev->device, "\tp = 0x%08x\n", dev->hw_link[i]->p);
439                 dev_dbg(dev->device, "\tnext = 0x%08x\n",
440                         dev->hw_link[i]->next);
441         }
442         dev_dbg(dev->device, "\n");
443 }
444 
445 static int sahara_hw_descriptor_create(struct sahara_dev *dev)
446 {
447         struct sahara_ctx *ctx = dev->ctx;
448         struct scatterlist *sg;
449         int ret;
450         int i, j;
451         int idx = 0;
452 
453         /* Copy new key if necessary */
454         if (ctx->flags & FLAGS_NEW_KEY) {
455                 memcpy(dev->key_base, ctx->key, ctx->keylen);
456                 ctx->flags &= ~FLAGS_NEW_KEY;
457 
458                 if (dev->flags & FLAGS_CBC) {
459                         dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE;
460                         dev->hw_desc[idx]->p1 = dev->iv_phys_base;
461                 } else {
462                         dev->hw_desc[idx]->len1 = 0;
463                         dev->hw_desc[idx]->p1 = 0;
464                 }
465                 dev->hw_desc[idx]->len2 = ctx->keylen;
466                 dev->hw_desc[idx]->p2 = dev->key_phys_base;
467                 dev->hw_desc[idx]->next = dev->hw_phys_desc[1];
468 
469                 dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev);
470 
471                 idx++;
472         }
473 
474         dev->nb_in_sg = sg_nents_for_len(dev->in_sg, dev->total);
475         if (dev->nb_in_sg < 0) {
476                 dev_err(dev->device, "Invalid numbers of src SG.\n");
477                 return dev->nb_in_sg;
478         }
479         dev->nb_out_sg = sg_nents_for_len(dev->out_sg, dev->total);
480         if (dev->nb_out_sg < 0) {
481                 dev_err(dev->device, "Invalid numbers of dst SG.\n");
482                 return dev->nb_out_sg;
483         }
484         if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) {
485                 dev_err(dev->device, "not enough hw links (%d)\n",
486                         dev->nb_in_sg + dev->nb_out_sg);
487                 return -EINVAL;
488         }
489 
490         ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg,
491                          DMA_TO_DEVICE);
492         if (ret != dev->nb_in_sg) {
493                 dev_err(dev->device, "couldn't map in sg\n");
494                 goto unmap_in;
495         }
496         ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg,
497                          DMA_FROM_DEVICE);
498         if (ret != dev->nb_out_sg) {
499                 dev_err(dev->device, "couldn't map out sg\n");
500                 goto unmap_out;
501         }
502 
503         /* Create input links */
504         dev->hw_desc[idx]->p1 = dev->hw_phys_link[0];
505         sg = dev->in_sg;
506         for (i = 0; i < dev->nb_in_sg; i++) {
507                 dev->hw_link[i]->len = sg->length;
508                 dev->hw_link[i]->p = sg->dma_address;
509                 if (i == (dev->nb_in_sg - 1)) {
510                         dev->hw_link[i]->next = 0;
511                 } else {
512                         dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
513                         sg = sg_next(sg);
514                 }
515         }
516 
517         /* Create output links */
518         dev->hw_desc[idx]->p2 = dev->hw_phys_link[i];
519         sg = dev->out_sg;
520         for (j = i; j < dev->nb_out_sg + i; j++) {
521                 dev->hw_link[j]->len = sg->length;
522                 dev->hw_link[j]->p = sg->dma_address;
523                 if (j == (dev->nb_out_sg + i - 1)) {
524                         dev->hw_link[j]->next = 0;
525                 } else {
526                         dev->hw_link[j]->next = dev->hw_phys_link[j + 1];
527                         sg = sg_next(sg);
528                 }
529         }
530 
531         /* Fill remaining fields of hw_desc[1] */
532         dev->hw_desc[idx]->hdr = sahara_aes_data_link_hdr(dev);
533         dev->hw_desc[idx]->len1 = dev->total;
534         dev->hw_desc[idx]->len2 = dev->total;
535         dev->hw_desc[idx]->next = 0;
536 
537         sahara_dump_descriptors(dev);
538         sahara_dump_links(dev);
539 
540         sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
541 
542         return 0;
543 
544 unmap_out:
545         dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
546                 DMA_TO_DEVICE);
547 unmap_in:
548         dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
549                 DMA_FROM_DEVICE);
550 
551         return -EINVAL;
552 }
553 
554 static int sahara_aes_process(struct ablkcipher_request *req)
555 {
556         struct sahara_dev *dev = dev_ptr;
557         struct sahara_ctx *ctx;
558         struct sahara_aes_reqctx *rctx;
559         int ret;
560         unsigned long timeout;
561 
562         /* Request is ready to be dispatched by the device */
563         dev_dbg(dev->device,
564                 "dispatch request (nbytes=%d, src=%p, dst=%p)\n",
565                 req->nbytes, req->src, req->dst);
566 
567         /* assign new request to device */
568         dev->total = req->nbytes;
569         dev->in_sg = req->src;
570         dev->out_sg = req->dst;
571 
572         rctx = ablkcipher_request_ctx(req);
573         ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
574         rctx->mode &= FLAGS_MODE_MASK;
575         dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode;
576 
577         if ((dev->flags & FLAGS_CBC) && req->info)
578                 memcpy(dev->iv_base, req->info, AES_KEYSIZE_128);
579 
580         /* assign new context to device */
581         dev->ctx = ctx;
582 
583         reinit_completion(&dev->dma_completion);
584 
585         ret = sahara_hw_descriptor_create(dev);
586         if (ret)
587                 return -EINVAL;
588 
589         timeout = wait_for_completion_timeout(&dev->dma_completion,
590                                 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
591         if (!timeout) {
592                 dev_err(dev->device, "AES timeout\n");
593                 return -ETIMEDOUT;
594         }
595 
596         dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
597                 DMA_TO_DEVICE);
598         dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
599                 DMA_FROM_DEVICE);
600 
601         return 0;
602 }
603 
604 static int sahara_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
605                              unsigned int keylen)
606 {
607         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(tfm);
608         int ret;
609 
610         ctx->keylen = keylen;
611 
612         /* SAHARA only supports 128bit keys */
613         if (keylen == AES_KEYSIZE_128) {
614                 memcpy(ctx->key, key, keylen);
615                 ctx->flags |= FLAGS_NEW_KEY;
616                 return 0;
617         }
618 
619         if (keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256)
620                 return -EINVAL;
621 
622         /*
623          * The requested key size is not supported by HW, do a fallback.
624          */
625         crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK);
626         crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags &
627                                                  CRYPTO_TFM_REQ_MASK);
628 
629         ret = crypto_skcipher_setkey(ctx->fallback, key, keylen);
630 
631         tfm->base.crt_flags &= ~CRYPTO_TFM_RES_MASK;
632         tfm->base.crt_flags |= crypto_skcipher_get_flags(ctx->fallback) &
633                                CRYPTO_TFM_RES_MASK;
634         return ret;
635 }
636 
637 static int sahara_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
638 {
639         struct sahara_aes_reqctx *rctx = ablkcipher_request_ctx(req);
640         struct sahara_dev *dev = dev_ptr;
641         int err = 0;
642 
643         dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n",
644                 req->nbytes, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC));
645 
646         if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
647                 dev_err(dev->device,
648                         "request size is not exact amount of AES blocks\n");
649                 return -EINVAL;
650         }
651 
652         rctx->mode = mode;
653 
654         mutex_lock(&dev->queue_mutex);
655         err = ablkcipher_enqueue_request(&dev->queue, req);
656         mutex_unlock(&dev->queue_mutex);
657 
658         wake_up_process(dev->kthread);
659 
660         return err;
661 }
662 
663 static int sahara_aes_ecb_encrypt(struct ablkcipher_request *req)
664 {
665         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
666                 crypto_ablkcipher_reqtfm(req));
667         int err;
668 
669         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
670                 SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
671 
672                 skcipher_request_set_tfm(subreq, ctx->fallback);
673                 skcipher_request_set_callback(subreq, req->base.flags,
674                                               NULL, NULL);
675                 skcipher_request_set_crypt(subreq, req->src, req->dst,
676                                            req->nbytes, req->info);
677                 err = crypto_skcipher_encrypt(subreq);
678                 skcipher_request_zero(subreq);
679                 return err;
680         }
681 
682         return sahara_aes_crypt(req, FLAGS_ENCRYPT);
683 }
684 
685 static int sahara_aes_ecb_decrypt(struct ablkcipher_request *req)
686 {
687         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
688                 crypto_ablkcipher_reqtfm(req));
689         int err;
690 
691         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
692                 SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
693 
694                 skcipher_request_set_tfm(subreq, ctx->fallback);
695                 skcipher_request_set_callback(subreq, req->base.flags,
696                                               NULL, NULL);
697                 skcipher_request_set_crypt(subreq, req->src, req->dst,
698                                            req->nbytes, req->info);
699                 err = crypto_skcipher_decrypt(subreq);
700                 skcipher_request_zero(subreq);
701                 return err;
702         }
703 
704         return sahara_aes_crypt(req, 0);
705 }
706 
707 static int sahara_aes_cbc_encrypt(struct ablkcipher_request *req)
708 {
709         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
710                 crypto_ablkcipher_reqtfm(req));
711         int err;
712 
713         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
714                 SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
715 
716                 skcipher_request_set_tfm(subreq, ctx->fallback);
717                 skcipher_request_set_callback(subreq, req->base.flags,
718                                               NULL, NULL);
719                 skcipher_request_set_crypt(subreq, req->src, req->dst,
720                                            req->nbytes, req->info);
721                 err = crypto_skcipher_encrypt(subreq);
722                 skcipher_request_zero(subreq);
723                 return err;
724         }
725 
726         return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
727 }
728 
729 static int sahara_aes_cbc_decrypt(struct ablkcipher_request *req)
730 {
731         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
732                 crypto_ablkcipher_reqtfm(req));
733         int err;
734 
735         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
736                 SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
737 
738                 skcipher_request_set_tfm(subreq, ctx->fallback);
739                 skcipher_request_set_callback(subreq, req->base.flags,
740                                               NULL, NULL);
741                 skcipher_request_set_crypt(subreq, req->src, req->dst,
742                                            req->nbytes, req->info);
743                 err = crypto_skcipher_decrypt(subreq);
744                 skcipher_request_zero(subreq);
745                 return err;
746         }
747 
748         return sahara_aes_crypt(req, FLAGS_CBC);
749 }
750 
751 static int sahara_aes_cra_init(struct crypto_tfm *tfm)
752 {
753         const char *name = crypto_tfm_alg_name(tfm);
754         struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
755 
756         ctx->fallback = crypto_alloc_skcipher(name, 0,
757                                               CRYPTO_ALG_ASYNC |
758                                               CRYPTO_ALG_NEED_FALLBACK);
759         if (IS_ERR(ctx->fallback)) {
760                 pr_err("Error allocating fallback algo %s\n", name);
761                 return PTR_ERR(ctx->fallback);
762         }
763 
764         tfm->crt_ablkcipher.reqsize = sizeof(struct sahara_aes_reqctx);
765 
766         return 0;
767 }
768 
769 static void sahara_aes_cra_exit(struct crypto_tfm *tfm)
770 {
771         struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
772 
773         crypto_free_skcipher(ctx->fallback);
774 }
775 
776 static u32 sahara_sha_init_hdr(struct sahara_dev *dev,
777                               struct sahara_sha_reqctx *rctx)
778 {
779         u32 hdr = 0;
780 
781         hdr = rctx->mode;
782 
783         if (rctx->first) {
784                 hdr |= SAHARA_HDR_MDHA_SET_MODE_HASH;
785                 hdr |= SAHARA_HDR_MDHA_INIT;
786         } else {
787                 hdr |= SAHARA_HDR_MDHA_SET_MODE_MD_KEY;
788         }
789 
790         if (rctx->last)
791                 hdr |= SAHARA_HDR_MDHA_PDATA;
792 
793         if (hweight_long(hdr) % 2 == 0)
794                 hdr |= SAHARA_HDR_PARITY_BIT;
795 
796         return hdr;
797 }
798 
799 static int sahara_sha_hw_links_create(struct sahara_dev *dev,
800                                        struct sahara_sha_reqctx *rctx,
801                                        int start)
802 {
803         struct scatterlist *sg;
804         unsigned int i;
805         int ret;
806 
807         dev->in_sg = rctx->in_sg;
808 
809         dev->nb_in_sg = sg_nents_for_len(dev->in_sg, rctx->total);
810         if (dev->nb_in_sg < 0) {
811                 dev_err(dev->device, "Invalid numbers of src SG.\n");
812                 return dev->nb_in_sg;
813         }
814         if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) {
815                 dev_err(dev->device, "not enough hw links (%d)\n",
816                         dev->nb_in_sg + dev->nb_out_sg);
817                 return -EINVAL;
818         }
819 
820         sg = dev->in_sg;
821         ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE);
822         if (!ret)
823                 return -EFAULT;
824 
825         for (i = start; i < dev->nb_in_sg + start; i++) {
826                 dev->hw_link[i]->len = sg->length;
827                 dev->hw_link[i]->p = sg->dma_address;
828                 if (i == (dev->nb_in_sg + start - 1)) {
829                         dev->hw_link[i]->next = 0;
830                 } else {
831                         dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
832                         sg = sg_next(sg);
833                 }
834         }
835 
836         return i;
837 }
838 
839 static int sahara_sha_hw_data_descriptor_create(struct sahara_dev *dev,
840                                                 struct sahara_sha_reqctx *rctx,
841                                                 struct ahash_request *req,
842                                                 int index)
843 {
844         unsigned result_len;
845         int i = index;
846 
847         if (rctx->first)
848                 /* Create initial descriptor: #8*/
849                 dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
850         else
851                 /* Create hash descriptor: #10. Must follow #6. */
852                 dev->hw_desc[index]->hdr = SAHARA_HDR_MDHA_HASH;
853 
854         dev->hw_desc[index]->len1 = rctx->total;
855         if (dev->hw_desc[index]->len1 == 0) {
856                 /* if len1 is 0, p1 must be 0, too */
857                 dev->hw_desc[index]->p1 = 0;
858                 rctx->sg_in_idx = 0;
859         } else {
860                 /* Create input links */
861                 dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
862                 i = sahara_sha_hw_links_create(dev, rctx, index);
863 
864                 rctx->sg_in_idx = index;
865                 if (i < 0)
866                         return i;
867         }
868 
869         dev->hw_desc[index]->p2 = dev->hw_phys_link[i];
870 
871         /* Save the context for the next operation */
872         result_len = rctx->context_size;
873         dev->hw_link[i]->p = dev->context_phys_base;
874 
875         dev->hw_link[i]->len = result_len;
876         dev->hw_desc[index]->len2 = result_len;
877 
878         dev->hw_link[i]->next = 0;
879 
880         return 0;
881 }
882 
883 /*
884  * Load descriptor aka #6
885  *
886  * To load a previously saved context back to the MDHA unit
887  *
888  * p1: Saved Context
889  * p2: NULL
890  *
891  */
892 static int sahara_sha_hw_context_descriptor_create(struct sahara_dev *dev,
893                                                 struct sahara_sha_reqctx *rctx,
894                                                 struct ahash_request *req,
895                                                 int index)
896 {
897         dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
898 
899         dev->hw_desc[index]->len1 = rctx->context_size;
900         dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
901         dev->hw_desc[index]->len2 = 0;
902         dev->hw_desc[index]->p2 = 0;
903 
904         dev->hw_link[index]->len = rctx->context_size;
905         dev->hw_link[index]->p = dev->context_phys_base;
906         dev->hw_link[index]->next = 0;
907 
908         return 0;
909 }
910 
911 static int sahara_walk_and_recalc(struct scatterlist *sg, unsigned int nbytes)
912 {
913         if (!sg || !sg->length)
914                 return nbytes;
915 
916         while (nbytes && sg) {
917                 if (nbytes <= sg->length) {
918                         sg->length = nbytes;
919                         sg_mark_end(sg);
920                         break;
921                 }
922                 nbytes -= sg->length;
923                 sg = sg_next(sg);
924         }
925 
926         return nbytes;
927 }
928 
929 static int sahara_sha_prepare_request(struct ahash_request *req)
930 {
931         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
932         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
933         unsigned int hash_later;
934         unsigned int block_size;
935         unsigned int len;
936 
937         block_size = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
938 
939         /* append bytes from previous operation */
940         len = rctx->buf_cnt + req->nbytes;
941 
942         /* only the last transfer can be padded in hardware */
943         if (!rctx->last && (len < block_size)) {
944                 /* to few data, save for next operation */
945                 scatterwalk_map_and_copy(rctx->buf + rctx->buf_cnt, req->src,
946                                          0, req->nbytes, 0);
947                 rctx->buf_cnt += req->nbytes;
948 
949                 return 0;
950         }
951 
952         /* add data from previous operation first */
953         if (rctx->buf_cnt)
954                 memcpy(rctx->rembuf, rctx->buf, rctx->buf_cnt);
955 
956         /* data must always be a multiple of block_size */
957         hash_later = rctx->last ? 0 : len & (block_size - 1);
958         if (hash_later) {
959                 unsigned int offset = req->nbytes - hash_later;
960                 /* Save remaining bytes for later use */
961                 scatterwalk_map_and_copy(rctx->buf, req->src, offset,
962                                         hash_later, 0);
963         }
964 
965         /* nbytes should now be multiple of blocksize */
966         req->nbytes = req->nbytes - hash_later;
967 
968         sahara_walk_and_recalc(req->src, req->nbytes);
969 
970         /* have data from previous operation and current */
971         if (rctx->buf_cnt && req->nbytes) {
972                 sg_init_table(rctx->in_sg_chain, 2);
973                 sg_set_buf(rctx->in_sg_chain, rctx->rembuf, rctx->buf_cnt);
974 
975                 sg_chain(rctx->in_sg_chain, 2, req->src);
976 
977                 rctx->total = req->nbytes + rctx->buf_cnt;
978                 rctx->in_sg = rctx->in_sg_chain;
979 
980                 req->src = rctx->in_sg_chain;
981         /* only data from previous operation */
982         } else if (rctx->buf_cnt) {
983                 if (req->src)
984                         rctx->in_sg = req->src;
985                 else
986                         rctx->in_sg = rctx->in_sg_chain;
987                 /* buf was copied into rembuf above */
988                 sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt);
989                 rctx->total = rctx->buf_cnt;
990         /* no data from previous operation */
991         } else {
992                 rctx->in_sg = req->src;
993                 rctx->total = req->nbytes;
994                 req->src = rctx->in_sg;
995         }
996 
997         /* on next call, we only have the remaining data in the buffer */
998         rctx->buf_cnt = hash_later;
999 
1000         return -EINPROGRESS;
1001 }
1002 
1003 static int sahara_sha_process(struct ahash_request *req)
1004 {
1005         struct sahara_dev *dev = dev_ptr;
1006         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1007         int ret;
1008         unsigned long timeout;
1009 
1010         ret = sahara_sha_prepare_request(req);
1011         if (!ret)
1012                 return ret;
1013 
1014         if (rctx->first) {
1015                 sahara_sha_hw_data_descriptor_create(dev, rctx, req, 0);
1016                 dev->hw_desc[0]->next = 0;
1017                 rctx->first = 0;
1018         } else {
1019                 memcpy(dev->context_base, rctx->context, rctx->context_size);
1020 
1021                 sahara_sha_hw_context_descriptor_create(dev, rctx, req, 0);
1022                 dev->hw_desc[0]->next = dev->hw_phys_desc[1];
1023                 sahara_sha_hw_data_descriptor_create(dev, rctx, req, 1);
1024                 dev->hw_desc[1]->next = 0;
1025         }
1026 
1027         sahara_dump_descriptors(dev);
1028         sahara_dump_links(dev);
1029 
1030         reinit_completion(&dev->dma_completion);
1031 
1032         sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
1033 
1034         timeout = wait_for_completion_timeout(&dev->dma_completion,
1035                                 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
1036         if (!timeout) {
1037                 dev_err(dev->device, "SHA timeout\n");
1038                 return -ETIMEDOUT;
1039         }
1040 
1041         if (rctx->sg_in_idx)
1042                 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
1043                              DMA_TO_DEVICE);
1044 
1045         memcpy(rctx->context, dev->context_base, rctx->context_size);
1046 
1047         if (req->result)
1048                 memcpy(req->result, rctx->context, rctx->digest_size);
1049 
1050         return 0;
1051 }
1052 
1053 static int sahara_queue_manage(void *data)
1054 {
1055         struct sahara_dev *dev = (struct sahara_dev *)data;
1056         struct crypto_async_request *async_req;
1057         struct crypto_async_request *backlog;
1058         int ret = 0;
1059 
1060         do {
1061                 __set_current_state(TASK_INTERRUPTIBLE);
1062 
1063                 mutex_lock(&dev->queue_mutex);
1064                 backlog = crypto_get_backlog(&dev->queue);
1065                 async_req = crypto_dequeue_request(&dev->queue);
1066                 mutex_unlock(&dev->queue_mutex);
1067 
1068                 if (backlog)
1069                         backlog->complete(backlog, -EINPROGRESS);
1070 
1071                 if (async_req) {
1072                         if (crypto_tfm_alg_type(async_req->tfm) ==
1073                             CRYPTO_ALG_TYPE_AHASH) {
1074                                 struct ahash_request *req =
1075                                         ahash_request_cast(async_req);
1076 
1077                                 ret = sahara_sha_process(req);
1078                         } else {
1079                                 struct ablkcipher_request *req =
1080                                         ablkcipher_request_cast(async_req);
1081 
1082                                 ret = sahara_aes_process(req);
1083                         }
1084 
1085                         async_req->complete(async_req, ret);
1086 
1087                         continue;
1088                 }
1089 
1090                 schedule();
1091         } while (!kthread_should_stop());
1092 
1093         return 0;
1094 }
1095 
1096 static int sahara_sha_enqueue(struct ahash_request *req, int last)
1097 {
1098         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1099         struct sahara_dev *dev = dev_ptr;
1100         int ret;
1101 
1102         if (!req->nbytes && !last)
1103                 return 0;
1104 
1105         rctx->last = last;
1106 
1107         if (!rctx->active) {
1108                 rctx->active = 1;
1109                 rctx->first = 1;
1110         }
1111 
1112         mutex_lock(&dev->queue_mutex);
1113         ret = crypto_enqueue_request(&dev->queue, &req->base);
1114         mutex_unlock(&dev->queue_mutex);
1115 
1116         wake_up_process(dev->kthread);
1117 
1118         return ret;
1119 }
1120 
1121 static int sahara_sha_init(struct ahash_request *req)
1122 {
1123         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1124         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1125 
1126         memset(rctx, 0, sizeof(*rctx));
1127 
1128         switch (crypto_ahash_digestsize(tfm)) {
1129         case SHA1_DIGEST_SIZE:
1130                 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA1;
1131                 rctx->digest_size = SHA1_DIGEST_SIZE;
1132                 break;
1133         case SHA256_DIGEST_SIZE:
1134                 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA256;
1135                 rctx->digest_size = SHA256_DIGEST_SIZE;
1136                 break;
1137         default:
1138                 return -EINVAL;
1139         }
1140 
1141         rctx->context_size = rctx->digest_size + 4;
1142         rctx->active = 0;
1143 
1144         return 0;
1145 }
1146 
1147 static int sahara_sha_update(struct ahash_request *req)
1148 {
1149         return sahara_sha_enqueue(req, 0);
1150 }
1151 
1152 static int sahara_sha_final(struct ahash_request *req)
1153 {
1154         req->nbytes = 0;
1155         return sahara_sha_enqueue(req, 1);
1156 }
1157 
1158 static int sahara_sha_finup(struct ahash_request *req)
1159 {
1160         return sahara_sha_enqueue(req, 1);
1161 }
1162 
1163 static int sahara_sha_digest(struct ahash_request *req)
1164 {
1165         sahara_sha_init(req);
1166 
1167         return sahara_sha_finup(req);
1168 }
1169 
1170 static int sahara_sha_export(struct ahash_request *req, void *out)
1171 {
1172         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1173 
1174         memcpy(out, rctx, sizeof(struct sahara_sha_reqctx));
1175 
1176         return 0;
1177 }
1178 
1179 static int sahara_sha_import(struct ahash_request *req, const void *in)
1180 {
1181         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1182 
1183         memcpy(rctx, in, sizeof(struct sahara_sha_reqctx));
1184 
1185         return 0;
1186 }
1187 
1188 static int sahara_sha_cra_init(struct crypto_tfm *tfm)
1189 {
1190         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1191                                  sizeof(struct sahara_sha_reqctx) +
1192                                  SHA_BUFFER_LEN + SHA256_BLOCK_SIZE);
1193 
1194         return 0;
1195 }
1196 
1197 static struct crypto_alg aes_algs[] = {
1198 {
1199         .cra_name               = "ecb(aes)",
1200         .cra_driver_name        = "sahara-ecb-aes",
1201         .cra_priority           = 300,
1202         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
1203                         CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1204         .cra_blocksize          = AES_BLOCK_SIZE,
1205         .cra_ctxsize            = sizeof(struct sahara_ctx),
1206         .cra_alignmask          = 0x0,
1207         .cra_type               = &crypto_ablkcipher_type,
1208         .cra_module             = THIS_MODULE,
1209         .cra_init               = sahara_aes_cra_init,
1210         .cra_exit               = sahara_aes_cra_exit,
1211         .cra_u.ablkcipher = {
1212                 .min_keysize    = AES_MIN_KEY_SIZE ,
1213                 .max_keysize    = AES_MAX_KEY_SIZE,
1214                 .setkey         = sahara_aes_setkey,
1215                 .encrypt        = sahara_aes_ecb_encrypt,
1216                 .decrypt        = sahara_aes_ecb_decrypt,
1217         }
1218 }, {
1219         .cra_name               = "cbc(aes)",
1220         .cra_driver_name        = "sahara-cbc-aes",
1221         .cra_priority           = 300,
1222         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
1223                         CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1224         .cra_blocksize          = AES_BLOCK_SIZE,
1225         .cra_ctxsize            = sizeof(struct sahara_ctx),
1226         .cra_alignmask          = 0x0,
1227         .cra_type               = &crypto_ablkcipher_type,
1228         .cra_module             = THIS_MODULE,
1229         .cra_init               = sahara_aes_cra_init,
1230         .cra_exit               = sahara_aes_cra_exit,
1231         .cra_u.ablkcipher = {
1232                 .min_keysize    = AES_MIN_KEY_SIZE ,
1233                 .max_keysize    = AES_MAX_KEY_SIZE,
1234                 .ivsize         = AES_BLOCK_SIZE,
1235                 .setkey         = sahara_aes_setkey,
1236                 .encrypt        = sahara_aes_cbc_encrypt,
1237                 .decrypt        = sahara_aes_cbc_decrypt,
1238         }
1239 }
1240 };
1241 
1242 static struct ahash_alg sha_v3_algs[] = {
1243 {
1244         .init           = sahara_sha_init,
1245         .update         = sahara_sha_update,
1246         .final          = sahara_sha_final,
1247         .finup          = sahara_sha_finup,
1248         .digest         = sahara_sha_digest,
1249         .export         = sahara_sha_export,
1250         .import         = sahara_sha_import,
1251         .halg.digestsize        = SHA1_DIGEST_SIZE,
1252         .halg.statesize         = sizeof(struct sahara_sha_reqctx),
1253         .halg.base      = {
1254                 .cra_name               = "sha1",
1255                 .cra_driver_name        = "sahara-sha1",
1256                 .cra_priority           = 300,
1257                 .cra_flags              = CRYPTO_ALG_TYPE_AHASH |
1258                                                 CRYPTO_ALG_ASYNC |
1259                                                 CRYPTO_ALG_NEED_FALLBACK,
1260                 .cra_blocksize          = SHA1_BLOCK_SIZE,
1261                 .cra_ctxsize            = sizeof(struct sahara_ctx),
1262                 .cra_alignmask          = 0,
1263                 .cra_module             = THIS_MODULE,
1264                 .cra_init               = sahara_sha_cra_init,
1265         }
1266 },
1267 };
1268 
1269 static struct ahash_alg sha_v4_algs[] = {
1270 {
1271         .init           = sahara_sha_init,
1272         .update         = sahara_sha_update,
1273         .final          = sahara_sha_final,
1274         .finup          = sahara_sha_finup,
1275         .digest         = sahara_sha_digest,
1276         .export         = sahara_sha_export,
1277         .import         = sahara_sha_import,
1278         .halg.digestsize        = SHA256_DIGEST_SIZE,
1279         .halg.statesize         = sizeof(struct sahara_sha_reqctx),
1280         .halg.base      = {
1281                 .cra_name               = "sha256",
1282                 .cra_driver_name        = "sahara-sha256",
1283                 .cra_priority           = 300,
1284                 .cra_flags              = CRYPTO_ALG_TYPE_AHASH |
1285                                                 CRYPTO_ALG_ASYNC |
1286                                                 CRYPTO_ALG_NEED_FALLBACK,
1287                 .cra_blocksize          = SHA256_BLOCK_SIZE,
1288                 .cra_ctxsize            = sizeof(struct sahara_ctx),
1289                 .cra_alignmask          = 0,
1290                 .cra_module             = THIS_MODULE,
1291                 .cra_init               = sahara_sha_cra_init,
1292         }
1293 },
1294 };
1295 
1296 static irqreturn_t sahara_irq_handler(int irq, void *data)
1297 {
1298         struct sahara_dev *dev = (struct sahara_dev *)data;
1299         unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS);
1300         unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS);
1301 
1302         sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR,
1303                      SAHARA_REG_CMD);
1304 
1305         sahara_decode_status(dev, stat);
1306 
1307         if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) {
1308                 return IRQ_NONE;
1309         } else if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_COMPLETE) {
1310                 dev->error = 0;
1311         } else {
1312                 sahara_decode_error(dev, err);
1313                 dev->error = -EINVAL;
1314         }
1315 
1316         complete(&dev->dma_completion);
1317 
1318         return IRQ_HANDLED;
1319 }
1320 
1321 
1322 static int sahara_register_algs(struct sahara_dev *dev)
1323 {
1324         int err;
1325         unsigned int i, j, k, l;
1326 
1327         for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
1328                 INIT_LIST_HEAD(&aes_algs[i].cra_list);
1329                 err = crypto_register_alg(&aes_algs[i]);
1330                 if (err)
1331                         goto err_aes_algs;
1332         }
1333 
1334         for (k = 0; k < ARRAY_SIZE(sha_v3_algs); k++) {
1335                 err = crypto_register_ahash(&sha_v3_algs[k]);
1336                 if (err)
1337                         goto err_sha_v3_algs;
1338         }
1339 
1340         if (dev->version > SAHARA_VERSION_3)
1341                 for (l = 0; l < ARRAY_SIZE(sha_v4_algs); l++) {
1342                         err = crypto_register_ahash(&sha_v4_algs[l]);
1343                         if (err)
1344                                 goto err_sha_v4_algs;
1345                 }
1346 
1347         return 0;
1348 
1349 err_sha_v4_algs:
1350         for (j = 0; j < l; j++)
1351                 crypto_unregister_ahash(&sha_v4_algs[j]);
1352 
1353 err_sha_v3_algs:
1354         for (j = 0; j < k; j++)
1355                 crypto_unregister_ahash(&sha_v4_algs[j]);
1356 
1357 err_aes_algs:
1358         for (j = 0; j < i; j++)
1359                 crypto_unregister_alg(&aes_algs[j]);
1360 
1361         return err;
1362 }
1363 
1364 static void sahara_unregister_algs(struct sahara_dev *dev)
1365 {
1366         unsigned int i;
1367 
1368         for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
1369                 crypto_unregister_alg(&aes_algs[i]);
1370 
1371         for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++)
1372                 crypto_unregister_ahash(&sha_v3_algs[i]);
1373 
1374         if (dev->version > SAHARA_VERSION_3)
1375                 for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++)
1376                         crypto_unregister_ahash(&sha_v4_algs[i]);
1377 }
1378 
1379 static struct platform_device_id sahara_platform_ids[] = {
1380         { .name = "sahara-imx27" },
1381         { /* sentinel */ }
1382 };
1383 MODULE_DEVICE_TABLE(platform, sahara_platform_ids);
1384 
1385 static struct of_device_id sahara_dt_ids[] = {
1386         { .compatible = "fsl,imx53-sahara" },
1387         { .compatible = "fsl,imx27-sahara" },
1388         { /* sentinel */ }
1389 };
1390 MODULE_DEVICE_TABLE(of, sahara_dt_ids);
1391 
1392 static int sahara_probe(struct platform_device *pdev)
1393 {
1394         struct sahara_dev *dev;
1395         struct resource *res;
1396         u32 version;
1397         int irq;
1398         int err;
1399         int i;
1400 
1401         dev = devm_kzalloc(&pdev->dev, sizeof(struct sahara_dev), GFP_KERNEL);
1402         if (dev == NULL) {
1403                 dev_err(&pdev->dev, "unable to alloc data struct.\n");
1404                 return -ENOMEM;
1405         }
1406 
1407         dev->device = &pdev->dev;
1408         platform_set_drvdata(pdev, dev);
1409 
1410         /* Get the base address */
1411         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1412         dev->regs_base = devm_ioremap_resource(&pdev->dev, res);
1413         if (IS_ERR(dev->regs_base))
1414                 return PTR_ERR(dev->regs_base);
1415 
1416         /* Get the IRQ */
1417         irq = platform_get_irq(pdev,  0);
1418         if (irq < 0) {
1419                 dev_err(&pdev->dev, "failed to get irq resource\n");
1420                 return irq;
1421         }
1422 
1423         err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler,
1424                                0, dev_name(&pdev->dev), dev);
1425         if (err) {
1426                 dev_err(&pdev->dev, "failed to request irq\n");
1427                 return err;
1428         }
1429 
1430         /* clocks */
1431         dev->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
1432         if (IS_ERR(dev->clk_ipg)) {
1433                 dev_err(&pdev->dev, "Could not get ipg clock\n");
1434                 return PTR_ERR(dev->clk_ipg);
1435         }
1436 
1437         dev->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
1438         if (IS_ERR(dev->clk_ahb)) {
1439                 dev_err(&pdev->dev, "Could not get ahb clock\n");
1440                 return PTR_ERR(dev->clk_ahb);
1441         }
1442 
1443         /* Allocate HW descriptors */
1444         dev->hw_desc[0] = dmam_alloc_coherent(&pdev->dev,
1445                         SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1446                         &dev->hw_phys_desc[0], GFP_KERNEL);
1447         if (!dev->hw_desc[0]) {
1448                 dev_err(&pdev->dev, "Could not allocate hw descriptors\n");
1449                 return -ENOMEM;
1450         }
1451         dev->hw_desc[1] = dev->hw_desc[0] + 1;
1452         dev->hw_phys_desc[1] = dev->hw_phys_desc[0] +
1453                                 sizeof(struct sahara_hw_desc);
1454 
1455         /* Allocate space for iv and key */
1456         dev->key_base = dmam_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128,
1457                                 &dev->key_phys_base, GFP_KERNEL);
1458         if (!dev->key_base) {
1459                 dev_err(&pdev->dev, "Could not allocate memory for key\n");
1460                 return -ENOMEM;
1461         }
1462         dev->iv_base = dev->key_base + AES_KEYSIZE_128;
1463         dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128;
1464 
1465         /* Allocate space for context: largest digest + message length field */
1466         dev->context_base = dmam_alloc_coherent(&pdev->dev,
1467                                         SHA256_DIGEST_SIZE + 4,
1468                                         &dev->context_phys_base, GFP_KERNEL);
1469         if (!dev->context_base) {
1470                 dev_err(&pdev->dev, "Could not allocate memory for MDHA context\n");
1471                 return -ENOMEM;
1472         }
1473 
1474         /* Allocate space for HW links */
1475         dev->hw_link[0] = dmam_alloc_coherent(&pdev->dev,
1476                         SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1477                         &dev->hw_phys_link[0], GFP_KERNEL);
1478         if (!dev->hw_link[0]) {
1479                 dev_err(&pdev->dev, "Could not allocate hw links\n");
1480                 return -ENOMEM;
1481         }
1482         for (i = 1; i < SAHARA_MAX_HW_LINK; i++) {
1483                 dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] +
1484                                         sizeof(struct sahara_hw_link);
1485                 dev->hw_link[i] = dev->hw_link[i - 1] + 1;
1486         }
1487 
1488         crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH);
1489 
1490         spin_lock_init(&dev->lock);
1491         mutex_init(&dev->queue_mutex);
1492 
1493         dev_ptr = dev;
1494 
1495         dev->kthread = kthread_run(sahara_queue_manage, dev, "sahara_crypto");
1496         if (IS_ERR(dev->kthread)) {
1497                 return PTR_ERR(dev->kthread);
1498         }
1499 
1500         init_completion(&dev->dma_completion);
1501 
1502         err = clk_prepare_enable(dev->clk_ipg);
1503         if (err)
1504                 return err;
1505         err = clk_prepare_enable(dev->clk_ahb);
1506         if (err)
1507                 goto clk_ipg_disable;
1508 
1509         version = sahara_read(dev, SAHARA_REG_VERSION);
1510         if (of_device_is_compatible(pdev->dev.of_node, "fsl,imx27-sahara")) {
1511                 if (version != SAHARA_VERSION_3)
1512                         err = -ENODEV;
1513         } else if (of_device_is_compatible(pdev->dev.of_node,
1514                         "fsl,imx53-sahara")) {
1515                 if (((version >> 8) & 0xff) != SAHARA_VERSION_4)
1516                         err = -ENODEV;
1517                 version = (version >> 8) & 0xff;
1518         }
1519         if (err == -ENODEV) {
1520                 dev_err(&pdev->dev, "SAHARA version %d not supported\n",
1521                                 version);
1522                 goto err_algs;
1523         }
1524 
1525         dev->version = version;
1526 
1527         sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH,
1528                      SAHARA_REG_CMD);
1529         sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) |
1530                         SAHARA_CONTROL_SET_MAXBURST(8) |
1531                         SAHARA_CONTROL_RNG_AUTORSD |
1532                         SAHARA_CONTROL_ENABLE_INT,
1533                         SAHARA_REG_CONTROL);
1534 
1535         err = sahara_register_algs(dev);
1536         if (err)
1537                 goto err_algs;
1538 
1539         dev_info(&pdev->dev, "SAHARA version %d initialized\n", version);
1540 
1541         return 0;
1542 
1543 err_algs:
1544         kthread_stop(dev->kthread);
1545         dev_ptr = NULL;
1546         clk_disable_unprepare(dev->clk_ahb);
1547 clk_ipg_disable:
1548         clk_disable_unprepare(dev->clk_ipg);
1549 
1550         return err;
1551 }
1552 
1553 static int sahara_remove(struct platform_device *pdev)
1554 {
1555         struct sahara_dev *dev = platform_get_drvdata(pdev);
1556 
1557         kthread_stop(dev->kthread);
1558 
1559         sahara_unregister_algs(dev);
1560 
1561         clk_disable_unprepare(dev->clk_ipg);
1562         clk_disable_unprepare(dev->clk_ahb);
1563 
1564         dev_ptr = NULL;
1565 
1566         return 0;
1567 }
1568 
1569 static struct platform_driver sahara_driver = {
1570         .probe          = sahara_probe,
1571         .remove         = sahara_remove,
1572         .driver         = {
1573                 .name   = SAHARA_NAME,
1574                 .of_match_table = sahara_dt_ids,
1575         },
1576         .id_table = sahara_platform_ids,
1577 };
1578 
1579 module_platform_driver(sahara_driver);
1580 
1581 MODULE_LICENSE("GPL");
1582 MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>");
1583 MODULE_AUTHOR("Steffen Trumtrar <s.trumtrar@pengutronix.de>");
1584 MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator");
1585 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us