Version:  2.0.40 2.2.26 2.4.37 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5

Linux/drivers/crypto/sahara.c

  1 /*
  2  * Cryptographic API.
  3  *
  4  * Support for SAHARA cryptographic accelerator.
  5  *
  6  * Copyright (c) 2014 Steffen Trumtrar <s.trumtrar@pengutronix.de>
  7  * Copyright (c) 2013 Vista Silicon S.L.
  8  * Author: Javier Martin <javier.martin@vista-silicon.com>
  9  *
 10  * This program is free software; you can redistribute it and/or modify
 11  * it under the terms of the GNU General Public License version 2 as published
 12  * by the Free Software Foundation.
 13  *
 14  * Based on omap-aes.c and tegra-aes.c
 15  */
 16 
 17 #include <crypto/algapi.h>
 18 #include <crypto/aes.h>
 19 #include <crypto/hash.h>
 20 #include <crypto/internal/hash.h>
 21 #include <crypto/scatterwalk.h>
 22 #include <crypto/sha.h>
 23 
 24 #include <linux/clk.h>
 25 #include <linux/crypto.h>
 26 #include <linux/interrupt.h>
 27 #include <linux/io.h>
 28 #include <linux/irq.h>
 29 #include <linux/kernel.h>
 30 #include <linux/kthread.h>
 31 #include <linux/module.h>
 32 #include <linux/mutex.h>
 33 #include <linux/of.h>
 34 #include <linux/of_device.h>
 35 #include <linux/platform_device.h>
 36 
 37 #define SHA_BUFFER_LEN          PAGE_SIZE
 38 #define SAHARA_MAX_SHA_BLOCK_SIZE       SHA256_BLOCK_SIZE
 39 
 40 #define SAHARA_NAME "sahara"
 41 #define SAHARA_VERSION_3        3
 42 #define SAHARA_VERSION_4        4
 43 #define SAHARA_TIMEOUT_MS       1000
 44 #define SAHARA_MAX_HW_DESC      2
 45 #define SAHARA_MAX_HW_LINK      20
 46 
 47 #define FLAGS_MODE_MASK         0x000f
 48 #define FLAGS_ENCRYPT           BIT(0)
 49 #define FLAGS_CBC               BIT(1)
 50 #define FLAGS_NEW_KEY           BIT(3)
 51 
 52 #define SAHARA_HDR_BASE                 0x00800000
 53 #define SAHARA_HDR_SKHA_ALG_AES 0
 54 #define SAHARA_HDR_SKHA_OP_ENC          (1 << 2)
 55 #define SAHARA_HDR_SKHA_MODE_ECB        (0 << 3)
 56 #define SAHARA_HDR_SKHA_MODE_CBC        (1 << 3)
 57 #define SAHARA_HDR_FORM_DATA            (5 << 16)
 58 #define SAHARA_HDR_FORM_KEY             (8 << 16)
 59 #define SAHARA_HDR_LLO                  (1 << 24)
 60 #define SAHARA_HDR_CHA_SKHA             (1 << 28)
 61 #define SAHARA_HDR_CHA_MDHA             (2 << 28)
 62 #define SAHARA_HDR_PARITY_BIT           (1 << 31)
 63 
 64 #define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000
 65 #define SAHARA_HDR_MDHA_SET_MODE_HASH   0x208D0000
 66 #define SAHARA_HDR_MDHA_HASH            0xA0850000
 67 #define SAHARA_HDR_MDHA_STORE_DIGEST    0x20820000
 68 #define SAHARA_HDR_MDHA_ALG_SHA1        0
 69 #define SAHARA_HDR_MDHA_ALG_MD5         1
 70 #define SAHARA_HDR_MDHA_ALG_SHA256      2
 71 #define SAHARA_HDR_MDHA_ALG_SHA224      3
 72 #define SAHARA_HDR_MDHA_PDATA           (1 << 2)
 73 #define SAHARA_HDR_MDHA_HMAC            (1 << 3)
 74 #define SAHARA_HDR_MDHA_INIT            (1 << 5)
 75 #define SAHARA_HDR_MDHA_IPAD            (1 << 6)
 76 #define SAHARA_HDR_MDHA_OPAD            (1 << 7)
 77 #define SAHARA_HDR_MDHA_SWAP            (1 << 8)
 78 #define SAHARA_HDR_MDHA_MAC_FULL        (1 << 9)
 79 #define SAHARA_HDR_MDHA_SSL             (1 << 10)
 80 
 81 /* SAHARA can only process one request at a time */
 82 #define SAHARA_QUEUE_LENGTH     1
 83 
 84 #define SAHARA_REG_VERSION      0x00
 85 #define SAHARA_REG_DAR          0x04
 86 #define SAHARA_REG_CONTROL      0x08
 87 #define         SAHARA_CONTROL_SET_THROTTLE(x)  (((x) & 0xff) << 24)
 88 #define         SAHARA_CONTROL_SET_MAXBURST(x)  (((x) & 0xff) << 16)
 89 #define         SAHARA_CONTROL_RNG_AUTORSD      (1 << 7)
 90 #define         SAHARA_CONTROL_ENABLE_INT       (1 << 4)
 91 #define SAHARA_REG_CMD          0x0C
 92 #define         SAHARA_CMD_RESET                (1 << 0)
 93 #define         SAHARA_CMD_CLEAR_INT            (1 << 8)
 94 #define         SAHARA_CMD_CLEAR_ERR            (1 << 9)
 95 #define         SAHARA_CMD_SINGLE_STEP          (1 << 10)
 96 #define         SAHARA_CMD_MODE_BATCH           (1 << 16)
 97 #define         SAHARA_CMD_MODE_DEBUG           (1 << 18)
 98 #define SAHARA_REG_STATUS       0x10
 99 #define         SAHARA_STATUS_GET_STATE(x)      ((x) & 0x7)
100 #define                 SAHARA_STATE_IDLE       0
101 #define                 SAHARA_STATE_BUSY       1
102 #define                 SAHARA_STATE_ERR        2
103 #define                 SAHARA_STATE_FAULT      3
104 #define                 SAHARA_STATE_COMPLETE   4
105 #define                 SAHARA_STATE_COMP_FLAG  (1 << 2)
106 #define         SAHARA_STATUS_DAR_FULL          (1 << 3)
107 #define         SAHARA_STATUS_ERROR             (1 << 4)
108 #define         SAHARA_STATUS_SECURE            (1 << 5)
109 #define         SAHARA_STATUS_FAIL              (1 << 6)
110 #define         SAHARA_STATUS_INIT              (1 << 7)
111 #define         SAHARA_STATUS_RNG_RESEED        (1 << 8)
112 #define         SAHARA_STATUS_ACTIVE_RNG        (1 << 9)
113 #define         SAHARA_STATUS_ACTIVE_MDHA       (1 << 10)
114 #define         SAHARA_STATUS_ACTIVE_SKHA       (1 << 11)
115 #define         SAHARA_STATUS_MODE_BATCH        (1 << 16)
116 #define         SAHARA_STATUS_MODE_DEDICATED    (1 << 17)
117 #define         SAHARA_STATUS_MODE_DEBUG        (1 << 18)
118 #define         SAHARA_STATUS_GET_ISTATE(x)     (((x) >> 24) & 0xff)
119 #define SAHARA_REG_ERRSTATUS    0x14
120 #define         SAHARA_ERRSTATUS_GET_SOURCE(x)  ((x) & 0xf)
121 #define                 SAHARA_ERRSOURCE_CHA    14
122 #define                 SAHARA_ERRSOURCE_DMA    15
123 #define         SAHARA_ERRSTATUS_DMA_DIR        (1 << 8)
124 #define         SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3)
125 #define         SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7)
126 #define         SAHARA_ERRSTATUS_GET_CHASRC(x)  (((x) >> 16) & 0xfff)
127 #define         SAHARA_ERRSTATUS_GET_CHAERR(x)  (((x) >> 28) & 0x3)
128 #define SAHARA_REG_FADDR        0x18
129 #define SAHARA_REG_CDAR         0x1C
130 #define SAHARA_REG_IDAR         0x20
131 
132 struct sahara_hw_desc {
133         u32     hdr;
134         u32     len1;
135         u32     p1;
136         u32     len2;
137         u32     p2;
138         u32     next;
139 };
140 
141 struct sahara_hw_link {
142         u32     len;
143         u32     p;
144         u32     next;
145 };
146 
147 struct sahara_ctx {
148         unsigned long flags;
149 
150         /* AES-specific context */
151         int keylen;
152         u8 key[AES_KEYSIZE_128];
153         struct crypto_ablkcipher *fallback;
154 
155         /* SHA-specific context */
156         struct crypto_shash *shash_fallback;
157 };
158 
159 struct sahara_aes_reqctx {
160         unsigned long mode;
161 };
162 
163 /*
164  * struct sahara_sha_reqctx - private data per request
165  * @buf: holds data for requests smaller than block_size
166  * @rembuf: used to prepare one block_size-aligned request
167  * @context: hw-specific context for request. Digest is extracted from this
168  * @mode: specifies what type of hw-descriptor needs to be built
169  * @digest_size: length of digest for this request
170  * @context_size: length of hw-context for this request.
171  *                Always digest_size + 4
172  * @buf_cnt: number of bytes saved in buf
173  * @sg_in_idx: number of hw links
174  * @in_sg: scatterlist for input data
175  * @in_sg_chain: scatterlists for chained input data
176  * @total: total number of bytes for transfer
177  * @last: is this the last block
178  * @first: is this the first block
179  * @active: inside a transfer
180  */
181 struct sahara_sha_reqctx {
182         u8                      buf[SAHARA_MAX_SHA_BLOCK_SIZE];
183         u8                      rembuf[SAHARA_MAX_SHA_BLOCK_SIZE];
184         u8                      context[SHA256_DIGEST_SIZE + 4];
185         struct mutex            mutex;
186         unsigned int            mode;
187         unsigned int            digest_size;
188         unsigned int            context_size;
189         unsigned int            buf_cnt;
190         unsigned int            sg_in_idx;
191         struct scatterlist      *in_sg;
192         struct scatterlist      in_sg_chain[2];
193         size_t                  total;
194         unsigned int            last;
195         unsigned int            first;
196         unsigned int            active;
197 };
198 
199 struct sahara_dev {
200         struct device           *device;
201         unsigned int            version;
202         void __iomem            *regs_base;
203         struct clk              *clk_ipg;
204         struct clk              *clk_ahb;
205         struct mutex            queue_mutex;
206         struct task_struct      *kthread;
207         struct completion       dma_completion;
208 
209         struct sahara_ctx       *ctx;
210         spinlock_t              lock;
211         struct crypto_queue     queue;
212         unsigned long           flags;
213 
214         struct sahara_hw_desc   *hw_desc[SAHARA_MAX_HW_DESC];
215         dma_addr_t              hw_phys_desc[SAHARA_MAX_HW_DESC];
216 
217         u8                      *key_base;
218         dma_addr_t              key_phys_base;
219 
220         u8                      *iv_base;
221         dma_addr_t              iv_phys_base;
222 
223         u8                      *context_base;
224         dma_addr_t              context_phys_base;
225 
226         struct sahara_hw_link   *hw_link[SAHARA_MAX_HW_LINK];
227         dma_addr_t              hw_phys_link[SAHARA_MAX_HW_LINK];
228 
229         size_t                  total;
230         struct scatterlist      *in_sg;
231         int             nb_in_sg;
232         struct scatterlist      *out_sg;
233         int             nb_out_sg;
234 
235         u32                     error;
236 };
237 
238 static struct sahara_dev *dev_ptr;
239 
240 static inline void sahara_write(struct sahara_dev *dev, u32 data, u32 reg)
241 {
242         writel(data, dev->regs_base + reg);
243 }
244 
245 static inline unsigned int sahara_read(struct sahara_dev *dev, u32 reg)
246 {
247         return readl(dev->regs_base + reg);
248 }
249 
250 static u32 sahara_aes_key_hdr(struct sahara_dev *dev)
251 {
252         u32 hdr = SAHARA_HDR_BASE | SAHARA_HDR_SKHA_ALG_AES |
253                         SAHARA_HDR_FORM_KEY | SAHARA_HDR_LLO |
254                         SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
255 
256         if (dev->flags & FLAGS_CBC) {
257                 hdr |= SAHARA_HDR_SKHA_MODE_CBC;
258                 hdr ^= SAHARA_HDR_PARITY_BIT;
259         }
260 
261         if (dev->flags & FLAGS_ENCRYPT) {
262                 hdr |= SAHARA_HDR_SKHA_OP_ENC;
263                 hdr ^= SAHARA_HDR_PARITY_BIT;
264         }
265 
266         return hdr;
267 }
268 
269 static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev)
270 {
271         return SAHARA_HDR_BASE | SAHARA_HDR_FORM_DATA |
272                         SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
273 }
274 
275 static const char *sahara_err_src[16] = {
276         "No error",
277         "Header error",
278         "Descriptor length error",
279         "Descriptor length or pointer error",
280         "Link length error",
281         "Link pointer error",
282         "Input buffer error",
283         "Output buffer error",
284         "Output buffer starvation",
285         "Internal state fault",
286         "General descriptor problem",
287         "Reserved",
288         "Descriptor address error",
289         "Link address error",
290         "CHA error",
291         "DMA error"
292 };
293 
294 static const char *sahara_err_dmasize[4] = {
295         "Byte transfer",
296         "Half-word transfer",
297         "Word transfer",
298         "Reserved"
299 };
300 
301 static const char *sahara_err_dmasrc[8] = {
302         "No error",
303         "AHB bus error",
304         "Internal IP bus error",
305         "Parity error",
306         "DMA crosses 256 byte boundary",
307         "DMA is busy",
308         "Reserved",
309         "DMA HW error"
310 };
311 
312 static const char *sahara_cha_errsrc[12] = {
313         "Input buffer non-empty",
314         "Illegal address",
315         "Illegal mode",
316         "Illegal data size",
317         "Illegal key size",
318         "Write during processing",
319         "CTX read during processing",
320         "HW error",
321         "Input buffer disabled/underflow",
322         "Output buffer disabled/overflow",
323         "DES key parity error",
324         "Reserved"
325 };
326 
327 static const char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" };
328 
329 static void sahara_decode_error(struct sahara_dev *dev, unsigned int error)
330 {
331         u8 source = SAHARA_ERRSTATUS_GET_SOURCE(error);
332         u16 chasrc = ffs(SAHARA_ERRSTATUS_GET_CHASRC(error));
333 
334         dev_err(dev->device, "%s: Error Register = 0x%08x\n", __func__, error);
335 
336         dev_err(dev->device, "  - %s.\n", sahara_err_src[source]);
337 
338         if (source == SAHARA_ERRSOURCE_DMA) {
339                 if (error & SAHARA_ERRSTATUS_DMA_DIR)
340                         dev_err(dev->device, "          * DMA read.\n");
341                 else
342                         dev_err(dev->device, "          * DMA write.\n");
343 
344                 dev_err(dev->device, "          * %s.\n",
345                        sahara_err_dmasize[SAHARA_ERRSTATUS_GET_DMASZ(error)]);
346                 dev_err(dev->device, "          * %s.\n",
347                        sahara_err_dmasrc[SAHARA_ERRSTATUS_GET_DMASRC(error)]);
348         } else if (source == SAHARA_ERRSOURCE_CHA) {
349                 dev_err(dev->device, "          * %s.\n",
350                         sahara_cha_errsrc[chasrc]);
351                 dev_err(dev->device, "          * %s.\n",
352                        sahara_cha_err[SAHARA_ERRSTATUS_GET_CHAERR(error)]);
353         }
354         dev_err(dev->device, "\n");
355 }
356 
357 static const char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" };
358 
359 static void sahara_decode_status(struct sahara_dev *dev, unsigned int status)
360 {
361         u8 state;
362 
363         if (!IS_ENABLED(DEBUG))
364                 return;
365 
366         state = SAHARA_STATUS_GET_STATE(status);
367 
368         dev_dbg(dev->device, "%s: Status Register = 0x%08x\n",
369                 __func__, status);
370 
371         dev_dbg(dev->device, "  - State = %d:\n", state);
372         if (state & SAHARA_STATE_COMP_FLAG)
373                 dev_dbg(dev->device, "          * Descriptor completed. IRQ pending.\n");
374 
375         dev_dbg(dev->device, "          * %s.\n",
376                sahara_state[state & ~SAHARA_STATE_COMP_FLAG]);
377 
378         if (status & SAHARA_STATUS_DAR_FULL)
379                 dev_dbg(dev->device, "  - DAR Full.\n");
380         if (status & SAHARA_STATUS_ERROR)
381                 dev_dbg(dev->device, "  - Error.\n");
382         if (status & SAHARA_STATUS_SECURE)
383                 dev_dbg(dev->device, "  - Secure.\n");
384         if (status & SAHARA_STATUS_FAIL)
385                 dev_dbg(dev->device, "  - Fail.\n");
386         if (status & SAHARA_STATUS_RNG_RESEED)
387                 dev_dbg(dev->device, "  - RNG Reseed Request.\n");
388         if (status & SAHARA_STATUS_ACTIVE_RNG)
389                 dev_dbg(dev->device, "  - RNG Active.\n");
390         if (status & SAHARA_STATUS_ACTIVE_MDHA)
391                 dev_dbg(dev->device, "  - MDHA Active.\n");
392         if (status & SAHARA_STATUS_ACTIVE_SKHA)
393                 dev_dbg(dev->device, "  - SKHA Active.\n");
394 
395         if (status & SAHARA_STATUS_MODE_BATCH)
396                 dev_dbg(dev->device, "  - Batch Mode.\n");
397         else if (status & SAHARA_STATUS_MODE_DEDICATED)
398                 dev_dbg(dev->device, "  - Decidated Mode.\n");
399         else if (status & SAHARA_STATUS_MODE_DEBUG)
400                 dev_dbg(dev->device, "  - Debug Mode.\n");
401 
402         dev_dbg(dev->device, "  - Internal state = 0x%02x\n",
403                SAHARA_STATUS_GET_ISTATE(status));
404 
405         dev_dbg(dev->device, "Current DAR: 0x%08x\n",
406                 sahara_read(dev, SAHARA_REG_CDAR));
407         dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n",
408                 sahara_read(dev, SAHARA_REG_IDAR));
409 }
410 
411 static void sahara_dump_descriptors(struct sahara_dev *dev)
412 {
413         int i;
414 
415         if (!IS_ENABLED(DEBUG))
416                 return;
417 
418         for (i = 0; i < SAHARA_MAX_HW_DESC; i++) {
419                 dev_dbg(dev->device, "Descriptor (%d) (%pad):\n",
420                         i, &dev->hw_phys_desc[i]);
421                 dev_dbg(dev->device, "\thdr = 0x%08x\n", dev->hw_desc[i]->hdr);
422                 dev_dbg(dev->device, "\tlen1 = %u\n", dev->hw_desc[i]->len1);
423                 dev_dbg(dev->device, "\tp1 = 0x%08x\n", dev->hw_desc[i]->p1);
424                 dev_dbg(dev->device, "\tlen2 = %u\n", dev->hw_desc[i]->len2);
425                 dev_dbg(dev->device, "\tp2 = 0x%08x\n", dev->hw_desc[i]->p2);
426                 dev_dbg(dev->device, "\tnext = 0x%08x\n",
427                         dev->hw_desc[i]->next);
428         }
429         dev_dbg(dev->device, "\n");
430 }
431 
432 static void sahara_dump_links(struct sahara_dev *dev)
433 {
434         int i;
435 
436         if (!IS_ENABLED(DEBUG))
437                 return;
438 
439         for (i = 0; i < SAHARA_MAX_HW_LINK; i++) {
440                 dev_dbg(dev->device, "Link (%d) (%pad):\n",
441                         i, &dev->hw_phys_link[i]);
442                 dev_dbg(dev->device, "\tlen = %u\n", dev->hw_link[i]->len);
443                 dev_dbg(dev->device, "\tp = 0x%08x\n", dev->hw_link[i]->p);
444                 dev_dbg(dev->device, "\tnext = 0x%08x\n",
445                         dev->hw_link[i]->next);
446         }
447         dev_dbg(dev->device, "\n");
448 }
449 
450 static int sahara_hw_descriptor_create(struct sahara_dev *dev)
451 {
452         struct sahara_ctx *ctx = dev->ctx;
453         struct scatterlist *sg;
454         int ret;
455         int i, j;
456         int idx = 0;
457 
458         /* Copy new key if necessary */
459         if (ctx->flags & FLAGS_NEW_KEY) {
460                 memcpy(dev->key_base, ctx->key, ctx->keylen);
461                 ctx->flags &= ~FLAGS_NEW_KEY;
462 
463                 if (dev->flags & FLAGS_CBC) {
464                         dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE;
465                         dev->hw_desc[idx]->p1 = dev->iv_phys_base;
466                 } else {
467                         dev->hw_desc[idx]->len1 = 0;
468                         dev->hw_desc[idx]->p1 = 0;
469                 }
470                 dev->hw_desc[idx]->len2 = ctx->keylen;
471                 dev->hw_desc[idx]->p2 = dev->key_phys_base;
472                 dev->hw_desc[idx]->next = dev->hw_phys_desc[1];
473 
474                 dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev);
475 
476                 idx++;
477         }
478 
479         dev->nb_in_sg = sg_nents_for_len(dev->in_sg, dev->total);
480         if (dev->nb_in_sg < 0) {
481                 dev_err(dev->device, "Invalid numbers of src SG.\n");
482                 return dev->nb_in_sg;
483         }
484         dev->nb_out_sg = sg_nents_for_len(dev->out_sg, dev->total);
485         if (dev->nb_out_sg < 0) {
486                 dev_err(dev->device, "Invalid numbers of dst SG.\n");
487                 return dev->nb_out_sg;
488         }
489         if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) {
490                 dev_err(dev->device, "not enough hw links (%d)\n",
491                         dev->nb_in_sg + dev->nb_out_sg);
492                 return -EINVAL;
493         }
494 
495         ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg,
496                          DMA_TO_DEVICE);
497         if (ret != dev->nb_in_sg) {
498                 dev_err(dev->device, "couldn't map in sg\n");
499                 goto unmap_in;
500         }
501         ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg,
502                          DMA_FROM_DEVICE);
503         if (ret != dev->nb_out_sg) {
504                 dev_err(dev->device, "couldn't map out sg\n");
505                 goto unmap_out;
506         }
507 
508         /* Create input links */
509         dev->hw_desc[idx]->p1 = dev->hw_phys_link[0];
510         sg = dev->in_sg;
511         for (i = 0; i < dev->nb_in_sg; i++) {
512                 dev->hw_link[i]->len = sg->length;
513                 dev->hw_link[i]->p = sg->dma_address;
514                 if (i == (dev->nb_in_sg - 1)) {
515                         dev->hw_link[i]->next = 0;
516                 } else {
517                         dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
518                         sg = sg_next(sg);
519                 }
520         }
521 
522         /* Create output links */
523         dev->hw_desc[idx]->p2 = dev->hw_phys_link[i];
524         sg = dev->out_sg;
525         for (j = i; j < dev->nb_out_sg + i; j++) {
526                 dev->hw_link[j]->len = sg->length;
527                 dev->hw_link[j]->p = sg->dma_address;
528                 if (j == (dev->nb_out_sg + i - 1)) {
529                         dev->hw_link[j]->next = 0;
530                 } else {
531                         dev->hw_link[j]->next = dev->hw_phys_link[j + 1];
532                         sg = sg_next(sg);
533                 }
534         }
535 
536         /* Fill remaining fields of hw_desc[1] */
537         dev->hw_desc[idx]->hdr = sahara_aes_data_link_hdr(dev);
538         dev->hw_desc[idx]->len1 = dev->total;
539         dev->hw_desc[idx]->len2 = dev->total;
540         dev->hw_desc[idx]->next = 0;
541 
542         sahara_dump_descriptors(dev);
543         sahara_dump_links(dev);
544 
545         sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
546 
547         return 0;
548 
549 unmap_out:
550         dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
551                 DMA_TO_DEVICE);
552 unmap_in:
553         dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
554                 DMA_FROM_DEVICE);
555 
556         return -EINVAL;
557 }
558 
559 static int sahara_aes_process(struct ablkcipher_request *req)
560 {
561         struct sahara_dev *dev = dev_ptr;
562         struct sahara_ctx *ctx;
563         struct sahara_aes_reqctx *rctx;
564         int ret;
565         unsigned long timeout;
566 
567         /* Request is ready to be dispatched by the device */
568         dev_dbg(dev->device,
569                 "dispatch request (nbytes=%d, src=%p, dst=%p)\n",
570                 req->nbytes, req->src, req->dst);
571 
572         /* assign new request to device */
573         dev->total = req->nbytes;
574         dev->in_sg = req->src;
575         dev->out_sg = req->dst;
576 
577         rctx = ablkcipher_request_ctx(req);
578         ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
579         rctx->mode &= FLAGS_MODE_MASK;
580         dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode;
581 
582         if ((dev->flags & FLAGS_CBC) && req->info)
583                 memcpy(dev->iv_base, req->info, AES_KEYSIZE_128);
584 
585         /* assign new context to device */
586         dev->ctx = ctx;
587 
588         reinit_completion(&dev->dma_completion);
589 
590         ret = sahara_hw_descriptor_create(dev);
591         if (ret)
592                 return -EINVAL;
593 
594         timeout = wait_for_completion_timeout(&dev->dma_completion,
595                                 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
596         if (!timeout) {
597                 dev_err(dev->device, "AES timeout\n");
598                 return -ETIMEDOUT;
599         }
600 
601         dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
602                 DMA_TO_DEVICE);
603         dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
604                 DMA_FROM_DEVICE);
605 
606         return 0;
607 }
608 
609 static int sahara_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
610                              unsigned int keylen)
611 {
612         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(tfm);
613         int ret;
614 
615         ctx->keylen = keylen;
616 
617         /* SAHARA only supports 128bit keys */
618         if (keylen == AES_KEYSIZE_128) {
619                 memcpy(ctx->key, key, keylen);
620                 ctx->flags |= FLAGS_NEW_KEY;
621                 return 0;
622         }
623 
624         if (keylen != AES_KEYSIZE_128 &&
625             keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256)
626                 return -EINVAL;
627 
628         /*
629          * The requested key size is not supported by HW, do a fallback.
630          */
631         ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
632         ctx->fallback->base.crt_flags |=
633                 (tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
634 
635         ret = crypto_ablkcipher_setkey(ctx->fallback, key, keylen);
636         if (ret) {
637                 struct crypto_tfm *tfm_aux = crypto_ablkcipher_tfm(tfm);
638 
639                 tfm_aux->crt_flags &= ~CRYPTO_TFM_RES_MASK;
640                 tfm_aux->crt_flags |=
641                         (ctx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK);
642         }
643         return ret;
644 }
645 
646 static int sahara_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
647 {
648         struct sahara_aes_reqctx *rctx = ablkcipher_request_ctx(req);
649         struct sahara_dev *dev = dev_ptr;
650         int err = 0;
651 
652         dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n",
653                 req->nbytes, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC));
654 
655         if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
656                 dev_err(dev->device,
657                         "request size is not exact amount of AES blocks\n");
658                 return -EINVAL;
659         }
660 
661         rctx->mode = mode;
662 
663         mutex_lock(&dev->queue_mutex);
664         err = ablkcipher_enqueue_request(&dev->queue, req);
665         mutex_unlock(&dev->queue_mutex);
666 
667         wake_up_process(dev->kthread);
668 
669         return err;
670 }
671 
672 static int sahara_aes_ecb_encrypt(struct ablkcipher_request *req)
673 {
674         struct crypto_tfm *tfm =
675                 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
676         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
677                 crypto_ablkcipher_reqtfm(req));
678         int err;
679 
680         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
681                 ablkcipher_request_set_tfm(req, ctx->fallback);
682                 err = crypto_ablkcipher_encrypt(req);
683                 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
684                 return err;
685         }
686 
687         return sahara_aes_crypt(req, FLAGS_ENCRYPT);
688 }
689 
690 static int sahara_aes_ecb_decrypt(struct ablkcipher_request *req)
691 {
692         struct crypto_tfm *tfm =
693                 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
694         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
695                 crypto_ablkcipher_reqtfm(req));
696         int err;
697 
698         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
699                 ablkcipher_request_set_tfm(req, ctx->fallback);
700                 err = crypto_ablkcipher_decrypt(req);
701                 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
702                 return err;
703         }
704 
705         return sahara_aes_crypt(req, 0);
706 }
707 
708 static int sahara_aes_cbc_encrypt(struct ablkcipher_request *req)
709 {
710         struct crypto_tfm *tfm =
711                 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
712         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
713                 crypto_ablkcipher_reqtfm(req));
714         int err;
715 
716         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
717                 ablkcipher_request_set_tfm(req, ctx->fallback);
718                 err = crypto_ablkcipher_encrypt(req);
719                 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
720                 return err;
721         }
722 
723         return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
724 }
725 
726 static int sahara_aes_cbc_decrypt(struct ablkcipher_request *req)
727 {
728         struct crypto_tfm *tfm =
729                 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
730         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
731                 crypto_ablkcipher_reqtfm(req));
732         int err;
733 
734         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
735                 ablkcipher_request_set_tfm(req, ctx->fallback);
736                 err = crypto_ablkcipher_decrypt(req);
737                 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
738                 return err;
739         }
740 
741         return sahara_aes_crypt(req, FLAGS_CBC);
742 }
743 
744 static int sahara_aes_cra_init(struct crypto_tfm *tfm)
745 {
746         const char *name = crypto_tfm_alg_name(tfm);
747         struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
748 
749         ctx->fallback = crypto_alloc_ablkcipher(name, 0,
750                                 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
751         if (IS_ERR(ctx->fallback)) {
752                 pr_err("Error allocating fallback algo %s\n", name);
753                 return PTR_ERR(ctx->fallback);
754         }
755 
756         tfm->crt_ablkcipher.reqsize = sizeof(struct sahara_aes_reqctx);
757 
758         return 0;
759 }
760 
761 static void sahara_aes_cra_exit(struct crypto_tfm *tfm)
762 {
763         struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
764 
765         if (ctx->fallback)
766                 crypto_free_ablkcipher(ctx->fallback);
767         ctx->fallback = NULL;
768 }
769 
770 static u32 sahara_sha_init_hdr(struct sahara_dev *dev,
771                               struct sahara_sha_reqctx *rctx)
772 {
773         u32 hdr = 0;
774 
775         hdr = rctx->mode;
776 
777         if (rctx->first) {
778                 hdr |= SAHARA_HDR_MDHA_SET_MODE_HASH;
779                 hdr |= SAHARA_HDR_MDHA_INIT;
780         } else {
781                 hdr |= SAHARA_HDR_MDHA_SET_MODE_MD_KEY;
782         }
783 
784         if (rctx->last)
785                 hdr |= SAHARA_HDR_MDHA_PDATA;
786 
787         if (hweight_long(hdr) % 2 == 0)
788                 hdr |= SAHARA_HDR_PARITY_BIT;
789 
790         return hdr;
791 }
792 
793 static int sahara_sha_hw_links_create(struct sahara_dev *dev,
794                                        struct sahara_sha_reqctx *rctx,
795                                        int start)
796 {
797         struct scatterlist *sg;
798         unsigned int i;
799         int ret;
800 
801         dev->in_sg = rctx->in_sg;
802 
803         dev->nb_in_sg = sg_nents_for_len(dev->in_sg, rctx->total);
804         if (dev->nb_in_sg < 0) {
805                 dev_err(dev->device, "Invalid numbers of src SG.\n");
806                 return dev->nb_in_sg;
807         }
808         if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) {
809                 dev_err(dev->device, "not enough hw links (%d)\n",
810                         dev->nb_in_sg + dev->nb_out_sg);
811                 return -EINVAL;
812         }
813 
814         sg = dev->in_sg;
815         ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE);
816         if (!ret)
817                 return -EFAULT;
818 
819         for (i = start; i < dev->nb_in_sg + start; i++) {
820                 dev->hw_link[i]->len = sg->length;
821                 dev->hw_link[i]->p = sg->dma_address;
822                 if (i == (dev->nb_in_sg + start - 1)) {
823                         dev->hw_link[i]->next = 0;
824                 } else {
825                         dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
826                         sg = sg_next(sg);
827                 }
828         }
829 
830         return i;
831 }
832 
833 static int sahara_sha_hw_data_descriptor_create(struct sahara_dev *dev,
834                                                 struct sahara_sha_reqctx *rctx,
835                                                 struct ahash_request *req,
836                                                 int index)
837 {
838         unsigned result_len;
839         int i = index;
840 
841         if (rctx->first)
842                 /* Create initial descriptor: #8*/
843                 dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
844         else
845                 /* Create hash descriptor: #10. Must follow #6. */
846                 dev->hw_desc[index]->hdr = SAHARA_HDR_MDHA_HASH;
847 
848         dev->hw_desc[index]->len1 = rctx->total;
849         if (dev->hw_desc[index]->len1 == 0) {
850                 /* if len1 is 0, p1 must be 0, too */
851                 dev->hw_desc[index]->p1 = 0;
852                 rctx->sg_in_idx = 0;
853         } else {
854                 /* Create input links */
855                 dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
856                 i = sahara_sha_hw_links_create(dev, rctx, index);
857 
858                 rctx->sg_in_idx = index;
859                 if (i < 0)
860                         return i;
861         }
862 
863         dev->hw_desc[index]->p2 = dev->hw_phys_link[i];
864 
865         /* Save the context for the next operation */
866         result_len = rctx->context_size;
867         dev->hw_link[i]->p = dev->context_phys_base;
868 
869         dev->hw_link[i]->len = result_len;
870         dev->hw_desc[index]->len2 = result_len;
871 
872         dev->hw_link[i]->next = 0;
873 
874         return 0;
875 }
876 
877 /*
878  * Load descriptor aka #6
879  *
880  * To load a previously saved context back to the MDHA unit
881  *
882  * p1: Saved Context
883  * p2: NULL
884  *
885  */
886 static int sahara_sha_hw_context_descriptor_create(struct sahara_dev *dev,
887                                                 struct sahara_sha_reqctx *rctx,
888                                                 struct ahash_request *req,
889                                                 int index)
890 {
891         dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
892 
893         dev->hw_desc[index]->len1 = rctx->context_size;
894         dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
895         dev->hw_desc[index]->len2 = 0;
896         dev->hw_desc[index]->p2 = 0;
897 
898         dev->hw_link[index]->len = rctx->context_size;
899         dev->hw_link[index]->p = dev->context_phys_base;
900         dev->hw_link[index]->next = 0;
901 
902         return 0;
903 }
904 
905 static int sahara_walk_and_recalc(struct scatterlist *sg, unsigned int nbytes)
906 {
907         if (!sg || !sg->length)
908                 return nbytes;
909 
910         while (nbytes && sg) {
911                 if (nbytes <= sg->length) {
912                         sg->length = nbytes;
913                         sg_mark_end(sg);
914                         break;
915                 }
916                 nbytes -= sg->length;
917                 sg = sg_next(sg);
918         }
919 
920         return nbytes;
921 }
922 
923 static int sahara_sha_prepare_request(struct ahash_request *req)
924 {
925         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
926         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
927         unsigned int hash_later;
928         unsigned int block_size;
929         unsigned int len;
930 
931         block_size = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
932 
933         /* append bytes from previous operation */
934         len = rctx->buf_cnt + req->nbytes;
935 
936         /* only the last transfer can be padded in hardware */
937         if (!rctx->last && (len < block_size)) {
938                 /* to few data, save for next operation */
939                 scatterwalk_map_and_copy(rctx->buf + rctx->buf_cnt, req->src,
940                                          0, req->nbytes, 0);
941                 rctx->buf_cnt += req->nbytes;
942 
943                 return 0;
944         }
945 
946         /* add data from previous operation first */
947         if (rctx->buf_cnt)
948                 memcpy(rctx->rembuf, rctx->buf, rctx->buf_cnt);
949 
950         /* data must always be a multiple of block_size */
951         hash_later = rctx->last ? 0 : len & (block_size - 1);
952         if (hash_later) {
953                 unsigned int offset = req->nbytes - hash_later;
954                 /* Save remaining bytes for later use */
955                 scatterwalk_map_and_copy(rctx->buf, req->src, offset,
956                                         hash_later, 0);
957         }
958 
959         /* nbytes should now be multiple of blocksize */
960         req->nbytes = req->nbytes - hash_later;
961 
962         sahara_walk_and_recalc(req->src, req->nbytes);
963 
964         /* have data from previous operation and current */
965         if (rctx->buf_cnt && req->nbytes) {
966                 sg_init_table(rctx->in_sg_chain, 2);
967                 sg_set_buf(rctx->in_sg_chain, rctx->rembuf, rctx->buf_cnt);
968 
969                 sg_chain(rctx->in_sg_chain, 2, req->src);
970 
971                 rctx->total = req->nbytes + rctx->buf_cnt;
972                 rctx->in_sg = rctx->in_sg_chain;
973 
974                 req->src = rctx->in_sg_chain;
975         /* only data from previous operation */
976         } else if (rctx->buf_cnt) {
977                 if (req->src)
978                         rctx->in_sg = req->src;
979                 else
980                         rctx->in_sg = rctx->in_sg_chain;
981                 /* buf was copied into rembuf above */
982                 sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt);
983                 rctx->total = rctx->buf_cnt;
984         /* no data from previous operation */
985         } else {
986                 rctx->in_sg = req->src;
987                 rctx->total = req->nbytes;
988                 req->src = rctx->in_sg;
989         }
990 
991         /* on next call, we only have the remaining data in the buffer */
992         rctx->buf_cnt = hash_later;
993 
994         return -EINPROGRESS;
995 }
996 
997 static int sahara_sha_process(struct ahash_request *req)
998 {
999         struct sahara_dev *dev = dev_ptr;
1000         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1001         int ret;
1002         unsigned long timeout;
1003 
1004         ret = sahara_sha_prepare_request(req);
1005         if (!ret)
1006                 return ret;
1007 
1008         if (rctx->first) {
1009                 sahara_sha_hw_data_descriptor_create(dev, rctx, req, 0);
1010                 dev->hw_desc[0]->next = 0;
1011                 rctx->first = 0;
1012         } else {
1013                 memcpy(dev->context_base, rctx->context, rctx->context_size);
1014 
1015                 sahara_sha_hw_context_descriptor_create(dev, rctx, req, 0);
1016                 dev->hw_desc[0]->next = dev->hw_phys_desc[1];
1017                 sahara_sha_hw_data_descriptor_create(dev, rctx, req, 1);
1018                 dev->hw_desc[1]->next = 0;
1019         }
1020 
1021         sahara_dump_descriptors(dev);
1022         sahara_dump_links(dev);
1023 
1024         reinit_completion(&dev->dma_completion);
1025 
1026         sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
1027 
1028         timeout = wait_for_completion_timeout(&dev->dma_completion,
1029                                 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
1030         if (!timeout) {
1031                 dev_err(dev->device, "SHA timeout\n");
1032                 return -ETIMEDOUT;
1033         }
1034 
1035         if (rctx->sg_in_idx)
1036                 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
1037                              DMA_TO_DEVICE);
1038 
1039         memcpy(rctx->context, dev->context_base, rctx->context_size);
1040 
1041         if (req->result)
1042                 memcpy(req->result, rctx->context, rctx->digest_size);
1043 
1044         return 0;
1045 }
1046 
1047 static int sahara_queue_manage(void *data)
1048 {
1049         struct sahara_dev *dev = (struct sahara_dev *)data;
1050         struct crypto_async_request *async_req;
1051         struct crypto_async_request *backlog;
1052         int ret = 0;
1053 
1054         do {
1055                 __set_current_state(TASK_INTERRUPTIBLE);
1056 
1057                 mutex_lock(&dev->queue_mutex);
1058                 backlog = crypto_get_backlog(&dev->queue);
1059                 async_req = crypto_dequeue_request(&dev->queue);
1060                 mutex_unlock(&dev->queue_mutex);
1061 
1062                 if (backlog)
1063                         backlog->complete(backlog, -EINPROGRESS);
1064 
1065                 if (async_req) {
1066                         if (crypto_tfm_alg_type(async_req->tfm) ==
1067                             CRYPTO_ALG_TYPE_AHASH) {
1068                                 struct ahash_request *req =
1069                                         ahash_request_cast(async_req);
1070 
1071                                 ret = sahara_sha_process(req);
1072                         } else {
1073                                 struct ablkcipher_request *req =
1074                                         ablkcipher_request_cast(async_req);
1075 
1076                                 ret = sahara_aes_process(req);
1077                         }
1078 
1079                         async_req->complete(async_req, ret);
1080 
1081                         continue;
1082                 }
1083 
1084                 schedule();
1085         } while (!kthread_should_stop());
1086 
1087         return 0;
1088 }
1089 
1090 static int sahara_sha_enqueue(struct ahash_request *req, int last)
1091 {
1092         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1093         struct sahara_dev *dev = dev_ptr;
1094         int ret;
1095 
1096         if (!req->nbytes && !last)
1097                 return 0;
1098 
1099         mutex_lock(&rctx->mutex);
1100         rctx->last = last;
1101 
1102         if (!rctx->active) {
1103                 rctx->active = 1;
1104                 rctx->first = 1;
1105         }
1106 
1107         mutex_lock(&dev->queue_mutex);
1108         ret = crypto_enqueue_request(&dev->queue, &req->base);
1109         mutex_unlock(&dev->queue_mutex);
1110 
1111         wake_up_process(dev->kthread);
1112         mutex_unlock(&rctx->mutex);
1113 
1114         return ret;
1115 }
1116 
1117 static int sahara_sha_init(struct ahash_request *req)
1118 {
1119         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1120         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1121 
1122         memset(rctx, 0, sizeof(*rctx));
1123 
1124         switch (crypto_ahash_digestsize(tfm)) {
1125         case SHA1_DIGEST_SIZE:
1126                 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA1;
1127                 rctx->digest_size = SHA1_DIGEST_SIZE;
1128                 break;
1129         case SHA256_DIGEST_SIZE:
1130                 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA256;
1131                 rctx->digest_size = SHA256_DIGEST_SIZE;
1132                 break;
1133         default:
1134                 return -EINVAL;
1135         }
1136 
1137         rctx->context_size = rctx->digest_size + 4;
1138         rctx->active = 0;
1139 
1140         mutex_init(&rctx->mutex);
1141 
1142         return 0;
1143 }
1144 
1145 static int sahara_sha_update(struct ahash_request *req)
1146 {
1147         return sahara_sha_enqueue(req, 0);
1148 }
1149 
1150 static int sahara_sha_final(struct ahash_request *req)
1151 {
1152         req->nbytes = 0;
1153         return sahara_sha_enqueue(req, 1);
1154 }
1155 
1156 static int sahara_sha_finup(struct ahash_request *req)
1157 {
1158         return sahara_sha_enqueue(req, 1);
1159 }
1160 
1161 static int sahara_sha_digest(struct ahash_request *req)
1162 {
1163         sahara_sha_init(req);
1164 
1165         return sahara_sha_finup(req);
1166 }
1167 
1168 static int sahara_sha_export(struct ahash_request *req, void *out)
1169 {
1170         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1171         struct sahara_ctx *ctx = crypto_ahash_ctx(ahash);
1172         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1173 
1174         memcpy(out, ctx, sizeof(struct sahara_ctx));
1175         memcpy(out + sizeof(struct sahara_sha_reqctx), rctx,
1176                sizeof(struct sahara_sha_reqctx));
1177 
1178         return 0;
1179 }
1180 
1181 static int sahara_sha_import(struct ahash_request *req, const void *in)
1182 {
1183         struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1184         struct sahara_ctx *ctx = crypto_ahash_ctx(ahash);
1185         struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1186 
1187         memcpy(ctx, in, sizeof(struct sahara_ctx));
1188         memcpy(rctx, in + sizeof(struct sahara_sha_reqctx),
1189                sizeof(struct sahara_sha_reqctx));
1190 
1191         return 0;
1192 }
1193 
1194 static int sahara_sha_cra_init(struct crypto_tfm *tfm)
1195 {
1196         const char *name = crypto_tfm_alg_name(tfm);
1197         struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
1198 
1199         ctx->shash_fallback = crypto_alloc_shash(name, 0,
1200                                         CRYPTO_ALG_NEED_FALLBACK);
1201         if (IS_ERR(ctx->shash_fallback)) {
1202                 pr_err("Error allocating fallback algo %s\n", name);
1203                 return PTR_ERR(ctx->shash_fallback);
1204         }
1205         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1206                                  sizeof(struct sahara_sha_reqctx) +
1207                                  SHA_BUFFER_LEN + SHA256_BLOCK_SIZE);
1208 
1209         return 0;
1210 }
1211 
1212 static void sahara_sha_cra_exit(struct crypto_tfm *tfm)
1213 {
1214         struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
1215 
1216         crypto_free_shash(ctx->shash_fallback);
1217         ctx->shash_fallback = NULL;
1218 }
1219 
1220 static struct crypto_alg aes_algs[] = {
1221 {
1222         .cra_name               = "ecb(aes)",
1223         .cra_driver_name        = "sahara-ecb-aes",
1224         .cra_priority           = 300,
1225         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
1226                         CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1227         .cra_blocksize          = AES_BLOCK_SIZE,
1228         .cra_ctxsize            = sizeof(struct sahara_ctx),
1229         .cra_alignmask          = 0x0,
1230         .cra_type               = &crypto_ablkcipher_type,
1231         .cra_module             = THIS_MODULE,
1232         .cra_init               = sahara_aes_cra_init,
1233         .cra_exit               = sahara_aes_cra_exit,
1234         .cra_u.ablkcipher = {
1235                 .min_keysize    = AES_MIN_KEY_SIZE ,
1236                 .max_keysize    = AES_MAX_KEY_SIZE,
1237                 .setkey         = sahara_aes_setkey,
1238                 .encrypt        = sahara_aes_ecb_encrypt,
1239                 .decrypt        = sahara_aes_ecb_decrypt,
1240         }
1241 }, {
1242         .cra_name               = "cbc(aes)",
1243         .cra_driver_name        = "sahara-cbc-aes",
1244         .cra_priority           = 300,
1245         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
1246                         CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1247         .cra_blocksize          = AES_BLOCK_SIZE,
1248         .cra_ctxsize            = sizeof(struct sahara_ctx),
1249         .cra_alignmask          = 0x0,
1250         .cra_type               = &crypto_ablkcipher_type,
1251         .cra_module             = THIS_MODULE,
1252         .cra_init               = sahara_aes_cra_init,
1253         .cra_exit               = sahara_aes_cra_exit,
1254         .cra_u.ablkcipher = {
1255                 .min_keysize    = AES_MIN_KEY_SIZE ,
1256                 .max_keysize    = AES_MAX_KEY_SIZE,
1257                 .ivsize         = AES_BLOCK_SIZE,
1258                 .setkey         = sahara_aes_setkey,
1259                 .encrypt        = sahara_aes_cbc_encrypt,
1260                 .decrypt        = sahara_aes_cbc_decrypt,
1261         }
1262 }
1263 };
1264 
1265 static struct ahash_alg sha_v3_algs[] = {
1266 {
1267         .init           = sahara_sha_init,
1268         .update         = sahara_sha_update,
1269         .final          = sahara_sha_final,
1270         .finup          = sahara_sha_finup,
1271         .digest         = sahara_sha_digest,
1272         .export         = sahara_sha_export,
1273         .import         = sahara_sha_import,
1274         .halg.digestsize        = SHA1_DIGEST_SIZE,
1275         .halg.base      = {
1276                 .cra_name               = "sha1",
1277                 .cra_driver_name        = "sahara-sha1",
1278                 .cra_priority           = 300,
1279                 .cra_flags              = CRYPTO_ALG_TYPE_AHASH |
1280                                                 CRYPTO_ALG_ASYNC |
1281                                                 CRYPTO_ALG_NEED_FALLBACK,
1282                 .cra_blocksize          = SHA1_BLOCK_SIZE,
1283                 .cra_ctxsize            = sizeof(struct sahara_ctx),
1284                 .cra_alignmask          = 0,
1285                 .cra_module             = THIS_MODULE,
1286                 .cra_init               = sahara_sha_cra_init,
1287                 .cra_exit               = sahara_sha_cra_exit,
1288         }
1289 },
1290 };
1291 
1292 static struct ahash_alg sha_v4_algs[] = {
1293 {
1294         .init           = sahara_sha_init,
1295         .update         = sahara_sha_update,
1296         .final          = sahara_sha_final,
1297         .finup          = sahara_sha_finup,
1298         .digest         = sahara_sha_digest,
1299         .export         = sahara_sha_export,
1300         .import         = sahara_sha_import,
1301         .halg.digestsize        = SHA256_DIGEST_SIZE,
1302         .halg.base      = {
1303                 .cra_name               = "sha256",
1304                 .cra_driver_name        = "sahara-sha256",
1305                 .cra_priority           = 300,
1306                 .cra_flags              = CRYPTO_ALG_TYPE_AHASH |
1307                                                 CRYPTO_ALG_ASYNC |
1308                                                 CRYPTO_ALG_NEED_FALLBACK,
1309                 .cra_blocksize          = SHA256_BLOCK_SIZE,
1310                 .cra_ctxsize            = sizeof(struct sahara_ctx),
1311                 .cra_alignmask          = 0,
1312                 .cra_module             = THIS_MODULE,
1313                 .cra_init               = sahara_sha_cra_init,
1314                 .cra_exit               = sahara_sha_cra_exit,
1315         }
1316 },
1317 };
1318 
1319 static irqreturn_t sahara_irq_handler(int irq, void *data)
1320 {
1321         struct sahara_dev *dev = (struct sahara_dev *)data;
1322         unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS);
1323         unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS);
1324 
1325         sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR,
1326                      SAHARA_REG_CMD);
1327 
1328         sahara_decode_status(dev, stat);
1329 
1330         if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) {
1331                 return IRQ_NONE;
1332         } else if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_COMPLETE) {
1333                 dev->error = 0;
1334         } else {
1335                 sahara_decode_error(dev, err);
1336                 dev->error = -EINVAL;
1337         }
1338 
1339         complete(&dev->dma_completion);
1340 
1341         return IRQ_HANDLED;
1342 }
1343 
1344 
1345 static int sahara_register_algs(struct sahara_dev *dev)
1346 {
1347         int err;
1348         unsigned int i, j, k, l;
1349 
1350         for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
1351                 INIT_LIST_HEAD(&aes_algs[i].cra_list);
1352                 err = crypto_register_alg(&aes_algs[i]);
1353                 if (err)
1354                         goto err_aes_algs;
1355         }
1356 
1357         for (k = 0; k < ARRAY_SIZE(sha_v3_algs); k++) {
1358                 err = crypto_register_ahash(&sha_v3_algs[k]);
1359                 if (err)
1360                         goto err_sha_v3_algs;
1361         }
1362 
1363         if (dev->version > SAHARA_VERSION_3)
1364                 for (l = 0; l < ARRAY_SIZE(sha_v4_algs); l++) {
1365                         err = crypto_register_ahash(&sha_v4_algs[l]);
1366                         if (err)
1367                                 goto err_sha_v4_algs;
1368                 }
1369 
1370         return 0;
1371 
1372 err_sha_v4_algs:
1373         for (j = 0; j < l; j++)
1374                 crypto_unregister_ahash(&sha_v4_algs[j]);
1375 
1376 err_sha_v3_algs:
1377         for (j = 0; j < k; j++)
1378                 crypto_unregister_ahash(&sha_v4_algs[j]);
1379 
1380 err_aes_algs:
1381         for (j = 0; j < i; j++)
1382                 crypto_unregister_alg(&aes_algs[j]);
1383 
1384         return err;
1385 }
1386 
1387 static void sahara_unregister_algs(struct sahara_dev *dev)
1388 {
1389         unsigned int i;
1390 
1391         for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
1392                 crypto_unregister_alg(&aes_algs[i]);
1393 
1394         for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++)
1395                 crypto_unregister_ahash(&sha_v3_algs[i]);
1396 
1397         if (dev->version > SAHARA_VERSION_3)
1398                 for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++)
1399                         crypto_unregister_ahash(&sha_v4_algs[i]);
1400 }
1401 
1402 static struct platform_device_id sahara_platform_ids[] = {
1403         { .name = "sahara-imx27" },
1404         { /* sentinel */ }
1405 };
1406 MODULE_DEVICE_TABLE(platform, sahara_platform_ids);
1407 
1408 static struct of_device_id sahara_dt_ids[] = {
1409         { .compatible = "fsl,imx53-sahara" },
1410         { .compatible = "fsl,imx27-sahara" },
1411         { /* sentinel */ }
1412 };
1413 MODULE_DEVICE_TABLE(of, sahara_dt_ids);
1414 
1415 static int sahara_probe(struct platform_device *pdev)
1416 {
1417         struct sahara_dev *dev;
1418         struct resource *res;
1419         u32 version;
1420         int irq;
1421         int err;
1422         int i;
1423 
1424         dev = devm_kzalloc(&pdev->dev, sizeof(struct sahara_dev), GFP_KERNEL);
1425         if (dev == NULL) {
1426                 dev_err(&pdev->dev, "unable to alloc data struct.\n");
1427                 return -ENOMEM;
1428         }
1429 
1430         dev->device = &pdev->dev;
1431         platform_set_drvdata(pdev, dev);
1432 
1433         /* Get the base address */
1434         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1435         dev->regs_base = devm_ioremap_resource(&pdev->dev, res);
1436         if (IS_ERR(dev->regs_base))
1437                 return PTR_ERR(dev->regs_base);
1438 
1439         /* Get the IRQ */
1440         irq = platform_get_irq(pdev,  0);
1441         if (irq < 0) {
1442                 dev_err(&pdev->dev, "failed to get irq resource\n");
1443                 return irq;
1444         }
1445 
1446         err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler,
1447                                0, dev_name(&pdev->dev), dev);
1448         if (err) {
1449                 dev_err(&pdev->dev, "failed to request irq\n");
1450                 return err;
1451         }
1452 
1453         /* clocks */
1454         dev->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
1455         if (IS_ERR(dev->clk_ipg)) {
1456                 dev_err(&pdev->dev, "Could not get ipg clock\n");
1457                 return PTR_ERR(dev->clk_ipg);
1458         }
1459 
1460         dev->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
1461         if (IS_ERR(dev->clk_ahb)) {
1462                 dev_err(&pdev->dev, "Could not get ahb clock\n");
1463                 return PTR_ERR(dev->clk_ahb);
1464         }
1465 
1466         /* Allocate HW descriptors */
1467         dev->hw_desc[0] = dmam_alloc_coherent(&pdev->dev,
1468                         SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1469                         &dev->hw_phys_desc[0], GFP_KERNEL);
1470         if (!dev->hw_desc[0]) {
1471                 dev_err(&pdev->dev, "Could not allocate hw descriptors\n");
1472                 return -ENOMEM;
1473         }
1474         dev->hw_desc[1] = dev->hw_desc[0] + 1;
1475         dev->hw_phys_desc[1] = dev->hw_phys_desc[0] +
1476                                 sizeof(struct sahara_hw_desc);
1477 
1478         /* Allocate space for iv and key */
1479         dev->key_base = dmam_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128,
1480                                 &dev->key_phys_base, GFP_KERNEL);
1481         if (!dev->key_base) {
1482                 dev_err(&pdev->dev, "Could not allocate memory for key\n");
1483                 return -ENOMEM;
1484         }
1485         dev->iv_base = dev->key_base + AES_KEYSIZE_128;
1486         dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128;
1487 
1488         /* Allocate space for context: largest digest + message length field */
1489         dev->context_base = dmam_alloc_coherent(&pdev->dev,
1490                                         SHA256_DIGEST_SIZE + 4,
1491                                         &dev->context_phys_base, GFP_KERNEL);
1492         if (!dev->context_base) {
1493                 dev_err(&pdev->dev, "Could not allocate memory for MDHA context\n");
1494                 return -ENOMEM;
1495         }
1496 
1497         /* Allocate space for HW links */
1498         dev->hw_link[0] = dmam_alloc_coherent(&pdev->dev,
1499                         SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1500                         &dev->hw_phys_link[0], GFP_KERNEL);
1501         if (!dev->hw_link[0]) {
1502                 dev_err(&pdev->dev, "Could not allocate hw links\n");
1503                 return -ENOMEM;
1504         }
1505         for (i = 1; i < SAHARA_MAX_HW_LINK; i++) {
1506                 dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] +
1507                                         sizeof(struct sahara_hw_link);
1508                 dev->hw_link[i] = dev->hw_link[i - 1] + 1;
1509         }
1510 
1511         crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH);
1512 
1513         spin_lock_init(&dev->lock);
1514         mutex_init(&dev->queue_mutex);
1515 
1516         dev_ptr = dev;
1517 
1518         dev->kthread = kthread_run(sahara_queue_manage, dev, "sahara_crypto");
1519         if (IS_ERR(dev->kthread)) {
1520                 return PTR_ERR(dev->kthread);
1521         }
1522 
1523         init_completion(&dev->dma_completion);
1524 
1525         err = clk_prepare_enable(dev->clk_ipg);
1526         if (err)
1527                 return err;
1528         err = clk_prepare_enable(dev->clk_ahb);
1529         if (err)
1530                 goto clk_ipg_disable;
1531 
1532         version = sahara_read(dev, SAHARA_REG_VERSION);
1533         if (of_device_is_compatible(pdev->dev.of_node, "fsl,imx27-sahara")) {
1534                 if (version != SAHARA_VERSION_3)
1535                         err = -ENODEV;
1536         } else if (of_device_is_compatible(pdev->dev.of_node,
1537                         "fsl,imx53-sahara")) {
1538                 if (((version >> 8) & 0xff) != SAHARA_VERSION_4)
1539                         err = -ENODEV;
1540                 version = (version >> 8) & 0xff;
1541         }
1542         if (err == -ENODEV) {
1543                 dev_err(&pdev->dev, "SAHARA version %d not supported\n",
1544                                 version);
1545                 goto err_algs;
1546         }
1547 
1548         dev->version = version;
1549 
1550         sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH,
1551                      SAHARA_REG_CMD);
1552         sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) |
1553                         SAHARA_CONTROL_SET_MAXBURST(8) |
1554                         SAHARA_CONTROL_RNG_AUTORSD |
1555                         SAHARA_CONTROL_ENABLE_INT,
1556                         SAHARA_REG_CONTROL);
1557 
1558         err = sahara_register_algs(dev);
1559         if (err)
1560                 goto err_algs;
1561 
1562         dev_info(&pdev->dev, "SAHARA version %d initialized\n", version);
1563 
1564         return 0;
1565 
1566 err_algs:
1567         kthread_stop(dev->kthread);
1568         dev_ptr = NULL;
1569         clk_disable_unprepare(dev->clk_ahb);
1570 clk_ipg_disable:
1571         clk_disable_unprepare(dev->clk_ipg);
1572 
1573         return err;
1574 }
1575 
1576 static int sahara_remove(struct platform_device *pdev)
1577 {
1578         struct sahara_dev *dev = platform_get_drvdata(pdev);
1579 
1580         kthread_stop(dev->kthread);
1581 
1582         sahara_unregister_algs(dev);
1583 
1584         clk_disable_unprepare(dev->clk_ipg);
1585         clk_disable_unprepare(dev->clk_ahb);
1586 
1587         dev_ptr = NULL;
1588 
1589         return 0;
1590 }
1591 
1592 static struct platform_driver sahara_driver = {
1593         .probe          = sahara_probe,
1594         .remove         = sahara_remove,
1595         .driver         = {
1596                 .name   = SAHARA_NAME,
1597                 .of_match_table = sahara_dt_ids,
1598         },
1599         .id_table = sahara_platform_ids,
1600 };
1601 
1602 module_platform_driver(sahara_driver);
1603 
1604 MODULE_LICENSE("GPL");
1605 MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>");
1606 MODULE_AUTHOR("Steffen Trumtrar <s.trumtrar@pengutronix.de>");
1607 MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator");
1608 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us