Version:  2.0.40 2.2.26 2.4.37 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18

Linux/drivers/crypto/sahara.c

  1 /*
  2  * Cryptographic API.
  3  *
  4  * Support for SAHARA cryptographic accelerator.
  5  *
  6  * Copyright (c) 2013 Vista Silicon S.L.
  7  * Author: Javier Martin <javier.martin@vista-silicon.com>
  8  *
  9  * This program is free software; you can redistribute it and/or modify
 10  * it under the terms of the GNU General Public License version 2 as published
 11  * by the Free Software Foundation.
 12  *
 13  * Based on omap-aes.c and tegra-aes.c
 14  */
 15 
 16 #include <crypto/algapi.h>
 17 #include <crypto/aes.h>
 18 
 19 #include <linux/clk.h>
 20 #include <linux/crypto.h>
 21 #include <linux/interrupt.h>
 22 #include <linux/io.h>
 23 #include <linux/irq.h>
 24 #include <linux/kernel.h>
 25 #include <linux/module.h>
 26 #include <linux/of.h>
 27 #include <linux/platform_device.h>
 28 
 29 #define SAHARA_NAME "sahara"
 30 #define SAHARA_VERSION_3        3
 31 #define SAHARA_TIMEOUT_MS       1000
 32 #define SAHARA_MAX_HW_DESC      2
 33 #define SAHARA_MAX_HW_LINK      20
 34 
 35 #define FLAGS_MODE_MASK         0x000f
 36 #define FLAGS_ENCRYPT           BIT(0)
 37 #define FLAGS_CBC               BIT(1)
 38 #define FLAGS_NEW_KEY           BIT(3)
 39 #define FLAGS_BUSY              4
 40 
 41 #define SAHARA_HDR_BASE                 0x00800000
 42 #define SAHARA_HDR_SKHA_ALG_AES 0
 43 #define SAHARA_HDR_SKHA_OP_ENC          (1 << 2)
 44 #define SAHARA_HDR_SKHA_MODE_ECB        (0 << 3)
 45 #define SAHARA_HDR_SKHA_MODE_CBC        (1 << 3)
 46 #define SAHARA_HDR_FORM_DATA            (5 << 16)
 47 #define SAHARA_HDR_FORM_KEY             (8 << 16)
 48 #define SAHARA_HDR_LLO                  (1 << 24)
 49 #define SAHARA_HDR_CHA_SKHA             (1 << 28)
 50 #define SAHARA_HDR_CHA_MDHA             (2 << 28)
 51 #define SAHARA_HDR_PARITY_BIT           (1 << 31)
 52 
 53 /* SAHARA can only process one request at a time */
 54 #define SAHARA_QUEUE_LENGTH     1
 55 
 56 #define SAHARA_REG_VERSION      0x00
 57 #define SAHARA_REG_DAR          0x04
 58 #define SAHARA_REG_CONTROL      0x08
 59 #define         SAHARA_CONTROL_SET_THROTTLE(x)  (((x) & 0xff) << 24)
 60 #define         SAHARA_CONTROL_SET_MAXBURST(x)  (((x) & 0xff) << 16)
 61 #define         SAHARA_CONTROL_RNG_AUTORSD      (1 << 7)
 62 #define         SAHARA_CONTROL_ENABLE_INT       (1 << 4)
 63 #define SAHARA_REG_CMD          0x0C
 64 #define         SAHARA_CMD_RESET                (1 << 0)
 65 #define         SAHARA_CMD_CLEAR_INT            (1 << 8)
 66 #define         SAHARA_CMD_CLEAR_ERR            (1 << 9)
 67 #define         SAHARA_CMD_SINGLE_STEP          (1 << 10)
 68 #define         SAHARA_CMD_MODE_BATCH           (1 << 16)
 69 #define         SAHARA_CMD_MODE_DEBUG           (1 << 18)
 70 #define SAHARA_REG_STATUS       0x10
 71 #define         SAHARA_STATUS_GET_STATE(x)      ((x) & 0x7)
 72 #define                 SAHARA_STATE_IDLE       0
 73 #define                 SAHARA_STATE_BUSY       1
 74 #define                 SAHARA_STATE_ERR        2
 75 #define                 SAHARA_STATE_FAULT      3
 76 #define                 SAHARA_STATE_COMPLETE   4
 77 #define                 SAHARA_STATE_COMP_FLAG  (1 << 2)
 78 #define         SAHARA_STATUS_DAR_FULL          (1 << 3)
 79 #define         SAHARA_STATUS_ERROR             (1 << 4)
 80 #define         SAHARA_STATUS_SECURE            (1 << 5)
 81 #define         SAHARA_STATUS_FAIL              (1 << 6)
 82 #define         SAHARA_STATUS_INIT              (1 << 7)
 83 #define         SAHARA_STATUS_RNG_RESEED        (1 << 8)
 84 #define         SAHARA_STATUS_ACTIVE_RNG        (1 << 9)
 85 #define         SAHARA_STATUS_ACTIVE_MDHA       (1 << 10)
 86 #define         SAHARA_STATUS_ACTIVE_SKHA       (1 << 11)
 87 #define         SAHARA_STATUS_MODE_BATCH        (1 << 16)
 88 #define         SAHARA_STATUS_MODE_DEDICATED    (1 << 17)
 89 #define         SAHARA_STATUS_MODE_DEBUG        (1 << 18)
 90 #define         SAHARA_STATUS_GET_ISTATE(x)     (((x) >> 24) & 0xff)
 91 #define SAHARA_REG_ERRSTATUS    0x14
 92 #define         SAHARA_ERRSTATUS_GET_SOURCE(x)  ((x) & 0xf)
 93 #define                 SAHARA_ERRSOURCE_CHA    14
 94 #define                 SAHARA_ERRSOURCE_DMA    15
 95 #define         SAHARA_ERRSTATUS_DMA_DIR        (1 << 8)
 96 #define         SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3)
 97 #define         SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7)
 98 #define         SAHARA_ERRSTATUS_GET_CHASRC(x)  (((x) >> 16) & 0xfff)
 99 #define         SAHARA_ERRSTATUS_GET_CHAERR(x)  (((x) >> 28) & 0x3)
100 #define SAHARA_REG_FADDR        0x18
101 #define SAHARA_REG_CDAR         0x1C
102 #define SAHARA_REG_IDAR         0x20
103 
104 struct sahara_hw_desc {
105         u32             hdr;
106         u32             len1;
107         dma_addr_t      p1;
108         u32             len2;
109         dma_addr_t      p2;
110         dma_addr_t      next;
111 };
112 
113 struct sahara_hw_link {
114         u32             len;
115         dma_addr_t      p;
116         dma_addr_t      next;
117 };
118 
119 struct sahara_ctx {
120         struct sahara_dev *dev;
121         unsigned long flags;
122         int keylen;
123         u8 key[AES_KEYSIZE_128];
124         struct crypto_ablkcipher *fallback;
125 };
126 
127 struct sahara_aes_reqctx {
128         unsigned long mode;
129 };
130 
131 struct sahara_dev {
132         struct device           *device;
133         void __iomem            *regs_base;
134         struct clk              *clk_ipg;
135         struct clk              *clk_ahb;
136 
137         struct sahara_ctx       *ctx;
138         spinlock_t              lock;
139         struct crypto_queue     queue;
140         unsigned long           flags;
141 
142         struct tasklet_struct   done_task;
143         struct tasklet_struct   queue_task;
144 
145         struct sahara_hw_desc   *hw_desc[SAHARA_MAX_HW_DESC];
146         dma_addr_t              hw_phys_desc[SAHARA_MAX_HW_DESC];
147 
148         u8                      *key_base;
149         dma_addr_t              key_phys_base;
150 
151         u8                      *iv_base;
152         dma_addr_t              iv_phys_base;
153 
154         struct sahara_hw_link   *hw_link[SAHARA_MAX_HW_LINK];
155         dma_addr_t              hw_phys_link[SAHARA_MAX_HW_LINK];
156 
157         struct ablkcipher_request *req;
158         size_t                  total;
159         struct scatterlist      *in_sg;
160         unsigned int            nb_in_sg;
161         struct scatterlist      *out_sg;
162         unsigned int            nb_out_sg;
163 
164         u32                     error;
165         struct timer_list       watchdog;
166 };
167 
168 static struct sahara_dev *dev_ptr;
169 
170 static inline void sahara_write(struct sahara_dev *dev, u32 data, u32 reg)
171 {
172         writel(data, dev->regs_base + reg);
173 }
174 
175 static inline unsigned int sahara_read(struct sahara_dev *dev, u32 reg)
176 {
177         return readl(dev->regs_base + reg);
178 }
179 
180 static u32 sahara_aes_key_hdr(struct sahara_dev *dev)
181 {
182         u32 hdr = SAHARA_HDR_BASE | SAHARA_HDR_SKHA_ALG_AES |
183                         SAHARA_HDR_FORM_KEY | SAHARA_HDR_LLO |
184                         SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
185 
186         if (dev->flags & FLAGS_CBC) {
187                 hdr |= SAHARA_HDR_SKHA_MODE_CBC;
188                 hdr ^= SAHARA_HDR_PARITY_BIT;
189         }
190 
191         if (dev->flags & FLAGS_ENCRYPT) {
192                 hdr |= SAHARA_HDR_SKHA_OP_ENC;
193                 hdr ^= SAHARA_HDR_PARITY_BIT;
194         }
195 
196         return hdr;
197 }
198 
199 static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev)
200 {
201         return SAHARA_HDR_BASE | SAHARA_HDR_FORM_DATA |
202                         SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
203 }
204 
205 static int sahara_sg_length(struct scatterlist *sg,
206                             unsigned int total)
207 {
208         int sg_nb;
209         unsigned int len;
210         struct scatterlist *sg_list;
211 
212         sg_nb = 0;
213         sg_list = sg;
214 
215         while (total) {
216                 len = min(sg_list->length, total);
217 
218                 sg_nb++;
219                 total -= len;
220 
221                 sg_list = sg_next(sg_list);
222                 if (!sg_list)
223                         total = 0;
224         }
225 
226         return sg_nb;
227 }
228 
229 static char *sahara_err_src[16] = {
230         "No error",
231         "Header error",
232         "Descriptor length error",
233         "Descriptor length or pointer error",
234         "Link length error",
235         "Link pointer error",
236         "Input buffer error",
237         "Output buffer error",
238         "Output buffer starvation",
239         "Internal state fault",
240         "General descriptor problem",
241         "Reserved",
242         "Descriptor address error",
243         "Link address error",
244         "CHA error",
245         "DMA error"
246 };
247 
248 static char *sahara_err_dmasize[4] = {
249         "Byte transfer",
250         "Half-word transfer",
251         "Word transfer",
252         "Reserved"
253 };
254 
255 static char *sahara_err_dmasrc[8] = {
256         "No error",
257         "AHB bus error",
258         "Internal IP bus error",
259         "Parity error",
260         "DMA crosses 256 byte boundary",
261         "DMA is busy",
262         "Reserved",
263         "DMA HW error"
264 };
265 
266 static char *sahara_cha_errsrc[12] = {
267         "Input buffer non-empty",
268         "Illegal address",
269         "Illegal mode",
270         "Illegal data size",
271         "Illegal key size",
272         "Write during processing",
273         "CTX read during processing",
274         "HW error",
275         "Input buffer disabled/underflow",
276         "Output buffer disabled/overflow",
277         "DES key parity error",
278         "Reserved"
279 };
280 
281 static char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" };
282 
283 static void sahara_decode_error(struct sahara_dev *dev, unsigned int error)
284 {
285         u8 source = SAHARA_ERRSTATUS_GET_SOURCE(error);
286         u16 chasrc = ffs(SAHARA_ERRSTATUS_GET_CHASRC(error));
287 
288         dev_err(dev->device, "%s: Error Register = 0x%08x\n", __func__, error);
289 
290         dev_err(dev->device, "  - %s.\n", sahara_err_src[source]);
291 
292         if (source == SAHARA_ERRSOURCE_DMA) {
293                 if (error & SAHARA_ERRSTATUS_DMA_DIR)
294                         dev_err(dev->device, "          * DMA read.\n");
295                 else
296                         dev_err(dev->device, "          * DMA write.\n");
297 
298                 dev_err(dev->device, "          * %s.\n",
299                        sahara_err_dmasize[SAHARA_ERRSTATUS_GET_DMASZ(error)]);
300                 dev_err(dev->device, "          * %s.\n",
301                        sahara_err_dmasrc[SAHARA_ERRSTATUS_GET_DMASRC(error)]);
302         } else if (source == SAHARA_ERRSOURCE_CHA) {
303                 dev_err(dev->device, "          * %s.\n",
304                         sahara_cha_errsrc[chasrc]);
305                 dev_err(dev->device, "          * %s.\n",
306                        sahara_cha_err[SAHARA_ERRSTATUS_GET_CHAERR(error)]);
307         }
308         dev_err(dev->device, "\n");
309 }
310 
311 static char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" };
312 
313 static void sahara_decode_status(struct sahara_dev *dev, unsigned int status)
314 {
315         u8 state;
316 
317         if (!IS_ENABLED(DEBUG))
318                 return;
319 
320         state = SAHARA_STATUS_GET_STATE(status);
321 
322         dev_dbg(dev->device, "%s: Status Register = 0x%08x\n",
323                 __func__, status);
324 
325         dev_dbg(dev->device, "  - State = %d:\n", state);
326         if (state & SAHARA_STATE_COMP_FLAG)
327                 dev_dbg(dev->device, "          * Descriptor completed. IRQ pending.\n");
328 
329         dev_dbg(dev->device, "          * %s.\n",
330                sahara_state[state & ~SAHARA_STATE_COMP_FLAG]);
331 
332         if (status & SAHARA_STATUS_DAR_FULL)
333                 dev_dbg(dev->device, "  - DAR Full.\n");
334         if (status & SAHARA_STATUS_ERROR)
335                 dev_dbg(dev->device, "  - Error.\n");
336         if (status & SAHARA_STATUS_SECURE)
337                 dev_dbg(dev->device, "  - Secure.\n");
338         if (status & SAHARA_STATUS_FAIL)
339                 dev_dbg(dev->device, "  - Fail.\n");
340         if (status & SAHARA_STATUS_RNG_RESEED)
341                 dev_dbg(dev->device, "  - RNG Reseed Request.\n");
342         if (status & SAHARA_STATUS_ACTIVE_RNG)
343                 dev_dbg(dev->device, "  - RNG Active.\n");
344         if (status & SAHARA_STATUS_ACTIVE_MDHA)
345                 dev_dbg(dev->device, "  - MDHA Active.\n");
346         if (status & SAHARA_STATUS_ACTIVE_SKHA)
347                 dev_dbg(dev->device, "  - SKHA Active.\n");
348 
349         if (status & SAHARA_STATUS_MODE_BATCH)
350                 dev_dbg(dev->device, "  - Batch Mode.\n");
351         else if (status & SAHARA_STATUS_MODE_DEDICATED)
352                 dev_dbg(dev->device, "  - Decidated Mode.\n");
353         else if (status & SAHARA_STATUS_MODE_DEBUG)
354                 dev_dbg(dev->device, "  - Debug Mode.\n");
355 
356         dev_dbg(dev->device, "  - Internal state = 0x%02x\n",
357                SAHARA_STATUS_GET_ISTATE(status));
358 
359         dev_dbg(dev->device, "Current DAR: 0x%08x\n",
360                 sahara_read(dev, SAHARA_REG_CDAR));
361         dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n",
362                 sahara_read(dev, SAHARA_REG_IDAR));
363 }
364 
365 static void sahara_dump_descriptors(struct sahara_dev *dev)
366 {
367         int i;
368 
369         if (!IS_ENABLED(DEBUG))
370                 return;
371 
372         for (i = 0; i < SAHARA_MAX_HW_DESC; i++) {
373                 dev_dbg(dev->device, "Descriptor (%d) (0x%08x):\n",
374                         i, dev->hw_phys_desc[i]);
375                 dev_dbg(dev->device, "\thdr = 0x%08x\n", dev->hw_desc[i]->hdr);
376                 dev_dbg(dev->device, "\tlen1 = %u\n", dev->hw_desc[i]->len1);
377                 dev_dbg(dev->device, "\tp1 = 0x%08x\n", dev->hw_desc[i]->p1);
378                 dev_dbg(dev->device, "\tlen2 = %u\n", dev->hw_desc[i]->len2);
379                 dev_dbg(dev->device, "\tp2 = 0x%08x\n", dev->hw_desc[i]->p2);
380                 dev_dbg(dev->device, "\tnext = 0x%08x\n",
381                         dev->hw_desc[i]->next);
382         }
383         dev_dbg(dev->device, "\n");
384 }
385 
386 static void sahara_dump_links(struct sahara_dev *dev)
387 {
388         int i;
389 
390         if (!IS_ENABLED(DEBUG))
391                 return;
392 
393         for (i = 0; i < SAHARA_MAX_HW_LINK; i++) {
394                 dev_dbg(dev->device, "Link (%d) (0x%08x):\n",
395                         i, dev->hw_phys_link[i]);
396                 dev_dbg(dev->device, "\tlen = %u\n", dev->hw_link[i]->len);
397                 dev_dbg(dev->device, "\tp = 0x%08x\n", dev->hw_link[i]->p);
398                 dev_dbg(dev->device, "\tnext = 0x%08x\n",
399                         dev->hw_link[i]->next);
400         }
401         dev_dbg(dev->device, "\n");
402 }
403 
404 static void sahara_aes_done_task(unsigned long data)
405 {
406         struct sahara_dev *dev = (struct sahara_dev *)data;
407 
408         dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
409                 DMA_TO_DEVICE);
410         dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
411                 DMA_FROM_DEVICE);
412 
413         spin_lock(&dev->lock);
414         clear_bit(FLAGS_BUSY, &dev->flags);
415         spin_unlock(&dev->lock);
416 
417         dev->req->base.complete(&dev->req->base, dev->error);
418 }
419 
420 static void sahara_watchdog(unsigned long data)
421 {
422         struct sahara_dev *dev = (struct sahara_dev *)data;
423         unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS);
424         unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS);
425 
426         sahara_decode_status(dev, stat);
427         sahara_decode_error(dev, err);
428         dev->error = -ETIMEDOUT;
429         sahara_aes_done_task(data);
430 }
431 
432 static int sahara_hw_descriptor_create(struct sahara_dev *dev)
433 {
434         struct sahara_ctx *ctx = dev->ctx;
435         struct scatterlist *sg;
436         int ret;
437         int i, j;
438 
439         /* Copy new key if necessary */
440         if (ctx->flags & FLAGS_NEW_KEY) {
441                 memcpy(dev->key_base, ctx->key, ctx->keylen);
442                 ctx->flags &= ~FLAGS_NEW_KEY;
443 
444                 if (dev->flags & FLAGS_CBC) {
445                         dev->hw_desc[0]->len1 = AES_BLOCK_SIZE;
446                         dev->hw_desc[0]->p1 = dev->iv_phys_base;
447                 } else {
448                         dev->hw_desc[0]->len1 = 0;
449                         dev->hw_desc[0]->p1 = 0;
450                 }
451                 dev->hw_desc[0]->len2 = ctx->keylen;
452                 dev->hw_desc[0]->p2 = dev->key_phys_base;
453                 dev->hw_desc[0]->next = dev->hw_phys_desc[1];
454         }
455         dev->hw_desc[0]->hdr = sahara_aes_key_hdr(dev);
456 
457         dev->nb_in_sg = sahara_sg_length(dev->in_sg, dev->total);
458         dev->nb_out_sg = sahara_sg_length(dev->out_sg, dev->total);
459         if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) {
460                 dev_err(dev->device, "not enough hw links (%d)\n",
461                         dev->nb_in_sg + dev->nb_out_sg);
462                 return -EINVAL;
463         }
464 
465         ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg,
466                          DMA_TO_DEVICE);
467         if (ret != dev->nb_in_sg) {
468                 dev_err(dev->device, "couldn't map in sg\n");
469                 goto unmap_in;
470         }
471         ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg,
472                          DMA_FROM_DEVICE);
473         if (ret != dev->nb_out_sg) {
474                 dev_err(dev->device, "couldn't map out sg\n");
475                 goto unmap_out;
476         }
477 
478         /* Create input links */
479         dev->hw_desc[1]->p1 = dev->hw_phys_link[0];
480         sg = dev->in_sg;
481         for (i = 0; i < dev->nb_in_sg; i++) {
482                 dev->hw_link[i]->len = sg->length;
483                 dev->hw_link[i]->p = sg->dma_address;
484                 if (i == (dev->nb_in_sg - 1)) {
485                         dev->hw_link[i]->next = 0;
486                 } else {
487                         dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
488                         sg = sg_next(sg);
489                 }
490         }
491 
492         /* Create output links */
493         dev->hw_desc[1]->p2 = dev->hw_phys_link[i];
494         sg = dev->out_sg;
495         for (j = i; j < dev->nb_out_sg + i; j++) {
496                 dev->hw_link[j]->len = sg->length;
497                 dev->hw_link[j]->p = sg->dma_address;
498                 if (j == (dev->nb_out_sg + i - 1)) {
499                         dev->hw_link[j]->next = 0;
500                 } else {
501                         dev->hw_link[j]->next = dev->hw_phys_link[j + 1];
502                         sg = sg_next(sg);
503                 }
504         }
505 
506         /* Fill remaining fields of hw_desc[1] */
507         dev->hw_desc[1]->hdr = sahara_aes_data_link_hdr(dev);
508         dev->hw_desc[1]->len1 = dev->total;
509         dev->hw_desc[1]->len2 = dev->total;
510         dev->hw_desc[1]->next = 0;
511 
512         sahara_dump_descriptors(dev);
513         sahara_dump_links(dev);
514 
515         /* Start processing descriptor chain. */
516         mod_timer(&dev->watchdog,
517                   jiffies + msecs_to_jiffies(SAHARA_TIMEOUT_MS));
518         sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
519 
520         return 0;
521 
522 unmap_out:
523         dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
524                 DMA_TO_DEVICE);
525 unmap_in:
526         dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
527                 DMA_FROM_DEVICE);
528 
529         return -EINVAL;
530 }
531 
532 static void sahara_aes_queue_task(unsigned long data)
533 {
534         struct sahara_dev *dev = (struct sahara_dev *)data;
535         struct crypto_async_request *async_req, *backlog;
536         struct sahara_ctx *ctx;
537         struct sahara_aes_reqctx *rctx;
538         struct ablkcipher_request *req;
539         int ret;
540 
541         spin_lock(&dev->lock);
542         backlog = crypto_get_backlog(&dev->queue);
543         async_req = crypto_dequeue_request(&dev->queue);
544         if (!async_req)
545                 clear_bit(FLAGS_BUSY, &dev->flags);
546         spin_unlock(&dev->lock);
547 
548         if (!async_req)
549                 return;
550 
551         if (backlog)
552                 backlog->complete(backlog, -EINPROGRESS);
553 
554         req = ablkcipher_request_cast(async_req);
555 
556         /* Request is ready to be dispatched by the device */
557         dev_dbg(dev->device,
558                 "dispatch request (nbytes=%d, src=%p, dst=%p)\n",
559                 req->nbytes, req->src, req->dst);
560 
561         /* assign new request to device */
562         dev->req = req;
563         dev->total = req->nbytes;
564         dev->in_sg = req->src;
565         dev->out_sg = req->dst;
566 
567         rctx = ablkcipher_request_ctx(req);
568         ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
569         rctx->mode &= FLAGS_MODE_MASK;
570         dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode;
571 
572         if ((dev->flags & FLAGS_CBC) && req->info)
573                 memcpy(dev->iv_base, req->info, AES_KEYSIZE_128);
574 
575         /* assign new context to device */
576         ctx->dev = dev;
577         dev->ctx = ctx;
578 
579         ret = sahara_hw_descriptor_create(dev);
580         if (ret < 0) {
581                 spin_lock(&dev->lock);
582                 clear_bit(FLAGS_BUSY, &dev->flags);
583                 spin_unlock(&dev->lock);
584                 dev->req->base.complete(&dev->req->base, ret);
585         }
586 }
587 
588 static int sahara_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
589                              unsigned int keylen)
590 {
591         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(tfm);
592         int ret;
593 
594         ctx->keylen = keylen;
595 
596         /* SAHARA only supports 128bit keys */
597         if (keylen == AES_KEYSIZE_128) {
598                 memcpy(ctx->key, key, keylen);
599                 ctx->flags |= FLAGS_NEW_KEY;
600                 return 0;
601         }
602 
603         if (keylen != AES_KEYSIZE_128 &&
604             keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256)
605                 return -EINVAL;
606 
607         /*
608          * The requested key size is not supported by HW, do a fallback.
609          */
610         ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
611         ctx->fallback->base.crt_flags |=
612                 (tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
613 
614         ret = crypto_ablkcipher_setkey(ctx->fallback, key, keylen);
615         if (ret) {
616                 struct crypto_tfm *tfm_aux = crypto_ablkcipher_tfm(tfm);
617 
618                 tfm_aux->crt_flags &= ~CRYPTO_TFM_RES_MASK;
619                 tfm_aux->crt_flags |=
620                         (ctx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK);
621         }
622         return ret;
623 }
624 
625 static int sahara_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
626 {
627         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
628                 crypto_ablkcipher_reqtfm(req));
629         struct sahara_aes_reqctx *rctx = ablkcipher_request_ctx(req);
630         struct sahara_dev *dev = dev_ptr;
631         int err = 0;
632         int busy;
633 
634         dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n",
635                 req->nbytes, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC));
636 
637         if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
638                 dev_err(dev->device,
639                         "request size is not exact amount of AES blocks\n");
640                 return -EINVAL;
641         }
642 
643         ctx->dev = dev;
644 
645         rctx->mode = mode;
646         spin_lock_bh(&dev->lock);
647         err = ablkcipher_enqueue_request(&dev->queue, req);
648         busy = test_and_set_bit(FLAGS_BUSY, &dev->flags);
649         spin_unlock_bh(&dev->lock);
650 
651         if (!busy)
652                 tasklet_schedule(&dev->queue_task);
653 
654         return err;
655 }
656 
657 static int sahara_aes_ecb_encrypt(struct ablkcipher_request *req)
658 {
659         struct crypto_tfm *tfm =
660                 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
661         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
662                 crypto_ablkcipher_reqtfm(req));
663         int err;
664 
665         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
666                 ablkcipher_request_set_tfm(req, ctx->fallback);
667                 err = crypto_ablkcipher_encrypt(req);
668                 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
669                 return err;
670         }
671 
672         return sahara_aes_crypt(req, FLAGS_ENCRYPT);
673 }
674 
675 static int sahara_aes_ecb_decrypt(struct ablkcipher_request *req)
676 {
677         struct crypto_tfm *tfm =
678                 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
679         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
680                 crypto_ablkcipher_reqtfm(req));
681         int err;
682 
683         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
684                 ablkcipher_request_set_tfm(req, ctx->fallback);
685                 err = crypto_ablkcipher_decrypt(req);
686                 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
687                 return err;
688         }
689 
690         return sahara_aes_crypt(req, 0);
691 }
692 
693 static int sahara_aes_cbc_encrypt(struct ablkcipher_request *req)
694 {
695         struct crypto_tfm *tfm =
696                 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
697         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
698                 crypto_ablkcipher_reqtfm(req));
699         int err;
700 
701         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
702                 ablkcipher_request_set_tfm(req, ctx->fallback);
703                 err = crypto_ablkcipher_encrypt(req);
704                 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
705                 return err;
706         }
707 
708         return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
709 }
710 
711 static int sahara_aes_cbc_decrypt(struct ablkcipher_request *req)
712 {
713         struct crypto_tfm *tfm =
714                 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
715         struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
716                 crypto_ablkcipher_reqtfm(req));
717         int err;
718 
719         if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
720                 ablkcipher_request_set_tfm(req, ctx->fallback);
721                 err = crypto_ablkcipher_decrypt(req);
722                 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
723                 return err;
724         }
725 
726         return sahara_aes_crypt(req, FLAGS_CBC);
727 }
728 
729 static int sahara_aes_cra_init(struct crypto_tfm *tfm)
730 {
731         const char *name = crypto_tfm_alg_name(tfm);
732         struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
733 
734         ctx->fallback = crypto_alloc_ablkcipher(name, 0,
735                                 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
736         if (IS_ERR(ctx->fallback)) {
737                 pr_err("Error allocating fallback algo %s\n", name);
738                 return PTR_ERR(ctx->fallback);
739         }
740 
741         tfm->crt_ablkcipher.reqsize = sizeof(struct sahara_aes_reqctx);
742 
743         return 0;
744 }
745 
746 static void sahara_aes_cra_exit(struct crypto_tfm *tfm)
747 {
748         struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
749 
750         if (ctx->fallback)
751                 crypto_free_ablkcipher(ctx->fallback);
752         ctx->fallback = NULL;
753 }
754 
755 static struct crypto_alg aes_algs[] = {
756 {
757         .cra_name               = "ecb(aes)",
758         .cra_driver_name        = "sahara-ecb-aes",
759         .cra_priority           = 300,
760         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
761                         CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
762         .cra_blocksize          = AES_BLOCK_SIZE,
763         .cra_ctxsize            = sizeof(struct sahara_ctx),
764         .cra_alignmask          = 0x0,
765         .cra_type               = &crypto_ablkcipher_type,
766         .cra_module             = THIS_MODULE,
767         .cra_init               = sahara_aes_cra_init,
768         .cra_exit               = sahara_aes_cra_exit,
769         .cra_u.ablkcipher = {
770                 .min_keysize    = AES_MIN_KEY_SIZE ,
771                 .max_keysize    = AES_MAX_KEY_SIZE,
772                 .setkey         = sahara_aes_setkey,
773                 .encrypt        = sahara_aes_ecb_encrypt,
774                 .decrypt        = sahara_aes_ecb_decrypt,
775         }
776 }, {
777         .cra_name               = "cbc(aes)",
778         .cra_driver_name        = "sahara-cbc-aes",
779         .cra_priority           = 300,
780         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER |
781                         CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
782         .cra_blocksize          = AES_BLOCK_SIZE,
783         .cra_ctxsize            = sizeof(struct sahara_ctx),
784         .cra_alignmask          = 0x0,
785         .cra_type               = &crypto_ablkcipher_type,
786         .cra_module             = THIS_MODULE,
787         .cra_init               = sahara_aes_cra_init,
788         .cra_exit               = sahara_aes_cra_exit,
789         .cra_u.ablkcipher = {
790                 .min_keysize    = AES_MIN_KEY_SIZE ,
791                 .max_keysize    = AES_MAX_KEY_SIZE,
792                 .ivsize         = AES_BLOCK_SIZE,
793                 .setkey         = sahara_aes_setkey,
794                 .encrypt        = sahara_aes_cbc_encrypt,
795                 .decrypt        = sahara_aes_cbc_decrypt,
796         }
797 }
798 };
799 
800 static irqreturn_t sahara_irq_handler(int irq, void *data)
801 {
802         struct sahara_dev *dev = (struct sahara_dev *)data;
803         unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS);
804         unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS);
805 
806         del_timer(&dev->watchdog);
807 
808         sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR,
809                      SAHARA_REG_CMD);
810 
811         sahara_decode_status(dev, stat);
812 
813         if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) {
814                 return IRQ_NONE;
815         } else if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_COMPLETE) {
816                 dev->error = 0;
817         } else {
818                 sahara_decode_error(dev, err);
819                 dev->error = -EINVAL;
820         }
821 
822         tasklet_schedule(&dev->done_task);
823 
824         return IRQ_HANDLED;
825 }
826 
827 
828 static int sahara_register_algs(struct sahara_dev *dev)
829 {
830         int err, i, j;
831 
832         for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
833                 INIT_LIST_HEAD(&aes_algs[i].cra_list);
834                 err = crypto_register_alg(&aes_algs[i]);
835                 if (err)
836                         goto err_aes_algs;
837         }
838 
839         return 0;
840 
841 err_aes_algs:
842         for (j = 0; j < i; j++)
843                 crypto_unregister_alg(&aes_algs[j]);
844 
845         return err;
846 }
847 
848 static void sahara_unregister_algs(struct sahara_dev *dev)
849 {
850         int i;
851 
852         for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
853                 crypto_unregister_alg(&aes_algs[i]);
854 }
855 
856 static struct platform_device_id sahara_platform_ids[] = {
857         { .name = "sahara-imx27" },
858         { /* sentinel */ }
859 };
860 MODULE_DEVICE_TABLE(platform, sahara_platform_ids);
861 
862 static struct of_device_id sahara_dt_ids[] = {
863         { .compatible = "fsl,imx27-sahara" },
864         { /* sentinel */ }
865 };
866 MODULE_DEVICE_TABLE(of, sahara_dt_ids);
867 
868 static int sahara_probe(struct platform_device *pdev)
869 {
870         struct sahara_dev *dev;
871         struct resource *res;
872         u32 version;
873         int irq;
874         int err;
875         int i;
876 
877         dev = devm_kzalloc(&pdev->dev, sizeof(struct sahara_dev), GFP_KERNEL);
878         if (dev == NULL) {
879                 dev_err(&pdev->dev, "unable to alloc data struct.\n");
880                 return -ENOMEM;
881         }
882 
883         dev->device = &pdev->dev;
884         platform_set_drvdata(pdev, dev);
885 
886         /* Get the base address */
887         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
888         dev->regs_base = devm_ioremap_resource(&pdev->dev, res);
889         if (IS_ERR(dev->regs_base))
890                 return PTR_ERR(dev->regs_base);
891 
892         /* Get the IRQ */
893         irq = platform_get_irq(pdev,  0);
894         if (irq < 0) {
895                 dev_err(&pdev->dev, "failed to get irq resource\n");
896                 return irq;
897         }
898 
899         err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler,
900                                0, dev_name(&pdev->dev), dev);
901         if (err) {
902                 dev_err(&pdev->dev, "failed to request irq\n");
903                 return err;
904         }
905 
906         /* clocks */
907         dev->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
908         if (IS_ERR(dev->clk_ipg)) {
909                 dev_err(&pdev->dev, "Could not get ipg clock\n");
910                 return PTR_ERR(dev->clk_ipg);
911         }
912 
913         dev->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
914         if (IS_ERR(dev->clk_ahb)) {
915                 dev_err(&pdev->dev, "Could not get ahb clock\n");
916                 return PTR_ERR(dev->clk_ahb);
917         }
918 
919         /* Allocate HW descriptors */
920         dev->hw_desc[0] = dma_alloc_coherent(&pdev->dev,
921                         SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
922                         &dev->hw_phys_desc[0], GFP_KERNEL);
923         if (!dev->hw_desc[0]) {
924                 dev_err(&pdev->dev, "Could not allocate hw descriptors\n");
925                 return -ENOMEM;
926         }
927         dev->hw_desc[1] = dev->hw_desc[0] + 1;
928         dev->hw_phys_desc[1] = dev->hw_phys_desc[0] +
929                                 sizeof(struct sahara_hw_desc);
930 
931         /* Allocate space for iv and key */
932         dev->key_base = dma_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128,
933                                 &dev->key_phys_base, GFP_KERNEL);
934         if (!dev->key_base) {
935                 dev_err(&pdev->dev, "Could not allocate memory for key\n");
936                 err = -ENOMEM;
937                 goto err_key;
938         }
939         dev->iv_base = dev->key_base + AES_KEYSIZE_128;
940         dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128;
941 
942         /* Allocate space for HW links */
943         dev->hw_link[0] = dma_alloc_coherent(&pdev->dev,
944                         SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
945                         &dev->hw_phys_link[0], GFP_KERNEL);
946         if (!dev->hw_link[0]) {
947                 dev_err(&pdev->dev, "Could not allocate hw links\n");
948                 err = -ENOMEM;
949                 goto err_link;
950         }
951         for (i = 1; i < SAHARA_MAX_HW_LINK; i++) {
952                 dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] +
953                                         sizeof(struct sahara_hw_link);
954                 dev->hw_link[i] = dev->hw_link[i - 1] + 1;
955         }
956 
957         crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH);
958 
959         dev_ptr = dev;
960 
961         tasklet_init(&dev->queue_task, sahara_aes_queue_task,
962                      (unsigned long)dev);
963         tasklet_init(&dev->done_task, sahara_aes_done_task,
964                      (unsigned long)dev);
965 
966         init_timer(&dev->watchdog);
967         dev->watchdog.function = &sahara_watchdog;
968         dev->watchdog.data = (unsigned long)dev;
969 
970         clk_prepare_enable(dev->clk_ipg);
971         clk_prepare_enable(dev->clk_ahb);
972 
973         version = sahara_read(dev, SAHARA_REG_VERSION);
974         if (version != SAHARA_VERSION_3) {
975                 dev_err(&pdev->dev, "SAHARA version %d not supported\n",
976                         version);
977                 err = -ENODEV;
978                 goto err_algs;
979         }
980 
981         sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH,
982                      SAHARA_REG_CMD);
983         sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) |
984                         SAHARA_CONTROL_SET_MAXBURST(8) |
985                         SAHARA_CONTROL_RNG_AUTORSD |
986                         SAHARA_CONTROL_ENABLE_INT,
987                         SAHARA_REG_CONTROL);
988 
989         err = sahara_register_algs(dev);
990         if (err)
991                 goto err_algs;
992 
993         dev_info(&pdev->dev, "SAHARA version %d initialized\n", version);
994 
995         return 0;
996 
997 err_algs:
998         dma_free_coherent(&pdev->dev,
999                           SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1000                           dev->hw_link[0], dev->hw_phys_link[0]);
1001         clk_disable_unprepare(dev->clk_ipg);
1002         clk_disable_unprepare(dev->clk_ahb);
1003         dev_ptr = NULL;
1004 err_link:
1005         dma_free_coherent(&pdev->dev,
1006                           2 * AES_KEYSIZE_128,
1007                           dev->key_base, dev->key_phys_base);
1008 err_key:
1009         dma_free_coherent(&pdev->dev,
1010                           SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1011                           dev->hw_desc[0], dev->hw_phys_desc[0]);
1012 
1013         return err;
1014 }
1015 
1016 static int sahara_remove(struct platform_device *pdev)
1017 {
1018         struct sahara_dev *dev = platform_get_drvdata(pdev);
1019 
1020         dma_free_coherent(&pdev->dev,
1021                           SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1022                           dev->hw_link[0], dev->hw_phys_link[0]);
1023         dma_free_coherent(&pdev->dev,
1024                           2 * AES_KEYSIZE_128,
1025                           dev->key_base, dev->key_phys_base);
1026         dma_free_coherent(&pdev->dev,
1027                           SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1028                           dev->hw_desc[0], dev->hw_phys_desc[0]);
1029 
1030         tasklet_kill(&dev->done_task);
1031         tasklet_kill(&dev->queue_task);
1032 
1033         sahara_unregister_algs(dev);
1034 
1035         clk_disable_unprepare(dev->clk_ipg);
1036         clk_disable_unprepare(dev->clk_ahb);
1037 
1038         dev_ptr = NULL;
1039 
1040         return 0;
1041 }
1042 
1043 static struct platform_driver sahara_driver = {
1044         .probe          = sahara_probe,
1045         .remove         = sahara_remove,
1046         .driver         = {
1047                 .name   = SAHARA_NAME,
1048                 .owner  = THIS_MODULE,
1049                 .of_match_table = sahara_dt_ids,
1050         },
1051         .id_table = sahara_platform_ids,
1052 };
1053 
1054 module_platform_driver(sahara_driver);
1055 
1056 MODULE_LICENSE("GPL");
1057 MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>");
1058 MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator");
1059 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us