Version:  2.0.40 2.2.26 2.4.37 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0

Linux/drivers/crypto/caam/jr.c

  1 /*
  2  * CAAM/SEC 4.x transport/backend driver
  3  * JobR backend functionality
  4  *
  5  * Copyright 2008-2012 Freescale Semiconductor, Inc.
  6  */
  7 
  8 #include <linux/of_irq.h>
  9 #include <linux/of_address.h>
 10 
 11 #include "compat.h"
 12 #include "regs.h"
 13 #include "jr.h"
 14 #include "desc.h"
 15 #include "intern.h"
 16 
 17 struct jr_driver_data {
 18         /* List of Physical JobR's with the Driver */
 19         struct list_head        jr_list;
 20         spinlock_t              jr_alloc_lock;  /* jr_list lock */
 21 } ____cacheline_aligned;
 22 
 23 static struct jr_driver_data driver_data;
 24 
 25 static int caam_reset_hw_jr(struct device *dev)
 26 {
 27         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
 28         unsigned int timeout = 100000;
 29 
 30         /*
 31          * mask interrupts since we are going to poll
 32          * for reset completion status
 33          */
 34         setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
 35 
 36         /* initiate flush (required prior to reset) */
 37         wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
 38         while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) ==
 39                 JRINT_ERR_HALT_INPROGRESS) && --timeout)
 40                 cpu_relax();
 41 
 42         if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) !=
 43             JRINT_ERR_HALT_COMPLETE || timeout == 0) {
 44                 dev_err(dev, "failed to flush job ring %d\n", jrp->ridx);
 45                 return -EIO;
 46         }
 47 
 48         /* initiate reset */
 49         timeout = 100000;
 50         wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
 51         while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout)
 52                 cpu_relax();
 53 
 54         if (timeout == 0) {
 55                 dev_err(dev, "failed to reset job ring %d\n", jrp->ridx);
 56                 return -EIO;
 57         }
 58 
 59         /* unmask interrupts */
 60         clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
 61 
 62         return 0;
 63 }
 64 
 65 /*
 66  * Shutdown JobR independent of platform property code
 67  */
 68 int caam_jr_shutdown(struct device *dev)
 69 {
 70         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
 71         dma_addr_t inpbusaddr, outbusaddr;
 72         int ret;
 73 
 74         ret = caam_reset_hw_jr(dev);
 75 
 76         tasklet_kill(&jrp->irqtask);
 77 
 78         /* Release interrupt */
 79         free_irq(jrp->irq, dev);
 80 
 81         /* Free rings */
 82         inpbusaddr = rd_reg64(&jrp->rregs->inpring_base);
 83         outbusaddr = rd_reg64(&jrp->rregs->outring_base);
 84         dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
 85                           jrp->inpring, inpbusaddr);
 86         dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
 87                           jrp->outring, outbusaddr);
 88         kfree(jrp->entinfo);
 89 
 90         return ret;
 91 }
 92 
 93 static int caam_jr_remove(struct platform_device *pdev)
 94 {
 95         int ret;
 96         struct device *jrdev;
 97         struct caam_drv_private_jr *jrpriv;
 98 
 99         jrdev = &pdev->dev;
100         jrpriv = dev_get_drvdata(jrdev);
101 
102         /*
103          * Return EBUSY if job ring already allocated.
104          */
105         if (atomic_read(&jrpriv->tfm_count)) {
106                 dev_err(jrdev, "Device is busy\n");
107                 return -EBUSY;
108         }
109 
110         /* Remove the node from Physical JobR list maintained by driver */
111         spin_lock(&driver_data.jr_alloc_lock);
112         list_del(&jrpriv->list_node);
113         spin_unlock(&driver_data.jr_alloc_lock);
114 
115         /* Release ring */
116         ret = caam_jr_shutdown(jrdev);
117         if (ret)
118                 dev_err(jrdev, "Failed to shut down job ring\n");
119         irq_dispose_mapping(jrpriv->irq);
120 
121         return ret;
122 }
123 
124 /* Main per-ring interrupt handler */
125 static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
126 {
127         struct device *dev = st_dev;
128         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
129         u32 irqstate;
130 
131         /*
132          * Check the output ring for ready responses, kick
133          * tasklet if jobs done.
134          */
135         irqstate = rd_reg32(&jrp->rregs->jrintstatus);
136         if (!irqstate)
137                 return IRQ_NONE;
138 
139         /*
140          * If JobR error, we got more development work to do
141          * Flag a bug now, but we really need to shut down and
142          * restart the queue (and fix code).
143          */
144         if (irqstate & JRINT_JR_ERROR) {
145                 dev_err(dev, "job ring error: irqstate: %08x\n", irqstate);
146                 BUG();
147         }
148 
149         /* mask valid interrupts */
150         setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
151 
152         /* Have valid interrupt at this point, just ACK and trigger */
153         wr_reg32(&jrp->rregs->jrintstatus, irqstate);
154 
155         preempt_disable();
156         tasklet_schedule(&jrp->irqtask);
157         preempt_enable();
158 
159         return IRQ_HANDLED;
160 }
161 
162 /* Deferred service handler, run as interrupt-fired tasklet */
163 static void caam_jr_dequeue(unsigned long devarg)
164 {
165         int hw_idx, sw_idx, i, head, tail;
166         struct device *dev = (struct device *)devarg;
167         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
168         void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
169         u32 *userdesc, userstatus;
170         void *userarg;
171 
172         while (rd_reg32(&jrp->rregs->outring_used)) {
173 
174                 head = ACCESS_ONCE(jrp->head);
175 
176                 spin_lock(&jrp->outlock);
177 
178                 sw_idx = tail = jrp->tail;
179                 hw_idx = jrp->out_ring_read_index;
180 
181                 for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) {
182                         sw_idx = (tail + i) & (JOBR_DEPTH - 1);
183 
184                         if (jrp->outring[hw_idx].desc ==
185                             jrp->entinfo[sw_idx].desc_addr_dma)
186                                 break; /* found */
187                 }
188                 /* we should never fail to find a matching descriptor */
189                 BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0);
190 
191                 /* Unmap just-run descriptor so we can post-process */
192                 dma_unmap_single(dev, jrp->outring[hw_idx].desc,
193                                  jrp->entinfo[sw_idx].desc_size,
194                                  DMA_TO_DEVICE);
195 
196                 /* mark completed, avoid matching on a recycled desc addr */
197                 jrp->entinfo[sw_idx].desc_addr_dma = 0;
198 
199                 /* Stash callback params for use outside of lock */
200                 usercall = jrp->entinfo[sw_idx].callbk;
201                 userarg = jrp->entinfo[sw_idx].cbkarg;
202                 userdesc = jrp->entinfo[sw_idx].desc_addr_virt;
203                 userstatus = jrp->outring[hw_idx].jrstatus;
204 
205                 /* set done */
206                 wr_reg32(&jrp->rregs->outring_rmvd, 1);
207 
208                 jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) &
209                                            (JOBR_DEPTH - 1);
210 
211                 /*
212                  * if this job completed out-of-order, do not increment
213                  * the tail.  Otherwise, increment tail by 1 plus the
214                  * number of subsequent jobs already completed out-of-order
215                  */
216                 if (sw_idx == tail) {
217                         do {
218                                 tail = (tail + 1) & (JOBR_DEPTH - 1);
219                         } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 &&
220                                  jrp->entinfo[tail].desc_addr_dma == 0);
221 
222                         jrp->tail = tail;
223                 }
224 
225                 spin_unlock(&jrp->outlock);
226 
227                 /* Finally, execute user's callback */
228                 usercall(dev, userdesc, userstatus, userarg);
229         }
230 
231         /* reenable / unmask IRQs */
232         clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
233 }
234 
235 /**
236  * caam_jr_alloc() - Alloc a job ring for someone to use as needed.
237  *
238  * returns :  pointer to the newly allocated physical
239  *            JobR dev can be written to if successful.
240  **/
241 struct device *caam_jr_alloc(void)
242 {
243         struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL;
244         struct device *dev = NULL;
245         int min_tfm_cnt = INT_MAX;
246         int tfm_cnt;
247 
248         spin_lock(&driver_data.jr_alloc_lock);
249 
250         if (list_empty(&driver_data.jr_list)) {
251                 spin_unlock(&driver_data.jr_alloc_lock);
252                 return ERR_PTR(-ENODEV);
253         }
254 
255         list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) {
256                 tfm_cnt = atomic_read(&jrpriv->tfm_count);
257                 if (tfm_cnt < min_tfm_cnt) {
258                         min_tfm_cnt = tfm_cnt;
259                         min_jrpriv = jrpriv;
260                 }
261                 if (!min_tfm_cnt)
262                         break;
263         }
264 
265         if (min_jrpriv) {
266                 atomic_inc(&min_jrpriv->tfm_count);
267                 dev = min_jrpriv->dev;
268         }
269         spin_unlock(&driver_data.jr_alloc_lock);
270 
271         return dev;
272 }
273 EXPORT_SYMBOL(caam_jr_alloc);
274 
275 /**
276  * caam_jr_free() - Free the Job Ring
277  * @rdev     - points to the dev that identifies the Job ring to
278  *             be released.
279  **/
280 void caam_jr_free(struct device *rdev)
281 {
282         struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev);
283 
284         atomic_dec(&jrpriv->tfm_count);
285 }
286 EXPORT_SYMBOL(caam_jr_free);
287 
288 /**
289  * caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK,
290  * -EBUSY if the queue is full, -EIO if it cannot map the caller's
291  * descriptor.
292  * @dev:  device of the job ring to be used. This device should have
293  *        been assigned prior by caam_jr_register().
294  * @desc: points to a job descriptor that execute our request. All
295  *        descriptors (and all referenced data) must be in a DMAable
296  *        region, and all data references must be physical addresses
297  *        accessible to CAAM (i.e. within a PAMU window granted
298  *        to it).
299  * @cbk:  pointer to a callback function to be invoked upon completion
300  *        of this request. This has the form:
301  *        callback(struct device *dev, u32 *desc, u32 stat, void *arg)
302  *        where:
303  *        @dev:    contains the job ring device that processed this
304  *                 response.
305  *        @desc:   descriptor that initiated the request, same as
306  *                 "desc" being argued to caam_jr_enqueue().
307  *        @status: untranslated status received from CAAM. See the
308  *                 reference manual for a detailed description of
309  *                 error meaning, or see the JRSTA definitions in the
310  *                 register header file
311  *        @areq:   optional pointer to an argument passed with the
312  *                 original request
313  * @areq: optional pointer to a user argument for use at callback
314  *        time.
315  **/
316 int caam_jr_enqueue(struct device *dev, u32 *desc,
317                     void (*cbk)(struct device *dev, u32 *desc,
318                                 u32 status, void *areq),
319                     void *areq)
320 {
321         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
322         struct caam_jrentry_info *head_entry;
323         int head, tail, desc_size;
324         dma_addr_t desc_dma;
325 
326         desc_size = (*desc & HDR_JD_LENGTH_MASK) * sizeof(u32);
327         desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE);
328         if (dma_mapping_error(dev, desc_dma)) {
329                 dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n");
330                 return -EIO;
331         }
332 
333         spin_lock_bh(&jrp->inplock);
334 
335         head = jrp->head;
336         tail = ACCESS_ONCE(jrp->tail);
337 
338         if (!rd_reg32(&jrp->rregs->inpring_avail) ||
339             CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) {
340                 spin_unlock_bh(&jrp->inplock);
341                 dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE);
342                 return -EBUSY;
343         }
344 
345         head_entry = &jrp->entinfo[head];
346         head_entry->desc_addr_virt = desc;
347         head_entry->desc_size = desc_size;
348         head_entry->callbk = (void *)cbk;
349         head_entry->cbkarg = areq;
350         head_entry->desc_addr_dma = desc_dma;
351 
352         jrp->inpring[jrp->inp_ring_write_index] = desc_dma;
353 
354         smp_wmb();
355 
356         jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) &
357                                     (JOBR_DEPTH - 1);
358         jrp->head = (head + 1) & (JOBR_DEPTH - 1);
359 
360         wr_reg32(&jrp->rregs->inpring_jobadd, 1);
361 
362         spin_unlock_bh(&jrp->inplock);
363 
364         return 0;
365 }
366 EXPORT_SYMBOL(caam_jr_enqueue);
367 
368 /*
369  * Init JobR independent of platform property detection
370  */
371 static int caam_jr_init(struct device *dev)
372 {
373         struct caam_drv_private_jr *jrp;
374         dma_addr_t inpbusaddr, outbusaddr;
375         int i, error;
376 
377         jrp = dev_get_drvdata(dev);
378 
379         tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
380 
381         /* Connect job ring interrupt handler. */
382         error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED,
383                             dev_name(dev), dev);
384         if (error) {
385                 dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
386                         jrp->ridx, jrp->irq);
387                 goto out_kill_deq;
388         }
389 
390         error = caam_reset_hw_jr(dev);
391         if (error)
392                 goto out_free_irq;
393 
394         error = -ENOMEM;
395         jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
396                                           &inpbusaddr, GFP_KERNEL);
397         if (!jrp->inpring)
398                 goto out_free_irq;
399 
400         jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) *
401                                           JOBR_DEPTH, &outbusaddr, GFP_KERNEL);
402         if (!jrp->outring)
403                 goto out_free_inpring;
404 
405         jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH,
406                                GFP_KERNEL);
407         if (!jrp->entinfo)
408                 goto out_free_outring;
409 
410         for (i = 0; i < JOBR_DEPTH; i++)
411                 jrp->entinfo[i].desc_addr_dma = !0;
412 
413         /* Setup rings */
414         jrp->inp_ring_write_index = 0;
415         jrp->out_ring_read_index = 0;
416         jrp->head = 0;
417         jrp->tail = 0;
418 
419         wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
420         wr_reg64(&jrp->rregs->outring_base, outbusaddr);
421         wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
422         wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
423 
424         jrp->ringsize = JOBR_DEPTH;
425 
426         spin_lock_init(&jrp->inplock);
427         spin_lock_init(&jrp->outlock);
428 
429         /* Select interrupt coalescing parameters */
430         setbits32(&jrp->rregs->rconfig_lo, JOBR_INTC |
431                   (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
432                   (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
433 
434         return 0;
435 
436 out_free_outring:
437         dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
438                           jrp->outring, outbusaddr);
439 out_free_inpring:
440         dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
441                           jrp->inpring, inpbusaddr);
442         dev_err(dev, "can't allocate job rings for %d\n", jrp->ridx);
443 out_free_irq:
444         free_irq(jrp->irq, dev);
445 out_kill_deq:
446         tasklet_kill(&jrp->irqtask);
447         return error;
448 }
449 
450 
451 /*
452  * Probe routine for each detected JobR subsystem.
453  */
454 static int caam_jr_probe(struct platform_device *pdev)
455 {
456         struct device *jrdev;
457         struct device_node *nprop;
458         struct caam_job_ring __iomem *ctrl;
459         struct caam_drv_private_jr *jrpriv;
460         static int total_jobrs;
461         int error;
462 
463         jrdev = &pdev->dev;
464         jrpriv = devm_kmalloc(jrdev, sizeof(struct caam_drv_private_jr),
465                               GFP_KERNEL);
466         if (!jrpriv)
467                 return -ENOMEM;
468 
469         dev_set_drvdata(jrdev, jrpriv);
470 
471         /* save ring identity relative to detection */
472         jrpriv->ridx = total_jobrs++;
473 
474         nprop = pdev->dev.of_node;
475         /* Get configuration properties from device tree */
476         /* First, get register page */
477         ctrl = of_iomap(nprop, 0);
478         if (!ctrl) {
479                 dev_err(jrdev, "of_iomap() failed\n");
480                 return -ENOMEM;
481         }
482 
483         jrpriv->rregs = (struct caam_job_ring __force *)ctrl;
484 
485         if (sizeof(dma_addr_t) == sizeof(u64))
486                 if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring"))
487                         dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(40));
488                 else
489                         dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(36));
490         else
491                 dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32));
492 
493         /* Identify the interrupt */
494         jrpriv->irq = irq_of_parse_and_map(nprop, 0);
495 
496         /* Now do the platform independent part */
497         error = caam_jr_init(jrdev); /* now turn on hardware */
498         if (error) {
499                 irq_dispose_mapping(jrpriv->irq);
500                 return error;
501         }
502 
503         jrpriv->dev = jrdev;
504         spin_lock(&driver_data.jr_alloc_lock);
505         list_add_tail(&jrpriv->list_node, &driver_data.jr_list);
506         spin_unlock(&driver_data.jr_alloc_lock);
507 
508         atomic_set(&jrpriv->tfm_count, 0);
509 
510         return 0;
511 }
512 
513 static struct of_device_id caam_jr_match[] = {
514         {
515                 .compatible = "fsl,sec-v4.0-job-ring",
516         },
517         {
518                 .compatible = "fsl,sec4.0-job-ring",
519         },
520         {},
521 };
522 MODULE_DEVICE_TABLE(of, caam_jr_match);
523 
524 static struct platform_driver caam_jr_driver = {
525         .driver = {
526                 .name = "caam_jr",
527                 .of_match_table = caam_jr_match,
528         },
529         .probe       = caam_jr_probe,
530         .remove      = caam_jr_remove,
531 };
532 
533 static int __init jr_driver_init(void)
534 {
535         spin_lock_init(&driver_data.jr_alloc_lock);
536         INIT_LIST_HEAD(&driver_data.jr_list);
537         return platform_driver_register(&caam_jr_driver);
538 }
539 
540 static void __exit jr_driver_exit(void)
541 {
542         platform_driver_unregister(&caam_jr_driver);
543 }
544 
545 module_init(jr_driver_init);
546 module_exit(jr_driver_exit);
547 
548 MODULE_LICENSE("GPL");
549 MODULE_DESCRIPTION("FSL CAAM JR request backend");
550 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
551 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us