Version:  2.0.40 2.2.26 2.4.37 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18 3.19

Linux/drivers/crypto/caam/jr.c

  1 /*
  2  * CAAM/SEC 4.x transport/backend driver
  3  * JobR backend functionality
  4  *
  5  * Copyright 2008-2012 Freescale Semiconductor, Inc.
  6  */
  7 
  8 #include <linux/of_irq.h>
  9 #include <linux/of_address.h>
 10 
 11 #include "compat.h"
 12 #include "regs.h"
 13 #include "jr.h"
 14 #include "desc.h"
 15 #include "intern.h"
 16 
 17 struct jr_driver_data {
 18         /* List of Physical JobR's with the Driver */
 19         struct list_head        jr_list;
 20         spinlock_t              jr_alloc_lock;  /* jr_list lock */
 21 } ____cacheline_aligned;
 22 
 23 static struct jr_driver_data driver_data;
 24 
 25 static int caam_reset_hw_jr(struct device *dev)
 26 {
 27         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
 28         unsigned int timeout = 100000;
 29 
 30         /*
 31          * mask interrupts since we are going to poll
 32          * for reset completion status
 33          */
 34         setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
 35 
 36         /* initiate flush (required prior to reset) */
 37         wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
 38         while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) ==
 39                 JRINT_ERR_HALT_INPROGRESS) && --timeout)
 40                 cpu_relax();
 41 
 42         if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) !=
 43             JRINT_ERR_HALT_COMPLETE || timeout == 0) {
 44                 dev_err(dev, "failed to flush job ring %d\n", jrp->ridx);
 45                 return -EIO;
 46         }
 47 
 48         /* initiate reset */
 49         timeout = 100000;
 50         wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
 51         while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout)
 52                 cpu_relax();
 53 
 54         if (timeout == 0) {
 55                 dev_err(dev, "failed to reset job ring %d\n", jrp->ridx);
 56                 return -EIO;
 57         }
 58 
 59         /* unmask interrupts */
 60         clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
 61 
 62         return 0;
 63 }
 64 
 65 /*
 66  * Shutdown JobR independent of platform property code
 67  */
 68 int caam_jr_shutdown(struct device *dev)
 69 {
 70         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
 71         dma_addr_t inpbusaddr, outbusaddr;
 72         int ret;
 73 
 74         ret = caam_reset_hw_jr(dev);
 75 
 76         tasklet_kill(&jrp->irqtask);
 77 
 78         /* Release interrupt */
 79         free_irq(jrp->irq, dev);
 80 
 81         /* Free rings */
 82         inpbusaddr = rd_reg64(&jrp->rregs->inpring_base);
 83         outbusaddr = rd_reg64(&jrp->rregs->outring_base);
 84         dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
 85                           jrp->inpring, inpbusaddr);
 86         dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
 87                           jrp->outring, outbusaddr);
 88         kfree(jrp->entinfo);
 89 
 90         return ret;
 91 }
 92 
 93 static int caam_jr_remove(struct platform_device *pdev)
 94 {
 95         int ret;
 96         struct device *jrdev;
 97         struct caam_drv_private_jr *jrpriv;
 98 
 99         jrdev = &pdev->dev;
100         jrpriv = dev_get_drvdata(jrdev);
101 
102         /*
103          * Return EBUSY if job ring already allocated.
104          */
105         if (atomic_read(&jrpriv->tfm_count)) {
106                 dev_err(jrdev, "Device is busy\n");
107                 return -EBUSY;
108         }
109 
110         /* Remove the node from Physical JobR list maintained by driver */
111         spin_lock(&driver_data.jr_alloc_lock);
112         list_del(&jrpriv->list_node);
113         spin_unlock(&driver_data.jr_alloc_lock);
114 
115         /* Release ring */
116         ret = caam_jr_shutdown(jrdev);
117         if (ret)
118                 dev_err(jrdev, "Failed to shut down job ring\n");
119         irq_dispose_mapping(jrpriv->irq);
120 
121         return ret;
122 }
123 
124 /* Main per-ring interrupt handler */
125 static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
126 {
127         struct device *dev = st_dev;
128         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
129         u32 irqstate;
130 
131         /*
132          * Check the output ring for ready responses, kick
133          * tasklet if jobs done.
134          */
135         irqstate = rd_reg32(&jrp->rregs->jrintstatus);
136         if (!irqstate)
137                 return IRQ_NONE;
138 
139         /*
140          * If JobR error, we got more development work to do
141          * Flag a bug now, but we really need to shut down and
142          * restart the queue (and fix code).
143          */
144         if (irqstate & JRINT_JR_ERROR) {
145                 dev_err(dev, "job ring error: irqstate: %08x\n", irqstate);
146                 BUG();
147         }
148 
149         /* mask valid interrupts */
150         setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
151 
152         /* Have valid interrupt at this point, just ACK and trigger */
153         wr_reg32(&jrp->rregs->jrintstatus, irqstate);
154 
155         preempt_disable();
156         tasklet_schedule(&jrp->irqtask);
157         preempt_enable();
158 
159         return IRQ_HANDLED;
160 }
161 
162 /* Deferred service handler, run as interrupt-fired tasklet */
163 static void caam_jr_dequeue(unsigned long devarg)
164 {
165         int hw_idx, sw_idx, i, head, tail;
166         struct device *dev = (struct device *)devarg;
167         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
168         void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
169         u32 *userdesc, userstatus;
170         void *userarg;
171 
172         while (rd_reg32(&jrp->rregs->outring_used)) {
173 
174                 head = ACCESS_ONCE(jrp->head);
175 
176                 spin_lock(&jrp->outlock);
177 
178                 sw_idx = tail = jrp->tail;
179                 hw_idx = jrp->out_ring_read_index;
180 
181                 for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) {
182                         sw_idx = (tail + i) & (JOBR_DEPTH - 1);
183 
184                         if (jrp->outring[hw_idx].desc ==
185                             jrp->entinfo[sw_idx].desc_addr_dma)
186                                 break; /* found */
187                 }
188                 /* we should never fail to find a matching descriptor */
189                 BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0);
190 
191                 /* Unmap just-run descriptor so we can post-process */
192                 dma_unmap_single(dev, jrp->outring[hw_idx].desc,
193                                  jrp->entinfo[sw_idx].desc_size,
194                                  DMA_TO_DEVICE);
195 
196                 /* mark completed, avoid matching on a recycled desc addr */
197                 jrp->entinfo[sw_idx].desc_addr_dma = 0;
198 
199                 /* Stash callback params for use outside of lock */
200                 usercall = jrp->entinfo[sw_idx].callbk;
201                 userarg = jrp->entinfo[sw_idx].cbkarg;
202                 userdesc = jrp->entinfo[sw_idx].desc_addr_virt;
203                 userstatus = jrp->outring[hw_idx].jrstatus;
204 
205                 /* set done */
206                 wr_reg32(&jrp->rregs->outring_rmvd, 1);
207 
208                 jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) &
209                                            (JOBR_DEPTH - 1);
210 
211                 /*
212                  * if this job completed out-of-order, do not increment
213                  * the tail.  Otherwise, increment tail by 1 plus the
214                  * number of subsequent jobs already completed out-of-order
215                  */
216                 if (sw_idx == tail) {
217                         do {
218                                 tail = (tail + 1) & (JOBR_DEPTH - 1);
219                         } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 &&
220                                  jrp->entinfo[tail].desc_addr_dma == 0);
221 
222                         jrp->tail = tail;
223                 }
224 
225                 spin_unlock(&jrp->outlock);
226 
227                 /* Finally, execute user's callback */
228                 usercall(dev, userdesc, userstatus, userarg);
229         }
230 
231         /* reenable / unmask IRQs */
232         clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
233 }
234 
235 /**
236  * caam_jr_alloc() - Alloc a job ring for someone to use as needed.
237  *
238  * returns :  pointer to the newly allocated physical
239  *            JobR dev can be written to if successful.
240  **/
241 struct device *caam_jr_alloc(void)
242 {
243         struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL;
244         struct device *dev = NULL;
245         int min_tfm_cnt = INT_MAX;
246         int tfm_cnt;
247 
248         spin_lock(&driver_data.jr_alloc_lock);
249 
250         if (list_empty(&driver_data.jr_list)) {
251                 spin_unlock(&driver_data.jr_alloc_lock);
252                 return ERR_PTR(-ENODEV);
253         }
254 
255         list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) {
256                 tfm_cnt = atomic_read(&jrpriv->tfm_count);
257                 if (tfm_cnt < min_tfm_cnt) {
258                         min_tfm_cnt = tfm_cnt;
259                         min_jrpriv = jrpriv;
260                 }
261                 if (!min_tfm_cnt)
262                         break;
263         }
264 
265         if (min_jrpriv) {
266                 atomic_inc(&min_jrpriv->tfm_count);
267                 dev = min_jrpriv->dev;
268         }
269         spin_unlock(&driver_data.jr_alloc_lock);
270 
271         return dev;
272 }
273 EXPORT_SYMBOL(caam_jr_alloc);
274 
275 /**
276  * caam_jr_free() - Free the Job Ring
277  * @rdev     - points to the dev that identifies the Job ring to
278  *             be released.
279  **/
280 void caam_jr_free(struct device *rdev)
281 {
282         struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev);
283 
284         atomic_dec(&jrpriv->tfm_count);
285 }
286 EXPORT_SYMBOL(caam_jr_free);
287 
288 /**
289  * caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK,
290  * -EBUSY if the queue is full, -EIO if it cannot map the caller's
291  * descriptor.
292  * @dev:  device of the job ring to be used. This device should have
293  *        been assigned prior by caam_jr_register().
294  * @desc: points to a job descriptor that execute our request. All
295  *        descriptors (and all referenced data) must be in a DMAable
296  *        region, and all data references must be physical addresses
297  *        accessible to CAAM (i.e. within a PAMU window granted
298  *        to it).
299  * @cbk:  pointer to a callback function to be invoked upon completion
300  *        of this request. This has the form:
301  *        callback(struct device *dev, u32 *desc, u32 stat, void *arg)
302  *        where:
303  *        @dev:    contains the job ring device that processed this
304  *                 response.
305  *        @desc:   descriptor that initiated the request, same as
306  *                 "desc" being argued to caam_jr_enqueue().
307  *        @status: untranslated status received from CAAM. See the
308  *                 reference manual for a detailed description of
309  *                 error meaning, or see the JRSTA definitions in the
310  *                 register header file
311  *        @areq:   optional pointer to an argument passed with the
312  *                 original request
313  * @areq: optional pointer to a user argument for use at callback
314  *        time.
315  **/
316 int caam_jr_enqueue(struct device *dev, u32 *desc,
317                     void (*cbk)(struct device *dev, u32 *desc,
318                                 u32 status, void *areq),
319                     void *areq)
320 {
321         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
322         struct caam_jrentry_info *head_entry;
323         int head, tail, desc_size;
324         dma_addr_t desc_dma;
325 
326         desc_size = (*desc & HDR_JD_LENGTH_MASK) * sizeof(u32);
327         desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE);
328         if (dma_mapping_error(dev, desc_dma)) {
329                 dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n");
330                 return -EIO;
331         }
332 
333         spin_lock_bh(&jrp->inplock);
334 
335         head = jrp->head;
336         tail = ACCESS_ONCE(jrp->tail);
337 
338         if (!rd_reg32(&jrp->rregs->inpring_avail) ||
339             CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) {
340                 spin_unlock_bh(&jrp->inplock);
341                 dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE);
342                 return -EBUSY;
343         }
344 
345         head_entry = &jrp->entinfo[head];
346         head_entry->desc_addr_virt = desc;
347         head_entry->desc_size = desc_size;
348         head_entry->callbk = (void *)cbk;
349         head_entry->cbkarg = areq;
350         head_entry->desc_addr_dma = desc_dma;
351 
352         jrp->inpring[jrp->inp_ring_write_index] = desc_dma;
353 
354         smp_wmb();
355 
356         jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) &
357                                     (JOBR_DEPTH - 1);
358         jrp->head = (head + 1) & (JOBR_DEPTH - 1);
359 
360         wr_reg32(&jrp->rregs->inpring_jobadd, 1);
361 
362         spin_unlock_bh(&jrp->inplock);
363 
364         return 0;
365 }
366 EXPORT_SYMBOL(caam_jr_enqueue);
367 
368 /*
369  * Init JobR independent of platform property detection
370  */
371 static int caam_jr_init(struct device *dev)
372 {
373         struct caam_drv_private_jr *jrp;
374         dma_addr_t inpbusaddr, outbusaddr;
375         int i, error;
376 
377         jrp = dev_get_drvdata(dev);
378 
379         tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
380 
381         /* Connect job ring interrupt handler. */
382         error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED,
383                             dev_name(dev), dev);
384         if (error) {
385                 dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
386                         jrp->ridx, jrp->irq);
387                 irq_dispose_mapping(jrp->irq);
388                 jrp->irq = 0;
389                 return -EINVAL;
390         }
391 
392         error = caam_reset_hw_jr(dev);
393         if (error)
394                 return error;
395 
396         jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
397                                           &inpbusaddr, GFP_KERNEL);
398 
399         jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) *
400                                           JOBR_DEPTH, &outbusaddr, GFP_KERNEL);
401 
402         jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH,
403                                GFP_KERNEL);
404 
405         if ((jrp->inpring == NULL) || (jrp->outring == NULL) ||
406             (jrp->entinfo == NULL)) {
407                 dev_err(dev, "can't allocate job rings for %d\n",
408                         jrp->ridx);
409                 return -ENOMEM;
410         }
411 
412         for (i = 0; i < JOBR_DEPTH; i++)
413                 jrp->entinfo[i].desc_addr_dma = !0;
414 
415         /* Setup rings */
416         jrp->inp_ring_write_index = 0;
417         jrp->out_ring_read_index = 0;
418         jrp->head = 0;
419         jrp->tail = 0;
420 
421         wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
422         wr_reg64(&jrp->rregs->outring_base, outbusaddr);
423         wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
424         wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
425 
426         jrp->ringsize = JOBR_DEPTH;
427 
428         spin_lock_init(&jrp->inplock);
429         spin_lock_init(&jrp->outlock);
430 
431         /* Select interrupt coalescing parameters */
432         setbits32(&jrp->rregs->rconfig_lo, JOBR_INTC |
433                   (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
434                   (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
435 
436         return 0;
437 }
438 
439 
440 /*
441  * Probe routine for each detected JobR subsystem.
442  */
443 static int caam_jr_probe(struct platform_device *pdev)
444 {
445         struct device *jrdev;
446         struct device_node *nprop;
447         struct caam_job_ring __iomem *ctrl;
448         struct caam_drv_private_jr *jrpriv;
449         static int total_jobrs;
450         int error;
451 
452         jrdev = &pdev->dev;
453         jrpriv = devm_kmalloc(jrdev, sizeof(struct caam_drv_private_jr),
454                               GFP_KERNEL);
455         if (!jrpriv)
456                 return -ENOMEM;
457 
458         dev_set_drvdata(jrdev, jrpriv);
459 
460         /* save ring identity relative to detection */
461         jrpriv->ridx = total_jobrs++;
462 
463         nprop = pdev->dev.of_node;
464         /* Get configuration properties from device tree */
465         /* First, get register page */
466         ctrl = of_iomap(nprop, 0);
467         if (!ctrl) {
468                 dev_err(jrdev, "of_iomap() failed\n");
469                 return -ENOMEM;
470         }
471 
472         jrpriv->rregs = (struct caam_job_ring __force *)ctrl;
473 
474         if (sizeof(dma_addr_t) == sizeof(u64))
475                 if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring"))
476                         dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(40));
477                 else
478                         dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(36));
479         else
480                 dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32));
481 
482         /* Identify the interrupt */
483         jrpriv->irq = irq_of_parse_and_map(nprop, 0);
484 
485         /* Now do the platform independent part */
486         error = caam_jr_init(jrdev); /* now turn on hardware */
487         if (error)
488                 return error;
489 
490         jrpriv->dev = jrdev;
491         spin_lock(&driver_data.jr_alloc_lock);
492         list_add_tail(&jrpriv->list_node, &driver_data.jr_list);
493         spin_unlock(&driver_data.jr_alloc_lock);
494 
495         atomic_set(&jrpriv->tfm_count, 0);
496 
497         return 0;
498 }
499 
500 static struct of_device_id caam_jr_match[] = {
501         {
502                 .compatible = "fsl,sec-v4.0-job-ring",
503         },
504         {
505                 .compatible = "fsl,sec4.0-job-ring",
506         },
507         {},
508 };
509 MODULE_DEVICE_TABLE(of, caam_jr_match);
510 
511 static struct platform_driver caam_jr_driver = {
512         .driver = {
513                 .name = "caam_jr",
514                 .of_match_table = caam_jr_match,
515         },
516         .probe       = caam_jr_probe,
517         .remove      = caam_jr_remove,
518 };
519 
520 static int __init jr_driver_init(void)
521 {
522         spin_lock_init(&driver_data.jr_alloc_lock);
523         INIT_LIST_HEAD(&driver_data.jr_list);
524         return platform_driver_register(&caam_jr_driver);
525 }
526 
527 static void __exit jr_driver_exit(void)
528 {
529         platform_driver_unregister(&caam_jr_driver);
530 }
531 
532 module_init(jr_driver_init);
533 module_exit(jr_driver_exit);
534 
535 MODULE_LICENSE("GPL");
536 MODULE_DESCRIPTION("FSL CAAM JR request backend");
537 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
538 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us