Version:  2.0.40 2.2.26 2.4.37 2.6.39 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15

Linux/drivers/crypto/caam/jr.c

  1 /*
  2  * CAAM/SEC 4.x transport/backend driver
  3  * JobR backend functionality
  4  *
  5  * Copyright 2008-2012 Freescale Semiconductor, Inc.
  6  */
  7 
  8 #include <linux/of_irq.h>
  9 #include <linux/of_address.h>
 10 
 11 #include "compat.h"
 12 #include "regs.h"
 13 #include "jr.h"
 14 #include "desc.h"
 15 #include "intern.h"
 16 
 17 struct jr_driver_data {
 18         /* List of Physical JobR's with the Driver */
 19         struct list_head        jr_list;
 20         spinlock_t              jr_alloc_lock;  /* jr_list lock */
 21 } ____cacheline_aligned;
 22 
 23 static struct jr_driver_data driver_data;
 24 
 25 static int caam_reset_hw_jr(struct device *dev)
 26 {
 27         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
 28         unsigned int timeout = 100000;
 29 
 30         /*
 31          * mask interrupts since we are going to poll
 32          * for reset completion status
 33          */
 34         setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
 35 
 36         /* initiate flush (required prior to reset) */
 37         wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
 38         while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) ==
 39                 JRINT_ERR_HALT_INPROGRESS) && --timeout)
 40                 cpu_relax();
 41 
 42         if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) !=
 43             JRINT_ERR_HALT_COMPLETE || timeout == 0) {
 44                 dev_err(dev, "failed to flush job ring %d\n", jrp->ridx);
 45                 return -EIO;
 46         }
 47 
 48         /* initiate reset */
 49         timeout = 100000;
 50         wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
 51         while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout)
 52                 cpu_relax();
 53 
 54         if (timeout == 0) {
 55                 dev_err(dev, "failed to reset job ring %d\n", jrp->ridx);
 56                 return -EIO;
 57         }
 58 
 59         /* unmask interrupts */
 60         clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
 61 
 62         return 0;
 63 }
 64 
 65 /*
 66  * Shutdown JobR independent of platform property code
 67  */
 68 int caam_jr_shutdown(struct device *dev)
 69 {
 70         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
 71         dma_addr_t inpbusaddr, outbusaddr;
 72         int ret;
 73 
 74         ret = caam_reset_hw_jr(dev);
 75 
 76         tasklet_kill(&jrp->irqtask);
 77 
 78         /* Release interrupt */
 79         free_irq(jrp->irq, dev);
 80 
 81         /* Free rings */
 82         inpbusaddr = rd_reg64(&jrp->rregs->inpring_base);
 83         outbusaddr = rd_reg64(&jrp->rregs->outring_base);
 84         dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
 85                           jrp->inpring, inpbusaddr);
 86         dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
 87                           jrp->outring, outbusaddr);
 88         kfree(jrp->entinfo);
 89 
 90         return ret;
 91 }
 92 
 93 static int caam_jr_remove(struct platform_device *pdev)
 94 {
 95         int ret;
 96         struct device *jrdev;
 97         struct caam_drv_private_jr *jrpriv;
 98 
 99         jrdev = &pdev->dev;
100         jrpriv = dev_get_drvdata(jrdev);
101 
102         /*
103          * Return EBUSY if job ring already allocated.
104          */
105         if (atomic_read(&jrpriv->tfm_count)) {
106                 dev_err(jrdev, "Device is busy\n");
107                 return -EBUSY;
108         }
109 
110         /* Remove the node from Physical JobR list maintained by driver */
111         spin_lock(&driver_data.jr_alloc_lock);
112         list_del(&jrpriv->list_node);
113         spin_unlock(&driver_data.jr_alloc_lock);
114 
115         /* Release ring */
116         ret = caam_jr_shutdown(jrdev);
117         if (ret)
118                 dev_err(jrdev, "Failed to shut down job ring\n");
119         irq_dispose_mapping(jrpriv->irq);
120 
121         return ret;
122 }
123 
124 /* Main per-ring interrupt handler */
125 static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
126 {
127         struct device *dev = st_dev;
128         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
129         u32 irqstate;
130 
131         /*
132          * Check the output ring for ready responses, kick
133          * tasklet if jobs done.
134          */
135         irqstate = rd_reg32(&jrp->rregs->jrintstatus);
136         if (!irqstate)
137                 return IRQ_NONE;
138 
139         /*
140          * If JobR error, we got more development work to do
141          * Flag a bug now, but we really need to shut down and
142          * restart the queue (and fix code).
143          */
144         if (irqstate & JRINT_JR_ERROR) {
145                 dev_err(dev, "job ring error: irqstate: %08x\n", irqstate);
146                 BUG();
147         }
148 
149         /* mask valid interrupts */
150         setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
151 
152         /* Have valid interrupt at this point, just ACK and trigger */
153         wr_reg32(&jrp->rregs->jrintstatus, irqstate);
154 
155         preempt_disable();
156         tasklet_schedule(&jrp->irqtask);
157         preempt_enable();
158 
159         return IRQ_HANDLED;
160 }
161 
162 /* Deferred service handler, run as interrupt-fired tasklet */
163 static void caam_jr_dequeue(unsigned long devarg)
164 {
165         int hw_idx, sw_idx, i, head, tail;
166         struct device *dev = (struct device *)devarg;
167         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
168         void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
169         u32 *userdesc, userstatus;
170         void *userarg;
171 
172         while (rd_reg32(&jrp->rregs->outring_used)) {
173 
174                 head = ACCESS_ONCE(jrp->head);
175 
176                 spin_lock(&jrp->outlock);
177 
178                 sw_idx = tail = jrp->tail;
179                 hw_idx = jrp->out_ring_read_index;
180 
181                 for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) {
182                         sw_idx = (tail + i) & (JOBR_DEPTH - 1);
183 
184                         smp_read_barrier_depends();
185 
186                         if (jrp->outring[hw_idx].desc ==
187                             jrp->entinfo[sw_idx].desc_addr_dma)
188                                 break; /* found */
189                 }
190                 /* we should never fail to find a matching descriptor */
191                 BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0);
192 
193                 /* Unmap just-run descriptor so we can post-process */
194                 dma_unmap_single(dev, jrp->outring[hw_idx].desc,
195                                  jrp->entinfo[sw_idx].desc_size,
196                                  DMA_TO_DEVICE);
197 
198                 /* mark completed, avoid matching on a recycled desc addr */
199                 jrp->entinfo[sw_idx].desc_addr_dma = 0;
200 
201                 /* Stash callback params for use outside of lock */
202                 usercall = jrp->entinfo[sw_idx].callbk;
203                 userarg = jrp->entinfo[sw_idx].cbkarg;
204                 userdesc = jrp->entinfo[sw_idx].desc_addr_virt;
205                 userstatus = jrp->outring[hw_idx].jrstatus;
206 
207                 /* set done */
208                 wr_reg32(&jrp->rregs->outring_rmvd, 1);
209 
210                 jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) &
211                                            (JOBR_DEPTH - 1);
212 
213                 /*
214                  * if this job completed out-of-order, do not increment
215                  * the tail.  Otherwise, increment tail by 1 plus the
216                  * number of subsequent jobs already completed out-of-order
217                  */
218                 if (sw_idx == tail) {
219                         do {
220                                 tail = (tail + 1) & (JOBR_DEPTH - 1);
221                                 smp_read_barrier_depends();
222                         } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 &&
223                                  jrp->entinfo[tail].desc_addr_dma == 0);
224 
225                         jrp->tail = tail;
226                 }
227 
228                 spin_unlock(&jrp->outlock);
229 
230                 /* Finally, execute user's callback */
231                 usercall(dev, userdesc, userstatus, userarg);
232         }
233 
234         /* reenable / unmask IRQs */
235         clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
236 }
237 
238 /**
239  * caam_jr_alloc() - Alloc a job ring for someone to use as needed.
240  *
241  * returns :  pointer to the newly allocated physical
242  *            JobR dev can be written to if successful.
243  **/
244 struct device *caam_jr_alloc(void)
245 {
246         struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL;
247         struct device *dev = NULL;
248         int min_tfm_cnt = INT_MAX;
249         int tfm_cnt;
250 
251         spin_lock(&driver_data.jr_alloc_lock);
252 
253         if (list_empty(&driver_data.jr_list)) {
254                 spin_unlock(&driver_data.jr_alloc_lock);
255                 return ERR_PTR(-ENODEV);
256         }
257 
258         list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) {
259                 tfm_cnt = atomic_read(&jrpriv->tfm_count);
260                 if (tfm_cnt < min_tfm_cnt) {
261                         min_tfm_cnt = tfm_cnt;
262                         min_jrpriv = jrpriv;
263                 }
264                 if (!min_tfm_cnt)
265                         break;
266         }
267 
268         if (min_jrpriv) {
269                 atomic_inc(&min_jrpriv->tfm_count);
270                 dev = min_jrpriv->dev;
271         }
272         spin_unlock(&driver_data.jr_alloc_lock);
273 
274         return dev;
275 }
276 EXPORT_SYMBOL(caam_jr_alloc);
277 
278 /**
279  * caam_jr_free() - Free the Job Ring
280  * @rdev     - points to the dev that identifies the Job ring to
281  *             be released.
282  **/
283 void caam_jr_free(struct device *rdev)
284 {
285         struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev);
286 
287         atomic_dec(&jrpriv->tfm_count);
288 }
289 EXPORT_SYMBOL(caam_jr_free);
290 
291 /**
292  * caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK,
293  * -EBUSY if the queue is full, -EIO if it cannot map the caller's
294  * descriptor.
295  * @dev:  device of the job ring to be used. This device should have
296  *        been assigned prior by caam_jr_register().
297  * @desc: points to a job descriptor that execute our request. All
298  *        descriptors (and all referenced data) must be in a DMAable
299  *        region, and all data references must be physical addresses
300  *        accessible to CAAM (i.e. within a PAMU window granted
301  *        to it).
302  * @cbk:  pointer to a callback function to be invoked upon completion
303  *        of this request. This has the form:
304  *        callback(struct device *dev, u32 *desc, u32 stat, void *arg)
305  *        where:
306  *        @dev:    contains the job ring device that processed this
307  *                 response.
308  *        @desc:   descriptor that initiated the request, same as
309  *                 "desc" being argued to caam_jr_enqueue().
310  *        @status: untranslated status received from CAAM. See the
311  *                 reference manual for a detailed description of
312  *                 error meaning, or see the JRSTA definitions in the
313  *                 register header file
314  *        @areq:   optional pointer to an argument passed with the
315  *                 original request
316  * @areq: optional pointer to a user argument for use at callback
317  *        time.
318  **/
319 int caam_jr_enqueue(struct device *dev, u32 *desc,
320                     void (*cbk)(struct device *dev, u32 *desc,
321                                 u32 status, void *areq),
322                     void *areq)
323 {
324         struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
325         struct caam_jrentry_info *head_entry;
326         int head, tail, desc_size;
327         dma_addr_t desc_dma;
328 
329         desc_size = (*desc & HDR_JD_LENGTH_MASK) * sizeof(u32);
330         desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE);
331         if (dma_mapping_error(dev, desc_dma)) {
332                 dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n");
333                 return -EIO;
334         }
335 
336         spin_lock_bh(&jrp->inplock);
337 
338         head = jrp->head;
339         tail = ACCESS_ONCE(jrp->tail);
340 
341         if (!rd_reg32(&jrp->rregs->inpring_avail) ||
342             CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) {
343                 spin_unlock_bh(&jrp->inplock);
344                 dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE);
345                 return -EBUSY;
346         }
347 
348         head_entry = &jrp->entinfo[head];
349         head_entry->desc_addr_virt = desc;
350         head_entry->desc_size = desc_size;
351         head_entry->callbk = (void *)cbk;
352         head_entry->cbkarg = areq;
353         head_entry->desc_addr_dma = desc_dma;
354 
355         jrp->inpring[jrp->inp_ring_write_index] = desc_dma;
356 
357         smp_wmb();
358 
359         jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) &
360                                     (JOBR_DEPTH - 1);
361         jrp->head = (head + 1) & (JOBR_DEPTH - 1);
362 
363         wr_reg32(&jrp->rregs->inpring_jobadd, 1);
364 
365         spin_unlock_bh(&jrp->inplock);
366 
367         return 0;
368 }
369 EXPORT_SYMBOL(caam_jr_enqueue);
370 
371 /*
372  * Init JobR independent of platform property detection
373  */
374 static int caam_jr_init(struct device *dev)
375 {
376         struct caam_drv_private_jr *jrp;
377         dma_addr_t inpbusaddr, outbusaddr;
378         int i, error;
379 
380         jrp = dev_get_drvdata(dev);
381 
382         tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
383 
384         /* Connect job ring interrupt handler. */
385         error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED,
386                             dev_name(dev), dev);
387         if (error) {
388                 dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
389                         jrp->ridx, jrp->irq);
390                 irq_dispose_mapping(jrp->irq);
391                 jrp->irq = 0;
392                 return -EINVAL;
393         }
394 
395         error = caam_reset_hw_jr(dev);
396         if (error)
397                 return error;
398 
399         jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
400                                           &inpbusaddr, GFP_KERNEL);
401 
402         jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) *
403                                           JOBR_DEPTH, &outbusaddr, GFP_KERNEL);
404 
405         jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH,
406                                GFP_KERNEL);
407 
408         if ((jrp->inpring == NULL) || (jrp->outring == NULL) ||
409             (jrp->entinfo == NULL)) {
410                 dev_err(dev, "can't allocate job rings for %d\n",
411                         jrp->ridx);
412                 return -ENOMEM;
413         }
414 
415         for (i = 0; i < JOBR_DEPTH; i++)
416                 jrp->entinfo[i].desc_addr_dma = !0;
417 
418         /* Setup rings */
419         jrp->inp_ring_write_index = 0;
420         jrp->out_ring_read_index = 0;
421         jrp->head = 0;
422         jrp->tail = 0;
423 
424         wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
425         wr_reg64(&jrp->rregs->outring_base, outbusaddr);
426         wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
427         wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
428 
429         jrp->ringsize = JOBR_DEPTH;
430 
431         spin_lock_init(&jrp->inplock);
432         spin_lock_init(&jrp->outlock);
433 
434         /* Select interrupt coalescing parameters */
435         setbits32(&jrp->rregs->rconfig_lo, JOBR_INTC |
436                   (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
437                   (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
438 
439         return 0;
440 }
441 
442 
443 /*
444  * Probe routine for each detected JobR subsystem.
445  */
446 static int caam_jr_probe(struct platform_device *pdev)
447 {
448         struct device *jrdev;
449         struct device_node *nprop;
450         struct caam_job_ring __iomem *ctrl;
451         struct caam_drv_private_jr *jrpriv;
452         static int total_jobrs;
453         int error;
454 
455         jrdev = &pdev->dev;
456         jrpriv = kmalloc(sizeof(struct caam_drv_private_jr),
457                          GFP_KERNEL);
458         if (!jrpriv)
459                 return -ENOMEM;
460 
461         dev_set_drvdata(jrdev, jrpriv);
462 
463         /* save ring identity relative to detection */
464         jrpriv->ridx = total_jobrs++;
465 
466         nprop = pdev->dev.of_node;
467         /* Get configuration properties from device tree */
468         /* First, get register page */
469         ctrl = of_iomap(nprop, 0);
470         if (!ctrl) {
471                 dev_err(jrdev, "of_iomap() failed\n");
472                 return -ENOMEM;
473         }
474 
475         jrpriv->rregs = (struct caam_job_ring __force *)ctrl;
476 
477         if (sizeof(dma_addr_t) == sizeof(u64))
478                 if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring"))
479                         dma_set_mask(jrdev, DMA_BIT_MASK(40));
480                 else
481                         dma_set_mask(jrdev, DMA_BIT_MASK(36));
482         else
483                 dma_set_mask(jrdev, DMA_BIT_MASK(32));
484 
485         /* Identify the interrupt */
486         jrpriv->irq = irq_of_parse_and_map(nprop, 0);
487 
488         /* Now do the platform independent part */
489         error = caam_jr_init(jrdev); /* now turn on hardware */
490         if (error) {
491                 kfree(jrpriv);
492                 return error;
493         }
494 
495         jrpriv->dev = jrdev;
496         spin_lock(&driver_data.jr_alloc_lock);
497         list_add_tail(&jrpriv->list_node, &driver_data.jr_list);
498         spin_unlock(&driver_data.jr_alloc_lock);
499 
500         atomic_set(&jrpriv->tfm_count, 0);
501 
502         return 0;
503 }
504 
505 static struct of_device_id caam_jr_match[] = {
506         {
507                 .compatible = "fsl,sec-v4.0-job-ring",
508         },
509         {
510                 .compatible = "fsl,sec4.0-job-ring",
511         },
512         {},
513 };
514 MODULE_DEVICE_TABLE(of, caam_jr_match);
515 
516 static struct platform_driver caam_jr_driver = {
517         .driver = {
518                 .name = "caam_jr",
519                 .owner = THIS_MODULE,
520                 .of_match_table = caam_jr_match,
521         },
522         .probe       = caam_jr_probe,
523         .remove      = caam_jr_remove,
524 };
525 
526 static int __init jr_driver_init(void)
527 {
528         spin_lock_init(&driver_data.jr_alloc_lock);
529         INIT_LIST_HEAD(&driver_data.jr_list);
530         return platform_driver_register(&caam_jr_driver);
531 }
532 
533 static void __exit jr_driver_exit(void)
534 {
535         platform_driver_unregister(&caam_jr_driver);
536 }
537 
538 module_init(jr_driver_init);
539 module_exit(jr_driver_exit);
540 
541 MODULE_LICENSE("GPL");
542 MODULE_DESCRIPTION("FSL CAAM JR request backend");
543 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
544 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us