Version:  2.0.40 2.2.26 2.4.37 3.2 3.3 3.4 3.5 3.6 3.7 3.8 3.9 3.10 3.11 3.12 3.13 3.14 3.15 3.16 3.17 3.18

Linux/drivers/crypto/n2_core.c

  1 /* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.
  2  *
  3  * Copyright (C) 2010, 2011 David S. Miller <davem@davemloft.net>
  4  */
  5 
  6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  7 
  8 #include <linux/kernel.h>
  9 #include <linux/module.h>
 10 #include <linux/of.h>
 11 #include <linux/of_device.h>
 12 #include <linux/cpumask.h>
 13 #include <linux/slab.h>
 14 #include <linux/interrupt.h>
 15 #include <linux/crypto.h>
 16 #include <crypto/md5.h>
 17 #include <crypto/sha.h>
 18 #include <crypto/aes.h>
 19 #include <crypto/des.h>
 20 #include <linux/mutex.h>
 21 #include <linux/delay.h>
 22 #include <linux/sched.h>
 23 
 24 #include <crypto/internal/hash.h>
 25 #include <crypto/scatterwalk.h>
 26 #include <crypto/algapi.h>
 27 
 28 #include <asm/hypervisor.h>
 29 #include <asm/mdesc.h>
 30 
 31 #include "n2_core.h"
 32 
 33 #define DRV_MODULE_NAME         "n2_crypto"
 34 #define DRV_MODULE_VERSION      "0.2"
 35 #define DRV_MODULE_RELDATE      "July 28, 2011"
 36 
 37 static char version[] =
 38         DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
 39 
 40 MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
 41 MODULE_DESCRIPTION("Niagara2 Crypto driver");
 42 MODULE_LICENSE("GPL");
 43 MODULE_VERSION(DRV_MODULE_VERSION);
 44 
 45 #define N2_CRA_PRIORITY         200
 46 
 47 static DEFINE_MUTEX(spu_lock);
 48 
 49 struct spu_queue {
 50         cpumask_t               sharing;
 51         unsigned long           qhandle;
 52 
 53         spinlock_t              lock;
 54         u8                      q_type;
 55         void                    *q;
 56         unsigned long           head;
 57         unsigned long           tail;
 58         struct list_head        jobs;
 59 
 60         unsigned long           devino;
 61 
 62         char                    irq_name[32];
 63         unsigned int            irq;
 64 
 65         struct list_head        list;
 66 };
 67 
 68 static struct spu_queue **cpu_to_cwq;
 69 static struct spu_queue **cpu_to_mau;
 70 
 71 static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off)
 72 {
 73         if (q->q_type == HV_NCS_QTYPE_MAU) {
 74                 off += MAU_ENTRY_SIZE;
 75                 if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES))
 76                         off = 0;
 77         } else {
 78                 off += CWQ_ENTRY_SIZE;
 79                 if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES))
 80                         off = 0;
 81         }
 82         return off;
 83 }
 84 
 85 struct n2_request_common {
 86         struct list_head        entry;
 87         unsigned int            offset;
 88 };
 89 #define OFFSET_NOT_RUNNING      (~(unsigned int)0)
 90 
 91 /* An async job request records the final tail value it used in
 92  * n2_request_common->offset, test to see if that offset is in
 93  * the range old_head, new_head, inclusive.
 94  */
 95 static inline bool job_finished(struct spu_queue *q, unsigned int offset,
 96                                 unsigned long old_head, unsigned long new_head)
 97 {
 98         if (old_head <= new_head) {
 99                 if (offset > old_head && offset <= new_head)
100                         return true;
101         } else {
102                 if (offset > old_head || offset <= new_head)
103                         return true;
104         }
105         return false;
106 }
107 
108 /* When the HEAD marker is unequal to the actual HEAD, we get
109  * a virtual device INO interrupt.  We should process the
110  * completed CWQ entries and adjust the HEAD marker to clear
111  * the IRQ.
112  */
113 static irqreturn_t cwq_intr(int irq, void *dev_id)
114 {
115         unsigned long off, new_head, hv_ret;
116         struct spu_queue *q = dev_id;
117 
118         pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",
119                smp_processor_id(), q->qhandle);
120 
121         spin_lock(&q->lock);
122 
123         hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head);
124 
125         pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",
126                smp_processor_id(), new_head, hv_ret);
127 
128         for (off = q->head; off != new_head; off = spu_next_offset(q, off)) {
129                 /* XXX ... XXX */
130         }
131 
132         hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head);
133         if (hv_ret == HV_EOK)
134                 q->head = new_head;
135 
136         spin_unlock(&q->lock);
137 
138         return IRQ_HANDLED;
139 }
140 
141 static irqreturn_t mau_intr(int irq, void *dev_id)
142 {
143         struct spu_queue *q = dev_id;
144         unsigned long head, hv_ret;
145 
146         spin_lock(&q->lock);
147 
148         pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",
149                smp_processor_id(), q->qhandle);
150 
151         hv_ret = sun4v_ncs_gethead(q->qhandle, &head);
152 
153         pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",
154                smp_processor_id(), head, hv_ret);
155 
156         sun4v_ncs_sethead_marker(q->qhandle, head);
157 
158         spin_unlock(&q->lock);
159 
160         return IRQ_HANDLED;
161 }
162 
163 static void *spu_queue_next(struct spu_queue *q, void *cur)
164 {
165         return q->q + spu_next_offset(q, cur - q->q);
166 }
167 
168 static int spu_queue_num_free(struct spu_queue *q)
169 {
170         unsigned long head = q->head;
171         unsigned long tail = q->tail;
172         unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES);
173         unsigned long diff;
174 
175         if (head > tail)
176                 diff = head - tail;
177         else
178                 diff = (end - tail) + head;
179 
180         return (diff / CWQ_ENTRY_SIZE) - 1;
181 }
182 
183 static void *spu_queue_alloc(struct spu_queue *q, int num_entries)
184 {
185         int avail = spu_queue_num_free(q);
186 
187         if (avail >= num_entries)
188                 return q->q + q->tail;
189 
190         return NULL;
191 }
192 
193 static unsigned long spu_queue_submit(struct spu_queue *q, void *last)
194 {
195         unsigned long hv_ret, new_tail;
196 
197         new_tail = spu_next_offset(q, last - q->q);
198 
199         hv_ret = sun4v_ncs_settail(q->qhandle, new_tail);
200         if (hv_ret == HV_EOK)
201                 q->tail = new_tail;
202         return hv_ret;
203 }
204 
205 static u64 control_word_base(unsigned int len, unsigned int hmac_key_len,
206                              int enc_type, int auth_type,
207                              unsigned int hash_len,
208                              bool sfas, bool sob, bool eob, bool encrypt,
209                              int opcode)
210 {
211         u64 word = (len - 1) & CONTROL_LEN;
212 
213         word |= ((u64) opcode << CONTROL_OPCODE_SHIFT);
214         word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT);
215         word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT);
216         if (sfas)
217                 word |= CONTROL_STORE_FINAL_AUTH_STATE;
218         if (sob)
219                 word |= CONTROL_START_OF_BLOCK;
220         if (eob)
221                 word |= CONTROL_END_OF_BLOCK;
222         if (encrypt)
223                 word |= CONTROL_ENCRYPT;
224         if (hmac_key_len)
225                 word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT;
226         if (hash_len)
227                 word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT;
228 
229         return word;
230 }
231 
232 #if 0
233 static inline bool n2_should_run_async(struct spu_queue *qp, int this_len)
234 {
235         if (this_len >= 64 ||
236             qp->head != qp->tail)
237                 return true;
238         return false;
239 }
240 #endif
241 
242 struct n2_ahash_alg {
243         struct list_head        entry;
244         const char              *hash_zero;
245         const u32               *hash_init;
246         u8                      hw_op_hashsz;
247         u8                      digest_size;
248         u8                      auth_type;
249         u8                      hmac_type;
250         struct ahash_alg        alg;
251 };
252 
253 static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm)
254 {
255         struct crypto_alg *alg = tfm->__crt_alg;
256         struct ahash_alg *ahash_alg;
257 
258         ahash_alg = container_of(alg, struct ahash_alg, halg.base);
259 
260         return container_of(ahash_alg, struct n2_ahash_alg, alg);
261 }
262 
263 struct n2_hmac_alg {
264         const char              *child_alg;
265         struct n2_ahash_alg     derived;
266 };
267 
268 static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm)
269 {
270         struct crypto_alg *alg = tfm->__crt_alg;
271         struct ahash_alg *ahash_alg;
272 
273         ahash_alg = container_of(alg, struct ahash_alg, halg.base);
274 
275         return container_of(ahash_alg, struct n2_hmac_alg, derived.alg);
276 }
277 
278 struct n2_hash_ctx {
279         struct crypto_ahash             *fallback_tfm;
280 };
281 
282 #define N2_HASH_KEY_MAX                 32 /* HW limit for all HMAC requests */
283 
284 struct n2_hmac_ctx {
285         struct n2_hash_ctx              base;
286 
287         struct crypto_shash             *child_shash;
288 
289         int                             hash_key_len;
290         unsigned char                   hash_key[N2_HASH_KEY_MAX];
291 };
292 
293 struct n2_hash_req_ctx {
294         union {
295                 struct md5_state        md5;
296                 struct sha1_state       sha1;
297                 struct sha256_state     sha256;
298         } u;
299 
300         struct ahash_request            fallback_req;
301 };
302 
303 static int n2_hash_async_init(struct ahash_request *req)
304 {
305         struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
306         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
307         struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
308 
309         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
310         rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
311 
312         return crypto_ahash_init(&rctx->fallback_req);
313 }
314 
315 static int n2_hash_async_update(struct ahash_request *req)
316 {
317         struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
318         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
319         struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
320 
321         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
322         rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
323         rctx->fallback_req.nbytes = req->nbytes;
324         rctx->fallback_req.src = req->src;
325 
326         return crypto_ahash_update(&rctx->fallback_req);
327 }
328 
329 static int n2_hash_async_final(struct ahash_request *req)
330 {
331         struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
332         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
333         struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
334 
335         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
336         rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
337         rctx->fallback_req.result = req->result;
338 
339         return crypto_ahash_final(&rctx->fallback_req);
340 }
341 
342 static int n2_hash_async_finup(struct ahash_request *req)
343 {
344         struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
345         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
346         struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
347 
348         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
349         rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
350         rctx->fallback_req.nbytes = req->nbytes;
351         rctx->fallback_req.src = req->src;
352         rctx->fallback_req.result = req->result;
353 
354         return crypto_ahash_finup(&rctx->fallback_req);
355 }
356 
357 static int n2_hash_cra_init(struct crypto_tfm *tfm)
358 {
359         const char *fallback_driver_name = crypto_tfm_alg_name(tfm);
360         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
361         struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
362         struct crypto_ahash *fallback_tfm;
363         int err;
364 
365         fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
366                                           CRYPTO_ALG_NEED_FALLBACK);
367         if (IS_ERR(fallback_tfm)) {
368                 pr_warning("Fallback driver '%s' could not be loaded!\n",
369                            fallback_driver_name);
370                 err = PTR_ERR(fallback_tfm);
371                 goto out;
372         }
373 
374         crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
375                                          crypto_ahash_reqsize(fallback_tfm)));
376 
377         ctx->fallback_tfm = fallback_tfm;
378         return 0;
379 
380 out:
381         return err;
382 }
383 
384 static void n2_hash_cra_exit(struct crypto_tfm *tfm)
385 {
386         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
387         struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
388 
389         crypto_free_ahash(ctx->fallback_tfm);
390 }
391 
392 static int n2_hmac_cra_init(struct crypto_tfm *tfm)
393 {
394         const char *fallback_driver_name = crypto_tfm_alg_name(tfm);
395         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
396         struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
397         struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm);
398         struct crypto_ahash *fallback_tfm;
399         struct crypto_shash *child_shash;
400         int err;
401 
402         fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
403                                           CRYPTO_ALG_NEED_FALLBACK);
404         if (IS_ERR(fallback_tfm)) {
405                 pr_warning("Fallback driver '%s' could not be loaded!\n",
406                            fallback_driver_name);
407                 err = PTR_ERR(fallback_tfm);
408                 goto out;
409         }
410 
411         child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0);
412         if (IS_ERR(child_shash)) {
413                 pr_warning("Child shash '%s' could not be loaded!\n",
414                            n2alg->child_alg);
415                 err = PTR_ERR(child_shash);
416                 goto out_free_fallback;
417         }
418 
419         crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
420                                          crypto_ahash_reqsize(fallback_tfm)));
421 
422         ctx->child_shash = child_shash;
423         ctx->base.fallback_tfm = fallback_tfm;
424         return 0;
425 
426 out_free_fallback:
427         crypto_free_ahash(fallback_tfm);
428 
429 out:
430         return err;
431 }
432 
433 static void n2_hmac_cra_exit(struct crypto_tfm *tfm)
434 {
435         struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
436         struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
437 
438         crypto_free_ahash(ctx->base.fallback_tfm);
439         crypto_free_shash(ctx->child_shash);
440 }
441 
442 static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key,
443                                 unsigned int keylen)
444 {
445         struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
446         struct crypto_shash *child_shash = ctx->child_shash;
447         struct crypto_ahash *fallback_tfm;
448         SHASH_DESC_ON_STACK(shash, child_shash);
449         int err, bs, ds;
450 
451         fallback_tfm = ctx->base.fallback_tfm;
452         err = crypto_ahash_setkey(fallback_tfm, key, keylen);
453         if (err)
454                 return err;
455 
456         shash->tfm = child_shash;
457         shash->flags = crypto_ahash_get_flags(tfm) &
458                 CRYPTO_TFM_REQ_MAY_SLEEP;
459 
460         bs = crypto_shash_blocksize(child_shash);
461         ds = crypto_shash_digestsize(child_shash);
462         BUG_ON(ds > N2_HASH_KEY_MAX);
463         if (keylen > bs) {
464                 err = crypto_shash_digest(shash, key, keylen,
465                                           ctx->hash_key);
466                 if (err)
467                         return err;
468                 keylen = ds;
469         } else if (keylen <= N2_HASH_KEY_MAX)
470                 memcpy(ctx->hash_key, key, keylen);
471 
472         ctx->hash_key_len = keylen;
473 
474         return err;
475 }
476 
477 static unsigned long wait_for_tail(struct spu_queue *qp)
478 {
479         unsigned long head, hv_ret;
480 
481         do {
482                 hv_ret = sun4v_ncs_gethead(qp->qhandle, &head);
483                 if (hv_ret != HV_EOK) {
484                         pr_err("Hypervisor error on gethead\n");
485                         break;
486                 }
487                 if (head == qp->tail) {
488                         qp->head = head;
489                         break;
490                 }
491         } while (1);
492         return hv_ret;
493 }
494 
495 static unsigned long submit_and_wait_for_tail(struct spu_queue *qp,
496                                               struct cwq_initial_entry *ent)
497 {
498         unsigned long hv_ret = spu_queue_submit(qp, ent);
499 
500         if (hv_ret == HV_EOK)
501                 hv_ret = wait_for_tail(qp);
502 
503         return hv_ret;
504 }
505 
506 static int n2_do_async_digest(struct ahash_request *req,
507                               unsigned int auth_type, unsigned int digest_size,
508                               unsigned int result_size, void *hash_loc,
509                               unsigned long auth_key, unsigned int auth_key_len)
510 {
511         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
512         struct cwq_initial_entry *ent;
513         struct crypto_hash_walk walk;
514         struct spu_queue *qp;
515         unsigned long flags;
516         int err = -ENODEV;
517         int nbytes, cpu;
518 
519         /* The total effective length of the operation may not
520          * exceed 2^16.
521          */
522         if (unlikely(req->nbytes > (1 << 16))) {
523                 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
524                 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
525 
526                 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
527                 rctx->fallback_req.base.flags =
528                         req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
529                 rctx->fallback_req.nbytes = req->nbytes;
530                 rctx->fallback_req.src = req->src;
531                 rctx->fallback_req.result = req->result;
532 
533                 return crypto_ahash_digest(&rctx->fallback_req);
534         }
535 
536         nbytes = crypto_hash_walk_first(req, &walk);
537 
538         cpu = get_cpu();
539         qp = cpu_to_cwq[cpu];
540         if (!qp)
541                 goto out;
542 
543         spin_lock_irqsave(&qp->lock, flags);
544 
545         /* XXX can do better, improve this later by doing a by-hand scatterlist
546          * XXX walk, etc.
547          */
548         ent = qp->q + qp->tail;
549 
550         ent->control = control_word_base(nbytes, auth_key_len, 0,
551                                          auth_type, digest_size,
552                                          false, true, false, false,
553                                          OPCODE_INPLACE_BIT |
554                                          OPCODE_AUTH_MAC);
555         ent->src_addr = __pa(walk.data);
556         ent->auth_key_addr = auth_key;
557         ent->auth_iv_addr = __pa(hash_loc);
558         ent->final_auth_state_addr = 0UL;
559         ent->enc_key_addr = 0UL;
560         ent->enc_iv_addr = 0UL;
561         ent->dest_addr = __pa(hash_loc);
562 
563         nbytes = crypto_hash_walk_done(&walk, 0);
564         while (nbytes > 0) {
565                 ent = spu_queue_next(qp, ent);
566 
567                 ent->control = (nbytes - 1);
568                 ent->src_addr = __pa(walk.data);
569                 ent->auth_key_addr = 0UL;
570                 ent->auth_iv_addr = 0UL;
571                 ent->final_auth_state_addr = 0UL;
572                 ent->enc_key_addr = 0UL;
573                 ent->enc_iv_addr = 0UL;
574                 ent->dest_addr = 0UL;
575 
576                 nbytes = crypto_hash_walk_done(&walk, 0);
577         }
578         ent->control |= CONTROL_END_OF_BLOCK;
579 
580         if (submit_and_wait_for_tail(qp, ent) != HV_EOK)
581                 err = -EINVAL;
582         else
583                 err = 0;
584 
585         spin_unlock_irqrestore(&qp->lock, flags);
586 
587         if (!err)
588                 memcpy(req->result, hash_loc, result_size);
589 out:
590         put_cpu();
591 
592         return err;
593 }
594 
595 static int n2_hash_async_digest(struct ahash_request *req)
596 {
597         struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm);
598         struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
599         int ds;
600 
601         ds = n2alg->digest_size;
602         if (unlikely(req->nbytes == 0)) {
603                 memcpy(req->result, n2alg->hash_zero, ds);
604                 return 0;
605         }
606         memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz);
607 
608         return n2_do_async_digest(req, n2alg->auth_type,
609                                   n2alg->hw_op_hashsz, ds,
610                                   &rctx->u, 0UL, 0);
611 }
612 
613 static int n2_hmac_async_digest(struct ahash_request *req)
614 {
615         struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm);
616         struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
617         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
618         struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
619         int ds;
620 
621         ds = n2alg->derived.digest_size;
622         if (unlikely(req->nbytes == 0) ||
623             unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) {
624                 struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
625                 struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
626 
627                 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
628                 rctx->fallback_req.base.flags =
629                         req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
630                 rctx->fallback_req.nbytes = req->nbytes;
631                 rctx->fallback_req.src = req->src;
632                 rctx->fallback_req.result = req->result;
633 
634                 return crypto_ahash_digest(&rctx->fallback_req);
635         }
636         memcpy(&rctx->u, n2alg->derived.hash_init,
637                n2alg->derived.hw_op_hashsz);
638 
639         return n2_do_async_digest(req, n2alg->derived.hmac_type,
640                                   n2alg->derived.hw_op_hashsz, ds,
641                                   &rctx->u,
642                                   __pa(&ctx->hash_key),
643                                   ctx->hash_key_len);
644 }
645 
646 struct n2_cipher_context {
647         int                     key_len;
648         int                     enc_type;
649         union {
650                 u8              aes[AES_MAX_KEY_SIZE];
651                 u8              des[DES_KEY_SIZE];
652                 u8              des3[3 * DES_KEY_SIZE];
653                 u8              arc4[258]; /* S-box, X, Y */
654         } key;
655 };
656 
657 #define N2_CHUNK_ARR_LEN        16
658 
659 struct n2_crypto_chunk {
660         struct list_head        entry;
661         unsigned long           iv_paddr : 44;
662         unsigned long           arr_len : 20;
663         unsigned long           dest_paddr;
664         unsigned long           dest_final;
665         struct {
666                 unsigned long   src_paddr : 44;
667                 unsigned long   src_len : 20;
668         } arr[N2_CHUNK_ARR_LEN];
669 };
670 
671 struct n2_request_context {
672         struct ablkcipher_walk  walk;
673         struct list_head        chunk_list;
674         struct n2_crypto_chunk  chunk;
675         u8                      temp_iv[16];
676 };
677 
678 /* The SPU allows some level of flexibility for partial cipher blocks
679  * being specified in a descriptor.
680  *
681  * It merely requires that every descriptor's length field is at least
682  * as large as the cipher block size.  This means that a cipher block
683  * can span at most 2 descriptors.  However, this does not allow a
684  * partial block to span into the final descriptor as that would
685  * violate the rule (since every descriptor's length must be at lest
686  * the block size).  So, for example, assuming an 8 byte block size:
687  *
688  *      0xe --> 0xa --> 0x8
689  *
690  * is a valid length sequence, whereas:
691  *
692  *      0xe --> 0xb --> 0x7
693  *
694  * is not a valid sequence.
695  */
696 
697 struct n2_cipher_alg {
698         struct list_head        entry;
699         u8                      enc_type;
700         struct crypto_alg       alg;
701 };
702 
703 static inline struct n2_cipher_alg *n2_cipher_alg(struct crypto_tfm *tfm)
704 {
705         struct crypto_alg *alg = tfm->__crt_alg;
706 
707         return container_of(alg, struct n2_cipher_alg, alg);
708 }
709 
710 struct n2_cipher_request_context {
711         struct ablkcipher_walk  walk;
712 };
713 
714 static int n2_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
715                          unsigned int keylen)
716 {
717         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
718         struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
719         struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
720 
721         ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK);
722 
723         switch (keylen) {
724         case AES_KEYSIZE_128:
725                 ctx->enc_type |= ENC_TYPE_ALG_AES128;
726                 break;
727         case AES_KEYSIZE_192:
728                 ctx->enc_type |= ENC_TYPE_ALG_AES192;
729                 break;
730         case AES_KEYSIZE_256:
731                 ctx->enc_type |= ENC_TYPE_ALG_AES256;
732                 break;
733         default:
734                 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
735                 return -EINVAL;
736         }
737 
738         ctx->key_len = keylen;
739         memcpy(ctx->key.aes, key, keylen);
740         return 0;
741 }
742 
743 static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
744                          unsigned int keylen)
745 {
746         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
747         struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
748         struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
749         u32 tmp[DES_EXPKEY_WORDS];
750         int err;
751 
752         ctx->enc_type = n2alg->enc_type;
753 
754         if (keylen != DES_KEY_SIZE) {
755                 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
756                 return -EINVAL;
757         }
758 
759         err = des_ekey(tmp, key);
760         if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
761                 tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
762                 return -EINVAL;
763         }
764 
765         ctx->key_len = keylen;
766         memcpy(ctx->key.des, key, keylen);
767         return 0;
768 }
769 
770 static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
771                           unsigned int keylen)
772 {
773         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
774         struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
775         struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
776 
777         ctx->enc_type = n2alg->enc_type;
778 
779         if (keylen != (3 * DES_KEY_SIZE)) {
780                 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
781                 return -EINVAL;
782         }
783         ctx->key_len = keylen;
784         memcpy(ctx->key.des3, key, keylen);
785         return 0;
786 }
787 
788 static int n2_arc4_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
789                           unsigned int keylen)
790 {
791         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
792         struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
793         struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
794         u8 *s = ctx->key.arc4;
795         u8 *x = s + 256;
796         u8 *y = x + 1;
797         int i, j, k;
798 
799         ctx->enc_type = n2alg->enc_type;
800 
801         j = k = 0;
802         *x = 0;
803         *y = 0;
804         for (i = 0; i < 256; i++)
805                 s[i] = i;
806         for (i = 0; i < 256; i++) {
807                 u8 a = s[i];
808                 j = (j + key[k] + a) & 0xff;
809                 s[i] = s[j];
810                 s[j] = a;
811                 if (++k >= keylen)
812                         k = 0;
813         }
814 
815         return 0;
816 }
817 
818 static inline int cipher_descriptor_len(int nbytes, unsigned int block_size)
819 {
820         int this_len = nbytes;
821 
822         this_len -= (nbytes & (block_size - 1));
823         return this_len > (1 << 16) ? (1 << 16) : this_len;
824 }
825 
826 static int __n2_crypt_chunk(struct crypto_tfm *tfm, struct n2_crypto_chunk *cp,
827                             struct spu_queue *qp, bool encrypt)
828 {
829         struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
830         struct cwq_initial_entry *ent;
831         bool in_place;
832         int i;
833 
834         ent = spu_queue_alloc(qp, cp->arr_len);
835         if (!ent) {
836                 pr_info("queue_alloc() of %d fails\n",
837                         cp->arr_len);
838                 return -EBUSY;
839         }
840 
841         in_place = (cp->dest_paddr == cp->arr[0].src_paddr);
842 
843         ent->control = control_word_base(cp->arr[0].src_len,
844                                          0, ctx->enc_type, 0, 0,
845                                          false, true, false, encrypt,
846                                          OPCODE_ENCRYPT |
847                                          (in_place ? OPCODE_INPLACE_BIT : 0));
848         ent->src_addr = cp->arr[0].src_paddr;
849         ent->auth_key_addr = 0UL;
850         ent->auth_iv_addr = 0UL;
851         ent->final_auth_state_addr = 0UL;
852         ent->enc_key_addr = __pa(&ctx->key);
853         ent->enc_iv_addr = cp->iv_paddr;
854         ent->dest_addr = (in_place ? 0UL : cp->dest_paddr);
855 
856         for (i = 1; i < cp->arr_len; i++) {
857                 ent = spu_queue_next(qp, ent);
858 
859                 ent->control = cp->arr[i].src_len - 1;
860                 ent->src_addr = cp->arr[i].src_paddr;
861                 ent->auth_key_addr = 0UL;
862                 ent->auth_iv_addr = 0UL;
863                 ent->final_auth_state_addr = 0UL;
864                 ent->enc_key_addr = 0UL;
865                 ent->enc_iv_addr = 0UL;
866                 ent->dest_addr = 0UL;
867         }
868         ent->control |= CONTROL_END_OF_BLOCK;
869 
870         return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0;
871 }
872 
873 static int n2_compute_chunks(struct ablkcipher_request *req)
874 {
875         struct n2_request_context *rctx = ablkcipher_request_ctx(req);
876         struct ablkcipher_walk *walk = &rctx->walk;
877         struct n2_crypto_chunk *chunk;
878         unsigned long dest_prev;
879         unsigned int tot_len;
880         bool prev_in_place;
881         int err, nbytes;
882 
883         ablkcipher_walk_init(walk, req->dst, req->src, req->nbytes);
884         err = ablkcipher_walk_phys(req, walk);
885         if (err)
886                 return err;
887 
888         INIT_LIST_HEAD(&rctx->chunk_list);
889 
890         chunk = &rctx->chunk;
891         INIT_LIST_HEAD(&chunk->entry);
892 
893         chunk->iv_paddr = 0UL;
894         chunk->arr_len = 0;
895         chunk->dest_paddr = 0UL;
896 
897         prev_in_place = false;
898         dest_prev = ~0UL;
899         tot_len = 0;
900 
901         while ((nbytes = walk->nbytes) != 0) {
902                 unsigned long dest_paddr, src_paddr;
903                 bool in_place;
904                 int this_len;
905 
906                 src_paddr = (page_to_phys(walk->src.page) +
907                              walk->src.offset);
908                 dest_paddr = (page_to_phys(walk->dst.page) +
909                               walk->dst.offset);
910                 in_place = (src_paddr == dest_paddr);
911                 this_len = cipher_descriptor_len(nbytes, walk->blocksize);
912 
913                 if (chunk->arr_len != 0) {
914                         if (in_place != prev_in_place ||
915                             (!prev_in_place &&
916                              dest_paddr != dest_prev) ||
917                             chunk->arr_len == N2_CHUNK_ARR_LEN ||
918                             tot_len + this_len > (1 << 16)) {
919                                 chunk->dest_final = dest_prev;
920                                 list_add_tail(&chunk->entry,
921                                               &rctx->chunk_list);
922                                 chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);
923                                 if (!chunk) {
924                                         err = -ENOMEM;
925                                         break;
926                                 }
927                                 INIT_LIST_HEAD(&chunk->entry);
928                         }
929                 }
930                 if (chunk->arr_len == 0) {
931                         chunk->dest_paddr = dest_paddr;
932                         tot_len = 0;
933                 }
934                 chunk->arr[chunk->arr_len].src_paddr = src_paddr;
935                 chunk->arr[chunk->arr_len].src_len = this_len;
936                 chunk->arr_len++;
937 
938                 dest_prev = dest_paddr + this_len;
939                 prev_in_place = in_place;
940                 tot_len += this_len;
941 
942                 err = ablkcipher_walk_done(req, walk, nbytes - this_len);
943                 if (err)
944                         break;
945         }
946         if (!err && chunk->arr_len != 0) {
947                 chunk->dest_final = dest_prev;
948                 list_add_tail(&chunk->entry, &rctx->chunk_list);
949         }
950 
951         return err;
952 }
953 
954 static void n2_chunk_complete(struct ablkcipher_request *req, void *final_iv)
955 {
956         struct n2_request_context *rctx = ablkcipher_request_ctx(req);
957         struct n2_crypto_chunk *c, *tmp;
958 
959         if (final_iv)
960                 memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize);
961 
962         ablkcipher_walk_complete(&rctx->walk);
963         list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
964                 list_del(&c->entry);
965                 if (unlikely(c != &rctx->chunk))
966                         kfree(c);
967         }
968 
969 }
970 
971 static int n2_do_ecb(struct ablkcipher_request *req, bool encrypt)
972 {
973         struct n2_request_context *rctx = ablkcipher_request_ctx(req);
974         struct crypto_tfm *tfm = req->base.tfm;
975         int err = n2_compute_chunks(req);
976         struct n2_crypto_chunk *c, *tmp;
977         unsigned long flags, hv_ret;
978         struct spu_queue *qp;
979 
980         if (err)
981                 return err;
982 
983         qp = cpu_to_cwq[get_cpu()];
984         err = -ENODEV;
985         if (!qp)
986                 goto out;
987 
988         spin_lock_irqsave(&qp->lock, flags);
989 
990         list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
991                 err = __n2_crypt_chunk(tfm, c, qp, encrypt);
992                 if (err)
993                         break;
994                 list_del(&c->entry);
995                 if (unlikely(c != &rctx->chunk))
996                         kfree(c);
997         }
998         if (!err) {
999                 hv_ret = wait_for_tail(qp);
1000                 if (hv_ret != HV_EOK)
1001                         err = -EINVAL;
1002         }
1003 
1004         spin_unlock_irqrestore(&qp->lock, flags);
1005 
1006 out:
1007         put_cpu();
1008 
1009         n2_chunk_complete(req, NULL);
1010         return err;
1011 }
1012 
1013 static int n2_encrypt_ecb(struct ablkcipher_request *req)
1014 {
1015         return n2_do_ecb(req, true);
1016 }
1017 
1018 static int n2_decrypt_ecb(struct ablkcipher_request *req)
1019 {
1020         return n2_do_ecb(req, false);
1021 }
1022 
1023 static int n2_do_chaining(struct ablkcipher_request *req, bool encrypt)
1024 {
1025         struct n2_request_context *rctx = ablkcipher_request_ctx(req);
1026         struct crypto_tfm *tfm = req->base.tfm;
1027         unsigned long flags, hv_ret, iv_paddr;
1028         int err = n2_compute_chunks(req);
1029         struct n2_crypto_chunk *c, *tmp;
1030         struct spu_queue *qp;
1031         void *final_iv_addr;
1032 
1033         final_iv_addr = NULL;
1034 
1035         if (err)
1036                 return err;
1037 
1038         qp = cpu_to_cwq[get_cpu()];
1039         err = -ENODEV;
1040         if (!qp)
1041                 goto out;
1042 
1043         spin_lock_irqsave(&qp->lock, flags);
1044 
1045         if (encrypt) {
1046                 iv_paddr = __pa(rctx->walk.iv);
1047                 list_for_each_entry_safe(c, tmp, &rctx->chunk_list,
1048                                          entry) {
1049                         c->iv_paddr = iv_paddr;
1050                         err = __n2_crypt_chunk(tfm, c, qp, true);
1051                         if (err)
1052                                 break;
1053                         iv_paddr = c->dest_final - rctx->walk.blocksize;
1054                         list_del(&c->entry);
1055                         if (unlikely(c != &rctx->chunk))
1056                                 kfree(c);
1057                 }
1058                 final_iv_addr = __va(iv_paddr);
1059         } else {
1060                 list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list,
1061                                                  entry) {
1062                         if (c == &rctx->chunk) {
1063                                 iv_paddr = __pa(rctx->walk.iv);
1064                         } else {
1065                                 iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr +
1066                                             tmp->arr[tmp->arr_len-1].src_len -
1067                                             rctx->walk.blocksize);
1068                         }
1069                         if (!final_iv_addr) {
1070                                 unsigned long pa;
1071 
1072                                 pa = (c->arr[c->arr_len-1].src_paddr +
1073                                       c->arr[c->arr_len-1].src_len -
1074                                       rctx->walk.blocksize);
1075                                 final_iv_addr = rctx->temp_iv;
1076                                 memcpy(rctx->temp_iv, __va(pa),
1077                                        rctx->walk.blocksize);
1078                         }
1079                         c->iv_paddr = iv_paddr;
1080                         err = __n2_crypt_chunk(tfm, c, qp, false);
1081                         if (err)
1082                                 break;
1083                         list_del(&c->entry);
1084                         if (unlikely(c != &rctx->chunk))
1085                                 kfree(c);
1086                 }
1087         }
1088         if (!err) {
1089                 hv_ret = wait_for_tail(qp);
1090                 if (hv_ret != HV_EOK)
1091                         err = -EINVAL;
1092         }
1093 
1094         spin_unlock_irqrestore(&qp->lock, flags);
1095 
1096 out:
1097         put_cpu();
1098 
1099         n2_chunk_complete(req, err ? NULL : final_iv_addr);
1100         return err;
1101 }
1102 
1103 static int n2_encrypt_chaining(struct ablkcipher_request *req)
1104 {
1105         return n2_do_chaining(req, true);
1106 }
1107 
1108 static int n2_decrypt_chaining(struct ablkcipher_request *req)
1109 {
1110         return n2_do_chaining(req, false);
1111 }
1112 
1113 struct n2_cipher_tmpl {
1114         const char              *name;
1115         const char              *drv_name;
1116         u8                      block_size;
1117         u8                      enc_type;
1118         struct ablkcipher_alg   ablkcipher;
1119 };
1120 
1121 static const struct n2_cipher_tmpl cipher_tmpls[] = {
1122         /* ARC4: only ECB is supported (chaining bits ignored) */
1123         {       .name           = "ecb(arc4)",
1124                 .drv_name       = "ecb-arc4",
1125                 .block_size     = 1,
1126                 .enc_type       = (ENC_TYPE_ALG_RC4_STREAM |
1127                                    ENC_TYPE_CHAINING_ECB),
1128                 .ablkcipher     = {
1129                         .min_keysize    = 1,
1130                         .max_keysize    = 256,
1131                         .setkey         = n2_arc4_setkey,
1132                         .encrypt        = n2_encrypt_ecb,
1133                         .decrypt        = n2_decrypt_ecb,
1134                 },
1135         },
1136 
1137         /* DES: ECB CBC and CFB are supported */
1138         {       .name           = "ecb(des)",
1139                 .drv_name       = "ecb-des",
1140                 .block_size     = DES_BLOCK_SIZE,
1141                 .enc_type       = (ENC_TYPE_ALG_DES |
1142                                    ENC_TYPE_CHAINING_ECB),
1143                 .ablkcipher     = {
1144                         .min_keysize    = DES_KEY_SIZE,
1145                         .max_keysize    = DES_KEY_SIZE,
1146                         .setkey         = n2_des_setkey,
1147                         .encrypt        = n2_encrypt_ecb,
1148                         .decrypt        = n2_decrypt_ecb,
1149                 },
1150         },
1151         {       .name           = "cbc(des)",
1152                 .drv_name       = "cbc-des",
1153                 .block_size     = DES_BLOCK_SIZE,
1154                 .enc_type       = (ENC_TYPE_ALG_DES |
1155                                    ENC_TYPE_CHAINING_CBC),
1156                 .ablkcipher     = {
1157                         .ivsize         = DES_BLOCK_SIZE,
1158                         .min_keysize    = DES_KEY_SIZE,
1159                         .max_keysize    = DES_KEY_SIZE,
1160                         .setkey         = n2_des_setkey,
1161                         .encrypt        = n2_encrypt_chaining,
1162                         .decrypt        = n2_decrypt_chaining,
1163                 },
1164         },
1165         {       .name           = "cfb(des)",
1166                 .drv_name       = "cfb-des",
1167                 .block_size     = DES_BLOCK_SIZE,
1168                 .enc_type       = (ENC_TYPE_ALG_DES |
1169                                    ENC_TYPE_CHAINING_CFB),
1170                 .ablkcipher     = {
1171                         .min_keysize    = DES_KEY_SIZE,
1172                         .max_keysize    = DES_KEY_SIZE,
1173                         .setkey         = n2_des_setkey,
1174                         .encrypt        = n2_encrypt_chaining,
1175                         .decrypt        = n2_decrypt_chaining,
1176                 },
1177         },
1178 
1179         /* 3DES: ECB CBC and CFB are supported */
1180         {       .name           = "ecb(des3_ede)",
1181                 .drv_name       = "ecb-3des",
1182                 .block_size     = DES_BLOCK_SIZE,
1183                 .enc_type       = (ENC_TYPE_ALG_3DES |
1184                                    ENC_TYPE_CHAINING_ECB),
1185                 .ablkcipher     = {
1186                         .min_keysize    = 3 * DES_KEY_SIZE,
1187                         .max_keysize    = 3 * DES_KEY_SIZE,
1188                         .setkey         = n2_3des_setkey,
1189                         .encrypt        = n2_encrypt_ecb,
1190                         .decrypt        = n2_decrypt_ecb,
1191                 },
1192         },
1193         {       .name           = "cbc(des3_ede)",
1194                 .drv_name       = "cbc-3des",
1195                 .block_size     = DES_BLOCK_SIZE,
1196                 .enc_type       = (ENC_TYPE_ALG_3DES |
1197                                    ENC_TYPE_CHAINING_CBC),
1198                 .ablkcipher     = {
1199                         .ivsize         = DES_BLOCK_SIZE,
1200                         .min_keysize    = 3 * DES_KEY_SIZE,
1201                         .max_keysize    = 3 * DES_KEY_SIZE,
1202                         .setkey         = n2_3des_setkey,
1203                         .encrypt        = n2_encrypt_chaining,
1204                         .decrypt        = n2_decrypt_chaining,
1205                 },
1206         },
1207         {       .name           = "cfb(des3_ede)",
1208                 .drv_name       = "cfb-3des",
1209                 .block_size     = DES_BLOCK_SIZE,
1210                 .enc_type       = (ENC_TYPE_ALG_3DES |
1211                                    ENC_TYPE_CHAINING_CFB),
1212                 .ablkcipher     = {
1213                         .min_keysize    = 3 * DES_KEY_SIZE,
1214                         .max_keysize    = 3 * DES_KEY_SIZE,
1215                         .setkey         = n2_3des_setkey,
1216                         .encrypt        = n2_encrypt_chaining,
1217                         .decrypt        = n2_decrypt_chaining,
1218                 },
1219         },
1220         /* AES: ECB CBC and CTR are supported */
1221         {       .name           = "ecb(aes)",
1222                 .drv_name       = "ecb-aes",
1223                 .block_size     = AES_BLOCK_SIZE,
1224                 .enc_type       = (ENC_TYPE_ALG_AES128 |
1225                                    ENC_TYPE_CHAINING_ECB),
1226                 .ablkcipher     = {
1227                         .min_keysize    = AES_MIN_KEY_SIZE,
1228                         .max_keysize    = AES_MAX_KEY_SIZE,
1229                         .setkey         = n2_aes_setkey,
1230                         .encrypt        = n2_encrypt_ecb,
1231                         .decrypt        = n2_decrypt_ecb,
1232                 },
1233         },
1234         {       .name           = "cbc(aes)",
1235                 .drv_name       = "cbc-aes",
1236                 .block_size     = AES_BLOCK_SIZE,
1237                 .enc_type       = (ENC_TYPE_ALG_AES128 |
1238                                    ENC_TYPE_CHAINING_CBC),
1239                 .ablkcipher     = {
1240                         .ivsize         = AES_BLOCK_SIZE,
1241                         .min_keysize    = AES_MIN_KEY_SIZE,
1242                         .max_keysize    = AES_MAX_KEY_SIZE,
1243                         .setkey         = n2_aes_setkey,
1244                         .encrypt        = n2_encrypt_chaining,
1245                         .decrypt        = n2_decrypt_chaining,
1246                 },
1247         },
1248         {       .name           = "ctr(aes)",
1249                 .drv_name       = "ctr-aes",
1250                 .block_size     = AES_BLOCK_SIZE,
1251                 .enc_type       = (ENC_TYPE_ALG_AES128 |
1252                                    ENC_TYPE_CHAINING_COUNTER),
1253                 .ablkcipher     = {
1254                         .ivsize         = AES_BLOCK_SIZE,
1255                         .min_keysize    = AES_MIN_KEY_SIZE,
1256                         .max_keysize    = AES_MAX_KEY_SIZE,
1257                         .setkey         = n2_aes_setkey,
1258                         .encrypt        = n2_encrypt_chaining,
1259                         .decrypt        = n2_encrypt_chaining,
1260                 },
1261         },
1262 
1263 };
1264 #define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls)
1265 
1266 static LIST_HEAD(cipher_algs);
1267 
1268 struct n2_hash_tmpl {
1269         const char      *name;
1270         const char      *hash_zero;
1271         const u32       *hash_init;
1272         u8              hw_op_hashsz;
1273         u8              digest_size;
1274         u8              block_size;
1275         u8              auth_type;
1276         u8              hmac_type;
1277 };
1278 
1279 static const char md5_zero[MD5_DIGEST_SIZE] = {
1280         0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,
1281         0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e,
1282 };
1283 static const u32 md5_init[MD5_HASH_WORDS] = {
1284         cpu_to_le32(0x67452301),
1285         cpu_to_le32(0xefcdab89),
1286         cpu_to_le32(0x98badcfe),
1287         cpu_to_le32(0x10325476),
1288 };
1289 static const char sha1_zero[SHA1_DIGEST_SIZE] = {
1290         0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32,
1291         0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8,
1292         0x07, 0x09
1293 };
1294 static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {
1295         SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
1296 };
1297 static const char sha256_zero[SHA256_DIGEST_SIZE] = {
1298         0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a,
1299         0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae,
1300         0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99,
1301         0x1b, 0x78, 0x52, 0xb8, 0x55
1302 };
1303 static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = {
1304         SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
1305         SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
1306 };
1307 static const char sha224_zero[SHA224_DIGEST_SIZE] = {
1308         0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,
1309         0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,
1310         0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4,
1311         0x2f
1312 };
1313 static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {
1314         SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
1315         SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
1316 };
1317 
1318 static const struct n2_hash_tmpl hash_tmpls[] = {
1319         { .name         = "md5",
1320           .hash_zero    = md5_zero,
1321           .hash_init    = md5_init,
1322           .auth_type    = AUTH_TYPE_MD5,
1323           .hmac_type    = AUTH_TYPE_HMAC_MD5,
1324           .hw_op_hashsz = MD5_DIGEST_SIZE,
1325           .digest_size  = MD5_DIGEST_SIZE,
1326           .block_size   = MD5_HMAC_BLOCK_SIZE },
1327         { .name         = "sha1",
1328           .hash_zero    = sha1_zero,
1329           .hash_init    = sha1_init,
1330           .auth_type    = AUTH_TYPE_SHA1,
1331           .hmac_type    = AUTH_TYPE_HMAC_SHA1,
1332           .hw_op_hashsz = SHA1_DIGEST_SIZE,
1333           .digest_size  = SHA1_DIGEST_SIZE,
1334           .block_size   = SHA1_BLOCK_SIZE },
1335         { .name         = "sha256",
1336           .hash_zero    = sha256_zero,
1337           .hash_init    = sha256_init,
1338           .auth_type    = AUTH_TYPE_SHA256,
1339           .hmac_type    = AUTH_TYPE_HMAC_SHA256,
1340           .hw_op_hashsz = SHA256_DIGEST_SIZE,
1341           .digest_size  = SHA256_DIGEST_SIZE,
1342           .block_size   = SHA256_BLOCK_SIZE },
1343         { .name         = "sha224",
1344           .hash_zero    = sha224_zero,
1345           .hash_init    = sha224_init,
1346           .auth_type    = AUTH_TYPE_SHA256,
1347           .hmac_type    = AUTH_TYPE_RESERVED,
1348           .hw_op_hashsz = SHA256_DIGEST_SIZE,
1349           .digest_size  = SHA224_DIGEST_SIZE,
1350           .block_size   = SHA224_BLOCK_SIZE },
1351 };
1352 #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)
1353 
1354 static LIST_HEAD(ahash_algs);
1355 static LIST_HEAD(hmac_algs);
1356 
1357 static int algs_registered;
1358 
1359 static void __n2_unregister_algs(void)
1360 {
1361         struct n2_cipher_alg *cipher, *cipher_tmp;
1362         struct n2_ahash_alg *alg, *alg_tmp;
1363         struct n2_hmac_alg *hmac, *hmac_tmp;
1364 
1365         list_for_each_entry_safe(cipher, cipher_tmp, &cipher_algs, entry) {
1366                 crypto_unregister_alg(&cipher->alg);
1367                 list_del(&cipher->entry);
1368                 kfree(cipher);
1369         }
1370         list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) {
1371                 crypto_unregister_ahash(&hmac->derived.alg);
1372                 list_del(&hmac->derived.entry);
1373                 kfree(hmac);
1374         }
1375         list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) {
1376                 crypto_unregister_ahash(&alg->alg);
1377                 list_del(&alg->entry);
1378                 kfree(alg);
1379         }
1380 }
1381 
1382 static int n2_cipher_cra_init(struct crypto_tfm *tfm)
1383 {
1384         tfm->crt_ablkcipher.reqsize = sizeof(struct n2_request_context);
1385         return 0;
1386 }
1387 
1388 static int __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl)
1389 {
1390         struct n2_cipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1391         struct crypto_alg *alg;
1392         int err;
1393 
1394         if (!p)
1395                 return -ENOMEM;
1396 
1397         alg = &p->alg;
1398 
1399         snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
1400         snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name);
1401         alg->cra_priority = N2_CRA_PRIORITY;
1402         alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1403                          CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC;
1404         alg->cra_blocksize = tmpl->block_size;
1405         p->enc_type = tmpl->enc_type;
1406         alg->cra_ctxsize = sizeof(struct n2_cipher_context);
1407         alg->cra_type = &crypto_ablkcipher_type;
1408         alg->cra_u.ablkcipher = tmpl->ablkcipher;
1409         alg->cra_init = n2_cipher_cra_init;
1410         alg->cra_module = THIS_MODULE;
1411 
1412         list_add(&p->entry, &cipher_algs);
1413         err = crypto_register_alg(alg);
1414         if (err) {
1415                 pr_err("%s alg registration failed\n", alg->cra_name);
1416                 list_del(&p->entry);
1417                 kfree(p);
1418         } else {
1419                 pr_info("%s alg registered\n", alg->cra_name);
1420         }
1421         return err;
1422 }
1423 
1424 static int __n2_register_one_hmac(struct n2_ahash_alg *n2ahash)
1425 {
1426         struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1427         struct ahash_alg *ahash;
1428         struct crypto_alg *base;
1429         int err;
1430 
1431         if (!p)
1432                 return -ENOMEM;
1433 
1434         p->child_alg = n2ahash->alg.halg.base.cra_name;
1435         memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg));
1436         INIT_LIST_HEAD(&p->derived.entry);
1437 
1438         ahash = &p->derived.alg;
1439         ahash->digest = n2_hmac_async_digest;
1440         ahash->setkey = n2_hmac_async_setkey;
1441 
1442         base = &ahash->halg.base;
1443         snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", p->child_alg);
1444         snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", p->child_alg);
1445 
1446         base->cra_ctxsize = sizeof(struct n2_hmac_ctx);
1447         base->cra_init = n2_hmac_cra_init;
1448         base->cra_exit = n2_hmac_cra_exit;
1449 
1450         list_add(&p->derived.entry, &hmac_algs);
1451         err = crypto_register_ahash(ahash);
1452         if (err) {
1453                 pr_err("%s alg registration failed\n", base->cra_name);
1454                 list_del(&p->derived.entry);
1455                 kfree(p);
1456         } else {
1457                 pr_info("%s alg registered\n", base->cra_name);
1458         }
1459         return err;
1460 }
1461 
1462 static int __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl)
1463 {
1464         struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
1465         struct hash_alg_common *halg;
1466         struct crypto_alg *base;
1467         struct ahash_alg *ahash;
1468         int err;
1469 
1470         if (!p)
1471                 return -ENOMEM;
1472 
1473         p->hash_zero = tmpl->hash_zero;
1474         p->hash_init = tmpl->hash_init;
1475         p->auth_type = tmpl->auth_type;
1476         p->hmac_type = tmpl->hmac_type;
1477         p->hw_op_hashsz = tmpl->hw_op_hashsz;
1478         p->digest_size = tmpl->digest_size;
1479 
1480         ahash = &p->alg;
1481         ahash->init = n2_hash_async_init;
1482         ahash->update = n2_hash_async_update;
1483         ahash->final = n2_hash_async_final;
1484         ahash->finup = n2_hash_async_finup;
1485         ahash->digest = n2_hash_async_digest;
1486 
1487         halg = &ahash->halg;
1488         halg->digestsize = tmpl->digest_size;
1489 
1490         base = &halg->base;
1491         snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
1492         snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name);
1493         base->cra_priority = N2_CRA_PRIORITY;
1494         base->cra_flags = CRYPTO_ALG_TYPE_AHASH |
1495                           CRYPTO_ALG_KERN_DRIVER_ONLY |
1496                           CRYPTO_ALG_NEED_FALLBACK;
1497         base->cra_blocksize = tmpl->block_size;
1498         base->cra_ctxsize = sizeof(struct n2_hash_ctx);
1499         base->cra_module = THIS_MODULE;
1500         base->cra_init = n2_hash_cra_init;
1501         base->cra_exit = n2_hash_cra_exit;
1502 
1503         list_add(&p->entry, &ahash_algs);
1504         err = crypto_register_ahash(ahash);
1505         if (err) {
1506                 pr_err("%s alg registration failed\n", base->cra_name);
1507                 list_del(&p->entry);
1508                 kfree(p);
1509         } else {
1510                 pr_info("%s alg registered\n", base->cra_name);
1511         }
1512         if (!err && p->hmac_type != AUTH_TYPE_RESERVED)
1513                 err = __n2_register_one_hmac(p);
1514         return err;
1515 }
1516 
1517 static int n2_register_algs(void)
1518 {
1519         int i, err = 0;
1520 
1521         mutex_lock(&spu_lock);
1522         if (algs_registered++)
1523                 goto out;
1524 
1525         for (i = 0; i < NUM_HASH_TMPLS; i++) {
1526                 err = __n2_register_one_ahash(&hash_tmpls[i]);
1527                 if (err) {
1528                         __n2_unregister_algs();
1529                         goto out;
1530                 }
1531         }
1532         for (i = 0; i < NUM_CIPHER_TMPLS; i++) {
1533                 err = __n2_register_one_cipher(&cipher_tmpls[i]);
1534                 if (err) {
1535                         __n2_unregister_algs();
1536                         goto out;
1537                 }
1538         }
1539 
1540 out:
1541         mutex_unlock(&spu_lock);
1542         return err;
1543 }
1544 
1545 static void n2_unregister_algs(void)
1546 {
1547         mutex_lock(&spu_lock);
1548         if (!--algs_registered)
1549                 __n2_unregister_algs();
1550         mutex_unlock(&spu_lock);
1551 }
1552 
1553 /* To map CWQ queues to interrupt sources, the hypervisor API provides
1554  * a devino.  This isn't very useful to us because all of the
1555  * interrupts listed in the device_node have been translated to
1556  * Linux virtual IRQ cookie numbers.
1557  *
1558  * So we have to back-translate, going through the 'intr' and 'ino'
1559  * property tables of the n2cp MDESC node, matching it with the OF
1560  * 'interrupts' property entries, in order to to figure out which
1561  * devino goes to which already-translated IRQ.
1562  */
1563 static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip,
1564                              unsigned long dev_ino)
1565 {
1566         const unsigned int *dev_intrs;
1567         unsigned int intr;
1568         int i;
1569 
1570         for (i = 0; i < ip->num_intrs; i++) {
1571                 if (ip->ino_table[i].ino == dev_ino)
1572                         break;
1573         }
1574         if (i == ip->num_intrs)
1575                 return -ENODEV;
1576 
1577         intr = ip->ino_table[i].intr;
1578 
1579         dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL);
1580         if (!dev_intrs)
1581                 return -ENODEV;
1582 
1583         for (i = 0; i < dev->archdata.num_irqs; i++) {
1584                 if (dev_intrs[i] == intr)
1585                         return i;
1586         }
1587 
1588         return -ENODEV;
1589 }
1590 
1591 static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip,
1592                        const char *irq_name, struct spu_queue *p,
1593                        irq_handler_t handler)
1594 {
1595         unsigned long herr;
1596         int index;
1597 
1598         herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino);
1599         if (herr)
1600                 return -EINVAL;
1601 
1602         index = find_devino_index(dev, ip, p->devino);
1603         if (index < 0)
1604                 return index;
1605 
1606         p->irq = dev->archdata.irqs[index];
1607 
1608         sprintf(p->irq_name, "%s-%d", irq_name, index);
1609 
1610         return request_irq(p->irq, handler, 0, p->irq_name, p);
1611 }
1612 
1613 static struct kmem_cache *queue_cache[2];
1614 
1615 static void *new_queue(unsigned long q_type)
1616 {
1617         return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL);
1618 }
1619 
1620 static void free_queue(void *p, unsigned long q_type)
1621 {
1622         return kmem_cache_free(queue_cache[q_type - 1], p);
1623 }
1624 
1625 static int queue_cache_init(void)
1626 {
1627         if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
1628                 queue_cache[HV_NCS_QTYPE_MAU - 1] =
1629                         kmem_cache_create("mau_queue",
1630                                           (MAU_NUM_ENTRIES *
1631                                            MAU_ENTRY_SIZE),
1632                                           MAU_ENTRY_SIZE, 0, NULL);
1633         if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
1634                 return -ENOMEM;
1635 
1636         if (!queue_cache[HV_NCS_QTYPE_CWQ - 1])
1637                 queue_cache[HV_NCS_QTYPE_CWQ - 1] =
1638                         kmem_cache_create("cwq_queue",
1639                                           (CWQ_NUM_ENTRIES *
1640                                            CWQ_ENTRY_SIZE),
1641                                           CWQ_ENTRY_SIZE, 0, NULL);
1642         if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) {
1643                 kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
1644                 return -ENOMEM;
1645         }
1646         return 0;
1647 }
1648 
1649 static void queue_cache_destroy(void)
1650 {
1651         kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
1652         kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]);
1653 }
1654 
1655 static int spu_queue_register(struct spu_queue *p, unsigned long q_type)
1656 {
1657         cpumask_var_t old_allowed;
1658         unsigned long hv_ret;
1659 
1660         if (cpumask_empty(&p->sharing))
1661                 return -EINVAL;
1662 
1663         if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL))
1664                 return -ENOMEM;
1665 
1666         cpumask_copy(old_allowed, &current->cpus_allowed);
1667 
1668         set_cpus_allowed_ptr(current, &p->sharing);
1669 
1670         hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q),
1671                                  CWQ_NUM_ENTRIES, &p->qhandle);
1672         if (!hv_ret)
1673                 sun4v_ncs_sethead_marker(p->qhandle, 0);
1674 
1675         set_cpus_allowed_ptr(current, old_allowed);
1676 
1677         free_cpumask_var(old_allowed);
1678 
1679         return (hv_ret ? -EINVAL : 0);
1680 }
1681 
1682 static int spu_queue_setup(struct spu_queue *p)
1683 {
1684         int err;
1685 
1686         p->q = new_queue(p->q_type);
1687         if (!p->q)
1688                 return -ENOMEM;
1689 
1690         err = spu_queue_register(p, p->q_type);
1691         if (err) {
1692                 free_queue(p->q, p->q_type);
1693                 p->q = NULL;
1694         }
1695 
1696         return err;
1697 }
1698 
1699 static void spu_queue_destroy(struct spu_queue *p)
1700 {
1701         unsigned long hv_ret;
1702 
1703         if (!p->q)
1704                 return;
1705 
1706         hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle);
1707 
1708         if (!hv_ret)
1709                 free_queue(p->q, p->q_type);
1710 }
1711 
1712 static void spu_list_destroy(struct list_head *list)
1713 {
1714         struct spu_queue *p, *n;
1715 
1716         list_for_each_entry_safe(p, n, list, list) {
1717                 int i;
1718 
1719                 for (i = 0; i < NR_CPUS; i++) {
1720                         if (cpu_to_cwq[i] == p)
1721                                 cpu_to_cwq[i] = NULL;
1722                 }
1723 
1724                 if (p->irq) {
1725                         free_irq(p->irq, p);
1726                         p->irq = 0;
1727                 }
1728                 spu_queue_destroy(p);
1729                 list_del(&p->list);
1730                 kfree(p);
1731         }
1732 }
1733 
1734 /* Walk the backward arcs of a CWQ 'exec-unit' node,
1735  * gathering cpu membership information.
1736  */
1737 static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,
1738                                struct platform_device *dev,
1739                                u64 node, struct spu_queue *p,
1740                                struct spu_queue **table)
1741 {
1742         u64 arc;
1743 
1744         mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) {
1745                 u64 tgt = mdesc_arc_target(mdesc, arc);
1746                 const char *name = mdesc_node_name(mdesc, tgt);
1747                 const u64 *id;
1748 
1749                 if (strcmp(name, "cpu"))
1750                         continue;
1751                 id = mdesc_get_property(mdesc, tgt, "id", NULL);
1752                 if (table[*id] != NULL) {
1753                         dev_err(&dev->dev, "%s: SPU cpu slot already set.\n",
1754                                 dev->dev.of_node->full_name);
1755                         return -EINVAL;
1756                 }
1757                 cpu_set(*id, p->sharing);
1758                 table[*id] = p;
1759         }
1760         return 0;
1761 }
1762 
1763 /* Process an 'exec-unit' MDESC node of type 'cwq'.  */
1764 static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,
1765                             struct platform_device *dev, struct mdesc_handle *mdesc,
1766                             u64 node, const char *iname, unsigned long q_type,
1767                             irq_handler_t handler, struct spu_queue **table)
1768 {
1769         struct spu_queue *p;
1770         int err;
1771 
1772         p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL);
1773         if (!p) {
1774                 dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n",
1775                         dev->dev.of_node->full_name);
1776                 return -ENOMEM;
1777         }
1778 
1779         cpus_clear(p->sharing);
1780         spin_lock_init(&p->lock);
1781         p->q_type = q_type;
1782         INIT_LIST_HEAD(&p->jobs);
1783         list_add(&p->list, list);
1784 
1785         err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table);
1786         if (err)
1787                 return err;
1788 
1789         err = spu_queue_setup(p);
1790         if (err)
1791                 return err;
1792 
1793         return spu_map_ino(dev, ip, iname, p, handler);
1794 }
1795 
1796 static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev,
1797                           struct spu_mdesc_info *ip, struct list_head *list,
1798                           const char *exec_name, unsigned long q_type,
1799                           irq_handler_t handler, struct spu_queue **table)
1800 {
1801         int err = 0;
1802         u64 node;
1803 
1804         mdesc_for_each_node_by_name(mdesc, node, "exec-unit") {
1805                 const char *type;
1806 
1807                 type = mdesc_get_property(mdesc, node, "type", NULL);
1808                 if (!type || strcmp(type, exec_name))
1809                         continue;
1810 
1811                 err = handle_exec_unit(ip, list, dev, mdesc, node,
1812                                        exec_name, q_type, handler, table);
1813                 if (err) {
1814                         spu_list_destroy(list);
1815                         break;
1816                 }
1817         }
1818 
1819         return err;
1820 }
1821 
1822 static int get_irq_props(struct mdesc_handle *mdesc, u64 node,
1823                          struct spu_mdesc_info *ip)
1824 {
1825         const u64 *ino;
1826         int ino_len;
1827         int i;
1828 
1829         ino = mdesc_get_property(mdesc, node, "ino", &ino_len);
1830         if (!ino) {
1831                 printk("NO 'ino'\n");
1832                 return -ENODEV;
1833         }
1834 
1835         ip->num_intrs = ino_len / sizeof(u64);
1836         ip->ino_table = kzalloc((sizeof(struct ino_blob) *
1837                                  ip->num_intrs),
1838                                 GFP_KERNEL);
1839         if (!ip->ino_table)
1840                 return -ENOMEM;
1841 
1842         for (i = 0; i < ip->num_intrs; i++) {
1843                 struct ino_blob *b = &ip->ino_table[i];
1844                 b->intr = i + 1;
1845                 b->ino = ino[i];
1846         }
1847 
1848         return 0;
1849 }
1850 
1851 static int grab_mdesc_irq_props(struct mdesc_handle *mdesc,
1852                                 struct platform_device *dev,
1853                                 struct spu_mdesc_info *ip,
1854                                 const char *node_name)
1855 {
1856         const unsigned int *reg;
1857         u64 node;
1858 
1859         reg = of_get_property(dev->dev.of_node, "reg", NULL);
1860         if (!reg)
1861                 return -ENODEV;
1862 
1863         mdesc_for_each_node_by_name(mdesc, node, "virtual-device") {
1864                 const char *name;
1865                 const u64 *chdl;
1866 
1867                 name = mdesc_get_property(mdesc, node, "name", NULL);
1868                 if (!name || strcmp(name, node_name))
1869                         continue;
1870                 chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL);
1871                 if (!chdl || (*chdl != *reg))
1872                         continue;
1873                 ip->cfg_handle = *chdl;
1874                 return get_irq_props(mdesc, node, ip);
1875         }
1876 
1877         return -ENODEV;
1878 }
1879 
1880 static unsigned long n2_spu_hvapi_major;
1881 static unsigned long n2_spu_hvapi_minor;
1882 
1883 static int n2_spu_hvapi_register(void)
1884 {
1885         int err;
1886 
1887         n2_spu_hvapi_major = 2;
1888         n2_spu_hvapi_minor = 0;
1889 
1890         err = sun4v_hvapi_register(HV_GRP_NCS,
1891                                    n2_spu_hvapi_major,
1892                                    &n2_spu_hvapi_minor);
1893 
1894         if (!err)
1895                 pr_info("Registered NCS HVAPI version %lu.%lu\n",
1896                         n2_spu_hvapi_major,
1897                         n2_spu_hvapi_minor);
1898 
1899         return err;
1900 }
1901 
1902 static void n2_spu_hvapi_unregister(void)
1903 {
1904         sun4v_hvapi_unregister(HV_GRP_NCS);
1905 }
1906 
1907 static int global_ref;
1908 
1909 static int grab_global_resources(void)
1910 {
1911         int err = 0;
1912 
1913         mutex_lock(&spu_lock);
1914 
1915         if (global_ref++)
1916                 goto out;
1917 
1918         err = n2_spu_hvapi_register();
1919         if (err)
1920                 goto out;
1921 
1922         err = queue_cache_init();
1923         if (err)
1924                 goto out_hvapi_release;
1925 
1926         err = -ENOMEM;
1927         cpu_to_cwq = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
1928                              GFP_KERNEL);
1929         if (!cpu_to_cwq)
1930                 goto out_queue_cache_destroy;
1931 
1932         cpu_to_mau = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
1933                              GFP_KERNEL);
1934         if (!cpu_to_mau)
1935                 goto out_free_cwq_table;
1936 
1937         err = 0;
1938 
1939 out:
1940         if (err)
1941                 global_ref--;
1942         mutex_unlock(&spu_lock);
1943         return err;
1944 
1945 out_free_cwq_table:
1946         kfree(cpu_to_cwq);
1947         cpu_to_cwq = NULL;
1948 
1949 out_queue_cache_destroy:
1950         queue_cache_destroy();
1951 
1952 out_hvapi_release:
1953         n2_spu_hvapi_unregister();
1954         goto out;
1955 }
1956 
1957 static void release_global_resources(void)
1958 {
1959         mutex_lock(&spu_lock);
1960         if (!--global_ref) {
1961                 kfree(cpu_to_cwq);
1962                 cpu_to_cwq = NULL;
1963 
1964                 kfree(cpu_to_mau);
1965                 cpu_to_mau = NULL;
1966 
1967                 queue_cache_destroy();
1968                 n2_spu_hvapi_unregister();
1969         }
1970         mutex_unlock(&spu_lock);
1971 }
1972 
1973 static struct n2_crypto *alloc_n2cp(void)
1974 {
1975         struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL);
1976 
1977         if (np)
1978                 INIT_LIST_HEAD(&np->cwq_list);
1979 
1980         return np;
1981 }
1982 
1983 static void free_n2cp(struct n2_crypto *np)
1984 {
1985         if (np->cwq_info.ino_table) {
1986                 kfree(np->cwq_info.ino_table);
1987                 np->cwq_info.ino_table = NULL;
1988         }
1989 
1990         kfree(np);
1991 }
1992 
1993 static void n2_spu_driver_version(void)
1994 {
1995         static int n2_spu_version_printed;
1996 
1997         if (n2_spu_version_printed++ == 0)
1998                 pr_info("%s", version);
1999 }
2000 
2001 static int n2_crypto_probe(struct platform_device *dev)
2002 {
2003         struct mdesc_handle *mdesc;
2004         const char *full_name;
2005         struct n2_crypto *np;
2006         int err;
2007 
2008         n2_spu_driver_version();
2009 
2010         full_name = dev->dev.of_node->full_name;
2011         pr_info("Found N2CP at %s\n", full_name);
2012 
2013         np = alloc_n2cp();
2014         if (!np) {
2015                 dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n",
2016                         full_name);
2017                 return -ENOMEM;
2018         }
2019 
2020         err = grab_global_resources();
2021         if (err) {
2022                 dev_err(&dev->dev, "%s: Unable to grab "
2023                         "global resources.\n", full_name);
2024                 goto out_free_n2cp;
2025         }
2026 
2027         mdesc = mdesc_grab();
2028 
2029         if (!mdesc) {
2030                 dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
2031                         full_name);
2032                 err = -ENODEV;
2033                 goto out_free_global;
2034         }
2035         err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp");
2036         if (err) {
2037                 dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
2038                         full_name);
2039                 mdesc_release(mdesc);
2040                 goto out_free_global;
2041         }
2042 
2043         err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list,
2044                              "cwq", HV_NCS_QTYPE_CWQ, cwq_intr,
2045                              cpu_to_cwq);
2046         mdesc_release(mdesc);
2047 
2048         if (err) {
2049                 dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n",
2050                         full_name);
2051                 goto out_free_global;
2052         }
2053 
2054         err = n2_register_algs();
2055         if (err) {
2056                 dev_err(&dev->dev, "%s: Unable to register algorithms.\n",
2057                         full_name);
2058                 goto out_free_spu_list;
2059         }
2060 
2061         dev_set_drvdata(&dev->dev, np);
2062 
2063         return 0;
2064 
2065 out_free_spu_list:
2066         spu_list_destroy(&np->cwq_list);
2067 
2068 out_free_global:
2069         release_global_resources();
2070 
2071 out_free_n2cp:
2072         free_n2cp(np);
2073 
2074         return err;
2075 }
2076 
2077 static int n2_crypto_remove(struct platform_device *dev)
2078 {
2079         struct n2_crypto *np = dev_get_drvdata(&dev->dev);
2080 
2081         n2_unregister_algs();
2082 
2083         spu_list_destroy(&np->cwq_list);
2084 
2085         release_global_resources();
2086 
2087         free_n2cp(np);
2088 
2089         return 0;
2090 }
2091 
2092 static struct n2_mau *alloc_ncp(void)
2093 {
2094         struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL);
2095 
2096         if (mp)
2097                 INIT_LIST_HEAD(&mp->mau_list);
2098 
2099         return mp;
2100 }
2101 
2102 static void free_ncp(struct n2_mau *mp)
2103 {
2104         if (mp->mau_info.ino_table) {
2105                 kfree(mp->mau_info.ino_table);
2106                 mp->mau_info.ino_table = NULL;
2107         }
2108 
2109         kfree(mp);
2110 }
2111 
2112 static int n2_mau_probe(struct platform_device *dev)
2113 {
2114         struct mdesc_handle *mdesc;
2115         const char *full_name;
2116         struct n2_mau *mp;
2117         int err;
2118 
2119         n2_spu_driver_version();
2120 
2121         full_name = dev->dev.of_node->full_name;
2122         pr_info("Found NCP at %s\n", full_name);
2123 
2124         mp = alloc_ncp();
2125         if (!mp) {
2126                 dev_err(&dev->dev, "%s: Unable to allocate ncp.\n",
2127                         full_name);
2128                 return -ENOMEM;
2129         }
2130 
2131         err = grab_global_resources();
2132         if (err) {
2133                 dev_err(&dev->dev, "%s: Unable to grab "
2134                         "global resources.\n", full_name);
2135                 goto out_free_ncp;
2136         }
2137 
2138         mdesc = mdesc_grab();
2139 
2140         if (!mdesc) {
2141                 dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
2142                         full_name);
2143                 err = -ENODEV;
2144                 goto out_free_global;
2145         }
2146 
2147         err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp");
2148         if (err) {
2149                 dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
2150                         full_name);
2151                 mdesc_release(mdesc);
2152                 goto out_free_global;
2153         }
2154 
2155         err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list,
2156                              "mau", HV_NCS_QTYPE_MAU, mau_intr,
2157                              cpu_to_mau);
2158         mdesc_release(mdesc);
2159 
2160         if (err) {
2161                 dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n",
2162                         full_name);
2163                 goto out_free_global;
2164         }
2165 
2166         dev_set_drvdata(&dev->dev, mp);
2167 
2168         return 0;
2169 
2170 out_free_global:
2171         release_global_resources();
2172 
2173 out_free_ncp:
2174         free_ncp(mp);
2175 
2176         return err;
2177 }
2178 
2179 static int n2_mau_remove(struct platform_device *dev)
2180 {
2181         struct n2_mau *mp = dev_get_drvdata(&dev->dev);
2182 
2183         spu_list_destroy(&mp->mau_list);
2184 
2185         release_global_resources();
2186 
2187         free_ncp(mp);
2188 
2189         return 0;
2190 }
2191 
2192 static struct of_device_id n2_crypto_match[] = {
2193         {
2194                 .name = "n2cp",
2195                 .compatible = "SUNW,n2-cwq",
2196         },
2197         {
2198                 .name = "n2cp",
2199                 .compatible = "SUNW,vf-cwq",
2200         },
2201         {
2202                 .name = "n2cp",
2203                 .compatible = "SUNW,kt-cwq",
2204         },
2205         {},
2206 };
2207 
2208 MODULE_DEVICE_TABLE(of, n2_crypto_match);
2209 
2210 static struct platform_driver n2_crypto_driver = {
2211         .driver = {
2212                 .name           =       "n2cp",
2213                 .owner          =       THIS_MODULE,
2214                 .of_match_table =       n2_crypto_match,
2215         },
2216         .probe          =       n2_crypto_probe,
2217         .remove         =       n2_crypto_remove,
2218 };
2219 
2220 static struct of_device_id n2_mau_match[] = {
2221         {
2222                 .name = "ncp",
2223                 .compatible = "SUNW,n2-mau",
2224         },
2225         {
2226                 .name = "ncp",
2227                 .compatible = "SUNW,vf-mau",
2228         },
2229         {
2230                 .name = "ncp",
2231                 .compatible = "SUNW,kt-mau",
2232         },
2233         {},
2234 };
2235 
2236 MODULE_DEVICE_TABLE(of, n2_mau_match);
2237 
2238 static struct platform_driver n2_mau_driver = {
2239         .driver = {
2240                 .name           =       "ncp",
2241                 .owner          =       THIS_MODULE,
2242                 .of_match_table =       n2_mau_match,
2243         },
2244         .probe          =       n2_mau_probe,
2245         .remove         =       n2_mau_remove,
2246 };
2247 
2248 static int __init n2_init(void)
2249 {
2250         int err = platform_driver_register(&n2_crypto_driver);
2251 
2252         if (!err) {
2253                 err = platform_driver_register(&n2_mau_driver);
2254                 if (err)
2255                         platform_driver_unregister(&n2_crypto_driver);
2256         }
2257         return err;
2258 }
2259 
2260 static void __exit n2_exit(void)
2261 {
2262         platform_driver_unregister(&n2_mau_driver);
2263         platform_driver_unregister(&n2_crypto_driver);
2264 }
2265 
2266 module_init(n2_init);
2267 module_exit(n2_exit);
2268 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us