Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/crypto/crypto_engine.c

  1 /*
  2  * Handle async block request by crypto hardware engine.
  3  *
  4  * Copyright (C) 2016 Linaro, Inc.
  5  *
  6  * Author: Baolin Wang <baolin.wang@linaro.org>
  7  *
  8  * This program is free software; you can redistribute it and/or modify it
  9  * under the terms of the GNU General Public License as published by the Free
 10  * Software Foundation; either version 2 of the License, or (at your option)
 11  * any later version.
 12  *
 13  */
 14 
 15 #include <linux/err.h>
 16 #include <linux/delay.h>
 17 #include <crypto/engine.h>
 18 #include <crypto/internal/hash.h>
 19 #include "internal.h"
 20 
 21 #define CRYPTO_ENGINE_MAX_QLEN 10
 22 
 23 /**
 24  * crypto_pump_requests - dequeue one request from engine queue to process
 25  * @engine: the hardware engine
 26  * @in_kthread: true if we are in the context of the request pump thread
 27  *
 28  * This function checks if there is any request in the engine queue that
 29  * needs processing and if so call out to the driver to initialize hardware
 30  * and handle each request.
 31  */
 32 static void crypto_pump_requests(struct crypto_engine *engine,
 33                                  bool in_kthread)
 34 {
 35         struct crypto_async_request *async_req, *backlog;
 36         struct ahash_request *hreq;
 37         struct ablkcipher_request *breq;
 38         unsigned long flags;
 39         bool was_busy = false;
 40         int ret, rtype;
 41 
 42         spin_lock_irqsave(&engine->queue_lock, flags);
 43 
 44         /* Make sure we are not already running a request */
 45         if (engine->cur_req)
 46                 goto out;
 47 
 48         /* If another context is idling then defer */
 49         if (engine->idling) {
 50                 kthread_queue_work(engine->kworker, &engine->pump_requests);
 51                 goto out;
 52         }
 53 
 54         /* Check if the engine queue is idle */
 55         if (!crypto_queue_len(&engine->queue) || !engine->running) {
 56                 if (!engine->busy)
 57                         goto out;
 58 
 59                 /* Only do teardown in the thread */
 60                 if (!in_kthread) {
 61                         kthread_queue_work(engine->kworker,
 62                                            &engine->pump_requests);
 63                         goto out;
 64                 }
 65 
 66                 engine->busy = false;
 67                 engine->idling = true;
 68                 spin_unlock_irqrestore(&engine->queue_lock, flags);
 69 
 70                 if (engine->unprepare_crypt_hardware &&
 71                     engine->unprepare_crypt_hardware(engine))
 72                         pr_err("failed to unprepare crypt hardware\n");
 73 
 74                 spin_lock_irqsave(&engine->queue_lock, flags);
 75                 engine->idling = false;
 76                 goto out;
 77         }
 78 
 79         /* Get the fist request from the engine queue to handle */
 80         backlog = crypto_get_backlog(&engine->queue);
 81         async_req = crypto_dequeue_request(&engine->queue);
 82         if (!async_req)
 83                 goto out;
 84 
 85         engine->cur_req = async_req;
 86         if (backlog)
 87                 backlog->complete(backlog, -EINPROGRESS);
 88 
 89         if (engine->busy)
 90                 was_busy = true;
 91         else
 92                 engine->busy = true;
 93 
 94         spin_unlock_irqrestore(&engine->queue_lock, flags);
 95 
 96         rtype = crypto_tfm_alg_type(engine->cur_req->tfm);
 97         /* Until here we get the request need to be encrypted successfully */
 98         if (!was_busy && engine->prepare_crypt_hardware) {
 99                 ret = engine->prepare_crypt_hardware(engine);
100                 if (ret) {
101                         pr_err("failed to prepare crypt hardware\n");
102                         goto req_err;
103                 }
104         }
105 
106         switch (rtype) {
107         case CRYPTO_ALG_TYPE_AHASH:
108                 hreq = ahash_request_cast(engine->cur_req);
109                 if (engine->prepare_hash_request) {
110                         ret = engine->prepare_hash_request(engine, hreq);
111                         if (ret) {
112                                 pr_err("failed to prepare request: %d\n", ret);
113                                 goto req_err;
114                         }
115                         engine->cur_req_prepared = true;
116                 }
117                 ret = engine->hash_one_request(engine, hreq);
118                 if (ret) {
119                         pr_err("failed to hash one request from queue\n");
120                         goto req_err;
121                 }
122                 return;
123         case CRYPTO_ALG_TYPE_ABLKCIPHER:
124                 breq = ablkcipher_request_cast(engine->cur_req);
125                 if (engine->prepare_cipher_request) {
126                         ret = engine->prepare_cipher_request(engine, breq);
127                         if (ret) {
128                                 pr_err("failed to prepare request: %d\n", ret);
129                                 goto req_err;
130                         }
131                         engine->cur_req_prepared = true;
132                 }
133                 ret = engine->cipher_one_request(engine, breq);
134                 if (ret) {
135                         pr_err("failed to cipher one request from queue\n");
136                         goto req_err;
137                 }
138                 return;
139         default:
140                 pr_err("failed to prepare request of unknown type\n");
141                 return;
142         }
143 
144 req_err:
145         switch (rtype) {
146         case CRYPTO_ALG_TYPE_AHASH:
147                 hreq = ahash_request_cast(engine->cur_req);
148                 crypto_finalize_hash_request(engine, hreq, ret);
149                 break;
150         case CRYPTO_ALG_TYPE_ABLKCIPHER:
151                 breq = ablkcipher_request_cast(engine->cur_req);
152                 crypto_finalize_cipher_request(engine, breq, ret);
153                 break;
154         }
155         return;
156 
157 out:
158         spin_unlock_irqrestore(&engine->queue_lock, flags);
159 }
160 
161 static void crypto_pump_work(struct kthread_work *work)
162 {
163         struct crypto_engine *engine =
164                 container_of(work, struct crypto_engine, pump_requests);
165 
166         crypto_pump_requests(engine, true);
167 }
168 
169 /**
170  * crypto_transfer_cipher_request - transfer the new request into the
171  * enginequeue
172  * @engine: the hardware engine
173  * @req: the request need to be listed into the engine queue
174  */
175 int crypto_transfer_cipher_request(struct crypto_engine *engine,
176                                    struct ablkcipher_request *req,
177                                    bool need_pump)
178 {
179         unsigned long flags;
180         int ret;
181 
182         spin_lock_irqsave(&engine->queue_lock, flags);
183 
184         if (!engine->running) {
185                 spin_unlock_irqrestore(&engine->queue_lock, flags);
186                 return -ESHUTDOWN;
187         }
188 
189         ret = ablkcipher_enqueue_request(&engine->queue, req);
190 
191         if (!engine->busy && need_pump)
192                 kthread_queue_work(engine->kworker, &engine->pump_requests);
193 
194         spin_unlock_irqrestore(&engine->queue_lock, flags);
195         return ret;
196 }
197 EXPORT_SYMBOL_GPL(crypto_transfer_cipher_request);
198 
199 /**
200  * crypto_transfer_cipher_request_to_engine - transfer one request to list
201  * into the engine queue
202  * @engine: the hardware engine
203  * @req: the request need to be listed into the engine queue
204  */
205 int crypto_transfer_cipher_request_to_engine(struct crypto_engine *engine,
206                                              struct ablkcipher_request *req)
207 {
208         return crypto_transfer_cipher_request(engine, req, true);
209 }
210 EXPORT_SYMBOL_GPL(crypto_transfer_cipher_request_to_engine);
211 
212 /**
213  * crypto_transfer_hash_request - transfer the new request into the
214  * enginequeue
215  * @engine: the hardware engine
216  * @req: the request need to be listed into the engine queue
217  */
218 int crypto_transfer_hash_request(struct crypto_engine *engine,
219                                  struct ahash_request *req, bool need_pump)
220 {
221         unsigned long flags;
222         int ret;
223 
224         spin_lock_irqsave(&engine->queue_lock, flags);
225 
226         if (!engine->running) {
227                 spin_unlock_irqrestore(&engine->queue_lock, flags);
228                 return -ESHUTDOWN;
229         }
230 
231         ret = ahash_enqueue_request(&engine->queue, req);
232 
233         if (!engine->busy && need_pump)
234                 kthread_queue_work(engine->kworker, &engine->pump_requests);
235 
236         spin_unlock_irqrestore(&engine->queue_lock, flags);
237         return ret;
238 }
239 EXPORT_SYMBOL_GPL(crypto_transfer_hash_request);
240 
241 /**
242  * crypto_transfer_hash_request_to_engine - transfer one request to list
243  * into the engine queue
244  * @engine: the hardware engine
245  * @req: the request need to be listed into the engine queue
246  */
247 int crypto_transfer_hash_request_to_engine(struct crypto_engine *engine,
248                                            struct ahash_request *req)
249 {
250         return crypto_transfer_hash_request(engine, req, true);
251 }
252 EXPORT_SYMBOL_GPL(crypto_transfer_hash_request_to_engine);
253 
254 /**
255  * crypto_finalize_cipher_request - finalize one request if the request is done
256  * @engine: the hardware engine
257  * @req: the request need to be finalized
258  * @err: error number
259  */
260 void crypto_finalize_cipher_request(struct crypto_engine *engine,
261                                     struct ablkcipher_request *req, int err)
262 {
263         unsigned long flags;
264         bool finalize_cur_req = false;
265         int ret;
266 
267         spin_lock_irqsave(&engine->queue_lock, flags);
268         if (engine->cur_req == &req->base)
269                 finalize_cur_req = true;
270         spin_unlock_irqrestore(&engine->queue_lock, flags);
271 
272         if (finalize_cur_req) {
273                 if (engine->cur_req_prepared &&
274                     engine->unprepare_cipher_request) {
275                         ret = engine->unprepare_cipher_request(engine, req);
276                         if (ret)
277                                 pr_err("failed to unprepare request\n");
278                 }
279                 spin_lock_irqsave(&engine->queue_lock, flags);
280                 engine->cur_req = NULL;
281                 engine->cur_req_prepared = false;
282                 spin_unlock_irqrestore(&engine->queue_lock, flags);
283         }
284 
285         req->base.complete(&req->base, err);
286 
287         kthread_queue_work(engine->kworker, &engine->pump_requests);
288 }
289 EXPORT_SYMBOL_GPL(crypto_finalize_cipher_request);
290 
291 /**
292  * crypto_finalize_hash_request - finalize one request if the request is done
293  * @engine: the hardware engine
294  * @req: the request need to be finalized
295  * @err: error number
296  */
297 void crypto_finalize_hash_request(struct crypto_engine *engine,
298                                   struct ahash_request *req, int err)
299 {
300         unsigned long flags;
301         bool finalize_cur_req = false;
302         int ret;
303 
304         spin_lock_irqsave(&engine->queue_lock, flags);
305         if (engine->cur_req == &req->base)
306                 finalize_cur_req = true;
307         spin_unlock_irqrestore(&engine->queue_lock, flags);
308 
309         if (finalize_cur_req) {
310                 if (engine->cur_req_prepared &&
311                     engine->unprepare_hash_request) {
312                         ret = engine->unprepare_hash_request(engine, req);
313                         if (ret)
314                                 pr_err("failed to unprepare request\n");
315                 }
316                 spin_lock_irqsave(&engine->queue_lock, flags);
317                 engine->cur_req = NULL;
318                 engine->cur_req_prepared = false;
319                 spin_unlock_irqrestore(&engine->queue_lock, flags);
320         }
321 
322         req->base.complete(&req->base, err);
323 
324         kthread_queue_work(engine->kworker, &engine->pump_requests);
325 }
326 EXPORT_SYMBOL_GPL(crypto_finalize_hash_request);
327 
328 /**
329  * crypto_engine_start - start the hardware engine
330  * @engine: the hardware engine need to be started
331  *
332  * Return 0 on success, else on fail.
333  */
334 int crypto_engine_start(struct crypto_engine *engine)
335 {
336         unsigned long flags;
337 
338         spin_lock_irqsave(&engine->queue_lock, flags);
339 
340         if (engine->running || engine->busy) {
341                 spin_unlock_irqrestore(&engine->queue_lock, flags);
342                 return -EBUSY;
343         }
344 
345         engine->running = true;
346         spin_unlock_irqrestore(&engine->queue_lock, flags);
347 
348         kthread_queue_work(engine->kworker, &engine->pump_requests);
349 
350         return 0;
351 }
352 EXPORT_SYMBOL_GPL(crypto_engine_start);
353 
354 /**
355  * crypto_engine_stop - stop the hardware engine
356  * @engine: the hardware engine need to be stopped
357  *
358  * Return 0 on success, else on fail.
359  */
360 int crypto_engine_stop(struct crypto_engine *engine)
361 {
362         unsigned long flags;
363         unsigned int limit = 500;
364         int ret = 0;
365 
366         spin_lock_irqsave(&engine->queue_lock, flags);
367 
368         /*
369          * If the engine queue is not empty or the engine is on busy state,
370          * we need to wait for a while to pump the requests of engine queue.
371          */
372         while ((crypto_queue_len(&engine->queue) || engine->busy) && limit--) {
373                 spin_unlock_irqrestore(&engine->queue_lock, flags);
374                 msleep(20);
375                 spin_lock_irqsave(&engine->queue_lock, flags);
376         }
377 
378         if (crypto_queue_len(&engine->queue) || engine->busy)
379                 ret = -EBUSY;
380         else
381                 engine->running = false;
382 
383         spin_unlock_irqrestore(&engine->queue_lock, flags);
384 
385         if (ret)
386                 pr_warn("could not stop engine\n");
387 
388         return ret;
389 }
390 EXPORT_SYMBOL_GPL(crypto_engine_stop);
391 
392 /**
393  * crypto_engine_alloc_init - allocate crypto hardware engine structure and
394  * initialize it.
395  * @dev: the device attached with one hardware engine
396  * @rt: whether this queue is set to run as a realtime task
397  *
398  * This must be called from context that can sleep.
399  * Return: the crypto engine structure on success, else NULL.
400  */
401 struct crypto_engine *crypto_engine_alloc_init(struct device *dev, bool rt)
402 {
403         struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
404         struct crypto_engine *engine;
405 
406         if (!dev)
407                 return NULL;
408 
409         engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
410         if (!engine)
411                 return NULL;
412 
413         engine->rt = rt;
414         engine->running = false;
415         engine->busy = false;
416         engine->idling = false;
417         engine->cur_req_prepared = false;
418         engine->priv_data = dev;
419         snprintf(engine->name, sizeof(engine->name),
420                  "%s-engine", dev_name(dev));
421 
422         crypto_init_queue(&engine->queue, CRYPTO_ENGINE_MAX_QLEN);
423         spin_lock_init(&engine->queue_lock);
424 
425         engine->kworker = kthread_create_worker(0, "%s", engine->name);
426         if (IS_ERR(engine->kworker)) {
427                 dev_err(dev, "failed to create crypto request pump task\n");
428                 return NULL;
429         }
430         kthread_init_work(&engine->pump_requests, crypto_pump_work);
431 
432         if (engine->rt) {
433                 dev_info(dev, "will run requests pump with realtime priority\n");
434                 sched_setscheduler(engine->kworker->task, SCHED_FIFO, &param);
435         }
436 
437         return engine;
438 }
439 EXPORT_SYMBOL_GPL(crypto_engine_alloc_init);
440 
441 /**
442  * crypto_engine_exit - free the resources of hardware engine when exit
443  * @engine: the hardware engine need to be freed
444  *
445  * Return 0 for success.
446  */
447 int crypto_engine_exit(struct crypto_engine *engine)
448 {
449         int ret;
450 
451         ret = crypto_engine_stop(engine);
452         if (ret)
453                 return ret;
454 
455         kthread_destroy_worker(engine->kworker);
456 
457         return 0;
458 }
459 EXPORT_SYMBOL_GPL(crypto_engine_exit);
460 
461 MODULE_LICENSE("GPL");
462 MODULE_DESCRIPTION("Crypto hardware engine framework");
463 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us