Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/mm/zswap.c

  1 /*
  2  * zswap.c - zswap driver file
  3  *
  4  * zswap is a backend for frontswap that takes pages that are in the process
  5  * of being swapped out and attempts to compress and store them in a
  6  * RAM-based memory pool.  This can result in a significant I/O reduction on
  7  * the swap device and, in the case where decompressing from RAM is faster
  8  * than reading from the swap device, can also improve workload performance.
  9  *
 10  * Copyright (C) 2012  Seth Jennings <sjenning@linux.vnet.ibm.com>
 11  *
 12  * This program is free software; you can redistribute it and/or
 13  * modify it under the terms of the GNU General Public License
 14  * as published by the Free Software Foundation; either version 2
 15  * of the License, or (at your option) any later version.
 16  *
 17  * This program is distributed in the hope that it will be useful,
 18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 20  * GNU General Public License for more details.
 21 */
 22 
 23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 24 
 25 #include <linux/module.h>
 26 #include <linux/cpu.h>
 27 #include <linux/highmem.h>
 28 #include <linux/slab.h>
 29 #include <linux/spinlock.h>
 30 #include <linux/types.h>
 31 #include <linux/atomic.h>
 32 #include <linux/frontswap.h>
 33 #include <linux/rbtree.h>
 34 #include <linux/swap.h>
 35 #include <linux/crypto.h>
 36 #include <linux/mempool.h>
 37 #include <linux/zpool.h>
 38 
 39 #include <linux/mm_types.h>
 40 #include <linux/page-flags.h>
 41 #include <linux/swapops.h>
 42 #include <linux/writeback.h>
 43 #include <linux/pagemap.h>
 44 
 45 /*********************************
 46 * statistics
 47 **********************************/
 48 /* Total bytes used by the compressed storage */
 49 static u64 zswap_pool_total_size;
 50 /* The number of compressed pages currently stored in zswap */
 51 static atomic_t zswap_stored_pages = ATOMIC_INIT(0);
 52 
 53 /*
 54  * The statistics below are not protected from concurrent access for
 55  * performance reasons so they may not be a 100% accurate.  However,
 56  * they do provide useful information on roughly how many times a
 57  * certain event is occurring.
 58 */
 59 
 60 /* Pool limit was hit (see zswap_max_pool_percent) */
 61 static u64 zswap_pool_limit_hit;
 62 /* Pages written back when pool limit was reached */
 63 static u64 zswap_written_back_pages;
 64 /* Store failed due to a reclaim failure after pool limit was reached */
 65 static u64 zswap_reject_reclaim_fail;
 66 /* Compressed page was too big for the allocator to (optimally) store */
 67 static u64 zswap_reject_compress_poor;
 68 /* Store failed because underlying allocator could not get memory */
 69 static u64 zswap_reject_alloc_fail;
 70 /* Store failed because the entry metadata could not be allocated (rare) */
 71 static u64 zswap_reject_kmemcache_fail;
 72 /* Duplicate store was encountered (rare) */
 73 static u64 zswap_duplicate_entry;
 74 
 75 /*********************************
 76 * tunables
 77 **********************************/
 78 
 79 /* Enable/disable zswap (disabled by default) */
 80 static bool zswap_enabled;
 81 static int zswap_enabled_param_set(const char *,
 82                                    const struct kernel_param *);
 83 static struct kernel_param_ops zswap_enabled_param_ops = {
 84         .set =          zswap_enabled_param_set,
 85         .get =          param_get_bool,
 86 };
 87 module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644);
 88 
 89 /* Crypto compressor to use */
 90 #define ZSWAP_COMPRESSOR_DEFAULT "lzo"
 91 static char *zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
 92 static int zswap_compressor_param_set(const char *,
 93                                       const struct kernel_param *);
 94 static struct kernel_param_ops zswap_compressor_param_ops = {
 95         .set =          zswap_compressor_param_set,
 96         .get =          param_get_charp,
 97         .free =         param_free_charp,
 98 };
 99 module_param_cb(compressor, &zswap_compressor_param_ops,
100                 &zswap_compressor, 0644);
101 
102 /* Compressed storage zpool to use */
103 #define ZSWAP_ZPOOL_DEFAULT "zbud"
104 static char *zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT;
105 static int zswap_zpool_param_set(const char *, const struct kernel_param *);
106 static struct kernel_param_ops zswap_zpool_param_ops = {
107         .set =          zswap_zpool_param_set,
108         .get =          param_get_charp,
109         .free =         param_free_charp,
110 };
111 module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644);
112 
113 /* The maximum percentage of memory that the compressed pool can occupy */
114 static unsigned int zswap_max_pool_percent = 20;
115 module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644);
116 
117 /*********************************
118 * data structures
119 **********************************/
120 
121 struct zswap_pool {
122         struct zpool *zpool;
123         struct crypto_comp * __percpu *tfm;
124         struct kref kref;
125         struct list_head list;
126         struct work_struct work;
127         struct hlist_node node;
128         char tfm_name[CRYPTO_MAX_ALG_NAME];
129 };
130 
131 /*
132  * struct zswap_entry
133  *
134  * This structure contains the metadata for tracking a single compressed
135  * page within zswap.
136  *
137  * rbnode - links the entry into red-black tree for the appropriate swap type
138  * offset - the swap offset for the entry.  Index into the red-black tree.
139  * refcount - the number of outstanding reference to the entry. This is needed
140  *            to protect against premature freeing of the entry by code
141  *            concurrent calls to load, invalidate, and writeback.  The lock
142  *            for the zswap_tree structure that contains the entry must
143  *            be held while changing the refcount.  Since the lock must
144  *            be held, there is no reason to also make refcount atomic.
145  * length - the length in bytes of the compressed page data.  Needed during
146  *          decompression
147  * pool - the zswap_pool the entry's data is in
148  * handle - zpool allocation handle that stores the compressed page data
149  */
150 struct zswap_entry {
151         struct rb_node rbnode;
152         pgoff_t offset;
153         int refcount;
154         unsigned int length;
155         struct zswap_pool *pool;
156         unsigned long handle;
157 };
158 
159 struct zswap_header {
160         swp_entry_t swpentry;
161 };
162 
163 /*
164  * The tree lock in the zswap_tree struct protects a few things:
165  * - the rbtree
166  * - the refcount field of each entry in the tree
167  */
168 struct zswap_tree {
169         struct rb_root rbroot;
170         spinlock_t lock;
171 };
172 
173 static struct zswap_tree *zswap_trees[MAX_SWAPFILES];
174 
175 /* RCU-protected iteration */
176 static LIST_HEAD(zswap_pools);
177 /* protects zswap_pools list modification */
178 static DEFINE_SPINLOCK(zswap_pools_lock);
179 /* pool counter to provide unique names to zpool */
180 static atomic_t zswap_pools_count = ATOMIC_INIT(0);
181 
182 /* used by param callback function */
183 static bool zswap_init_started;
184 
185 /* fatal error during init */
186 static bool zswap_init_failed;
187 
188 /*********************************
189 * helpers and fwd declarations
190 **********************************/
191 
192 #define zswap_pool_debug(msg, p)                                \
193         pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name,         \
194                  zpool_get_type((p)->zpool))
195 
196 static int zswap_writeback_entry(struct zpool *pool, unsigned long handle);
197 static int zswap_pool_get(struct zswap_pool *pool);
198 static void zswap_pool_put(struct zswap_pool *pool);
199 
200 static const struct zpool_ops zswap_zpool_ops = {
201         .evict = zswap_writeback_entry
202 };
203 
204 static bool zswap_is_full(void)
205 {
206         return totalram_pages * zswap_max_pool_percent / 100 <
207                 DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE);
208 }
209 
210 static void zswap_update_total_size(void)
211 {
212         struct zswap_pool *pool;
213         u64 total = 0;
214 
215         rcu_read_lock();
216 
217         list_for_each_entry_rcu(pool, &zswap_pools, list)
218                 total += zpool_get_total_size(pool->zpool);
219 
220         rcu_read_unlock();
221 
222         zswap_pool_total_size = total;
223 }
224 
225 /*********************************
226 * zswap entry functions
227 **********************************/
228 static struct kmem_cache *zswap_entry_cache;
229 
230 static int __init zswap_entry_cache_create(void)
231 {
232         zswap_entry_cache = KMEM_CACHE(zswap_entry, 0);
233         return zswap_entry_cache == NULL;
234 }
235 
236 static void __init zswap_entry_cache_destroy(void)
237 {
238         kmem_cache_destroy(zswap_entry_cache);
239 }
240 
241 static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp)
242 {
243         struct zswap_entry *entry;
244         entry = kmem_cache_alloc(zswap_entry_cache, gfp);
245         if (!entry)
246                 return NULL;
247         entry->refcount = 1;
248         RB_CLEAR_NODE(&entry->rbnode);
249         return entry;
250 }
251 
252 static void zswap_entry_cache_free(struct zswap_entry *entry)
253 {
254         kmem_cache_free(zswap_entry_cache, entry);
255 }
256 
257 /*********************************
258 * rbtree functions
259 **********************************/
260 static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset)
261 {
262         struct rb_node *node = root->rb_node;
263         struct zswap_entry *entry;
264 
265         while (node) {
266                 entry = rb_entry(node, struct zswap_entry, rbnode);
267                 if (entry->offset > offset)
268                         node = node->rb_left;
269                 else if (entry->offset < offset)
270                         node = node->rb_right;
271                 else
272                         return entry;
273         }
274         return NULL;
275 }
276 
277 /*
278  * In the case that a entry with the same offset is found, a pointer to
279  * the existing entry is stored in dupentry and the function returns -EEXIST
280  */
281 static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry,
282                         struct zswap_entry **dupentry)
283 {
284         struct rb_node **link = &root->rb_node, *parent = NULL;
285         struct zswap_entry *myentry;
286 
287         while (*link) {
288                 parent = *link;
289                 myentry = rb_entry(parent, struct zswap_entry, rbnode);
290                 if (myentry->offset > entry->offset)
291                         link = &(*link)->rb_left;
292                 else if (myentry->offset < entry->offset)
293                         link = &(*link)->rb_right;
294                 else {
295                         *dupentry = myentry;
296                         return -EEXIST;
297                 }
298         }
299         rb_link_node(&entry->rbnode, parent, link);
300         rb_insert_color(&entry->rbnode, root);
301         return 0;
302 }
303 
304 static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry)
305 {
306         if (!RB_EMPTY_NODE(&entry->rbnode)) {
307                 rb_erase(&entry->rbnode, root);
308                 RB_CLEAR_NODE(&entry->rbnode);
309         }
310 }
311 
312 /*
313  * Carries out the common pattern of freeing and entry's zpool allocation,
314  * freeing the entry itself, and decrementing the number of stored pages.
315  */
316 static void zswap_free_entry(struct zswap_entry *entry)
317 {
318         zpool_free(entry->pool->zpool, entry->handle);
319         zswap_pool_put(entry->pool);
320         zswap_entry_cache_free(entry);
321         atomic_dec(&zswap_stored_pages);
322         zswap_update_total_size();
323 }
324 
325 /* caller must hold the tree lock */
326 static void zswap_entry_get(struct zswap_entry *entry)
327 {
328         entry->refcount++;
329 }
330 
331 /* caller must hold the tree lock
332 * remove from the tree and free it, if nobody reference the entry
333 */
334 static void zswap_entry_put(struct zswap_tree *tree,
335                         struct zswap_entry *entry)
336 {
337         int refcount = --entry->refcount;
338 
339         BUG_ON(refcount < 0);
340         if (refcount == 0) {
341                 zswap_rb_erase(&tree->rbroot, entry);
342                 zswap_free_entry(entry);
343         }
344 }
345 
346 /* caller must hold the tree lock */
347 static struct zswap_entry *zswap_entry_find_get(struct rb_root *root,
348                                 pgoff_t offset)
349 {
350         struct zswap_entry *entry;
351 
352         entry = zswap_rb_search(root, offset);
353         if (entry)
354                 zswap_entry_get(entry);
355 
356         return entry;
357 }
358 
359 /*********************************
360 * per-cpu code
361 **********************************/
362 static DEFINE_PER_CPU(u8 *, zswap_dstmem);
363 
364 static int zswap_dstmem_prepare(unsigned int cpu)
365 {
366         u8 *dst;
367 
368         dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
369         if (!dst) {
370                 pr_err("can't allocate compressor buffer\n");
371                 return -ENOMEM;
372         }
373         per_cpu(zswap_dstmem, cpu) = dst;
374         return 0;
375 }
376 
377 static int zswap_dstmem_dead(unsigned int cpu)
378 {
379         u8 *dst;
380 
381         dst = per_cpu(zswap_dstmem, cpu);
382         kfree(dst);
383         per_cpu(zswap_dstmem, cpu) = NULL;
384 
385         return 0;
386 }
387 
388 static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
389 {
390         struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
391         struct crypto_comp *tfm;
392 
393         if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu)))
394                 return 0;
395 
396         tfm = crypto_alloc_comp(pool->tfm_name, 0, 0);
397         if (IS_ERR_OR_NULL(tfm)) {
398                 pr_err("could not alloc crypto comp %s : %ld\n",
399                        pool->tfm_name, PTR_ERR(tfm));
400                 return -ENOMEM;
401         }
402         *per_cpu_ptr(pool->tfm, cpu) = tfm;
403         return 0;
404 }
405 
406 static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node)
407 {
408         struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
409         struct crypto_comp *tfm;
410 
411         tfm = *per_cpu_ptr(pool->tfm, cpu);
412         if (!IS_ERR_OR_NULL(tfm))
413                 crypto_free_comp(tfm);
414         *per_cpu_ptr(pool->tfm, cpu) = NULL;
415         return 0;
416 }
417 
418 /*********************************
419 * pool functions
420 **********************************/
421 
422 static struct zswap_pool *__zswap_pool_current(void)
423 {
424         struct zswap_pool *pool;
425 
426         pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
427         WARN_ON(!pool);
428 
429         return pool;
430 }
431 
432 static struct zswap_pool *zswap_pool_current(void)
433 {
434         assert_spin_locked(&zswap_pools_lock);
435 
436         return __zswap_pool_current();
437 }
438 
439 static struct zswap_pool *zswap_pool_current_get(void)
440 {
441         struct zswap_pool *pool;
442 
443         rcu_read_lock();
444 
445         pool = __zswap_pool_current();
446         if (!pool || !zswap_pool_get(pool))
447                 pool = NULL;
448 
449         rcu_read_unlock();
450 
451         return pool;
452 }
453 
454 static struct zswap_pool *zswap_pool_last_get(void)
455 {
456         struct zswap_pool *pool, *last = NULL;
457 
458         rcu_read_lock();
459 
460         list_for_each_entry_rcu(pool, &zswap_pools, list)
461                 last = pool;
462         if (!WARN_ON(!last) && !zswap_pool_get(last))
463                 last = NULL;
464 
465         rcu_read_unlock();
466 
467         return last;
468 }
469 
470 /* type and compressor must be null-terminated */
471 static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor)
472 {
473         struct zswap_pool *pool;
474 
475         assert_spin_locked(&zswap_pools_lock);
476 
477         list_for_each_entry_rcu(pool, &zswap_pools, list) {
478                 if (strcmp(pool->tfm_name, compressor))
479                         continue;
480                 if (strcmp(zpool_get_type(pool->zpool), type))
481                         continue;
482                 /* if we can't get it, it's about to be destroyed */
483                 if (!zswap_pool_get(pool))
484                         continue;
485                 return pool;
486         }
487 
488         return NULL;
489 }
490 
491 static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
492 {
493         struct zswap_pool *pool;
494         char name[38]; /* 'zswap' + 32 char (max) num + \0 */
495         gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
496         int ret;
497 
498         pool = kzalloc(sizeof(*pool), GFP_KERNEL);
499         if (!pool) {
500                 pr_err("pool alloc failed\n");
501                 return NULL;
502         }
503 
504         /* unique name for each pool specifically required by zsmalloc */
505         snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count));
506 
507         pool->zpool = zpool_create_pool(type, name, gfp, &zswap_zpool_ops);
508         if (!pool->zpool) {
509                 pr_err("%s zpool not available\n", type);
510                 goto error;
511         }
512         pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));
513 
514         strlcpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));
515         pool->tfm = alloc_percpu(struct crypto_comp *);
516         if (!pool->tfm) {
517                 pr_err("percpu alloc failed\n");
518                 goto error;
519         }
520 
521         ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE,
522                                        &pool->node);
523         if (ret)
524                 goto error;
525         pr_debug("using %s compressor\n", pool->tfm_name);
526 
527         /* being the current pool takes 1 ref; this func expects the
528          * caller to always add the new pool as the current pool
529          */
530         kref_init(&pool->kref);
531         INIT_LIST_HEAD(&pool->list);
532 
533         zswap_pool_debug("created", pool);
534 
535         return pool;
536 
537 error:
538         free_percpu(pool->tfm);
539         if (pool->zpool)
540                 zpool_destroy_pool(pool->zpool);
541         kfree(pool);
542         return NULL;
543 }
544 
545 static __init struct zswap_pool *__zswap_pool_create_fallback(void)
546 {
547         if (!crypto_has_comp(zswap_compressor, 0, 0)) {
548                 if (!strcmp(zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT)) {
549                         pr_err("default compressor %s not available\n",
550                                zswap_compressor);
551                         return NULL;
552                 }
553                 pr_err("compressor %s not available, using default %s\n",
554                        zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT);
555                 param_free_charp(&zswap_compressor);
556                 zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
557         }
558         if (!zpool_has_pool(zswap_zpool_type)) {
559                 if (!strcmp(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT)) {
560                         pr_err("default zpool %s not available\n",
561                                zswap_zpool_type);
562                         return NULL;
563                 }
564                 pr_err("zpool %s not available, using default %s\n",
565                        zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT);
566                 param_free_charp(&zswap_zpool_type);
567                 zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT;
568         }
569 
570         return zswap_pool_create(zswap_zpool_type, zswap_compressor);
571 }
572 
573 static void zswap_pool_destroy(struct zswap_pool *pool)
574 {
575         zswap_pool_debug("destroying", pool);
576 
577         cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);
578         free_percpu(pool->tfm);
579         zpool_destroy_pool(pool->zpool);
580         kfree(pool);
581 }
582 
583 static int __must_check zswap_pool_get(struct zswap_pool *pool)
584 {
585         return kref_get_unless_zero(&pool->kref);
586 }
587 
588 static void __zswap_pool_release(struct work_struct *work)
589 {
590         struct zswap_pool *pool = container_of(work, typeof(*pool), work);
591 
592         synchronize_rcu();
593 
594         /* nobody should have been able to get a kref... */
595         WARN_ON(kref_get_unless_zero(&pool->kref));
596 
597         /* pool is now off zswap_pools list and has no references. */
598         zswap_pool_destroy(pool);
599 }
600 
601 static void __zswap_pool_empty(struct kref *kref)
602 {
603         struct zswap_pool *pool;
604 
605         pool = container_of(kref, typeof(*pool), kref);
606 
607         spin_lock(&zswap_pools_lock);
608 
609         WARN_ON(pool == zswap_pool_current());
610 
611         list_del_rcu(&pool->list);
612 
613         INIT_WORK(&pool->work, __zswap_pool_release);
614         schedule_work(&pool->work);
615 
616         spin_unlock(&zswap_pools_lock);
617 }
618 
619 static void zswap_pool_put(struct zswap_pool *pool)
620 {
621         kref_put(&pool->kref, __zswap_pool_empty);
622 }
623 
624 /*********************************
625 * param callbacks
626 **********************************/
627 
628 /* val must be a null-terminated string */
629 static int __zswap_param_set(const char *val, const struct kernel_param *kp,
630                              char *type, char *compressor)
631 {
632         struct zswap_pool *pool, *put_pool = NULL;
633         char *s = strstrip((char *)val);
634         int ret;
635 
636         if (zswap_init_failed) {
637                 pr_err("can't set param, initialization failed\n");
638                 return -ENODEV;
639         }
640 
641         /* no change required */
642         if (!strcmp(s, *(char **)kp->arg))
643                 return 0;
644 
645         /* if this is load-time (pre-init) param setting,
646          * don't create a pool; that's done during init.
647          */
648         if (!zswap_init_started)
649                 return param_set_charp(s, kp);
650 
651         if (!type) {
652                 if (!zpool_has_pool(s)) {
653                         pr_err("zpool %s not available\n", s);
654                         return -ENOENT;
655                 }
656                 type = s;
657         } else if (!compressor) {
658                 if (!crypto_has_comp(s, 0, 0)) {
659                         pr_err("compressor %s not available\n", s);
660                         return -ENOENT;
661                 }
662                 compressor = s;
663         } else {
664                 WARN_ON(1);
665                 return -EINVAL;
666         }
667 
668         spin_lock(&zswap_pools_lock);
669 
670         pool = zswap_pool_find_get(type, compressor);
671         if (pool) {
672                 zswap_pool_debug("using existing", pool);
673                 list_del_rcu(&pool->list);
674         } else {
675                 spin_unlock(&zswap_pools_lock);
676                 pool = zswap_pool_create(type, compressor);
677                 spin_lock(&zswap_pools_lock);
678         }
679 
680         if (pool)
681                 ret = param_set_charp(s, kp);
682         else
683                 ret = -EINVAL;
684 
685         if (!ret) {
686                 put_pool = zswap_pool_current();
687                 list_add_rcu(&pool->list, &zswap_pools);
688         } else if (pool) {
689                 /* add the possibly pre-existing pool to the end of the pools
690                  * list; if it's new (and empty) then it'll be removed and
691                  * destroyed by the put after we drop the lock
692                  */
693                 list_add_tail_rcu(&pool->list, &zswap_pools);
694                 put_pool = pool;
695         }
696 
697         spin_unlock(&zswap_pools_lock);
698 
699         /* drop the ref from either the old current pool,
700          * or the new pool we failed to add
701          */
702         if (put_pool)
703                 zswap_pool_put(put_pool);
704 
705         return ret;
706 }
707 
708 static int zswap_compressor_param_set(const char *val,
709                                       const struct kernel_param *kp)
710 {
711         return __zswap_param_set(val, kp, zswap_zpool_type, NULL);
712 }
713 
714 static int zswap_zpool_param_set(const char *val,
715                                  const struct kernel_param *kp)
716 {
717         return __zswap_param_set(val, kp, NULL, zswap_compressor);
718 }
719 
720 static int zswap_enabled_param_set(const char *val,
721                                    const struct kernel_param *kp)
722 {
723         if (zswap_init_failed) {
724                 pr_err("can't enable, initialization failed\n");
725                 return -ENODEV;
726         }
727 
728         return param_set_bool(val, kp);
729 }
730 
731 /*********************************
732 * writeback code
733 **********************************/
734 /* return enum for zswap_get_swap_cache_page */
735 enum zswap_get_swap_ret {
736         ZSWAP_SWAPCACHE_NEW,
737         ZSWAP_SWAPCACHE_EXIST,
738         ZSWAP_SWAPCACHE_FAIL,
739 };
740 
741 /*
742  * zswap_get_swap_cache_page
743  *
744  * This is an adaption of read_swap_cache_async()
745  *
746  * This function tries to find a page with the given swap entry
747  * in the swapper_space address space (the swap cache).  If the page
748  * is found, it is returned in retpage.  Otherwise, a page is allocated,
749  * added to the swap cache, and returned in retpage.
750  *
751  * If success, the swap cache page is returned in retpage
752  * Returns ZSWAP_SWAPCACHE_EXIST if page was already in the swap cache
753  * Returns ZSWAP_SWAPCACHE_NEW if the new page needs to be populated,
754  *     the new page is added to swapcache and locked
755  * Returns ZSWAP_SWAPCACHE_FAIL on error
756  */
757 static int zswap_get_swap_cache_page(swp_entry_t entry,
758                                 struct page **retpage)
759 {
760         bool page_was_allocated;
761 
762         *retpage = __read_swap_cache_async(entry, GFP_KERNEL,
763                         NULL, 0, &page_was_allocated);
764         if (page_was_allocated)
765                 return ZSWAP_SWAPCACHE_NEW;
766         if (!*retpage)
767                 return ZSWAP_SWAPCACHE_FAIL;
768         return ZSWAP_SWAPCACHE_EXIST;
769 }
770 
771 /*
772  * Attempts to free an entry by adding a page to the swap cache,
773  * decompressing the entry data into the page, and issuing a
774  * bio write to write the page back to the swap device.
775  *
776  * This can be thought of as a "resumed writeback" of the page
777  * to the swap device.  We are basically resuming the same swap
778  * writeback path that was intercepted with the frontswap_store()
779  * in the first place.  After the page has been decompressed into
780  * the swap cache, the compressed version stored by zswap can be
781  * freed.
782  */
783 static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)
784 {
785         struct zswap_header *zhdr;
786         swp_entry_t swpentry;
787         struct zswap_tree *tree;
788         pgoff_t offset;
789         struct zswap_entry *entry;
790         struct page *page;
791         struct crypto_comp *tfm;
792         u8 *src, *dst;
793         unsigned int dlen;
794         int ret;
795         struct writeback_control wbc = {
796                 .sync_mode = WB_SYNC_NONE,
797         };
798 
799         /* extract swpentry from data */
800         zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO);
801         swpentry = zhdr->swpentry; /* here */
802         zpool_unmap_handle(pool, handle);
803         tree = zswap_trees[swp_type(swpentry)];
804         offset = swp_offset(swpentry);
805 
806         /* find and ref zswap entry */
807         spin_lock(&tree->lock);
808         entry = zswap_entry_find_get(&tree->rbroot, offset);
809         if (!entry) {
810                 /* entry was invalidated */
811                 spin_unlock(&tree->lock);
812                 return 0;
813         }
814         spin_unlock(&tree->lock);
815         BUG_ON(offset != entry->offset);
816 
817         /* try to allocate swap cache page */
818         switch (zswap_get_swap_cache_page(swpentry, &page)) {
819         case ZSWAP_SWAPCACHE_FAIL: /* no memory or invalidate happened */
820                 ret = -ENOMEM;
821                 goto fail;
822 
823         case ZSWAP_SWAPCACHE_EXIST:
824                 /* page is already in the swap cache, ignore for now */
825                 put_page(page);
826                 ret = -EEXIST;
827                 goto fail;
828 
829         case ZSWAP_SWAPCACHE_NEW: /* page is locked */
830                 /* decompress */
831                 dlen = PAGE_SIZE;
832                 src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,
833                                 ZPOOL_MM_RO) + sizeof(struct zswap_header);
834                 dst = kmap_atomic(page);
835                 tfm = *get_cpu_ptr(entry->pool->tfm);
836                 ret = crypto_comp_decompress(tfm, src, entry->length,
837                                              dst, &dlen);
838                 put_cpu_ptr(entry->pool->tfm);
839                 kunmap_atomic(dst);
840                 zpool_unmap_handle(entry->pool->zpool, entry->handle);
841                 BUG_ON(ret);
842                 BUG_ON(dlen != PAGE_SIZE);
843 
844                 /* page is up to date */
845                 SetPageUptodate(page);
846         }
847 
848         /* move it to the tail of the inactive list after end_writeback */
849         SetPageReclaim(page);
850 
851         /* start writeback */
852         __swap_writepage(page, &wbc, end_swap_bio_write);
853         put_page(page);
854         zswap_written_back_pages++;
855 
856         spin_lock(&tree->lock);
857         /* drop local reference */
858         zswap_entry_put(tree, entry);
859 
860         /*
861         * There are two possible situations for entry here:
862         * (1) refcount is 1(normal case),  entry is valid and on the tree
863         * (2) refcount is 0, entry is freed and not on the tree
864         *     because invalidate happened during writeback
865         *  search the tree and free the entry if find entry
866         */
867         if (entry == zswap_rb_search(&tree->rbroot, offset))
868                 zswap_entry_put(tree, entry);
869         spin_unlock(&tree->lock);
870 
871         goto end;
872 
873         /*
874         * if we get here due to ZSWAP_SWAPCACHE_EXIST
875         * a load may happening concurrently
876         * it is safe and okay to not free the entry
877         * if we free the entry in the following put
878         * it it either okay to return !0
879         */
880 fail:
881         spin_lock(&tree->lock);
882         zswap_entry_put(tree, entry);
883         spin_unlock(&tree->lock);
884 
885 end:
886         return ret;
887 }
888 
889 static int zswap_shrink(void)
890 {
891         struct zswap_pool *pool;
892         int ret;
893 
894         pool = zswap_pool_last_get();
895         if (!pool)
896                 return -ENOENT;
897 
898         ret = zpool_shrink(pool->zpool, 1, NULL);
899 
900         zswap_pool_put(pool);
901 
902         return ret;
903 }
904 
905 /*********************************
906 * frontswap hooks
907 **********************************/
908 /* attempts to compress and store an single page */
909 static int zswap_frontswap_store(unsigned type, pgoff_t offset,
910                                 struct page *page)
911 {
912         struct zswap_tree *tree = zswap_trees[type];
913         struct zswap_entry *entry, *dupentry;
914         struct crypto_comp *tfm;
915         int ret;
916         unsigned int dlen = PAGE_SIZE, len;
917         unsigned long handle;
918         char *buf;
919         u8 *src, *dst;
920         struct zswap_header *zhdr;
921 
922         if (!zswap_enabled || !tree) {
923                 ret = -ENODEV;
924                 goto reject;
925         }
926 
927         /* reclaim space if needed */
928         if (zswap_is_full()) {
929                 zswap_pool_limit_hit++;
930                 if (zswap_shrink()) {
931                         zswap_reject_reclaim_fail++;
932                         ret = -ENOMEM;
933                         goto reject;
934                 }
935         }
936 
937         /* allocate entry */
938         entry = zswap_entry_cache_alloc(GFP_KERNEL);
939         if (!entry) {
940                 zswap_reject_kmemcache_fail++;
941                 ret = -ENOMEM;
942                 goto reject;
943         }
944 
945         /* if entry is successfully added, it keeps the reference */
946         entry->pool = zswap_pool_current_get();
947         if (!entry->pool) {
948                 ret = -EINVAL;
949                 goto freepage;
950         }
951 
952         /* compress */
953         dst = get_cpu_var(zswap_dstmem);
954         tfm = *get_cpu_ptr(entry->pool->tfm);
955         src = kmap_atomic(page);
956         ret = crypto_comp_compress(tfm, src, PAGE_SIZE, dst, &dlen);
957         kunmap_atomic(src);
958         put_cpu_ptr(entry->pool->tfm);
959         if (ret) {
960                 ret = -EINVAL;
961                 goto put_dstmem;
962         }
963 
964         /* store */
965         len = dlen + sizeof(struct zswap_header);
966         ret = zpool_malloc(entry->pool->zpool, len,
967                            __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM,
968                            &handle);
969         if (ret == -ENOSPC) {
970                 zswap_reject_compress_poor++;
971                 goto put_dstmem;
972         }
973         if (ret) {
974                 zswap_reject_alloc_fail++;
975                 goto put_dstmem;
976         }
977         zhdr = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_RW);
978         zhdr->swpentry = swp_entry(type, offset);
979         buf = (u8 *)(zhdr + 1);
980         memcpy(buf, dst, dlen);
981         zpool_unmap_handle(entry->pool->zpool, handle);
982         put_cpu_var(zswap_dstmem);
983 
984         /* populate entry */
985         entry->offset = offset;
986         entry->handle = handle;
987         entry->length = dlen;
988 
989         /* map */
990         spin_lock(&tree->lock);
991         do {
992                 ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry);
993                 if (ret == -EEXIST) {
994                         zswap_duplicate_entry++;
995                         /* remove from rbtree */
996                         zswap_rb_erase(&tree->rbroot, dupentry);
997                         zswap_entry_put(tree, dupentry);
998                 }
999         } while (ret == -EEXIST);
1000         spin_unlock(&tree->lock);
1001 
1002         /* update stats */
1003         atomic_inc(&zswap_stored_pages);
1004         zswap_update_total_size();
1005 
1006         return 0;
1007 
1008 put_dstmem:
1009         put_cpu_var(zswap_dstmem);
1010         zswap_pool_put(entry->pool);
1011 freepage:
1012         zswap_entry_cache_free(entry);
1013 reject:
1014         return ret;
1015 }
1016 
1017 /*
1018  * returns 0 if the page was successfully decompressed
1019  * return -1 on entry not found or error
1020 */
1021 static int zswap_frontswap_load(unsigned type, pgoff_t offset,
1022                                 struct page *page)
1023 {
1024         struct zswap_tree *tree = zswap_trees[type];
1025         struct zswap_entry *entry;
1026         struct crypto_comp *tfm;
1027         u8 *src, *dst;
1028         unsigned int dlen;
1029         int ret;
1030 
1031         /* find */
1032         spin_lock(&tree->lock);
1033         entry = zswap_entry_find_get(&tree->rbroot, offset);
1034         if (!entry) {
1035                 /* entry was written back */
1036                 spin_unlock(&tree->lock);
1037                 return -1;
1038         }
1039         spin_unlock(&tree->lock);
1040 
1041         /* decompress */
1042         dlen = PAGE_SIZE;
1043         src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,
1044                         ZPOOL_MM_RO) + sizeof(struct zswap_header);
1045         dst = kmap_atomic(page);
1046         tfm = *get_cpu_ptr(entry->pool->tfm);
1047         ret = crypto_comp_decompress(tfm, src, entry->length, dst, &dlen);
1048         put_cpu_ptr(entry->pool->tfm);
1049         kunmap_atomic(dst);
1050         zpool_unmap_handle(entry->pool->zpool, entry->handle);
1051         BUG_ON(ret);
1052 
1053         spin_lock(&tree->lock);
1054         zswap_entry_put(tree, entry);
1055         spin_unlock(&tree->lock);
1056 
1057         return 0;
1058 }
1059 
1060 /* frees an entry in zswap */
1061 static void zswap_frontswap_invalidate_page(unsigned type, pgoff_t offset)
1062 {
1063         struct zswap_tree *tree = zswap_trees[type];
1064         struct zswap_entry *entry;
1065 
1066         /* find */
1067         spin_lock(&tree->lock);
1068         entry = zswap_rb_search(&tree->rbroot, offset);
1069         if (!entry) {
1070                 /* entry was written back */
1071                 spin_unlock(&tree->lock);
1072                 return;
1073         }
1074 
1075         /* remove from rbtree */
1076         zswap_rb_erase(&tree->rbroot, entry);
1077 
1078         /* drop the initial reference from entry creation */
1079         zswap_entry_put(tree, entry);
1080 
1081         spin_unlock(&tree->lock);
1082 }
1083 
1084 /* frees all zswap entries for the given swap type */
1085 static void zswap_frontswap_invalidate_area(unsigned type)
1086 {
1087         struct zswap_tree *tree = zswap_trees[type];
1088         struct zswap_entry *entry, *n;
1089 
1090         if (!tree)
1091                 return;
1092 
1093         /* walk the tree and free everything */
1094         spin_lock(&tree->lock);
1095         rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode)
1096                 zswap_free_entry(entry);
1097         tree->rbroot = RB_ROOT;
1098         spin_unlock(&tree->lock);
1099         kfree(tree);
1100         zswap_trees[type] = NULL;
1101 }
1102 
1103 static void zswap_frontswap_init(unsigned type)
1104 {
1105         struct zswap_tree *tree;
1106 
1107         tree = kzalloc(sizeof(struct zswap_tree), GFP_KERNEL);
1108         if (!tree) {
1109                 pr_err("alloc failed, zswap disabled for swap type %d\n", type);
1110                 return;
1111         }
1112 
1113         tree->rbroot = RB_ROOT;
1114         spin_lock_init(&tree->lock);
1115         zswap_trees[type] = tree;
1116 }
1117 
1118 static struct frontswap_ops zswap_frontswap_ops = {
1119         .store = zswap_frontswap_store,
1120         .load = zswap_frontswap_load,
1121         .invalidate_page = zswap_frontswap_invalidate_page,
1122         .invalidate_area = zswap_frontswap_invalidate_area,
1123         .init = zswap_frontswap_init
1124 };
1125 
1126 /*********************************
1127 * debugfs functions
1128 **********************************/
1129 #ifdef CONFIG_DEBUG_FS
1130 #include <linux/debugfs.h>
1131 
1132 static struct dentry *zswap_debugfs_root;
1133 
1134 static int __init zswap_debugfs_init(void)
1135 {
1136         if (!debugfs_initialized())
1137                 return -ENODEV;
1138 
1139         zswap_debugfs_root = debugfs_create_dir("zswap", NULL);
1140         if (!zswap_debugfs_root)
1141                 return -ENOMEM;
1142 
1143         debugfs_create_u64("pool_limit_hit", S_IRUGO,
1144                         zswap_debugfs_root, &zswap_pool_limit_hit);
1145         debugfs_create_u64("reject_reclaim_fail", S_IRUGO,
1146                         zswap_debugfs_root, &zswap_reject_reclaim_fail);
1147         debugfs_create_u64("reject_alloc_fail", S_IRUGO,
1148                         zswap_debugfs_root, &zswap_reject_alloc_fail);
1149         debugfs_create_u64("reject_kmemcache_fail", S_IRUGO,
1150                         zswap_debugfs_root, &zswap_reject_kmemcache_fail);
1151         debugfs_create_u64("reject_compress_poor", S_IRUGO,
1152                         zswap_debugfs_root, &zswap_reject_compress_poor);
1153         debugfs_create_u64("written_back_pages", S_IRUGO,
1154                         zswap_debugfs_root, &zswap_written_back_pages);
1155         debugfs_create_u64("duplicate_entry", S_IRUGO,
1156                         zswap_debugfs_root, &zswap_duplicate_entry);
1157         debugfs_create_u64("pool_total_size", S_IRUGO,
1158                         zswap_debugfs_root, &zswap_pool_total_size);
1159         debugfs_create_atomic_t("stored_pages", S_IRUGO,
1160                         zswap_debugfs_root, &zswap_stored_pages);
1161 
1162         return 0;
1163 }
1164 
1165 static void __exit zswap_debugfs_exit(void)
1166 {
1167         debugfs_remove_recursive(zswap_debugfs_root);
1168 }
1169 #else
1170 static int __init zswap_debugfs_init(void)
1171 {
1172         return 0;
1173 }
1174 
1175 static void __exit zswap_debugfs_exit(void) { }
1176 #endif
1177 
1178 /*********************************
1179 * module init and exit
1180 **********************************/
1181 static int __init init_zswap(void)
1182 {
1183         struct zswap_pool *pool;
1184         int ret;
1185 
1186         zswap_init_started = true;
1187 
1188         if (zswap_entry_cache_create()) {
1189                 pr_err("entry cache creation failed\n");
1190                 goto cache_fail;
1191         }
1192 
1193         ret = cpuhp_setup_state(CPUHP_MM_ZSWP_MEM_PREPARE, "mm/zswap:prepare",
1194                                 zswap_dstmem_prepare, zswap_dstmem_dead);
1195         if (ret) {
1196                 pr_err("dstmem alloc failed\n");
1197                 goto dstmem_fail;
1198         }
1199 
1200         ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE,
1201                                       "mm/zswap_pool:prepare",
1202                                       zswap_cpu_comp_prepare,
1203                                       zswap_cpu_comp_dead);
1204         if (ret)
1205                 goto hp_fail;
1206 
1207         pool = __zswap_pool_create_fallback();
1208         if (!pool) {
1209                 pr_err("pool creation failed\n");
1210                 goto pool_fail;
1211         }
1212         pr_info("loaded using pool %s/%s\n", pool->tfm_name,
1213                 zpool_get_type(pool->zpool));
1214 
1215         list_add(&pool->list, &zswap_pools);
1216 
1217         frontswap_register_ops(&zswap_frontswap_ops);
1218         if (zswap_debugfs_init())
1219                 pr_warn("debugfs initialization failed\n");
1220         return 0;
1221 
1222 pool_fail:
1223         cpuhp_remove_state_nocalls(CPUHP_MM_ZSWP_POOL_PREPARE);
1224 hp_fail:
1225         cpuhp_remove_state(CPUHP_MM_ZSWP_MEM_PREPARE);
1226 dstmem_fail:
1227         zswap_entry_cache_destroy();
1228 cache_fail:
1229         /* if built-in, we aren't unloaded on failure; don't allow use */
1230         zswap_init_failed = true;
1231         zswap_enabled = false;
1232         return -ENOMEM;
1233 }
1234 /* must be late so crypto has time to come up */
1235 late_initcall(init_zswap);
1236 
1237 MODULE_LICENSE("GPL");
1238 MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>");
1239 MODULE_DESCRIPTION("Compressed cache for swap pages");
1240 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us