Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/mm/page_io.c

  1 /*
  2  *  linux/mm/page_io.c
  3  *
  4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
  5  *
  6  *  Swap reorganised 29.12.95, 
  7  *  Asynchronous swapping added 30.12.95. Stephen Tweedie
  8  *  Removed race in async swapping. 14.4.1996. Bruno Haible
  9  *  Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie
 10  *  Always use brw_page, life becomes simpler. 12 May 1998 Eric Biederman
 11  */
 12 
 13 #include <linux/mm.h>
 14 #include <linux/kernel_stat.h>
 15 #include <linux/gfp.h>
 16 #include <linux/pagemap.h>
 17 #include <linux/swap.h>
 18 #include <linux/bio.h>
 19 #include <linux/swapops.h>
 20 #include <linux/buffer_head.h>
 21 #include <linux/writeback.h>
 22 #include <linux/frontswap.h>
 23 #include <linux/blkdev.h>
 24 #include <linux/uio.h>
 25 #include <asm/pgtable.h>
 26 
 27 static struct bio *get_swap_bio(gfp_t gfp_flags,
 28                                 struct page *page, bio_end_io_t end_io)
 29 {
 30         struct bio *bio;
 31 
 32         bio = bio_alloc(gfp_flags, 1);
 33         if (bio) {
 34                 bio->bi_iter.bi_sector = map_swap_page(page, &bio->bi_bdev);
 35                 bio->bi_iter.bi_sector <<= PAGE_SHIFT - 9;
 36                 bio->bi_end_io = end_io;
 37 
 38                 bio_add_page(bio, page, PAGE_SIZE, 0);
 39                 BUG_ON(bio->bi_iter.bi_size != PAGE_SIZE);
 40         }
 41         return bio;
 42 }
 43 
 44 void end_swap_bio_write(struct bio *bio)
 45 {
 46         struct page *page = bio->bi_io_vec[0].bv_page;
 47 
 48         if (bio->bi_error) {
 49                 SetPageError(page);
 50                 /*
 51                  * We failed to write the page out to swap-space.
 52                  * Re-dirty the page in order to avoid it being reclaimed.
 53                  * Also print a dire warning that things will go BAD (tm)
 54                  * very quickly.
 55                  *
 56                  * Also clear PG_reclaim to avoid rotate_reclaimable_page()
 57                  */
 58                 set_page_dirty(page);
 59                 pr_alert("Write-error on swap-device (%u:%u:%llu)\n",
 60                          imajor(bio->bi_bdev->bd_inode),
 61                          iminor(bio->bi_bdev->bd_inode),
 62                          (unsigned long long)bio->bi_iter.bi_sector);
 63                 ClearPageReclaim(page);
 64         }
 65         end_page_writeback(page);
 66         bio_put(bio);
 67 }
 68 
 69 static void swap_slot_free_notify(struct page *page)
 70 {
 71         struct swap_info_struct *sis;
 72         struct gendisk *disk;
 73 
 74         /*
 75          * There is no guarantee that the page is in swap cache - the software
 76          * suspend code (at least) uses end_swap_bio_read() against a non-
 77          * swapcache page.  So we must check PG_swapcache before proceeding with
 78          * this optimization.
 79          */
 80         if (unlikely(!PageSwapCache(page)))
 81                 return;
 82 
 83         sis = page_swap_info(page);
 84         if (!(sis->flags & SWP_BLKDEV))
 85                 return;
 86 
 87         /*
 88          * The swap subsystem performs lazy swap slot freeing,
 89          * expecting that the page will be swapped out again.
 90          * So we can avoid an unnecessary write if the page
 91          * isn't redirtied.
 92          * This is good for real swap storage because we can
 93          * reduce unnecessary I/O and enhance wear-leveling
 94          * if an SSD is used as the as swap device.
 95          * But if in-memory swap device (eg zram) is used,
 96          * this causes a duplicated copy between uncompressed
 97          * data in VM-owned memory and compressed data in
 98          * zram-owned memory.  So let's free zram-owned memory
 99          * and make the VM-owned decompressed page *dirty*,
100          * so the page should be swapped out somewhere again if
101          * we again wish to reclaim it.
102          */
103         disk = sis->bdev->bd_disk;
104         if (disk->fops->swap_slot_free_notify) {
105                 swp_entry_t entry;
106                 unsigned long offset;
107 
108                 entry.val = page_private(page);
109                 offset = swp_offset(entry);
110 
111                 SetPageDirty(page);
112                 disk->fops->swap_slot_free_notify(sis->bdev,
113                                 offset);
114         }
115 }
116 
117 static void end_swap_bio_read(struct bio *bio)
118 {
119         struct page *page = bio->bi_io_vec[0].bv_page;
120 
121         if (bio->bi_error) {
122                 SetPageError(page);
123                 ClearPageUptodate(page);
124                 pr_alert("Read-error on swap-device (%u:%u:%llu)\n",
125                          imajor(bio->bi_bdev->bd_inode),
126                          iminor(bio->bi_bdev->bd_inode),
127                          (unsigned long long)bio->bi_iter.bi_sector);
128                 goto out;
129         }
130 
131         SetPageUptodate(page);
132         swap_slot_free_notify(page);
133 out:
134         unlock_page(page);
135         bio_put(bio);
136 }
137 
138 int generic_swapfile_activate(struct swap_info_struct *sis,
139                                 struct file *swap_file,
140                                 sector_t *span)
141 {
142         struct address_space *mapping = swap_file->f_mapping;
143         struct inode *inode = mapping->host;
144         unsigned blocks_per_page;
145         unsigned long page_no;
146         unsigned blkbits;
147         sector_t probe_block;
148         sector_t last_block;
149         sector_t lowest_block = -1;
150         sector_t highest_block = 0;
151         int nr_extents = 0;
152         int ret;
153 
154         blkbits = inode->i_blkbits;
155         blocks_per_page = PAGE_SIZE >> blkbits;
156 
157         /*
158          * Map all the blocks into the extent list.  This code doesn't try
159          * to be very smart.
160          */
161         probe_block = 0;
162         page_no = 0;
163         last_block = i_size_read(inode) >> blkbits;
164         while ((probe_block + blocks_per_page) <= last_block &&
165                         page_no < sis->max) {
166                 unsigned block_in_page;
167                 sector_t first_block;
168 
169                 cond_resched();
170 
171                 first_block = bmap(inode, probe_block);
172                 if (first_block == 0)
173                         goto bad_bmap;
174 
175                 /*
176                  * It must be PAGE_SIZE aligned on-disk
177                  */
178                 if (first_block & (blocks_per_page - 1)) {
179                         probe_block++;
180                         goto reprobe;
181                 }
182 
183                 for (block_in_page = 1; block_in_page < blocks_per_page;
184                                         block_in_page++) {
185                         sector_t block;
186 
187                         block = bmap(inode, probe_block + block_in_page);
188                         if (block == 0)
189                                 goto bad_bmap;
190                         if (block != first_block + block_in_page) {
191                                 /* Discontiguity */
192                                 probe_block++;
193                                 goto reprobe;
194                         }
195                 }
196 
197                 first_block >>= (PAGE_SHIFT - blkbits);
198                 if (page_no) {  /* exclude the header page */
199                         if (first_block < lowest_block)
200                                 lowest_block = first_block;
201                         if (first_block > highest_block)
202                                 highest_block = first_block;
203                 }
204 
205                 /*
206                  * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
207                  */
208                 ret = add_swap_extent(sis, page_no, 1, first_block);
209                 if (ret < 0)
210                         goto out;
211                 nr_extents += ret;
212                 page_no++;
213                 probe_block += blocks_per_page;
214 reprobe:
215                 continue;
216         }
217         ret = nr_extents;
218         *span = 1 + highest_block - lowest_block;
219         if (page_no == 0)
220                 page_no = 1;    /* force Empty message */
221         sis->max = page_no;
222         sis->pages = page_no - 1;
223         sis->highest_bit = page_no - 1;
224 out:
225         return ret;
226 bad_bmap:
227         pr_err("swapon: swapfile has holes\n");
228         ret = -EINVAL;
229         goto out;
230 }
231 
232 /*
233  * We may have stale swap cache pages in memory: notice
234  * them here and get rid of the unnecessary final write.
235  */
236 int swap_writepage(struct page *page, struct writeback_control *wbc)
237 {
238         int ret = 0;
239 
240         if (try_to_free_swap(page)) {
241                 unlock_page(page);
242                 goto out;
243         }
244         if (frontswap_store(page) == 0) {
245                 set_page_writeback(page);
246                 unlock_page(page);
247                 end_page_writeback(page);
248                 goto out;
249         }
250         ret = __swap_writepage(page, wbc, end_swap_bio_write);
251 out:
252         return ret;
253 }
254 
255 static sector_t swap_page_sector(struct page *page)
256 {
257         return (sector_t)__page_file_index(page) << (PAGE_SHIFT - 9);
258 }
259 
260 int __swap_writepage(struct page *page, struct writeback_control *wbc,
261                 bio_end_io_t end_write_func)
262 {
263         struct bio *bio;
264         int ret;
265         struct swap_info_struct *sis = page_swap_info(page);
266 
267         VM_BUG_ON_PAGE(!PageSwapCache(page), page);
268         if (sis->flags & SWP_FILE) {
269                 struct kiocb kiocb;
270                 struct file *swap_file = sis->swap_file;
271                 struct address_space *mapping = swap_file->f_mapping;
272                 struct bio_vec bv = {
273                         .bv_page = page,
274                         .bv_len  = PAGE_SIZE,
275                         .bv_offset = 0
276                 };
277                 struct iov_iter from;
278 
279                 iov_iter_bvec(&from, ITER_BVEC | WRITE, &bv, 1, PAGE_SIZE);
280                 init_sync_kiocb(&kiocb, swap_file);
281                 kiocb.ki_pos = page_file_offset(page);
282 
283                 set_page_writeback(page);
284                 unlock_page(page);
285                 ret = mapping->a_ops->direct_IO(&kiocb, &from);
286                 if (ret == PAGE_SIZE) {
287                         count_vm_event(PSWPOUT);
288                         ret = 0;
289                 } else {
290                         /*
291                          * In the case of swap-over-nfs, this can be a
292                          * temporary failure if the system has limited
293                          * memory for allocating transmit buffers.
294                          * Mark the page dirty and avoid
295                          * rotate_reclaimable_page but rate-limit the
296                          * messages but do not flag PageError like
297                          * the normal direct-to-bio case as it could
298                          * be temporary.
299                          */
300                         set_page_dirty(page);
301                         ClearPageReclaim(page);
302                         pr_err_ratelimited("Write error on dio swapfile (%llu)\n",
303                                            page_file_offset(page));
304                 }
305                 end_page_writeback(page);
306                 return ret;
307         }
308 
309         ret = bdev_write_page(sis->bdev, swap_page_sector(page), page, wbc);
310         if (!ret) {
311                 count_vm_event(PSWPOUT);
312                 return 0;
313         }
314 
315         ret = 0;
316         bio = get_swap_bio(GFP_NOIO, page, end_write_func);
317         if (bio == NULL) {
318                 set_page_dirty(page);
319                 unlock_page(page);
320                 ret = -ENOMEM;
321                 goto out;
322         }
323         bio->bi_opf = REQ_OP_WRITE | wbc_to_write_flags(wbc);
324         count_vm_event(PSWPOUT);
325         set_page_writeback(page);
326         unlock_page(page);
327         submit_bio(bio);
328 out:
329         return ret;
330 }
331 
332 int swap_readpage(struct page *page)
333 {
334         struct bio *bio;
335         int ret = 0;
336         struct swap_info_struct *sis = page_swap_info(page);
337 
338         VM_BUG_ON_PAGE(!PageSwapCache(page), page);
339         VM_BUG_ON_PAGE(!PageLocked(page), page);
340         VM_BUG_ON_PAGE(PageUptodate(page), page);
341         if (frontswap_load(page) == 0) {
342                 SetPageUptodate(page);
343                 unlock_page(page);
344                 goto out;
345         }
346 
347         if (sis->flags & SWP_FILE) {
348                 struct file *swap_file = sis->swap_file;
349                 struct address_space *mapping = swap_file->f_mapping;
350 
351                 ret = mapping->a_ops->readpage(swap_file, page);
352                 if (!ret)
353                         count_vm_event(PSWPIN);
354                 return ret;
355         }
356 
357         ret = bdev_read_page(sis->bdev, swap_page_sector(page), page);
358         if (!ret) {
359                 if (trylock_page(page)) {
360                         swap_slot_free_notify(page);
361                         unlock_page(page);
362                 }
363 
364                 count_vm_event(PSWPIN);
365                 return 0;
366         }
367 
368         ret = 0;
369         bio = get_swap_bio(GFP_KERNEL, page, end_swap_bio_read);
370         if (bio == NULL) {
371                 unlock_page(page);
372                 ret = -ENOMEM;
373                 goto out;
374         }
375         bio_set_op_attrs(bio, REQ_OP_READ, 0);
376         count_vm_event(PSWPIN);
377         submit_bio(bio);
378 out:
379         return ret;
380 }
381 
382 int swap_set_page_dirty(struct page *page)
383 {
384         struct swap_info_struct *sis = page_swap_info(page);
385 
386         if (sis->flags & SWP_FILE) {
387                 struct address_space *mapping = sis->swap_file->f_mapping;
388 
389                 VM_BUG_ON_PAGE(!PageSwapCache(page), page);
390                 return mapping->a_ops->set_page_dirty(page);
391         } else {
392                 return __set_page_dirty_no_writeback(page);
393         }
394 }
395 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us