Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/fs/read_write.c

  1 /*
  2  *  linux/fs/read_write.c
  3  *
  4  *  Copyright (C) 1991, 1992  Linus Torvalds
  5  */
  6 
  7 #include <linux/slab.h> 
  8 #include <linux/stat.h>
  9 #include <linux/fcntl.h>
 10 #include <linux/file.h>
 11 #include <linux/uio.h>
 12 #include <linux/aio.h>
 13 #include <linux/fsnotify.h>
 14 #include <linux/security.h>
 15 #include <linux/export.h>
 16 #include <linux/syscalls.h>
 17 #include <linux/pagemap.h>
 18 #include <linux/splice.h>
 19 #include <linux/compat.h>
 20 #include "internal.h"
 21 
 22 #include <asm/uaccess.h>
 23 #include <asm/unistd.h>
 24 
 25 typedef ssize_t (*io_fn_t)(struct file *, char __user *, size_t, loff_t *);
 26 typedef ssize_t (*iov_fn_t)(struct kiocb *, const struct iovec *,
 27                 unsigned long, loff_t);
 28 typedef ssize_t (*iter_fn_t)(struct kiocb *, struct iov_iter *);
 29 
 30 const struct file_operations generic_ro_fops = {
 31         .llseek         = generic_file_llseek,
 32         .read           = new_sync_read,
 33         .read_iter      = generic_file_read_iter,
 34         .mmap           = generic_file_readonly_mmap,
 35         .splice_read    = generic_file_splice_read,
 36 };
 37 
 38 EXPORT_SYMBOL(generic_ro_fops);
 39 
 40 static inline int unsigned_offsets(struct file *file)
 41 {
 42         return file->f_mode & FMODE_UNSIGNED_OFFSET;
 43 }
 44 
 45 /**
 46  * vfs_setpos - update the file offset for lseek
 47  * @file:       file structure in question
 48  * @offset:     file offset to seek to
 49  * @maxsize:    maximum file size
 50  *
 51  * This is a low-level filesystem helper for updating the file offset to
 52  * the value specified by @offset if the given offset is valid and it is
 53  * not equal to the current file offset.
 54  *
 55  * Return the specified offset on success and -EINVAL on invalid offset.
 56  */
 57 loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize)
 58 {
 59         if (offset < 0 && !unsigned_offsets(file))
 60                 return -EINVAL;
 61         if (offset > maxsize)
 62                 return -EINVAL;
 63 
 64         if (offset != file->f_pos) {
 65                 file->f_pos = offset;
 66                 file->f_version = 0;
 67         }
 68         return offset;
 69 }
 70 EXPORT_SYMBOL(vfs_setpos);
 71 
 72 /**
 73  * generic_file_llseek_size - generic llseek implementation for regular files
 74  * @file:       file structure to seek on
 75  * @offset:     file offset to seek to
 76  * @whence:     type of seek
 77  * @size:       max size of this file in file system
 78  * @eof:        offset used for SEEK_END position
 79  *
 80  * This is a variant of generic_file_llseek that allows passing in a custom
 81  * maximum file size and a custom EOF position, for e.g. hashed directories
 82  *
 83  * Synchronization:
 84  * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
 85  * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
 86  * read/writes behave like SEEK_SET against seeks.
 87  */
 88 loff_t
 89 generic_file_llseek_size(struct file *file, loff_t offset, int whence,
 90                 loff_t maxsize, loff_t eof)
 91 {
 92         switch (whence) {
 93         case SEEK_END:
 94                 offset += eof;
 95                 break;
 96         case SEEK_CUR:
 97                 /*
 98                  * Here we special-case the lseek(fd, 0, SEEK_CUR)
 99                  * position-querying operation.  Avoid rewriting the "same"
100                  * f_pos value back to the file because a concurrent read(),
101                  * write() or lseek() might have altered it
102                  */
103                 if (offset == 0)
104                         return file->f_pos;
105                 /*
106                  * f_lock protects against read/modify/write race with other
107                  * SEEK_CURs. Note that parallel writes and reads behave
108                  * like SEEK_SET.
109                  */
110                 spin_lock(&file->f_lock);
111                 offset = vfs_setpos(file, file->f_pos + offset, maxsize);
112                 spin_unlock(&file->f_lock);
113                 return offset;
114         case SEEK_DATA:
115                 /*
116                  * In the generic case the entire file is data, so as long as
117                  * offset isn't at the end of the file then the offset is data.
118                  */
119                 if (offset >= eof)
120                         return -ENXIO;
121                 break;
122         case SEEK_HOLE:
123                 /*
124                  * There is a virtual hole at the end of the file, so as long as
125                  * offset isn't i_size or larger, return i_size.
126                  */
127                 if (offset >= eof)
128                         return -ENXIO;
129                 offset = eof;
130                 break;
131         }
132 
133         return vfs_setpos(file, offset, maxsize);
134 }
135 EXPORT_SYMBOL(generic_file_llseek_size);
136 
137 /**
138  * generic_file_llseek - generic llseek implementation for regular files
139  * @file:       file structure to seek on
140  * @offset:     file offset to seek to
141  * @whence:     type of seek
142  *
143  * This is a generic implemenation of ->llseek useable for all normal local
144  * filesystems.  It just updates the file offset to the value specified by
145  * @offset and @whence.
146  */
147 loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
148 {
149         struct inode *inode = file->f_mapping->host;
150 
151         return generic_file_llseek_size(file, offset, whence,
152                                         inode->i_sb->s_maxbytes,
153                                         i_size_read(inode));
154 }
155 EXPORT_SYMBOL(generic_file_llseek);
156 
157 /**
158  * fixed_size_llseek - llseek implementation for fixed-sized devices
159  * @file:       file structure to seek on
160  * @offset:     file offset to seek to
161  * @whence:     type of seek
162  * @size:       size of the file
163  *
164  */
165 loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size)
166 {
167         switch (whence) {
168         case SEEK_SET: case SEEK_CUR: case SEEK_END:
169                 return generic_file_llseek_size(file, offset, whence,
170                                                 size, size);
171         default:
172                 return -EINVAL;
173         }
174 }
175 EXPORT_SYMBOL(fixed_size_llseek);
176 
177 /**
178  * noop_llseek - No Operation Performed llseek implementation
179  * @file:       file structure to seek on
180  * @offset:     file offset to seek to
181  * @whence:     type of seek
182  *
183  * This is an implementation of ->llseek useable for the rare special case when
184  * userspace expects the seek to succeed but the (device) file is actually not
185  * able to perform the seek. In this case you use noop_llseek() instead of
186  * falling back to the default implementation of ->llseek.
187  */
188 loff_t noop_llseek(struct file *file, loff_t offset, int whence)
189 {
190         return file->f_pos;
191 }
192 EXPORT_SYMBOL(noop_llseek);
193 
194 loff_t no_llseek(struct file *file, loff_t offset, int whence)
195 {
196         return -ESPIPE;
197 }
198 EXPORT_SYMBOL(no_llseek);
199 
200 loff_t default_llseek(struct file *file, loff_t offset, int whence)
201 {
202         struct inode *inode = file_inode(file);
203         loff_t retval;
204 
205         mutex_lock(&inode->i_mutex);
206         switch (whence) {
207                 case SEEK_END:
208                         offset += i_size_read(inode);
209                         break;
210                 case SEEK_CUR:
211                         if (offset == 0) {
212                                 retval = file->f_pos;
213                                 goto out;
214                         }
215                         offset += file->f_pos;
216                         break;
217                 case SEEK_DATA:
218                         /*
219                          * In the generic case the entire file is data, so as
220                          * long as offset isn't at the end of the file then the
221                          * offset is data.
222                          */
223                         if (offset >= inode->i_size) {
224                                 retval = -ENXIO;
225                                 goto out;
226                         }
227                         break;
228                 case SEEK_HOLE:
229                         /*
230                          * There is a virtual hole at the end of the file, so
231                          * as long as offset isn't i_size or larger, return
232                          * i_size.
233                          */
234                         if (offset >= inode->i_size) {
235                                 retval = -ENXIO;
236                                 goto out;
237                         }
238                         offset = inode->i_size;
239                         break;
240         }
241         retval = -EINVAL;
242         if (offset >= 0 || unsigned_offsets(file)) {
243                 if (offset != file->f_pos) {
244                         file->f_pos = offset;
245                         file->f_version = 0;
246                 }
247                 retval = offset;
248         }
249 out:
250         mutex_unlock(&inode->i_mutex);
251         return retval;
252 }
253 EXPORT_SYMBOL(default_llseek);
254 
255 loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
256 {
257         loff_t (*fn)(struct file *, loff_t, int);
258 
259         fn = no_llseek;
260         if (file->f_mode & FMODE_LSEEK) {
261                 if (file->f_op->llseek)
262                         fn = file->f_op->llseek;
263         }
264         return fn(file, offset, whence);
265 }
266 EXPORT_SYMBOL(vfs_llseek);
267 
268 static inline struct fd fdget_pos(int fd)
269 {
270         return __to_fd(__fdget_pos(fd));
271 }
272 
273 static inline void fdput_pos(struct fd f)
274 {
275         if (f.flags & FDPUT_POS_UNLOCK)
276                 mutex_unlock(&f.file->f_pos_lock);
277         fdput(f);
278 }
279 
280 SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
281 {
282         off_t retval;
283         struct fd f = fdget_pos(fd);
284         if (!f.file)
285                 return -EBADF;
286 
287         retval = -EINVAL;
288         if (whence <= SEEK_MAX) {
289                 loff_t res = vfs_llseek(f.file, offset, whence);
290                 retval = res;
291                 if (res != (loff_t)retval)
292                         retval = -EOVERFLOW;    /* LFS: should only happen on 32 bit platforms */
293         }
294         fdput_pos(f);
295         return retval;
296 }
297 
298 #ifdef CONFIG_COMPAT
299 COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence)
300 {
301         return sys_lseek(fd, offset, whence);
302 }
303 #endif
304 
305 #ifdef __ARCH_WANT_SYS_LLSEEK
306 SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
307                 unsigned long, offset_low, loff_t __user *, result,
308                 unsigned int, whence)
309 {
310         int retval;
311         struct fd f = fdget_pos(fd);
312         loff_t offset;
313 
314         if (!f.file)
315                 return -EBADF;
316 
317         retval = -EINVAL;
318         if (whence > SEEK_MAX)
319                 goto out_putf;
320 
321         offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low,
322                         whence);
323 
324         retval = (int)offset;
325         if (offset >= 0) {
326                 retval = -EFAULT;
327                 if (!copy_to_user(result, &offset, sizeof(offset)))
328                         retval = 0;
329         }
330 out_putf:
331         fdput_pos(f);
332         return retval;
333 }
334 #endif
335 
336 /*
337  * rw_verify_area doesn't like huge counts. We limit
338  * them to something that fits in "int" so that others
339  * won't have to do range checks all the time.
340  */
341 int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count)
342 {
343         struct inode *inode;
344         loff_t pos;
345         int retval = -EINVAL;
346 
347         inode = file_inode(file);
348         if (unlikely((ssize_t) count < 0))
349                 return retval;
350         pos = *ppos;
351         if (unlikely(pos < 0)) {
352                 if (!unsigned_offsets(file))
353                         return retval;
354                 if (count >= -pos) /* both values are in 0..LLONG_MAX */
355                         return -EOVERFLOW;
356         } else if (unlikely((loff_t) (pos + count) < 0)) {
357                 if (!unsigned_offsets(file))
358                         return retval;
359         }
360 
361         if (unlikely(inode->i_flock && mandatory_lock(inode))) {
362                 retval = locks_mandatory_area(
363                         read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
364                         inode, file, pos, count);
365                 if (retval < 0)
366                         return retval;
367         }
368         retval = security_file_permission(file,
369                                 read_write == READ ? MAY_READ : MAY_WRITE);
370         if (retval)
371                 return retval;
372         return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
373 }
374 
375 ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
376 {
377         struct iovec iov = { .iov_base = buf, .iov_len = len };
378         struct kiocb kiocb;
379         ssize_t ret;
380 
381         init_sync_kiocb(&kiocb, filp);
382         kiocb.ki_pos = *ppos;
383         kiocb.ki_nbytes = len;
384 
385         ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos);
386         if (-EIOCBQUEUED == ret)
387                 ret = wait_on_sync_kiocb(&kiocb);
388         *ppos = kiocb.ki_pos;
389         return ret;
390 }
391 
392 EXPORT_SYMBOL(do_sync_read);
393 
394 ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
395 {
396         struct iovec iov = { .iov_base = buf, .iov_len = len };
397         struct kiocb kiocb;
398         struct iov_iter iter;
399         ssize_t ret;
400 
401         init_sync_kiocb(&kiocb, filp);
402         kiocb.ki_pos = *ppos;
403         kiocb.ki_nbytes = len;
404         iov_iter_init(&iter, READ, &iov, 1, len);
405 
406         ret = filp->f_op->read_iter(&kiocb, &iter);
407         if (-EIOCBQUEUED == ret)
408                 ret = wait_on_sync_kiocb(&kiocb);
409         *ppos = kiocb.ki_pos;
410         return ret;
411 }
412 
413 EXPORT_SYMBOL(new_sync_read);
414 
415 ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
416 {
417         ssize_t ret;
418 
419         if (!(file->f_mode & FMODE_READ))
420                 return -EBADF;
421         if (!(file->f_mode & FMODE_CAN_READ))
422                 return -EINVAL;
423         if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
424                 return -EFAULT;
425 
426         ret = rw_verify_area(READ, file, pos, count);
427         if (ret >= 0) {
428                 count = ret;
429                 if (file->f_op->read)
430                         ret = file->f_op->read(file, buf, count, pos);
431                 else if (file->f_op->aio_read)
432                         ret = do_sync_read(file, buf, count, pos);
433                 else
434                         ret = new_sync_read(file, buf, count, pos);
435                 if (ret > 0) {
436                         fsnotify_access(file);
437                         add_rchar(current, ret);
438                 }
439                 inc_syscr(current);
440         }
441 
442         return ret;
443 }
444 
445 EXPORT_SYMBOL(vfs_read);
446 
447 ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
448 {
449         struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
450         struct kiocb kiocb;
451         ssize_t ret;
452 
453         init_sync_kiocb(&kiocb, filp);
454         kiocb.ki_pos = *ppos;
455         kiocb.ki_nbytes = len;
456 
457         ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos);
458         if (-EIOCBQUEUED == ret)
459                 ret = wait_on_sync_kiocb(&kiocb);
460         *ppos = kiocb.ki_pos;
461         return ret;
462 }
463 
464 EXPORT_SYMBOL(do_sync_write);
465 
466 ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
467 {
468         struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
469         struct kiocb kiocb;
470         struct iov_iter iter;
471         ssize_t ret;
472 
473         init_sync_kiocb(&kiocb, filp);
474         kiocb.ki_pos = *ppos;
475         kiocb.ki_nbytes = len;
476         iov_iter_init(&iter, WRITE, &iov, 1, len);
477 
478         ret = filp->f_op->write_iter(&kiocb, &iter);
479         if (-EIOCBQUEUED == ret)
480                 ret = wait_on_sync_kiocb(&kiocb);
481         *ppos = kiocb.ki_pos;
482         return ret;
483 }
484 
485 EXPORT_SYMBOL(new_sync_write);
486 
487 ssize_t __kernel_write(struct file *file, const char *buf, size_t count, loff_t *pos)
488 {
489         mm_segment_t old_fs;
490         const char __user *p;
491         ssize_t ret;
492 
493         if (!(file->f_mode & FMODE_CAN_WRITE))
494                 return -EINVAL;
495 
496         old_fs = get_fs();
497         set_fs(get_ds());
498         p = (__force const char __user *)buf;
499         if (count > MAX_RW_COUNT)
500                 count =  MAX_RW_COUNT;
501         if (file->f_op->write)
502                 ret = file->f_op->write(file, p, count, pos);
503         else if (file->f_op->aio_write)
504                 ret = do_sync_write(file, p, count, pos);
505         else
506                 ret = new_sync_write(file, p, count, pos);
507         set_fs(old_fs);
508         if (ret > 0) {
509                 fsnotify_modify(file);
510                 add_wchar(current, ret);
511         }
512         inc_syscw(current);
513         return ret;
514 }
515 
516 EXPORT_SYMBOL(__kernel_write);
517 
518 ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
519 {
520         ssize_t ret;
521 
522         if (!(file->f_mode & FMODE_WRITE))
523                 return -EBADF;
524         if (!(file->f_mode & FMODE_CAN_WRITE))
525                 return -EINVAL;
526         if (unlikely(!access_ok(VERIFY_READ, buf, count)))
527                 return -EFAULT;
528 
529         ret = rw_verify_area(WRITE, file, pos, count);
530         if (ret >= 0) {
531                 count = ret;
532                 file_start_write(file);
533                 if (file->f_op->write)
534                         ret = file->f_op->write(file, buf, count, pos);
535                 else if (file->f_op->aio_write)
536                         ret = do_sync_write(file, buf, count, pos);
537                 else
538                         ret = new_sync_write(file, buf, count, pos);
539                 if (ret > 0) {
540                         fsnotify_modify(file);
541                         add_wchar(current, ret);
542                 }
543                 inc_syscw(current);
544                 file_end_write(file);
545         }
546 
547         return ret;
548 }
549 
550 EXPORT_SYMBOL(vfs_write);
551 
552 static inline loff_t file_pos_read(struct file *file)
553 {
554         return file->f_pos;
555 }
556 
557 static inline void file_pos_write(struct file *file, loff_t pos)
558 {
559         file->f_pos = pos;
560 }
561 
562 SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
563 {
564         struct fd f = fdget_pos(fd);
565         ssize_t ret = -EBADF;
566 
567         if (f.file) {
568                 loff_t pos = file_pos_read(f.file);
569                 ret = vfs_read(f.file, buf, count, &pos);
570                 if (ret >= 0)
571                         file_pos_write(f.file, pos);
572                 fdput_pos(f);
573         }
574         return ret;
575 }
576 
577 SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
578                 size_t, count)
579 {
580         struct fd f = fdget_pos(fd);
581         ssize_t ret = -EBADF;
582 
583         if (f.file) {
584                 loff_t pos = file_pos_read(f.file);
585                 ret = vfs_write(f.file, buf, count, &pos);
586                 if (ret >= 0)
587                         file_pos_write(f.file, pos);
588                 fdput_pos(f);
589         }
590 
591         return ret;
592 }
593 
594 SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf,
595                         size_t, count, loff_t, pos)
596 {
597         struct fd f;
598         ssize_t ret = -EBADF;
599 
600         if (pos < 0)
601                 return -EINVAL;
602 
603         f = fdget(fd);
604         if (f.file) {
605                 ret = -ESPIPE;
606                 if (f.file->f_mode & FMODE_PREAD)
607                         ret = vfs_read(f.file, buf, count, &pos);
608                 fdput(f);
609         }
610 
611         return ret;
612 }
613 
614 SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf,
615                          size_t, count, loff_t, pos)
616 {
617         struct fd f;
618         ssize_t ret = -EBADF;
619 
620         if (pos < 0)
621                 return -EINVAL;
622 
623         f = fdget(fd);
624         if (f.file) {
625                 ret = -ESPIPE;
626                 if (f.file->f_mode & FMODE_PWRITE)  
627                         ret = vfs_write(f.file, buf, count, &pos);
628                 fdput(f);
629         }
630 
631         return ret;
632 }
633 
634 /*
635  * Reduce an iovec's length in-place.  Return the resulting number of segments
636  */
637 unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
638 {
639         unsigned long seg = 0;
640         size_t len = 0;
641 
642         while (seg < nr_segs) {
643                 seg++;
644                 if (len + iov->iov_len >= to) {
645                         iov->iov_len = to - len;
646                         break;
647                 }
648                 len += iov->iov_len;
649                 iov++;
650         }
651         return seg;
652 }
653 EXPORT_SYMBOL(iov_shorten);
654 
655 static ssize_t do_iter_readv_writev(struct file *filp, int rw, const struct iovec *iov,
656                 unsigned long nr_segs, size_t len, loff_t *ppos, iter_fn_t fn)
657 {
658         struct kiocb kiocb;
659         struct iov_iter iter;
660         ssize_t ret;
661 
662         init_sync_kiocb(&kiocb, filp);
663         kiocb.ki_pos = *ppos;
664         kiocb.ki_nbytes = len;
665 
666         iov_iter_init(&iter, rw, iov, nr_segs, len);
667         ret = fn(&kiocb, &iter);
668         if (ret == -EIOCBQUEUED)
669                 ret = wait_on_sync_kiocb(&kiocb);
670         *ppos = kiocb.ki_pos;
671         return ret;
672 }
673 
674 static ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
675                 unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
676 {
677         struct kiocb kiocb;
678         ssize_t ret;
679 
680         init_sync_kiocb(&kiocb, filp);
681         kiocb.ki_pos = *ppos;
682         kiocb.ki_nbytes = len;
683 
684         ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos);
685         if (ret == -EIOCBQUEUED)
686                 ret = wait_on_sync_kiocb(&kiocb);
687         *ppos = kiocb.ki_pos;
688         return ret;
689 }
690 
691 /* Do it by hand, with file-ops */
692 static ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
693                 unsigned long nr_segs, loff_t *ppos, io_fn_t fn)
694 {
695         struct iovec *vector = iov;
696         ssize_t ret = 0;
697 
698         while (nr_segs > 0) {
699                 void __user *base;
700                 size_t len;
701                 ssize_t nr;
702 
703                 base = vector->iov_base;
704                 len = vector->iov_len;
705                 vector++;
706                 nr_segs--;
707 
708                 nr = fn(filp, base, len, ppos);
709 
710                 if (nr < 0) {
711                         if (!ret)
712                                 ret = nr;
713                         break;
714                 }
715                 ret += nr;
716                 if (nr != len)
717                         break;
718         }
719 
720         return ret;
721 }
722 
723 /* A write operation does a read from user space and vice versa */
724 #define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
725 
726 ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
727                               unsigned long nr_segs, unsigned long fast_segs,
728                               struct iovec *fast_pointer,
729                               struct iovec **ret_pointer)
730 {
731         unsigned long seg;
732         ssize_t ret;
733         struct iovec *iov = fast_pointer;
734 
735         /*
736          * SuS says "The readv() function *may* fail if the iovcnt argument
737          * was less than or equal to 0, or greater than {IOV_MAX}.  Linux has
738          * traditionally returned zero for zero segments, so...
739          */
740         if (nr_segs == 0) {
741                 ret = 0;
742                 goto out;
743         }
744 
745         /*
746          * First get the "struct iovec" from user memory and
747          * verify all the pointers
748          */
749         if (nr_segs > UIO_MAXIOV) {
750                 ret = -EINVAL;
751                 goto out;
752         }
753         if (nr_segs > fast_segs) {
754                 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
755                 if (iov == NULL) {
756                         ret = -ENOMEM;
757                         goto out;
758                 }
759         }
760         if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
761                 ret = -EFAULT;
762                 goto out;
763         }
764 
765         /*
766          * According to the Single Unix Specification we should return EINVAL
767          * if an element length is < 0 when cast to ssize_t or if the
768          * total length would overflow the ssize_t return value of the
769          * system call.
770          *
771          * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the
772          * overflow case.
773          */
774         ret = 0;
775         for (seg = 0; seg < nr_segs; seg++) {
776                 void __user *buf = iov[seg].iov_base;
777                 ssize_t len = (ssize_t)iov[seg].iov_len;
778 
779                 /* see if we we're about to use an invalid len or if
780                  * it's about to overflow ssize_t */
781                 if (len < 0) {
782                         ret = -EINVAL;
783                         goto out;
784                 }
785                 if (type >= 0
786                     && unlikely(!access_ok(vrfy_dir(type), buf, len))) {
787                         ret = -EFAULT;
788                         goto out;
789                 }
790                 if (len > MAX_RW_COUNT - ret) {
791                         len = MAX_RW_COUNT - ret;
792                         iov[seg].iov_len = len;
793                 }
794                 ret += len;
795         }
796 out:
797         *ret_pointer = iov;
798         return ret;
799 }
800 
801 static ssize_t do_readv_writev(int type, struct file *file,
802                                const struct iovec __user * uvector,
803                                unsigned long nr_segs, loff_t *pos)
804 {
805         size_t tot_len;
806         struct iovec iovstack[UIO_FASTIOV];
807         struct iovec *iov = iovstack;
808         ssize_t ret;
809         io_fn_t fn;
810         iov_fn_t fnv;
811         iter_fn_t iter_fn;
812 
813         ret = rw_copy_check_uvector(type, uvector, nr_segs,
814                                     ARRAY_SIZE(iovstack), iovstack, &iov);
815         if (ret <= 0)
816                 goto out;
817 
818         tot_len = ret;
819         ret = rw_verify_area(type, file, pos, tot_len);
820         if (ret < 0)
821                 goto out;
822 
823         fnv = NULL;
824         if (type == READ) {
825                 fn = file->f_op->read;
826                 fnv = file->f_op->aio_read;
827                 iter_fn = file->f_op->read_iter;
828         } else {
829                 fn = (io_fn_t)file->f_op->write;
830                 fnv = file->f_op->aio_write;
831                 iter_fn = file->f_op->write_iter;
832                 file_start_write(file);
833         }
834 
835         if (iter_fn)
836                 ret = do_iter_readv_writev(file, type, iov, nr_segs, tot_len,
837                                                 pos, iter_fn);
838         else if (fnv)
839                 ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
840                                                 pos, fnv);
841         else
842                 ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
843 
844         if (type != READ)
845                 file_end_write(file);
846 
847 out:
848         if (iov != iovstack)
849                 kfree(iov);
850         if ((ret + (type == READ)) > 0) {
851                 if (type == READ)
852                         fsnotify_access(file);
853                 else
854                         fsnotify_modify(file);
855         }
856         return ret;
857 }
858 
859 ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
860                   unsigned long vlen, loff_t *pos)
861 {
862         if (!(file->f_mode & FMODE_READ))
863                 return -EBADF;
864         if (!(file->f_mode & FMODE_CAN_READ))
865                 return -EINVAL;
866 
867         return do_readv_writev(READ, file, vec, vlen, pos);
868 }
869 
870 EXPORT_SYMBOL(vfs_readv);
871 
872 ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
873                    unsigned long vlen, loff_t *pos)
874 {
875         if (!(file->f_mode & FMODE_WRITE))
876                 return -EBADF;
877         if (!(file->f_mode & FMODE_CAN_WRITE))
878                 return -EINVAL;
879 
880         return do_readv_writev(WRITE, file, vec, vlen, pos);
881 }
882 
883 EXPORT_SYMBOL(vfs_writev);
884 
885 SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
886                 unsigned long, vlen)
887 {
888         struct fd f = fdget_pos(fd);
889         ssize_t ret = -EBADF;
890 
891         if (f.file) {
892                 loff_t pos = file_pos_read(f.file);
893                 ret = vfs_readv(f.file, vec, vlen, &pos);
894                 if (ret >= 0)
895                         file_pos_write(f.file, pos);
896                 fdput_pos(f);
897         }
898 
899         if (ret > 0)
900                 add_rchar(current, ret);
901         inc_syscr(current);
902         return ret;
903 }
904 
905 SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
906                 unsigned long, vlen)
907 {
908         struct fd f = fdget_pos(fd);
909         ssize_t ret = -EBADF;
910 
911         if (f.file) {
912                 loff_t pos = file_pos_read(f.file);
913                 ret = vfs_writev(f.file, vec, vlen, &pos);
914                 if (ret >= 0)
915                         file_pos_write(f.file, pos);
916                 fdput_pos(f);
917         }
918 
919         if (ret > 0)
920                 add_wchar(current, ret);
921         inc_syscw(current);
922         return ret;
923 }
924 
925 static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
926 {
927 #define HALF_LONG_BITS (BITS_PER_LONG / 2)
928         return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
929 }
930 
931 SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
932                 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
933 {
934         loff_t pos = pos_from_hilo(pos_h, pos_l);
935         struct fd f;
936         ssize_t ret = -EBADF;
937 
938         if (pos < 0)
939                 return -EINVAL;
940 
941         f = fdget(fd);
942         if (f.file) {
943                 ret = -ESPIPE;
944                 if (f.file->f_mode & FMODE_PREAD)
945                         ret = vfs_readv(f.file, vec, vlen, &pos);
946                 fdput(f);
947         }
948 
949         if (ret > 0)
950                 add_rchar(current, ret);
951         inc_syscr(current);
952         return ret;
953 }
954 
955 SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
956                 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
957 {
958         loff_t pos = pos_from_hilo(pos_h, pos_l);
959         struct fd f;
960         ssize_t ret = -EBADF;
961 
962         if (pos < 0)
963                 return -EINVAL;
964 
965         f = fdget(fd);
966         if (f.file) {
967                 ret = -ESPIPE;
968                 if (f.file->f_mode & FMODE_PWRITE)
969                         ret = vfs_writev(f.file, vec, vlen, &pos);
970                 fdput(f);
971         }
972 
973         if (ret > 0)
974                 add_wchar(current, ret);
975         inc_syscw(current);
976         return ret;
977 }
978 
979 #ifdef CONFIG_COMPAT
980 
981 static ssize_t compat_do_readv_writev(int type, struct file *file,
982                                const struct compat_iovec __user *uvector,
983                                unsigned long nr_segs, loff_t *pos)
984 {
985         compat_ssize_t tot_len;
986         struct iovec iovstack[UIO_FASTIOV];
987         struct iovec *iov = iovstack;
988         ssize_t ret;
989         io_fn_t fn;
990         iov_fn_t fnv;
991         iter_fn_t iter_fn;
992 
993         ret = compat_rw_copy_check_uvector(type, uvector, nr_segs,
994                                                UIO_FASTIOV, iovstack, &iov);
995         if (ret <= 0)
996                 goto out;
997 
998         tot_len = ret;
999         ret = rw_verify_area(type, file, pos, tot_len);
1000         if (ret < 0)
1001                 goto out;
1002 
1003         fnv = NULL;
1004         if (type == READ) {
1005                 fn = file->f_op->read;
1006                 fnv = file->f_op->aio_read;
1007                 iter_fn = file->f_op->read_iter;
1008         } else {
1009                 fn = (io_fn_t)file->f_op->write;
1010                 fnv = file->f_op->aio_write;
1011                 iter_fn = file->f_op->write_iter;
1012                 file_start_write(file);
1013         }
1014 
1015         if (iter_fn)
1016                 ret = do_iter_readv_writev(file, type, iov, nr_segs, tot_len,
1017                                                 pos, iter_fn);
1018         else if (fnv)
1019                 ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
1020                                                 pos, fnv);
1021         else
1022                 ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
1023 
1024         if (type != READ)
1025                 file_end_write(file);
1026 
1027 out:
1028         if (iov != iovstack)
1029                 kfree(iov);
1030         if ((ret + (type == READ)) > 0) {
1031                 if (type == READ)
1032                         fsnotify_access(file);
1033                 else
1034                         fsnotify_modify(file);
1035         }
1036         return ret;
1037 }
1038 
1039 static size_t compat_readv(struct file *file,
1040                            const struct compat_iovec __user *vec,
1041                            unsigned long vlen, loff_t *pos)
1042 {
1043         ssize_t ret = -EBADF;
1044 
1045         if (!(file->f_mode & FMODE_READ))
1046                 goto out;
1047 
1048         ret = -EINVAL;
1049         if (!(file->f_mode & FMODE_CAN_READ))
1050                 goto out;
1051 
1052         ret = compat_do_readv_writev(READ, file, vec, vlen, pos);
1053 
1054 out:
1055         if (ret > 0)
1056                 add_rchar(current, ret);
1057         inc_syscr(current);
1058         return ret;
1059 }
1060 
1061 COMPAT_SYSCALL_DEFINE3(readv, compat_ulong_t, fd,
1062                 const struct compat_iovec __user *,vec,
1063                 compat_ulong_t, vlen)
1064 {
1065         struct fd f = fdget_pos(fd);
1066         ssize_t ret;
1067         loff_t pos;
1068 
1069         if (!f.file)
1070                 return -EBADF;
1071         pos = f.file->f_pos;
1072         ret = compat_readv(f.file, vec, vlen, &pos);
1073         if (ret >= 0)
1074                 f.file->f_pos = pos;
1075         fdput_pos(f);
1076         return ret;
1077 }
1078 
1079 static long __compat_sys_preadv64(unsigned long fd,
1080                                   const struct compat_iovec __user *vec,
1081                                   unsigned long vlen, loff_t pos)
1082 {
1083         struct fd f;
1084         ssize_t ret;
1085 
1086         if (pos < 0)
1087                 return -EINVAL;
1088         f = fdget(fd);
1089         if (!f.file)
1090                 return -EBADF;
1091         ret = -ESPIPE;
1092         if (f.file->f_mode & FMODE_PREAD)
1093                 ret = compat_readv(f.file, vec, vlen, &pos);
1094         fdput(f);
1095         return ret;
1096 }
1097 
1098 #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64
1099 COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
1100                 const struct compat_iovec __user *,vec,
1101                 unsigned long, vlen, loff_t, pos)
1102 {
1103         return __compat_sys_preadv64(fd, vec, vlen, pos);
1104 }
1105 #endif
1106 
1107 COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd,
1108                 const struct compat_iovec __user *,vec,
1109                 compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1110 {
1111         loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1112 
1113         return __compat_sys_preadv64(fd, vec, vlen, pos);
1114 }
1115 
1116 static size_t compat_writev(struct file *file,
1117                             const struct compat_iovec __user *vec,
1118                             unsigned long vlen, loff_t *pos)
1119 {
1120         ssize_t ret = -EBADF;
1121 
1122         if (!(file->f_mode & FMODE_WRITE))
1123                 goto out;
1124 
1125         ret = -EINVAL;
1126         if (!(file->f_mode & FMODE_CAN_WRITE))
1127                 goto out;
1128 
1129         ret = compat_do_readv_writev(WRITE, file, vec, vlen, pos);
1130 
1131 out:
1132         if (ret > 0)
1133                 add_wchar(current, ret);
1134         inc_syscw(current);
1135         return ret;
1136 }
1137 
1138 COMPAT_SYSCALL_DEFINE3(writev, compat_ulong_t, fd,
1139                 const struct compat_iovec __user *, vec,
1140                 compat_ulong_t, vlen)
1141 {
1142         struct fd f = fdget_pos(fd);
1143         ssize_t ret;
1144         loff_t pos;
1145 
1146         if (!f.file)
1147                 return -EBADF;
1148         pos = f.file->f_pos;
1149         ret = compat_writev(f.file, vec, vlen, &pos);
1150         if (ret >= 0)
1151                 f.file->f_pos = pos;
1152         fdput_pos(f);
1153         return ret;
1154 }
1155 
1156 static long __compat_sys_pwritev64(unsigned long fd,
1157                                    const struct compat_iovec __user *vec,
1158                                    unsigned long vlen, loff_t pos)
1159 {
1160         struct fd f;
1161         ssize_t ret;
1162 
1163         if (pos < 0)
1164                 return -EINVAL;
1165         f = fdget(fd);
1166         if (!f.file)
1167                 return -EBADF;
1168         ret = -ESPIPE;
1169         if (f.file->f_mode & FMODE_PWRITE)
1170                 ret = compat_writev(f.file, vec, vlen, &pos);
1171         fdput(f);
1172         return ret;
1173 }
1174 
1175 #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64
1176 COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
1177                 const struct compat_iovec __user *,vec,
1178                 unsigned long, vlen, loff_t, pos)
1179 {
1180         return __compat_sys_pwritev64(fd, vec, vlen, pos);
1181 }
1182 #endif
1183 
1184 COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd,
1185                 const struct compat_iovec __user *,vec,
1186                 compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
1187 {
1188         loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1189 
1190         return __compat_sys_pwritev64(fd, vec, vlen, pos);
1191 }
1192 #endif
1193 
1194 static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
1195                            size_t count, loff_t max)
1196 {
1197         struct fd in, out;
1198         struct inode *in_inode, *out_inode;
1199         loff_t pos;
1200         loff_t out_pos;
1201         ssize_t retval;
1202         int fl;
1203 
1204         /*
1205          * Get input file, and verify that it is ok..
1206          */
1207         retval = -EBADF;
1208         in = fdget(in_fd);
1209         if (!in.file)
1210                 goto out;
1211         if (!(in.file->f_mode & FMODE_READ))
1212                 goto fput_in;
1213         retval = -ESPIPE;
1214         if (!ppos) {
1215                 pos = in.file->f_pos;
1216         } else {
1217                 pos = *ppos;
1218                 if (!(in.file->f_mode & FMODE_PREAD))
1219                         goto fput_in;
1220         }
1221         retval = rw_verify_area(READ, in.file, &pos, count);
1222         if (retval < 0)
1223                 goto fput_in;
1224         count = retval;
1225 
1226         /*
1227          * Get output file, and verify that it is ok..
1228          */
1229         retval = -EBADF;
1230         out = fdget(out_fd);
1231         if (!out.file)
1232                 goto fput_in;
1233         if (!(out.file->f_mode & FMODE_WRITE))
1234                 goto fput_out;
1235         retval = -EINVAL;
1236         in_inode = file_inode(in.file);
1237         out_inode = file_inode(out.file);
1238         out_pos = out.file->f_pos;
1239         retval = rw_verify_area(WRITE, out.file, &out_pos, count);
1240         if (retval < 0)
1241                 goto fput_out;
1242         count = retval;
1243 
1244         if (!max)
1245                 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
1246 
1247         if (unlikely(pos + count > max)) {
1248                 retval = -EOVERFLOW;
1249                 if (pos >= max)
1250                         goto fput_out;
1251                 count = max - pos;
1252         }
1253 
1254         fl = 0;
1255 #if 0
1256         /*
1257          * We need to debate whether we can enable this or not. The
1258          * man page documents EAGAIN return for the output at least,
1259          * and the application is arguably buggy if it doesn't expect
1260          * EAGAIN on a non-blocking file descriptor.
1261          */
1262         if (in.file->f_flags & O_NONBLOCK)
1263                 fl = SPLICE_F_NONBLOCK;
1264 #endif
1265         file_start_write(out.file);
1266         retval = do_splice_direct(in.file, &pos, out.file, &out_pos, count, fl);
1267         file_end_write(out.file);
1268 
1269         if (retval > 0) {
1270                 add_rchar(current, retval);
1271                 add_wchar(current, retval);
1272                 fsnotify_access(in.file);
1273                 fsnotify_modify(out.file);
1274                 out.file->f_pos = out_pos;
1275                 if (ppos)
1276                         *ppos = pos;
1277                 else
1278                         in.file->f_pos = pos;
1279         }
1280 
1281         inc_syscr(current);
1282         inc_syscw(current);
1283         if (pos > max)
1284                 retval = -EOVERFLOW;
1285 
1286 fput_out:
1287         fdput(out);
1288 fput_in:
1289         fdput(in);
1290 out:
1291         return retval;
1292 }
1293 
1294 SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
1295 {
1296         loff_t pos;
1297         off_t off;
1298         ssize_t ret;
1299 
1300         if (offset) {
1301                 if (unlikely(get_user(off, offset)))
1302                         return -EFAULT;
1303                 pos = off;
1304                 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1305                 if (unlikely(put_user(pos, offset)))
1306                         return -EFAULT;
1307                 return ret;
1308         }
1309 
1310         return do_sendfile(out_fd, in_fd, NULL, count, 0);
1311 }
1312 
1313 SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
1314 {
1315         loff_t pos;
1316         ssize_t ret;
1317 
1318         if (offset) {
1319                 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1320                         return -EFAULT;
1321                 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1322                 if (unlikely(put_user(pos, offset)))
1323                         return -EFAULT;
1324                 return ret;
1325         }
1326 
1327         return do_sendfile(out_fd, in_fd, NULL, count, 0);
1328 }
1329 
1330 #ifdef CONFIG_COMPAT
1331 COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd,
1332                 compat_off_t __user *, offset, compat_size_t, count)
1333 {
1334         loff_t pos;
1335         off_t off;
1336         ssize_t ret;
1337 
1338         if (offset) {
1339                 if (unlikely(get_user(off, offset)))
1340                         return -EFAULT;
1341                 pos = off;
1342                 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
1343                 if (unlikely(put_user(pos, offset)))
1344                         return -EFAULT;
1345                 return ret;
1346         }
1347 
1348         return do_sendfile(out_fd, in_fd, NULL, count, 0);
1349 }
1350 
1351 COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd,
1352                 compat_loff_t __user *, offset, compat_size_t, count)
1353 {
1354         loff_t pos;
1355         ssize_t ret;
1356 
1357         if (offset) {
1358                 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
1359                         return -EFAULT;
1360                 ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
1361                 if (unlikely(put_user(pos, offset)))
1362                         return -EFAULT;
1363                 return ret;
1364         }
1365 
1366         return do_sendfile(out_fd, in_fd, NULL, count, 0);
1367 }
1368 #endif
1369 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us