Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/mm/page_counter.c

  1 /*
  2  * Lockless hierarchical page accounting & limiting
  3  *
  4  * Copyright (C) 2014 Red Hat, Inc., Johannes Weiner
  5  */
  6 
  7 #include <linux/page_counter.h>
  8 #include <linux/atomic.h>
  9 #include <linux/kernel.h>
 10 #include <linux/string.h>
 11 #include <linux/sched.h>
 12 #include <linux/bug.h>
 13 #include <asm/page.h>
 14 
 15 /**
 16  * page_counter_cancel - take pages out of the local counter
 17  * @counter: counter
 18  * @nr_pages: number of pages to cancel
 19  */
 20 void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages)
 21 {
 22         long new;
 23 
 24         new = atomic_long_sub_return(nr_pages, &counter->count);
 25         /* More uncharges than charges? */
 26         WARN_ON_ONCE(new < 0);
 27 }
 28 
 29 /**
 30  * page_counter_charge - hierarchically charge pages
 31  * @counter: counter
 32  * @nr_pages: number of pages to charge
 33  *
 34  * NOTE: This does not consider any configured counter limits.
 35  */
 36 void page_counter_charge(struct page_counter *counter, unsigned long nr_pages)
 37 {
 38         struct page_counter *c;
 39 
 40         for (c = counter; c; c = c->parent) {
 41                 long new;
 42 
 43                 new = atomic_long_add_return(nr_pages, &c->count);
 44                 /*
 45                  * This is indeed racy, but we can live with some
 46                  * inaccuracy in the watermark.
 47                  */
 48                 if (new > c->watermark)
 49                         c->watermark = new;
 50         }
 51 }
 52 
 53 /**
 54  * page_counter_try_charge - try to hierarchically charge pages
 55  * @counter: counter
 56  * @nr_pages: number of pages to charge
 57  * @fail: points first counter to hit its limit, if any
 58  *
 59  * Returns %true on success, or %false and @fail if the counter or one
 60  * of its ancestors has hit its configured limit.
 61  */
 62 bool page_counter_try_charge(struct page_counter *counter,
 63                              unsigned long nr_pages,
 64                              struct page_counter **fail)
 65 {
 66         struct page_counter *c;
 67 
 68         for (c = counter; c; c = c->parent) {
 69                 long new;
 70                 /*
 71                  * Charge speculatively to avoid an expensive CAS.  If
 72                  * a bigger charge fails, it might falsely lock out a
 73                  * racing smaller charge and send it into reclaim
 74                  * early, but the error is limited to the difference
 75                  * between the two sizes, which is less than 2M/4M in
 76                  * case of a THP locking out a regular page charge.
 77                  *
 78                  * The atomic_long_add_return() implies a full memory
 79                  * barrier between incrementing the count and reading
 80                  * the limit.  When racing with page_counter_limit(),
 81                  * we either see the new limit or the setter sees the
 82                  * counter has changed and retries.
 83                  */
 84                 new = atomic_long_add_return(nr_pages, &c->count);
 85                 if (new > c->limit) {
 86                         atomic_long_sub(nr_pages, &c->count);
 87                         /*
 88                          * This is racy, but we can live with some
 89                          * inaccuracy in the failcnt.
 90                          */
 91                         c->failcnt++;
 92                         *fail = c;
 93                         goto failed;
 94                 }
 95                 /*
 96                  * Just like with failcnt, we can live with some
 97                  * inaccuracy in the watermark.
 98                  */
 99                 if (new > c->watermark)
100                         c->watermark = new;
101         }
102         return true;
103 
104 failed:
105         for (c = counter; c != *fail; c = c->parent)
106                 page_counter_cancel(c, nr_pages);
107 
108         return false;
109 }
110 
111 /**
112  * page_counter_uncharge - hierarchically uncharge pages
113  * @counter: counter
114  * @nr_pages: number of pages to uncharge
115  */
116 void page_counter_uncharge(struct page_counter *counter, unsigned long nr_pages)
117 {
118         struct page_counter *c;
119 
120         for (c = counter; c; c = c->parent)
121                 page_counter_cancel(c, nr_pages);
122 }
123 
124 /**
125  * page_counter_limit - limit the number of pages allowed
126  * @counter: counter
127  * @limit: limit to set
128  *
129  * Returns 0 on success, -EBUSY if the current number of pages on the
130  * counter already exceeds the specified limit.
131  *
132  * The caller must serialize invocations on the same counter.
133  */
134 int page_counter_limit(struct page_counter *counter, unsigned long limit)
135 {
136         for (;;) {
137                 unsigned long old;
138                 long count;
139 
140                 /*
141                  * Update the limit while making sure that it's not
142                  * below the concurrently-changing counter value.
143                  *
144                  * The xchg implies two full memory barriers before
145                  * and after, so the read-swap-read is ordered and
146                  * ensures coherency with page_counter_try_charge():
147                  * that function modifies the count before checking
148                  * the limit, so if it sees the old limit, we see the
149                  * modified counter and retry.
150                  */
151                 count = atomic_long_read(&counter->count);
152 
153                 if (count > limit)
154                         return -EBUSY;
155 
156                 old = xchg(&counter->limit, limit);
157 
158                 if (atomic_long_read(&counter->count) <= count)
159                         return 0;
160 
161                 counter->limit = old;
162                 cond_resched();
163         }
164 }
165 
166 /**
167  * page_counter_memparse - memparse() for page counter limits
168  * @buf: string to parse
169  * @max: string meaning maximum possible value
170  * @nr_pages: returns the result in number of pages
171  *
172  * Returns -EINVAL, or 0 and @nr_pages on success.  @nr_pages will be
173  * limited to %PAGE_COUNTER_MAX.
174  */
175 int page_counter_memparse(const char *buf, const char *max,
176                           unsigned long *nr_pages)
177 {
178         char *end;
179         u64 bytes;
180 
181         if (!strcmp(buf, max)) {
182                 *nr_pages = PAGE_COUNTER_MAX;
183                 return 0;
184         }
185 
186         bytes = memparse(buf, &end);
187         if (*end != '\0')
188                 return -EINVAL;
189 
190         *nr_pages = min(bytes / PAGE_SIZE, (u64)PAGE_COUNTER_MAX);
191 
192         return 0;
193 }
194 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us