Version:  2.0.40 2.2.26 2.4.37 3.13 3.14 3.15 3.16 3.17 3.18 3.19 4.0 4.1 4.2 4.3 4.4 4.5 4.6 4.7 4.8 4.9 4.10

Linux/lib/atomic64.c

  1 /*
  2  * Generic implementation of 64-bit atomics using spinlocks,
  3  * useful on processors that don't have 64-bit atomic instructions.
  4  *
  5  * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
  6  *
  7  * This program is free software; you can redistribute it and/or
  8  * modify it under the terms of the GNU General Public License
  9  * as published by the Free Software Foundation; either version
 10  * 2 of the License, or (at your option) any later version.
 11  */
 12 #include <linux/types.h>
 13 #include <linux/cache.h>
 14 #include <linux/spinlock.h>
 15 #include <linux/init.h>
 16 #include <linux/export.h>
 17 #include <linux/atomic.h>
 18 
 19 /*
 20  * We use a hashed array of spinlocks to provide exclusive access
 21  * to each atomic64_t variable.  Since this is expected to used on
 22  * systems with small numbers of CPUs (<= 4 or so), we use a
 23  * relatively small array of 16 spinlocks to avoid wasting too much
 24  * memory on the spinlock array.
 25  */
 26 #define NR_LOCKS        16
 27 
 28 /*
 29  * Ensure each lock is in a separate cacheline.
 30  */
 31 static union {
 32         raw_spinlock_t lock;
 33         char pad[L1_CACHE_BYTES];
 34 } atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = {
 35         [0 ... (NR_LOCKS - 1)] = {
 36                 .lock =  __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock),
 37         },
 38 };
 39 
 40 static inline raw_spinlock_t *lock_addr(const atomic64_t *v)
 41 {
 42         unsigned long addr = (unsigned long) v;
 43 
 44         addr >>= L1_CACHE_SHIFT;
 45         addr ^= (addr >> 8) ^ (addr >> 16);
 46         return &atomic64_lock[addr & (NR_LOCKS - 1)].lock;
 47 }
 48 
 49 long long atomic64_read(const atomic64_t *v)
 50 {
 51         unsigned long flags;
 52         raw_spinlock_t *lock = lock_addr(v);
 53         long long val;
 54 
 55         raw_spin_lock_irqsave(lock, flags);
 56         val = v->counter;
 57         raw_spin_unlock_irqrestore(lock, flags);
 58         return val;
 59 }
 60 EXPORT_SYMBOL(atomic64_read);
 61 
 62 void atomic64_set(atomic64_t *v, long long i)
 63 {
 64         unsigned long flags;
 65         raw_spinlock_t *lock = lock_addr(v);
 66 
 67         raw_spin_lock_irqsave(lock, flags);
 68         v->counter = i;
 69         raw_spin_unlock_irqrestore(lock, flags);
 70 }
 71 EXPORT_SYMBOL(atomic64_set);
 72 
 73 #define ATOMIC64_OP(op, c_op)                                           \
 74 void atomic64_##op(long long a, atomic64_t *v)                          \
 75 {                                                                       \
 76         unsigned long flags;                                            \
 77         raw_spinlock_t *lock = lock_addr(v);                            \
 78                                                                         \
 79         raw_spin_lock_irqsave(lock, flags);                             \
 80         v->counter c_op a;                                              \
 81         raw_spin_unlock_irqrestore(lock, flags);                        \
 82 }                                                                       \
 83 EXPORT_SYMBOL(atomic64_##op);
 84 
 85 #define ATOMIC64_OP_RETURN(op, c_op)                                    \
 86 long long atomic64_##op##_return(long long a, atomic64_t *v)            \
 87 {                                                                       \
 88         unsigned long flags;                                            \
 89         raw_spinlock_t *lock = lock_addr(v);                            \
 90         long long val;                                                  \
 91                                                                         \
 92         raw_spin_lock_irqsave(lock, flags);                             \
 93         val = (v->counter c_op a);                                      \
 94         raw_spin_unlock_irqrestore(lock, flags);                        \
 95         return val;                                                     \
 96 }                                                                       \
 97 EXPORT_SYMBOL(atomic64_##op##_return);
 98 
 99 #define ATOMIC64_FETCH_OP(op, c_op)                                     \
100 long long atomic64_fetch_##op(long long a, atomic64_t *v)               \
101 {                                                                       \
102         unsigned long flags;                                            \
103         raw_spinlock_t *lock = lock_addr(v);                            \
104         long long val;                                                  \
105                                                                         \
106         raw_spin_lock_irqsave(lock, flags);                             \
107         val = v->counter;                                               \
108         v->counter c_op a;                                              \
109         raw_spin_unlock_irqrestore(lock, flags);                        \
110         return val;                                                     \
111 }                                                                       \
112 EXPORT_SYMBOL(atomic64_fetch_##op);
113 
114 #define ATOMIC64_OPS(op, c_op)                                          \
115         ATOMIC64_OP(op, c_op)                                           \
116         ATOMIC64_OP_RETURN(op, c_op)                                    \
117         ATOMIC64_FETCH_OP(op, c_op)
118 
119 ATOMIC64_OPS(add, +=)
120 ATOMIC64_OPS(sub, -=)
121 
122 #undef ATOMIC64_OPS
123 #define ATOMIC64_OPS(op, c_op)                                          \
124         ATOMIC64_OP(op, c_op)                                           \
125         ATOMIC64_OP_RETURN(op, c_op)                                    \
126         ATOMIC64_FETCH_OP(op, c_op)
127 
128 ATOMIC64_OPS(and, &=)
129 ATOMIC64_OPS(or, |=)
130 ATOMIC64_OPS(xor, ^=)
131 
132 #undef ATOMIC64_OPS
133 #undef ATOMIC64_FETCH_OP
134 #undef ATOMIC64_OP_RETURN
135 #undef ATOMIC64_OP
136 
137 long long atomic64_dec_if_positive(atomic64_t *v)
138 {
139         unsigned long flags;
140         raw_spinlock_t *lock = lock_addr(v);
141         long long val;
142 
143         raw_spin_lock_irqsave(lock, flags);
144         val = v->counter - 1;
145         if (val >= 0)
146                 v->counter = val;
147         raw_spin_unlock_irqrestore(lock, flags);
148         return val;
149 }
150 EXPORT_SYMBOL(atomic64_dec_if_positive);
151 
152 long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n)
153 {
154         unsigned long flags;
155         raw_spinlock_t *lock = lock_addr(v);
156         long long val;
157 
158         raw_spin_lock_irqsave(lock, flags);
159         val = v->counter;
160         if (val == o)
161                 v->counter = n;
162         raw_spin_unlock_irqrestore(lock, flags);
163         return val;
164 }
165 EXPORT_SYMBOL(atomic64_cmpxchg);
166 
167 long long atomic64_xchg(atomic64_t *v, long long new)
168 {
169         unsigned long flags;
170         raw_spinlock_t *lock = lock_addr(v);
171         long long val;
172 
173         raw_spin_lock_irqsave(lock, flags);
174         val = v->counter;
175         v->counter = new;
176         raw_spin_unlock_irqrestore(lock, flags);
177         return val;
178 }
179 EXPORT_SYMBOL(atomic64_xchg);
180 
181 int atomic64_add_unless(atomic64_t *v, long long a, long long u)
182 {
183         unsigned long flags;
184         raw_spinlock_t *lock = lock_addr(v);
185         int ret = 0;
186 
187         raw_spin_lock_irqsave(lock, flags);
188         if (v->counter != u) {
189                 v->counter += a;
190                 ret = 1;
191         }
192         raw_spin_unlock_irqrestore(lock, flags);
193         return ret;
194 }
195 EXPORT_SYMBOL(atomic64_add_unless);
196 

This page was automatically generated by LXR 0.3.1 (source).  •  Linux is a registered trademark of Linus Torvalds  •  Contact us