blob: 18299057402f1bf9015cff936f9f2a37c01a5896 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef __LINUX_SEQLOCK_H
2#define __LINUX_SEQLOCK_H
3/*
4 * Reader/writer consistent mechanism without starving writers. This type of
Robert P. J. Dayd08df602007-02-17 19:07:33 +01005 * lock for data where the reader wants a consistent set of information
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 * and is willing to retry if the information changes. Readers never
7 * block but they may have to retry if a writer is in
8 * progress. Writers do not wait for readers.
9 *
10 * This is not as cache friendly as brlock. Also, this will not work
11 * for data that contains pointers, because any writer could
12 * invalidate a pointer that a reader was following.
13 *
14 * Expected reader usage:
15 * do {
16 * seq = read_seqbegin(&foo);
17 * ...
18 * } while (read_seqretry(&foo, seq));
19 *
20 *
21 * On non-SMP the spin locks disappear but the writer still needs
22 * to increment the sequence variables because an interrupt routine could
23 * change the state of the data.
24 *
25 * Based on x86_64 vsyscall gettimeofday
26 * by Keith Owens and Andrea Arcangeli
27 */
28
Linus Torvalds1da177e2005-04-16 15:20:36 -070029#include <linux/spinlock.h>
30#include <linux/preempt.h>
David Howells56a21052011-06-11 12:29:58 +010031#include <asm/processor.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070032
Linus Torvalds1da177e2005-04-16 15:20:36 -070033/*
34 * Version using sequence counter only.
35 * This can be used when code has its own mutex protecting the
36 * updating starting before the write_seqcountbeqin() and ending
37 * after the write_seqcount_end().
38 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070039typedef struct seqcount {
40 unsigned sequence;
41} seqcount_t;
42
43#define SEQCNT_ZERO { 0 }
44#define seqcount_init(x) do { *(x) = (seqcount_t) SEQCNT_ZERO; } while (0)
45
Nick Piggin3c22cd52011-01-07 17:49:51 +110046/**
47 * __read_seqcount_begin - begin a seq-read critical section (without barrier)
48 * @s: pointer to seqcount_t
49 * Returns: count to be passed to read_seqcount_retry
50 *
51 * __read_seqcount_begin is like read_seqcount_begin, but has no smp_rmb()
52 * barrier. Callers should ensure that smp_rmb() or equivalent ordering is
53 * provided before actually loading any of the variables that are to be
54 * protected in this critical section.
55 *
56 * Use carefully, only in critical code, and comment how the barrier is
57 * provided.
58 */
59static inline unsigned __read_seqcount_begin(const seqcount_t *s)
Linus Torvalds1da177e2005-04-16 15:20:36 -070060{
Ingo Molnar88a411c2008-04-03 09:06:13 +020061 unsigned ret;
62
63repeat:
Linus Torvalds2f624272012-05-04 14:46:02 -070064 ret = ACCESS_ONCE(s->sequence);
Ingo Molnar88a411c2008-04-03 09:06:13 +020065 if (unlikely(ret & 1)) {
66 cpu_relax();
67 goto repeat;
68 }
Linus Torvalds1da177e2005-04-16 15:20:36 -070069 return ret;
70}
71
Nick Piggin3c22cd52011-01-07 17:49:51 +110072/**
73 * read_seqcount_begin - begin a seq-read critical section
74 * @s: pointer to seqcount_t
75 * Returns: count to be passed to read_seqcount_retry
76 *
77 * read_seqcount_begin opens a read critical section of the given seqcount.
78 * Validity of the critical section is tested by checking read_seqcount_retry
79 * function.
80 */
81static inline unsigned read_seqcount_begin(const seqcount_t *s)
82{
83 unsigned ret = __read_seqcount_begin(s);
84 smp_rmb();
85 return ret;
86}
87
88/**
Linus Torvalds4f988f12012-05-04 15:13:54 -070089 * raw_seqcount_begin - begin a seq-read critical section
90 * @s: pointer to seqcount_t
91 * Returns: count to be passed to read_seqcount_retry
92 *
93 * raw_seqcount_begin opens a read critical section of the given seqcount.
94 * Validity of the critical section is tested by checking read_seqcount_retry
95 * function.
96 *
97 * Unlike read_seqcount_begin(), this function will not wait for the count
98 * to stabilize. If a writer is active when we begin, we will fail the
99 * read_seqcount_retry() instead of stabilizing at the beginning of the
100 * critical section.
101 */
102static inline unsigned raw_seqcount_begin(const seqcount_t *s)
103{
104 unsigned ret = ACCESS_ONCE(s->sequence);
105 smp_rmb();
106 return ret & ~1;
107}
108
109/**
Nick Piggin3c22cd52011-01-07 17:49:51 +1100110 * __read_seqcount_retry - end a seq-read critical section (without barrier)
111 * @s: pointer to seqcount_t
112 * @start: count, from read_seqcount_begin
113 * Returns: 1 if retry is required, else 0
114 *
115 * __read_seqcount_retry is like read_seqcount_retry, but has no smp_rmb()
116 * barrier. Callers should ensure that smp_rmb() or equivalent ordering is
117 * provided before actually loading any of the variables that are to be
118 * protected in this critical section.
119 *
120 * Use carefully, only in critical code, and comment how the barrier is
121 * provided.
122 */
123static inline int __read_seqcount_retry(const seqcount_t *s, unsigned start)
124{
125 return unlikely(s->sequence != start);
126}
127
128/**
129 * read_seqcount_retry - end a seq-read critical section
130 * @s: pointer to seqcount_t
131 * @start: count, from read_seqcount_begin
132 * Returns: 1 if retry is required, else 0
133 *
134 * read_seqcount_retry closes a read critical section of the given seqcount.
135 * If the critical section was invalid, it must be ignored (and typically
136 * retried).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700137 */
Ingo Molnar88a411c2008-04-03 09:06:13 +0200138static inline int read_seqcount_retry(const seqcount_t *s, unsigned start)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139{
140 smp_rmb();
Nick Piggin3c22cd52011-01-07 17:49:51 +1100141 return __read_seqcount_retry(s, start);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700142}
143
144
145/*
146 * Sequence counter only version assumes that callers are using their
147 * own mutexing.
148 */
149static inline void write_seqcount_begin(seqcount_t *s)
150{
151 s->sequence++;
152 smp_wmb();
153}
154
155static inline void write_seqcount_end(seqcount_t *s)
156{
157 smp_wmb();
158 s->sequence++;
159}
160
Nick Piggin3c22cd52011-01-07 17:49:51 +1100161/**
162 * write_seqcount_barrier - invalidate in-progress read-side seq operations
163 * @s: pointer to seqcount_t
164 *
165 * After write_seqcount_barrier, no read-side seq operations will complete
166 * successfully and see data older than this.
167 */
168static inline void write_seqcount_barrier(seqcount_t *s)
169{
170 smp_wmb();
171 s->sequence+=2;
172}
173
Thomas Gleixner6617fec2011-07-16 18:40:26 +0200174typedef struct {
175 struct seqcount seqcount;
176 spinlock_t lock;
177} seqlock_t;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700178
Thomas Gleixner6617fec2011-07-16 18:40:26 +0200179/*
180 * These macros triggered gcc-3.x compile-time problems. We think these are
181 * OK now. Be cautious.
182 */
183#define __SEQLOCK_UNLOCKED(lockname) \
184 { \
185 .seqcount = SEQCNT_ZERO, \
186 .lock = __SPIN_LOCK_UNLOCKED(lockname) \
187 }
188
189#define seqlock_init(x) \
190 do { \
191 seqcount_init(&(x)->seqcount); \
192 spin_lock_init(&(x)->lock); \
193 } while (0)
194
195#define DEFINE_SEQLOCK(x) \
196 seqlock_t x = __SEQLOCK_UNLOCKED(x)
197
198/*
199 * Read side functions for starting and finalizing a read side section.
200 */
201static inline unsigned read_seqbegin(const seqlock_t *sl)
202{
203 return read_seqcount_begin(&sl->seqcount);
204}
205
206static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start)
207{
208 return read_seqcount_retry(&sl->seqcount, start);
209}
210
211/*
212 * Lock out other writers and update the count.
213 * Acts like a normal spin_lock/unlock.
214 * Don't need preempt_disable() because that is in the spin_lock already.
215 */
216static inline void write_seqlock(seqlock_t *sl)
217{
218 spin_lock(&sl->lock);
219 write_seqcount_begin(&sl->seqcount);
220}
221
222static inline void write_sequnlock(seqlock_t *sl)
223{
224 write_seqcount_end(&sl->seqcount);
225 spin_unlock(&sl->lock);
226}
227
228static inline void write_seqlock_bh(seqlock_t *sl)
229{
230 spin_lock_bh(&sl->lock);
231 write_seqcount_begin(&sl->seqcount);
232}
233
234static inline void write_sequnlock_bh(seqlock_t *sl)
235{
236 write_seqcount_end(&sl->seqcount);
237 spin_unlock_bh(&sl->lock);
238}
239
240static inline void write_seqlock_irq(seqlock_t *sl)
241{
242 spin_lock_irq(&sl->lock);
243 write_seqcount_begin(&sl->seqcount);
244}
245
246static inline void write_sequnlock_irq(seqlock_t *sl)
247{
248 write_seqcount_end(&sl->seqcount);
249 spin_unlock_irq(&sl->lock);
250}
251
252static inline unsigned long __write_seqlock_irqsave(seqlock_t *sl)
253{
254 unsigned long flags;
255
256 spin_lock_irqsave(&sl->lock, flags);
257 write_seqcount_begin(&sl->seqcount);
258 return flags;
259}
260
261#define write_seqlock_irqsave(lock, flags) \
262 do { flags = __write_seqlock_irqsave(lock); } while (0)
263
264static inline void
265write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags)
266{
267 write_seqcount_end(&sl->seqcount);
268 spin_unlock_irqrestore(&sl->lock, flags);
269}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700270
Linus Torvalds1da177e2005-04-16 15:20:36 -0700271#endif /* __LINUX_SEQLOCK_H */