NVBIO
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
atomics.h
Go to the documentation of this file.
1 /*
2  * nvbio
3  * Copyright (c) 2011-2014, NVIDIA CORPORATION. All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7  * * Redistributions of source code must retain the above copyright
8  * notice, this list of conditions and the following disclaimer.
9  * * Redistributions in binary form must reproduce the above copyright
10  * notice, this list of conditions and the following disclaimer in the
11  * documentation and/or other materials provided with the distribution.
12  * * Neither the name of the NVIDIA CORPORATION nor the
13  * names of its contributors may be used to endorse or promote products
14  * derived from this software without specific prior written permission.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19  * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
20  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  */
27 
28 #pragma once
29 
30 #include <nvbio/basic/types.h>
31 
32 #ifndef WIN32
33 #ifdef __INTEL_COMPILER
34 #include <ia32intrin.h> // ia32intrin.h
35 #else
36 //#warning atomics.h BROKEN on GCC!
37 // Intrinsics docs at http://gcc.gnu.org/onlinedocs/gcc-4.3.2/gcc/Atomic-Builtins.html
38 #endif
39 #endif
40 
41 namespace nvbio {
42 
50 
53 
57 
58 void host_release_fence();
59 void host_acquire_fence();
60 
61 int32 host_atomic_add( int32* value, const int32 op);
62 uint32 host_atomic_add(uint32* value, const uint32 op);
63 int64 host_atomic_add( int64* value, const int64 op);
64 uint64 host_atomic_add(uint64* value, const uint64 op);
65 
66 int32 host_atomic_sub( int32* value, const int32 op);
67 uint32 host_atomic_sub(uint32* value, const uint32 op);
68 int64 host_atomic_sub( int64* value, const int64 op);
69 uint64 host_atomic_sub(uint64* value, const uint64 op);
70 
71 uint32 host_atomic_or(uint32* value, const uint32 op);
72 uint64 host_atomic_or(uint64* value, const uint64 op);
73 
75 int32 atomic_add(int32* value, const int32 op)
76 {
77  #if defined(NVBIO_DEVICE_COMPILATION)
78  return atomicAdd( value, op );
79  #else
80  return host_atomic_add( value, op );
81  #endif
82 }
83 
85 uint32 atomic_add(uint32* value, const uint32 op)
86 {
87  #if defined(NVBIO_DEVICE_COMPILATION)
88  return atomicAdd( value, op );
89  #else
90  return host_atomic_add( value, op );
91  #endif
92 }
93 
95 uint64 atomic_add(uint64* value, const uint64 op)
96 {
97  #if defined(NVBIO_DEVICE_COMPILATION)
98  return atomicAdd( value, op );
99  #else
100  return host_atomic_add( value, op );
101  #endif
102 }
103 
105 int32 atomic_sub(int32* value, const int32 op)
106 {
107  #if defined(NVBIO_DEVICE_COMPILATION)
108  return atomicSub( value, op );
109  #else
110  return host_atomic_sub( value, op );
111  #endif
112 }
113 
115 uint32 atomic_sub(uint32* value, const uint32 op)
116 {
117  #if defined(NVBIO_DEVICE_COMPILATION)
118  return atomicSub( value, op );
119  #else
120  return host_atomic_sub( value, op );
121  #endif
122 }
123 
125 uint32 atomic_or(uint32* value, const uint32 op)
126 {
127  #if defined(NVBIO_DEVICE_COMPILATION)
128  return atomicOr( value, op );
129  #else
130  return host_atomic_or( value, op );
131  #endif
132 }
133 
135 uint64 atomic_or(uint64* value, const uint64 op)
136 {
137  #if defined(NVBIO_DEVICE_COMPILATION)
138  return atomicOr( value, op );
139  #else
140  return host_atomic_or( value, op );
141  #endif
142 }
143 
144 #if defined(WIN32)
145 
146 int32 atomic_increment(int32 volatile *value);
147 int64 atomic_increment(int64 volatile *value);
148 
149 int32 atomic_decrement(int32 volatile *value);
150 int64 atomic_decrement(int64 volatile *value);
151 
154 template<typename intT>
155 struct AtomicInt
156 {
158  AtomicInt() : m_value(0) {}
159 
161  AtomicInt(const intT value) : m_value(value) {}
162 
164  intT increment()
165  {
166  return atomic_increment( &m_value );
167  }
168 
170  intT decrement()
171  {
172  return atomic_decrement( &m_value );
173  }
174 
176  intT operator++(int) { return increment()-1u; }
177 
179  intT operator--(int) { return decrement()+1u; }
180 
182  intT operator++() { return increment(); }
183 
185  intT operator--() { return decrement(); }
186 
188  intT operator+=(const intT v) { return m_value += v; }
190  intT operator-=(const intT v) { return m_value -= v; }
191 
193  bool operator==(const intT value) { return m_value == value; }
194  bool operator!=(const intT value) { return m_value != value; }
195  bool operator>=(const intT value) { return m_value >= value; }
196  bool operator<=(const intT value) { return m_value <= value; }
197  bool operator>(const intT value) { return m_value > value; }
198  bool operator<(const intT value) { return m_value < value; }
199 
200  volatile intT m_value;
201 };
202 
203 typedef AtomicInt<int> AtomicInt32;
204 typedef AtomicInt<int64> AtomicInt64;
205 
206 #else
207 
208 #define ASM_ATOMICS
209 
212 {
215 
217  AtomicInt32(const int value) : m_value(value) {}
218 
219 #if defined(PLATFORM_X86) && defined(ASM_ATOMICS)
220  // look here
221  // http://www.memoryhole.net/kyle/2007/05/atomic_incrementing.html
222  // and also figure if this could use xaddq for the 64bit version
224  int operator++(int) {
225  int tmp;
226 
227  __asm__ __volatile__ ("lock; xaddl %0, %1"
228  : "=r" (tmp), "=m" (m_value)
229  : "0" (1), "m" (m_value));
230 
231  return tmp;
232  }
233 
235  int operator--(int) {
236  int tmp;
237 
238  __asm__ __volatile__ ("lock; xaddl %0, %1"
239  : "=r" (tmp), "=m" (m_value)
240  : "0" (-1), "m" (m_value));
241 
242  return tmp;
243  }
244 
246  int operator++() {
247  int tmp;
248 
249  __asm__ __volatile__ ("lock; xaddl %0, %1"
250  : "=r" (tmp), "=m" (m_value)
251  : "0" (1), "m" (m_value));
252 
253  return tmp + 1;
254  }
255 
257  int operator--() {
258  int tmp;
259 
260  __asm__ __volatile__ ("lock; xaddl %0, %1"
261  : "=r" (tmp), "=m" (m_value)
262  : "0" (-1), "m" (m_value));
263 
264  return tmp - 1;
265  }
266 
268  int operator+=(int value) {
269  register int v asm ("eax") = value;
270 
271  __asm__ __volatile__ ("lock; xaddl %0, %1"
272  : "=r" (v), "=m" (m_value)
273  : "r" (v), "m" (m_value));
274 
275  return v + value;
276  }
277 
279  int operator-=(int value ) {
280  register int v asm ("eax") = -value;
281 
282  __asm__ __volatile__ ("lock; xaddl %0, %1"
283  : "=r" (v), "=m" (m_value)
284  : "r" (v), "m" (m_value));
285 
286  return v - value;
287  }
288 #elif defined(__INTEL_COMPILER)
289  int operator++(int) { return _InterlockedIncrement( (void*)&m_value )-1u; }
291 
293  int operator--(int) { return _InterlockedDecrement( (void*)&m_value )+1u; }
294 
296  int operator++() { return _InterlockedIncrement( (void*)&m_value ); }
297 
299  int operator--() { return _InterlockedDecrement( (void*)&m_value ); }
300 
302  int operator+=(int value) { return __sync_add_and_fetch(&m_value, value); }
303 
305  int operator-=(int value) { return __sync_sub_and_fetch(&m_value, value); }
306 #else
307  int operator++(int) { return __sync_fetch_and_add(&m_value, 1); }
309 
311  int operator--(int) { return __sync_fetch_and_add(&m_value, -1); }
312 
314  int operator++() { return __sync_add_and_fetch(&m_value, 1); }
315 
317  int operator--() { return __sync_add_and_fetch(&m_value, -1); }
318 
320  int operator+=(int value) { return __sync_add_and_fetch(&m_value, value); }
321 
323  int operator-=(int value) { return __sync_sub_and_fetch(&m_value, value); }
324 #endif
325 
327  bool operator==(const int value) { return m_value == value; }
328  bool operator!=(const int value) { return m_value != value; }
329  bool operator>=(const int value) { return m_value >= value; }
330  bool operator<=(const int value) { return m_value <= value; }
331  bool operator>(const int value) { return m_value > value; }
332  bool operator<(const int value) { return m_value < value; }
333 
334  volatile int m_value;
335 };
336 
338 {
341 
343  AtomicInt64(const long value) : m_value(value) {}
344 
345 #if defined(PLATFORM_X86) && defined(ASM_ATOMICS)
346  // look here
347  // http://www.memoryhole.net/kyle/2007/05/atomic_incrementing.html
348  // and also figure if this could use xaddq for the 64bit version
350  long operator++(int) {
351  register long v asm ("eax");
352 
353  __asm__ __volatile__ ("lock; xaddq %0, %1"
354  : "=r" (v), "=m" (m_value)
355  : "0" (1), "m" (m_value));
356 
357  return v;
358  }
359 
361  long operator--(int) {
362  register long v asm ("eax");
363 
364  __asm__ __volatile__ ("lock; xaddq %0, %1"
365  : "=r" (v), "=m" (m_value)
366  : "0" (-1), "m" (m_value));
367 
368  return v;
369  }
370 
372  long operator++() {
373  register long v asm ("eax");
374 
375  __asm__ __volatile__ ("lock; xaddq %0, %1"
376  : "=r" (v), "=m" (m_value)
377  : "0" (1), "m" (m_value));
378 
379  return v + 1;
380  }
381 
383  long operator--() {
384  register long v asm ("eax");
385 
386  __asm__ __volatile__ ("lock; xaddq %0, %1"
387  : "=r" (v), "=m" (m_value)
388  : "0" (-1), "m" (m_value));
389 
390  return v - 1;
391  }
392 
394  long operator+=(long value) {
395  register long v asm ("eax") = value;
396 
397  __asm__ __volatile__ ("lock; xaddq %0, %1"
398  : "=r" (v), "=m" (m_value)
399  : "r" (v), "m" (m_value));
400 
401  return v + value;
402  }
403 
405  long operator-=(long value ) {
406  register long v asm ("eax") = -value;
407 
408  __asm__ __volatile__ ("lock; xaddq %0, %1"
409  : "=r" (v), "=m" (m_value)
410  : "r" (v), "m" (m_value));
411 
412  return v - value;
413  }
414 #elif defined(__INTEL_COMPILER)
415 #error TODO!
416 #else
417  long operator++(int) { return __sync_fetch_and_add(&m_value, 1); }
419 
421  long operator--(int) { return __sync_fetch_and_add(&m_value, -1); }
422 
424  long operator++() { return __sync_add_and_fetch(&m_value, 1); }
425 
427  long operator--() { return __sync_add_and_fetch(&m_value, -1); }
428 
430  long operator+=(long value) { return __sync_add_and_fetch(&m_value, value); }
431 
433  long operator-=(long value) { return __sync_sub_and_fetch(&m_value, value); }
434 #endif
435 
437  bool operator==(const long value) { return m_value == value; }
438  bool operator!=(const long value) { return m_value != value; }
439  bool operator>=(const long value) { return m_value >= value; }
440  bool operator<=(const long value) { return m_value <= value; }
441  bool operator>(const long value) { return m_value > value; }
442  bool operator<(const long value) { return m_value < value; }
443 
444  volatile long m_value;
445 };
446 
447 #endif
448 
450 inline bool operator==(const long a, const AtomicInt64& b) { return a == b.m_value; }
451 inline bool operator!=(const long a, const AtomicInt64& b) { return a != b.m_value; }
452 inline bool operator>=(const long a, const AtomicInt64& b) { return a >= b.m_value; }
453 inline bool operator<=(const long a, const AtomicInt64& b) { return a <= b.m_value; }
454 inline bool operator> (const long a, const AtomicInt64& b) { return a > b.m_value; }
455 inline bool operator< (const long a, const AtomicInt64& b) { return a < b.m_value; }
456 
457 inline bool operator==(const int a, const AtomicInt32& b) { return a == b.m_value; }
458 inline bool operator!=(const int a, const AtomicInt32& b) { return a != b.m_value; }
459 inline bool operator>=(const int a, const AtomicInt32& b) { return a >= b.m_value; }
460 inline bool operator<=(const int a, const AtomicInt32& b) { return a <= b.m_value; }
461 inline bool operator> (const int a, const AtomicInt32& b) { return a > b.m_value; }
462 inline bool operator< (const int a, const AtomicInt32& b) { return a < b.m_value; }
463 
466 
467 } // namespace nvbio