StarPU Handbook - StarPU Performances
starpu_util.h
Go to the documentation of this file.
1 /* StarPU --- Runtime system for heterogeneous multicore architectures.
2  *
3  * Copyright (C) 2008-2021 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria
4  *
5  * StarPU is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU Lesser General Public License as published by
7  * the Free Software Foundation; either version 2.1 of the License, or (at
8  * your option) any later version.
9  *
10  * StarPU is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13  *
14  * See the GNU Lesser General Public License in COPYING.LGPL for more details.
15  */
16 
17 #ifndef __STARPU_UTIL_H__
18 #define __STARPU_UTIL_H__
19 
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <stdint.h>
23 #include <string.h>
24 #include <assert.h>
25 
26 #include <starpu_config.h>
27 
28 #ifdef __GLIBC__
29 #include <execinfo.h>
30 #endif
31 
32 #ifdef STARPU_SIMGRID_MC
33 #include <simgrid/modelchecker.h>
34 #endif
35 
36 #ifdef __cplusplus
37 extern "C" {
38 #endif
39 
51 #if defined __GNUC__ && defined __GNUC_MINOR__
52 #define STARPU_GNUC_PREREQ(maj, min) \
53  ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
54 #else
55 #define STARPU_GNUC_PREREQ(maj, min) 0
56 #endif
57 
62 #ifdef __GNUC__
63 #define STARPU_UNLIKELY(expr) (__builtin_expect(!!(expr), 0))
64 #else
65 #define STARPU_UNLIKELY(expr) (expr)
66 #endif
67 
72 #ifdef __GNUC__
73 #define STARPU_LIKELY(expr) (__builtin_expect(!!(expr), 1))
74 #else
75 #define STARPU_LIKELY(expr) (expr)
76 #endif
77 
81 #ifdef __GNUC__
82 #define STARPU_ATTRIBUTE_UNUSED __attribute__((unused))
83 #else
84 #define STARPU_ATTRIBUTE_UNUSED
85 #endif
86 
90 #ifdef __GNUC__
91 #define STARPU_ATTRIBUTE_NORETURN __attribute__((noreturn))
92 #else
93 #define STARPU_ATTRIBUTE_NORETURN
94 #endif
95 
99 #ifdef __GNUC__
100 #define STARPU_ATTRIBUTE_VISIBILITY_DEFAULT __attribute__((visibility("default")))
101 #else
102 #define STARPU_ATTRIBUTE_VISIBILITY_DEFAULT
103 #endif
104 
108 #ifdef __GNUC__
109 #define STARPU_VISIBILITY_PUSH_HIDDEN #pragma GCC visibility push(hidden)
110 #else
111 #define STARPU_VISIBILITY_PUSH_HIDDEN
112 #endif
113 
117 #ifdef __GNUC__
118 #define STARPU_VISIBILITY_POP #pragma GCC visibility pop
119 #else
120 #define STARPU_VISIBILITY_POP
121 #endif
122 
126 #ifdef __GNUC__
127 #define STARPU_ATTRIBUTE_MALLOC __attribute__((malloc))
128 #else
129 #define STARPU_ATTRIBUTE_MALLOC
130 #endif
131 
135 #ifdef __GNUC__
136 #define STARPU_ATTRIBUTE_WARN_UNUSED_RESULT __attribute__((warn_unused_result))
137 #else
138 #define STARPU_ATTRIBUTE_WARN_UNUSED_RESULT
139 #endif
140 
144 #ifdef __GNUC__
145 #define STARPU_ATTRIBUTE_PURE __attribute__((pure))
146 #else
147 #define STARPU_ATTRIBUTE_PURE
148 #endif
149 
153 #ifdef __GNUC__
154 #define STARPU_ATTRIBUTE_ALIGNED(size) __attribute__((aligned(size)))
155 #else
156 #define STARPU_ATTRIBUTE_ALIGNED(size)
157 #endif
158 
159 #ifdef __GNUC__
160 #define STARPU_ATTRIBUTE_FORMAT(type, string, first) __attribute__((format(type, string, first)))
161 #else
162 #define STARPU_ATTRIBUTE_FORMAT(type, string, first)
163 #endif
164 
165 /* Note that if we're compiling C++, then just use the "inline"
166  keyword, since it's part of C++ */
167 #if defined(c_plusplus) || defined(__cplusplus)
168 #define STARPU_INLINE inline
169 #elif defined(_MSC_VER) || defined(__HP_cc)
170 #define STARPU_INLINE __inline
171 #else
172 #define STARPU_INLINE __inline__
173 #endif
174 
175 #if STARPU_GNUC_PREREQ(4, 3)
176 #define STARPU_ATTRIBUTE_CALLOC_SIZE(num, size) __attribute__((alloc_size(num, size)))
177 #define STARPU_ATTRIBUTE_ALLOC_SIZE(size) __attribute__((alloc_size(size)))
178 #else
179 #define STARPU_ATTRIBUTE_CALLOC_SIZE(num, size)
180 #define STARPU_ATTRIBUTE_ALLOC_SIZE(size)
181 #endif
182 
183 #if STARPU_GNUC_PREREQ(3, 1) && !defined(BUILDING_STARPU) && !defined(STARPU_USE_DEPRECATED_API) && !defined(STARPU_USE_DEPRECATED_ONE_ZERO_API)
184 #define STARPU_DEPRECATED __attribute__((__deprecated__))
185 #else
186 #define STARPU_DEPRECATED
187 #endif /* __GNUC__ */
188 
189 #if STARPU_GNUC_PREREQ(3, 3)
190 #define STARPU_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__))
191 #else
192 #define STARPU_WARN_UNUSED_RESULT
193 #endif /* __GNUC__ */
194 
195 #define STARPU_BACKTRACE_LENGTH 32
196 #ifdef __GLIBC__
197 #define STARPU_DUMP_BACKTRACE() \
198  do { \
199  void *__ptrs[STARPU_BACKTRACE_LENGTH]; \
200  int __n = backtrace(__ptrs, STARPU_BACKTRACE_LENGTH); \
201  backtrace_symbols_fd(__ptrs, __n, 2); \
202  } \
203  while (0)
204 #else
205 #define STARPU_DUMP_BACKTRACE() \
206  do { \
207  } \
208  while (0)
209 #endif
210 
211 #ifdef STARPU_SIMGRID_MC
212 #define STARPU_SIMGRID_ASSERT(x) MC_assert(!!(x))
213 #else
214 #define STARPU_SIMGRID_ASSERT(x)
215 #endif
216 
221 #ifdef STARPU_NO_ASSERT
222 #define STARPU_ASSERT(x) \
223  do { \
224  if (0) { (void)(x); } \
225  } \
226  while (0)
227 #else
228 #if defined(__CUDACC__) || defined(STARPU_HAVE_WINDOWS)
229 #define STARPU_ASSERT(x) \
230  do { \
231  if (STARPU_UNLIKELY(!(x))) \
232  { \
233  STARPU_DUMP_BACKTRACE(); \
234  STARPU_SIMGRID_ASSERT(0 && #x); \
235  *(int *)NULL = 0; \
236  } \
237  } \
238  while (0)
239 #else
240 #define STARPU_ASSERT(x) \
241  do { \
242  if (STARPU_UNLIKELY(!(x))) \
243  { \
244  STARPU_DUMP_BACKTRACE(); \
245  STARPU_SIMGRID_ASSERT(0 && #x); \
246  assert(0 && #x); \
247  } \
248  } \
249  while (0)
250 #endif
251 #endif
252 
258 #ifdef STARPU_NO_ASSERT
259 #define STARPU_ASSERT_ACCESSIBLE(x) \
260  do { \
261  if (0) { (void)(x); } \
262  } \
263  while (0)
264 #else
265 #define STARPU_ASSERT_ACCESSIBLE(ptr) \
266  do { \
267  volatile char __c STARPU_ATTRIBUTE_UNUSED = *(char *)(ptr); \
268  } \
269  while (0)
270 #endif
271 
275 #if STARPU_GNUC_PREREQ(4, 6) && !defined __cplusplus && !defined(__STRICT_ANSI__)
276 #define STARPU_STATIC_ASSERT(x) _Static_assert(x, #x)
277 #else
278 #define STARPU_STATIC_ASSERT(x) STARPU_ASSERT(x)
279 #endif
280 
286 #ifdef STARPU_NO_ASSERT
287 #define STARPU_ASSERT_MSG(x, msg, ...) \
288  do { \
289  if (0) \
290  { \
291  (void)(x); \
292  (void)msg; \
293  } \
294  } \
295  while (0)
296 #else
297 #if defined(__INTEL_COMPILER)
298 #pragma warning disable 279 // otherwise icc triggers "warning #279: controlling expression is constant" (probably because of assert(0 && #x))
299 #endif
300 #if defined(__CUDACC__) || defined(STARPU_HAVE_WINDOWS)
301 #define STARPU_ASSERT_MSG(x, msg, ...) \
302  do { \
303  if (STARPU_UNLIKELY(!(x))) \
304  { \
305  STARPU_DUMP_BACKTRACE(); \
306  fprintf(stderr, "\n[starpu][%s][assert failure] " msg "\n\n", __starpu_func__, ##__VA_ARGS__); \
307  STARPU_SIMGRID_ASSERT(0 && #x); \
308  *(int *)NULL = 0; \
309  } \
310  } \
311  while (0)
312 #else
313 #define STARPU_ASSERT_MSG(x, msg, ...) \
314  do { \
315  if (STARPU_UNLIKELY(!(x))) \
316  { \
317  STARPU_DUMP_BACKTRACE(); \
318  fprintf(stderr, "\n[starpu][%s][assert failure] " msg "\n\n", __starpu_func__, ##__VA_ARGS__); \
319  STARPU_SIMGRID_ASSERT(0 && #x); \
320  assert(0 && #x); \
321  abort(); \
322  *(int *)NULL = 0; \
323  } \
324  } \
325  while (0)
326 #endif
327 #endif
328 
329 #ifdef __APPLE_CC__
330 #ifdef __clang_analyzer__
331 #define _starpu_abort() exit(42)
332 #else
333 #define _starpu_abort() *(volatile int *)NULL = 0
334 #endif
335 #else
336 #define _starpu_abort() abort()
337 #endif
338 
342 #define STARPU_ABORT() \
343  do { \
344  STARPU_DUMP_BACKTRACE(); \
345  fprintf(stderr, "[starpu][abort][%s()@%s:%d]\n", __starpu_func__, __FILE__, __LINE__); \
346  _starpu_abort(); \
347  } \
348  while (0)
349 
355 #define STARPU_ABORT_MSG(msg, ...) \
356  do { \
357  STARPU_DUMP_BACKTRACE(); \
358  fprintf(stderr, "[starpu][abort][%s()@%s:%d] " msg "\n", __starpu_func__, __FILE__, __LINE__, ##__VA_ARGS__); \
359  _starpu_abort(); \
360  } \
361  while (0)
362 
363 #if defined(_MSC_VER)
364 #undef STARPU_HAVE_STRERROR_R
365 #endif
366 
367 #if defined(STARPU_HAVE_STRERROR_R)
368 #if (!defined(__GLIBC__) || !__GLIBC__) || ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && (!defined(_GNU_SOURCE)))
369 /* XSI-compliant version of strerror_r returns an int */
370 #define starpu_strerror_r(errnum, buf, buflen) \
371  do \
372  { \
373  int _ret = strerror_r((errnum), (buf), (buflen)); \
374  STARPU_ASSERT(_ret == 0); \
375  } \
376  while (0)
377 #else
378 /* GNU-specific version of strerror_r returns a char * */
379 #define starpu_strerror_r(errnum, buf, buflen) \
380  do \
381  { \
382  char *const _user_buf = (buf); \
383  const size_t _user_buflen = (buflen); \
384  /* the GNU-specific behaviour when 'buf' == NULL cannot be emulated with the XSI-compliant version */ \
385  STARPU_ASSERT((buf) != NULL); \
386  char *_tmp_buf = strerror_r((errnum), _user_buf, _user_buflen); \
387  if (_tmp_buf != _user_buf) \
388  { \
389  if (_user_buflen > 0) \
390  { \
391  strncpy(_user_buf, _tmp_buf, _user_buflen - 1); \
392  _user_buf[_user_buflen - 1] = '\0'; \
393  } \
394  } \
395  } \
396  while (0)
397 #endif /* strerror_r ABI version */
398 #endif /* STARPU_HAVE_STRERROR_R */
399 
404 #if defined(STARPU_HAVE_STRERROR_R)
405 #define STARPU_CHECK_RETURN_VALUE(err, message, ...) \
406  { \
407  if (STARPU_UNLIKELY(err != 0)) \
408  { \
409  char xmessage[256]; \
410  starpu_strerror_r(-err, xmessage, 256); \
411  fprintf(stderr, "[starpu] Unexpected value: <%d:%s> returned for " message "\n", err, xmessage, ##__VA_ARGS__); \
412  STARPU_ABORT(); \
413  } \
414  }
415 #else
416 #define STARPU_CHECK_RETURN_VALUE(err, message, ...) \
417  { \
418  if (STARPU_UNLIKELY(err != 0)) \
419  { \
420  fprintf(stderr, "[starpu] Unexpected value: <%d> returned for " message "\n", err, ##__VA_ARGS__); \
421  STARPU_ABORT(); \
422  } \
423  }
424 #endif
425 
430 #if defined(STARPU_HAVE_STRERROR_R)
431 #define STARPU_CHECK_RETURN_VALUE_IS(err, value, message, ...) \
432  { \
433  if (STARPU_UNLIKELY(err != value)) \
434  { \
435  char xmessage[256]; \
436  starpu_strerror_r(-err, xmessage, 256); \
437  fprintf(stderr, "[starpu] Unexpected value: <%d!=%d:%s> returned for " message "\n", err, value, xmessage, ##__VA_ARGS__); \
438  STARPU_ABORT(); \
439  } \
440  }
441 #else
442 #define STARPU_CHECK_RETURN_VALUE_IS(err, value, message, ...) \
443  { \
444  if (STARPU_UNLIKELY(err != value)) \
445  { \
446  fprintf(stderr, "[starpu] Unexpected value: <%d != %d> returned for " message "\n", err, value, ##__VA_ARGS__); \
447  STARPU_ABORT(); \
448  } \
449  }
450 #endif
451 
452 /* Note: do not use _starpu_cmpxchg / _starpu_xchg / _starpu_cmpxchgl /
453  * _starpu_xchgl / _starpu_cmpxchg64 / _starpu_xchg64, which only
454  * assembly-hand-written fallbacks used when building with an old gcc.
455  * Rather use STARPU_VAL_COMPARE_AND_SWAP and STARPU_VAL_EXCHANGE available on
456  * all platforms with a recent-enough gcc */
457 
458 #if defined(__i386__) || defined(__x86_64__)
459 static __starpu_inline unsigned _starpu_cmpxchg(unsigned *ptr, unsigned old, unsigned next)
460 {
461  __asm__ __volatile__("lock cmpxchgl %2,%1"
462  : "+a"(old), "+m"(*ptr)
463  : "q"(next)
464  : "memory");
465  return old;
466 }
467 #define STARPU_HAVE_CMPXCHG
468 static __starpu_inline unsigned _starpu_xchg(unsigned *ptr, unsigned next)
469 {
470  /* Note: xchg is always locked already */
471  __asm__ __volatile__("xchgl %1,%0"
472  : "+m"(*ptr), "+q"(next)
473  :
474  : "memory");
475  return next;
476 }
477 #define STARPU_HAVE_XCHG
478 
479 static __starpu_inline uint32_t _starpu_cmpxchg32(uint32_t *ptr, uint32_t old, uint32_t next)
480 {
481  __asm__ __volatile__("lock cmpxchgl %2,%1"
482  : "+a"(old), "+m"(*ptr)
483  : "q"(next)
484  : "memory");
485  return old;
486 }
487 #define STARPU_HAVE_CMPXCHG32
488 static __starpu_inline uint32_t _starpu_xchg32(uint32_t *ptr, uint32_t next)
489 {
490  /* Note: xchg is always locked already */
491  __asm__ __volatile__("xchgl %1,%0"
492  : "+m"(*ptr), "+q"(next)
493  :
494  : "memory");
495  return next;
496 }
497 #define STARPU_HAVE_XCHG32
498 
499 #if defined(__i386__)
500 static __starpu_inline unsigned long _starpu_cmpxchgl(unsigned long *ptr, unsigned long old, unsigned long next)
501 {
502  __asm__ __volatile__("lock cmpxchgl %2,%1"
503  : "+a"(old), "+m"(*ptr)
504  : "q"(next)
505  : "memory");
506  return old;
507 }
508 #define STARPU_HAVE_CMPXCHGL
509 static __starpu_inline unsigned long _starpu_xchgl(unsigned long *ptr, unsigned long next)
510 {
511  /* Note: xchg is always locked already */
512  __asm__ __volatile__("xchgl %1,%0"
513  : "+m"(*ptr), "+q"(next)
514  :
515  : "memory");
516  return next;
517 }
518 #define STARPU_HAVE_XCHGL
519 #endif
520 
521 #if defined(__x86_64__)
522 static __starpu_inline unsigned long _starpu_cmpxchgl(unsigned long *ptr, unsigned long old, unsigned long next)
523 {
524  __asm__ __volatile__("lock cmpxchgq %2,%1"
525  : "+a"(old), "+m"(*ptr)
526  : "q"(next)
527  : "memory");
528  return old;
529 }
530 #define STARPU_HAVE_CMPXCHGL
531 static __starpu_inline unsigned long _starpu_xchgl(unsigned long *ptr, unsigned long next)
532 {
533  /* Note: xchg is always locked already */
534  __asm__ __volatile__("xchgq %1,%0"
535  : "+m"(*ptr), "+q"(next)
536  :
537  : "memory");
538  return next;
539 }
540 #define STARPU_HAVE_XCHGL
541 #endif
542 
543 #if defined(__i386__)
544 static __starpu_inline uint64_t _starpu_cmpxchg64(uint64_t *ptr, uint64_t old, uint64_t next)
545 {
546  uint32_t next_hi = next >> 32;
547  uint32_t next_lo = next & 0xfffffffful;
548  __asm__ __volatile__("lock cmpxchg8b %1"
549  : "+A"(old), "+m"(*ptr)
550  : "c"(next_hi), "b"(next_lo)
551  : "memory");
552  return old;
553 }
554 #define STARPU_HAVE_CMPXCHG64
555 #endif
556 
557 #if defined(__x86_64__)
558 static __starpu_inline uint64_t _starpu_cmpxchg64(uint64_t *ptr, uint64_t old, uint64_t next)
559 {
560  __asm__ __volatile__("lock cmpxchgq %2,%1"
561  : "+a"(old), "+m"(*ptr)
562  : "q"(next)
563  : "memory");
564  return old;
565 }
566 #define STARPU_HAVE_CMPXCHG64
567 static __starpu_inline uint64_t _starpu_xchg64(uint64_t *ptr, uint64_t next)
568 {
569  /* Note: xchg is always locked already */
570  __asm__ __volatile__("xchgq %1,%0"
571  : "+m"(*ptr), "+q"(next)
572  :
573  : "memory");
574  return next;
575 }
576 #define STARPU_HAVE_XCHG64
577 #endif
578 
579 #endif
580 
581 #define STARPU_ATOMIC_SOMETHING(name, expr) \
582  static __starpu_inline unsigned starpu_atomic_##name(unsigned *ptr, unsigned value) \
583  { \
584  unsigned old, next; \
585  while (1) \
586  { \
587  old = *ptr; \
588  next = expr; \
589  if (_starpu_cmpxchg(ptr, old, next) == old) \
590  break; \
591  }; \
592  return expr; \
593  }
594 #define STARPU_ATOMIC_SOMETHINGL(name, expr) \
595  static __starpu_inline unsigned long starpu_atomic_##name##l(unsigned long *ptr, unsigned long value) \
596  { \
597  unsigned long old, next; \
598  while (1) \
599  { \
600  old = *ptr; \
601  next = expr; \
602  if (_starpu_cmpxchgl(ptr, old, next) == old) \
603  break; \
604  }; \
605  return expr; \
606  }
607 #define STARPU_ATOMIC_SOMETHING64(name, expr) \
608  static __starpu_inline uint64_t starpu_atomic_##name##64(uint64_t * ptr, uint64_t value) \
609  { \
610  uint64_t old, next; \
611  while (1) \
612  { \
613  old = *ptr; \
614  next = expr; \
615  if (_starpu_cmpxchg64(ptr, old, next) == old) \
616  break; \
617  }; \
618  return expr; \
619  }
620 
621 /* Returns the new value */
622 #ifdef STARPU_HAVE_SYNC_FETCH_AND_ADD
623 #define STARPU_ATOMIC_ADD(ptr, value) (__sync_fetch_and_add((ptr), (value)) + (value))
624 #define STARPU_ATOMIC_ADDL(ptr, value) (__sync_fetch_and_add((ptr), (value)) + (value))
625 #define STARPU_ATOMIC_ADD64(ptr, value) (__sync_fetch_and_add((ptr), (value)) + (value))
626 #else
627 #if defined(STARPU_HAVE_CMPXCHG)
628 STARPU_ATOMIC_SOMETHING(add, old + value)
629 #define STARPU_ATOMIC_ADD(ptr, value) starpu_atomic_add(ptr, value)
630 #endif
631 #if defined(STARPU_HAVE_CMPXCHGL)
632 STARPU_ATOMIC_SOMETHINGL(add, old + value)
633 #define STARPU_ATOMIC_ADDL(ptr, value) starpu_atomic_addl(ptr, value)
634 #endif
635 #if defined(STARPU_HAVE_CMPXCHG64)
636 STARPU_ATOMIC_SOMETHING64(add, old + value)
637 #define STARPU_ATOMIC_ADD64(ptr, value) starpu_atomic_add64(ptr, value)
638 #endif
639 #endif
640 
641 #ifdef STARPU_HAVE_SYNC_FETCH_AND_OR
642 #define STARPU_ATOMIC_OR(ptr, value) (__sync_fetch_and_or((ptr), (value)))
643 #define STARPU_ATOMIC_ORL(ptr, value) (__sync_fetch_and_or((ptr), (value)))
644 #define STARPU_ATOMIC_OR64(ptr, value) (__sync_fetch_and_or((ptr), (value)))
645 #else
646 #if defined(STARPU_HAVE_CMPXCHG)
647 STARPU_ATOMIC_SOMETHING(or, old | value)
648 #define STARPU_ATOMIC_OR(ptr, value) starpu_atomic_or(ptr, value)
649 #endif
650 #if defined(STARPU_HAVE_CMPXCHGL)
651 STARPU_ATOMIC_SOMETHINGL(or, old | value)
652 #define STARPU_ATOMIC_ORL(ptr, value) starpu_atomic_orl(ptr, value)
653 #endif
654 #if defined(STARPU_HAVE_CMPXCHG64)
655 STARPU_ATOMIC_SOMETHING64(or, old | value)
656 #define STARPU_ATOMIC_OR64(ptr, value) starpu_atomic_or64(ptr, value)
657 #endif
658 #endif
659 
660 /* Try to replace `old' with `value' at `ptr'. Returns true iff the swap was successful. */
661 #ifdef STARPU_HAVE_SYNC_BOOL_COMPARE_AND_SWAP
662 #define STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value) (__sync_bool_compare_and_swap((ptr), (old), (value)))
663 #define STARPU_BOOL_COMPARE_AND_SWAP32(ptr, old, value) STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value)
664 #define STARPU_BOOL_COMPARE_AND_SWAP64(ptr, old, value) STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value)
665 #else
666 #ifdef STARPU_HAVE_CMPXCHG
667 #define STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value) (_starpu_cmpxchg((ptr), (old), (value)) == (old))
668 #endif
669 #ifdef STARPU_HAVE_CMPXCHG32
670 #define STARPU_BOOL_COMPARE_AND_SWAP32(ptr, old, value) (_starpu_cmpxchg32((ptr), (old), (value)) == (old))
671 #endif
672 #ifdef STARPU_HAVE_CMPXCHG64
673 #define STARPU_BOOL_COMPARE_AND_SWAP64(ptr, old, value) (_starpu_cmpxchg64((ptr), (old), (value)) == (old))
674 #endif
675 #endif
676 
677 #if UINTPTR_MAX == UINT64_MAX
678 #define STARPU_BOOL_COMPARE_AND_SWAP_PTR(ptr, old, value) STARPU_BOOL_COMPARE_AND_SWAP64(ptr, old, value)
679 #else
680 #define STARPU_BOOL_COMPARE_AND_SWAP_PTR(ptr, old, value) STARPU_BOOL_COMPARE_AND_SWAP32(ptr, old, value)
681 #endif
682 
683 /* Try to replace `old' with `value' at `ptr'. Returns the value actually seen at `ptr'. */
684 #ifdef STARPU_HAVE_SYNC_VAL_COMPARE_AND_SWAP
685 #define STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value) (__sync_val_compare_and_swap((ptr), (old), (value)))
686 #define STARPU_VAL_COMPARE_AND_SWAP32(ptr, old, value) STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value)
687 #define STARPU_VAL_COMPARE_AND_SWAP64(ptr, old, value) STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value)
688 #else
689 #ifdef STARPU_HAVE_CMPXCHG
690 #define STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value) (_starpu_cmpxchg((ptr), (old), (value)))
691 #endif
692 #ifdef STARPU_HAVE_CMPXCHG32
693 #define STARPU_VAL_COMPARE_AND_SWAP32(ptr, old, value) (_starpu_cmpxchg32((ptr), (old), (value)))
694 #endif
695 #ifdef STARPU_HAVE_CMPXCHG64
696 #define STARPU_VAL_COMPARE_AND_SWAP64(ptr, old, value) (_starpu_cmpxchg64((ptr), (old), (value)))
697 #endif
698 #endif
699 
700 #if UINTPTR_MAX == UINT64_MAX
701 #define STARPU_VAL_COMPARE_AND_SWAP_PTR(ptr, old, value) STARPU_VAL_COMPARE_AND_SWAP64(ptr, old, value)
702 #else
703 #define STARPU_VAL_COMPARE_AND_SWAP_PTR(ptr, old, value) STARPU_VAL_COMPARE_AND_SWAP32(ptr, old, value)
704 #endif
705 
706 #ifdef STARPU_HAVE_ATOMIC_EXCHANGE_N
707 #define STARPU_VAL_EXCHANGE(ptr, value) (__atomic_exchange_n((ptr), (value), __ATOMIC_SEQ_CST))
708 #define STARPU_VAL_EXCHANGEL(ptr, value) STARPU_VAL_EXCHANGE((ptr)(value))
709 #define STARPU_VAL_EXCHANGE32(ptr, value) STARPU_VAL_EXCHANGE((ptr)(value))
710 #define STARPU_VAL_EXCHANGE64(ptr, value) STARPU_VAL_EXCHANGE((ptr)(value))
711 #else
712 #ifdef STARPU_HAVE_XCHG
713 #define STARPU_VAL_EXCHANGE(ptr, value) (_starpu_xchg((ptr), (value)))
714 #endif
715 #ifdef STARPU_HAVE_XCHGL
716 #define STARPU_VAL_EXCHANGEL(ptr, value) (_starpu_xchgl((ptr), (value)))
717 #endif
718 #ifdef STARPU_HAVE_XCHG32
719 #define STARPU_VAL_EXCHANGE32(ptr, value) (_starpu_xchg32((ptr), (value)))
720 #endif
721 #ifdef STARPU_HAVE_XCHG64
722 #define STARPU_VAL_EXCHANGE64(ptr, value) (_starpu_xchg64((ptr), (value)))
723 #endif
724 #endif
725 
726 /* Returns the previous value */
727 #ifdef STARPU_HAVE_SYNC_LOCK_TEST_AND_SET
728 #define STARPU_TEST_AND_SET(ptr, value) (__sync_lock_test_and_set((ptr), (value)))
729 #define STARPU_RELEASE(ptr) (__sync_lock_release((ptr)))
730 #elif defined(STARPU_HAVE_XCHG)
731 #define STARPU_TEST_AND_SET(ptr, value) (_starpu_xchg((ptr), (value)))
732 #define STARPU_RELEASE(ptr) (_starpu_xchg((ptr), 0))
733 #endif
734 
735 #ifdef STARPU_HAVE_SYNC_SYNCHRONIZE
736 #define STARPU_SYNCHRONIZE() __sync_synchronize()
737 #elif defined(__i386__)
738 #define STARPU_SYNCHRONIZE() __asm__ __volatile__("lock; addl $0,0(%%esp)" :: \
739  : "memory")
740 #elif defined(__KNC__) || defined(__KNF__)
741 #define STARPU_SYNCHRONIZE() __asm__ __volatile__("lock; addl $0,0(%%rsp)" :: \
742  : "memory")
743 #elif defined(__x86_64__)
744 #define STARPU_SYNCHRONIZE() __asm__ __volatile__("mfence" :: \
745  : "memory")
746 #elif defined(__ppc__) || defined(__ppc64__)
747 #define STARPU_SYNCHRONIZE() __asm__ __volatile__("sync" :: \
748  : "memory")
749 #endif
750 
754 #if defined(__x86_64__)
755 #define STARPU_RMB() __asm__ __volatile__("lfence" :: \
756  : "memory")
757 #elif defined(__aarch64__)
758 #define STARPU_RMB() __asm__ __volatile__("dsb ld" :: \
759  : "memory")
760 #else
761 #define STARPU_RMB() STARPU_SYNCHRONIZE()
762 #endif
763 
767 #if defined(__x86_64__)
768 #define STARPU_WMB() __asm__ __volatile__("sfence" :: \
769  : "memory")
770 #elif defined(__aarch64__)
771 #define STARPU_WMB() __asm__ __volatile__("dsb st" :: \
772  : "memory")
773 #else
774 #define STARPU_WMB() STARPU_SYNCHRONIZE()
775 #endif
776 
777 #if defined(__i386__) || defined(__x86_64__)
778 #define STARPU_CACHELINE_SIZE 64
779 #elif defined(__ppc__) || defined(__ppc64__) || defined(__ia64__)
780 #define STARPU_CACHELINE_SIZE 128
781 #elif defined(__s390__) || defined(__s390x__)
782 #define STARPU_CACHELINE_SIZE 256
783 #else
784 /* Conservative default */
785 #define STARPU_CACHELINE_SIZE 1024
786 #endif
787 
788 #ifdef _WIN32
789 /* Try to fetch the system definition of timespec */
790 #include <sys/types.h>
791 #include <sys/stat.h>
792 #ifdef HAVE_UNISTD_H
793 #include <unistd.h>
794 #endif
795 #include <time.h>
796 #if !defined(_MSC_VER) || defined(BUILDING_STARPU)
797 #include <pthread.h>
798 #endif
799 #if !defined(STARPU_HAVE_STRUCT_TIMESPEC) || (defined(_MSC_VER) && _MSC_VER < 1900)
800 /* If it didn't get defined in the standard places, then define it ourself */
801 #ifndef STARPU_TIMESPEC_DEFINED
802 #define STARPU_TIMESPEC_DEFINED 1
803 struct timespec
804 {
805  time_t tv_sec; /* Seconds */
806  long tv_nsec; /* Nanoseconds */
807 };
808 #endif /* STARPU_TIMESPEC_DEFINED */
809 #endif /* STARPU_HAVE_STRUCT_TIMESPEC */
810 /* Fetch gettimeofday on mingw/cygwin */
811 #if defined(__MINGW32__) || defined(__CYGWIN__)
812 #include <sys/time.h>
813 #endif
814 #else
815 #include <sys/time.h>
816 #endif /* _WIN32 */
817 
820 #ifdef __cplusplus
821 }
822 #endif
823 
824 #endif /* __STARPU_UTIL_H__ */