16 #include "kmp_atomic.h" 19 typedef unsigned char uchar;
20 typedef unsigned short ushort;
543 #ifndef KMP_GOMP_COMPAT 544 int __kmp_atomic_mode = 1;
546 int __kmp_atomic_mode = 2;
551 kmp_atomic_lock_t __kmp_atomic_lock;
552 kmp_atomic_lock_t __kmp_atomic_lock_1i;
553 kmp_atomic_lock_t __kmp_atomic_lock_2i;
554 kmp_atomic_lock_t __kmp_atomic_lock_4i;
555 kmp_atomic_lock_t __kmp_atomic_lock_4r;
556 kmp_atomic_lock_t __kmp_atomic_lock_8i;
557 kmp_atomic_lock_t __kmp_atomic_lock_8r;
558 kmp_atomic_lock_t __kmp_atomic_lock_8c;
559 kmp_atomic_lock_t __kmp_atomic_lock_10r;
560 kmp_atomic_lock_t __kmp_atomic_lock_16r;
561 kmp_atomic_lock_t __kmp_atomic_lock_16c;
562 kmp_atomic_lock_t __kmp_atomic_lock_20c;
563 kmp_atomic_lock_t __kmp_atomic_lock_32c;
573 #define KMP_ATOMIC_VOLATILE volatile 575 #if ( KMP_ARCH_X86 ) && KMP_HAVE_QUAD 577 static inline void operator +=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q += rhs.q; };
578 static inline void operator -=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q -= rhs.q; };
579 static inline void operator *=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q *= rhs.q; };
580 static inline void operator /=( Quad_a4_t & lhs, Quad_a4_t & rhs ) { lhs.q /= rhs.q; };
581 static inline bool operator < ( Quad_a4_t & lhs, Quad_a4_t & rhs ) {
return lhs.q < rhs.q; }
582 static inline bool operator > ( Quad_a4_t & lhs, Quad_a4_t & rhs ) {
return lhs.q > rhs.q; }
584 static inline void operator +=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q += rhs.q; };
585 static inline void operator -=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q -= rhs.q; };
586 static inline void operator *=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q *= rhs.q; };
587 static inline void operator /=( Quad_a16_t & lhs, Quad_a16_t & rhs ) { lhs.q /= rhs.q; };
588 static inline bool operator < ( Quad_a16_t & lhs, Quad_a16_t & rhs ) {
return lhs.q < rhs.q; }
589 static inline bool operator > ( Quad_a16_t & lhs, Quad_a16_t & rhs ) {
return lhs.q > rhs.q; }
591 static inline void operator +=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q += rhs.q; };
592 static inline void operator -=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q -= rhs.q; };
593 static inline void operator *=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q *= rhs.q; };
594 static inline void operator /=( kmp_cmplx128_a4_t & lhs, kmp_cmplx128_a4_t & rhs ) { lhs.q /= rhs.q; };
596 static inline void operator +=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q += rhs.q; };
597 static inline void operator -=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q -= rhs.q; };
598 static inline void operator *=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q *= rhs.q; };
599 static inline void operator /=( kmp_cmplx128_a16_t & lhs, kmp_cmplx128_a16_t & rhs ) { lhs.q /= rhs.q; };
612 #define KMP_CHECK_GTID \ 613 if ( gtid == KMP_GTID_UNKNOWN ) { \ 614 gtid = __kmp_entry_gtid(); \ 615 } // check and get gtid when needed 621 #define ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE, RET_TYPE) \ 622 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \ 624 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 625 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid )); 629 #define ATOMIC_LOCK0 __kmp_atomic_lock // all types, for Gnu compat 630 #define ATOMIC_LOCK1i __kmp_atomic_lock_1i // char 631 #define ATOMIC_LOCK2i __kmp_atomic_lock_2i // short 632 #define ATOMIC_LOCK4i __kmp_atomic_lock_4i // long int 633 #define ATOMIC_LOCK4r __kmp_atomic_lock_4r // float 634 #define ATOMIC_LOCK8i __kmp_atomic_lock_8i // long long int 635 #define ATOMIC_LOCK8r __kmp_atomic_lock_8r // double 636 #define ATOMIC_LOCK8c __kmp_atomic_lock_8c // float complex 637 #define ATOMIC_LOCK10r __kmp_atomic_lock_10r // long double 638 #define ATOMIC_LOCK16r __kmp_atomic_lock_16r // _Quad 639 #define ATOMIC_LOCK16c __kmp_atomic_lock_16c // double complex 640 #define ATOMIC_LOCK20c __kmp_atomic_lock_20c // long double complex 641 #define ATOMIC_LOCK32c __kmp_atomic_lock_32c // _Quad complex 649 #define OP_CRITICAL(OP,LCK_ID) \ 650 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 654 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); 679 #ifdef KMP_GOMP_COMPAT 680 # define OP_GOMP_CRITICAL(OP,FLAG) \ 681 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 683 OP_CRITICAL( OP, 0 ); \ 687 # define OP_GOMP_CRITICAL(OP,FLAG) 691 # define KMP_DO_PAUSE _mm_delay_32( 1 ) 693 # define KMP_DO_PAUSE KMP_CPU_PAUSE() 701 #define OP_CMPXCHG(TYPE,BITS,OP) \ 703 TYPE old_value, new_value; \ 704 old_value = *(TYPE volatile *)lhs; \ 705 new_value = old_value OP rhs; \ 706 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 707 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 708 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \ 712 old_value = *(TYPE volatile *)lhs; \ 713 new_value = old_value OP rhs; \ 725 #define OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \ 729 kmp_int##BITS *vvv; \ 731 struct _sss old_value, new_value; \ 732 old_value.vvv = ( kmp_int##BITS * )&old_value.cmp; \ 733 new_value.vvv = ( kmp_int##BITS * )&new_value.cmp; \ 734 *old_value.vvv = * ( volatile kmp_int##BITS * ) lhs; \ 735 new_value.cmp = old_value.cmp OP rhs; \ 736 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 737 *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \ 738 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv ) ) \ 742 *old_value.vvv = * ( volatile kmp_int##BITS * ) lhs; \ 743 new_value.cmp = old_value.cmp OP rhs; \ 747 #endif // USE_CMPXCHG_FIX 749 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 753 #define ATOMIC_FIXED_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 754 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 755 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 757 KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \ 760 #define ATOMIC_CMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 761 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 762 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 763 OP_CMPXCHG(TYPE,BITS,OP) \ 768 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 769 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 770 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 771 OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \ 779 #define ATOMIC_FIXED_ADD(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 780 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 781 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 782 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 784 KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \ 787 OP_CRITICAL(OP##=,LCK_ID) \ 791 #define ATOMIC_CMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 792 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 793 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 794 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 795 OP_CMPXCHG(TYPE,BITS,OP) \ 798 OP_CRITICAL(OP##=,LCK_ID) \ 804 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 805 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 806 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 807 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 808 OP_CMPXCHG(TYPE,BITS,OP) \ 811 OP_CRITICAL(OP##=,LCK_ID) \ 815 #endif // USE_CMPXCHG_FIX 819 ATOMIC_FIXED_ADD( fixed4, add, kmp_int32, 32, +, 4i, 3, 0 )
820 ATOMIC_FIXED_ADD( fixed4, sub, kmp_int32, 32, -, 4i, 3, 0 )
822 ATOMIC_CMPXCHG( float4, add, kmp_real32, 32, +, 4r, 3, KMP_ARCH_X86 )
823 ATOMIC_CMPXCHG( float4, sub, kmp_real32, 32, -, 4r, 3, KMP_ARCH_X86 )
826 ATOMIC_FIXED_ADD( fixed8, add, kmp_int64, 64, +, 8i, 7, KMP_ARCH_X86 )
827 ATOMIC_FIXED_ADD( fixed8, sub, kmp_int64, 64, -, 8i, 7, KMP_ARCH_X86 )
829 ATOMIC_CMPXCHG( float8, add, kmp_real64, 64, +, 8r, 7, KMP_ARCH_X86 )
830 ATOMIC_CMPXCHG( float8, sub, kmp_real64, 64, -, 8r, 7, KMP_ARCH_X86 )
847 ATOMIC_CMPXCHG( fixed1, add, kmp_int8, 8, +, 1i, 0, KMP_ARCH_X86 )
848 ATOMIC_CMPXCHG( fixed1, andb, kmp_int8, 8, &, 1i, 0, 0 )
849 ATOMIC_CMPXCHG( fixed1, div, kmp_int8, 8, /, 1i, 0, KMP_ARCH_X86 )
850 ATOMIC_CMPXCHG( fixed1u, div, kmp_uint8, 8, /, 1i, 0, KMP_ARCH_X86 )
851 ATOMIC_CMPXCHG( fixed1, mul, kmp_int8, 8, *, 1i, 0, KMP_ARCH_X86 )
852 ATOMIC_CMPXCHG( fixed1, orb, kmp_int8, 8, |, 1i, 0, 0 )
853 ATOMIC_CMPXCHG( fixed1, shl, kmp_int8, 8, <<, 1i, 0, KMP_ARCH_X86 )
854 ATOMIC_CMPXCHG( fixed1, shr, kmp_int8, 8, >>, 1i, 0, KMP_ARCH_X86 )
855 ATOMIC_CMPXCHG( fixed1u, shr, kmp_uint8, 8, >>, 1i, 0, KMP_ARCH_X86 )
856 ATOMIC_CMPXCHG( fixed1, sub, kmp_int8, 8, -, 1i, 0, KMP_ARCH_X86 )
857 ATOMIC_CMPXCHG( fixed1, xor, kmp_int8, 8, ^, 1i, 0, 0 )
858 ATOMIC_CMPXCHG( fixed2, add, kmp_int16, 16, +, 2i, 1, KMP_ARCH_X86 )
859 ATOMIC_CMPXCHG( fixed2, andb, kmp_int16, 16, &, 2i, 1, 0 )
860 ATOMIC_CMPXCHG( fixed2, div, kmp_int16, 16, /, 2i, 1, KMP_ARCH_X86 )
861 ATOMIC_CMPXCHG( fixed2u, div, kmp_uint16, 16, /, 2i, 1, KMP_ARCH_X86 )
862 ATOMIC_CMPXCHG( fixed2, mul, kmp_int16, 16, *, 2i, 1, KMP_ARCH_X86 )
863 ATOMIC_CMPXCHG( fixed2, orb, kmp_int16, 16, |, 2i, 1, 0 )
864 ATOMIC_CMPXCHG( fixed2, shl, kmp_int16, 16, <<, 2i, 1, KMP_ARCH_X86 )
865 ATOMIC_CMPXCHG( fixed2, shr, kmp_int16, 16, >>, 2i, 1, KMP_ARCH_X86 )
866 ATOMIC_CMPXCHG( fixed2u, shr, kmp_uint16, 16, >>, 2i, 1, KMP_ARCH_X86 )
867 ATOMIC_CMPXCHG( fixed2, sub, kmp_int16, 16, -, 2i, 1, KMP_ARCH_X86 )
868 ATOMIC_CMPXCHG( fixed2, xor, kmp_int16, 16, ^, 2i, 1, 0 )
869 ATOMIC_CMPXCHG( fixed4, andb, kmp_int32, 32, &, 4i, 3, 0 )
870 ATOMIC_CMPXCHG( fixed4, div, kmp_int32, 32, /, 4i, 3, KMP_ARCH_X86 )
871 ATOMIC_CMPXCHG( fixed4u, div, kmp_uint32, 32, /, 4i, 3, KMP_ARCH_X86 )
872 ATOMIC_CMPXCHG( fixed4, mul, kmp_int32, 32, *, 4i, 3, KMP_ARCH_X86 )
873 ATOMIC_CMPXCHG( fixed4, orb, kmp_int32, 32, |, 4i, 3, 0 )
874 ATOMIC_CMPXCHG( fixed4, shl, kmp_int32, 32, <<, 4i, 3, KMP_ARCH_X86 )
875 ATOMIC_CMPXCHG( fixed4, shr, kmp_int32, 32, >>, 4i, 3, KMP_ARCH_X86 )
876 ATOMIC_CMPXCHG( fixed4u, shr, kmp_uint32, 32, >>, 4i, 3, KMP_ARCH_X86 )
877 ATOMIC_CMPXCHG( fixed4, xor, kmp_int32, 32, ^, 4i, 3, 0 )
878 ATOMIC_CMPXCHG( fixed8, andb, kmp_int64, 64, &, 8i, 7, KMP_ARCH_X86 )
879 ATOMIC_CMPXCHG( fixed8, div, kmp_int64, 64, /, 8i, 7, KMP_ARCH_X86 )
880 ATOMIC_CMPXCHG( fixed8u, div, kmp_uint64, 64, /, 8i, 7, KMP_ARCH_X86 )
881 ATOMIC_CMPXCHG( fixed8, mul, kmp_int64, 64, *, 8i, 7, KMP_ARCH_X86 )
882 ATOMIC_CMPXCHG( fixed8, orb, kmp_int64, 64, |, 8i, 7, KMP_ARCH_X86 )
883 ATOMIC_CMPXCHG( fixed8, shl, kmp_int64, 64, <<, 8i, 7, KMP_ARCH_X86 )
884 ATOMIC_CMPXCHG( fixed8, shr, kmp_int64, 64, >>, 8i, 7, KMP_ARCH_X86 )
885 ATOMIC_CMPXCHG( fixed8u, shr, kmp_uint64, 64, >>, 8i, 7, KMP_ARCH_X86 )
886 ATOMIC_CMPXCHG( fixed8, xor, kmp_int64, 64, ^, 8i, 7, KMP_ARCH_X86 )
887 ATOMIC_CMPXCHG( float4, div, kmp_real32, 32, /, 4r, 3, KMP_ARCH_X86 )
888 ATOMIC_CMPXCHG( float4, mul, kmp_real32, 32, *, 4r, 3, KMP_ARCH_X86 )
889 ATOMIC_CMPXCHG( float8, div, kmp_real64, 64, /, 8r, 7, KMP_ARCH_X86 )
890 ATOMIC_CMPXCHG( float8, mul, kmp_real64, 64, *, 8r, 7, KMP_ARCH_X86 )
901 #define ATOMIC_CRIT_L(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 902 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 903 OP_GOMP_CRITICAL( = *lhs OP, GOMP_FLAG ) \ 904 OP_CRITICAL( = *lhs OP, LCK_ID ) \ 907 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 911 #define ATOMIC_CMPX_L(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 912 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 913 OP_GOMP_CRITICAL( = *lhs OP, GOMP_FLAG ) \ 914 OP_CMPXCHG(TYPE,BITS,OP) \ 920 #define ATOMIC_CMPX_L(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 921 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 922 OP_GOMP_CRITICAL(= *lhs OP,GOMP_FLAG) \ 923 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 924 OP_CMPXCHG(TYPE,BITS,OP) \ 927 OP_CRITICAL(= *lhs OP,LCK_ID) \ 932 ATOMIC_CMPX_L( fixed1, andl,
char, 8, &&, 1i, 0, KMP_ARCH_X86 )
933 ATOMIC_CMPX_L( fixed1, orl,
char, 8, ||, 1i, 0, KMP_ARCH_X86 )
934 ATOMIC_CMPX_L( fixed2, andl,
short, 16, &&, 2i, 1, KMP_ARCH_X86 )
935 ATOMIC_CMPX_L( fixed2, orl,
short, 16, ||, 2i, 1, KMP_ARCH_X86 )
936 ATOMIC_CMPX_L( fixed4, andl, kmp_int32, 32, &&, 4i, 3, 0 )
937 ATOMIC_CMPX_L( fixed4, orl, kmp_int32, 32, ||, 4i, 3, 0 )
938 ATOMIC_CMPX_L( fixed8, andl, kmp_int64, 64, &&, 8i, 7, KMP_ARCH_X86 )
939 ATOMIC_CMPX_L( fixed8, orl, kmp_int64, 64, ||, 8i, 7, KMP_ARCH_X86 )
952 #define MIN_MAX_CRITSECT(OP,LCK_ID) \ 953 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 955 if ( *lhs OP rhs ) { \ 958 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); 961 #ifdef KMP_GOMP_COMPAT 962 #define GOMP_MIN_MAX_CRITSECT(OP,FLAG) \ 963 if (( FLAG ) && ( __kmp_atomic_mode == 2 )) { \ 965 MIN_MAX_CRITSECT( OP, 0 ); \ 969 #define GOMP_MIN_MAX_CRITSECT(OP,FLAG) 973 #define MIN_MAX_CMPXCHG(TYPE,BITS,OP) \ 975 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 978 old_value = temp_val; \ 979 while ( old_value OP rhs && \ 980 ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 981 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 982 *VOLATILE_CAST(kmp_int##BITS *) &rhs ) ) \ 986 old_value = temp_val; \ 992 #define MIN_MAX_CRITICAL(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 993 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 994 if ( *lhs OP rhs ) { \ 995 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \ 996 MIN_MAX_CRITSECT(OP,LCK_ID) \ 1000 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1004 #define MIN_MAX_COMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 1005 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1006 if ( *lhs OP rhs ) { \ 1007 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \ 1008 MIN_MAX_CMPXCHG(TYPE,BITS,OP) \ 1015 #define MIN_MAX_COMPXCHG(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 1016 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1017 if ( *lhs OP rhs ) { \ 1018 GOMP_MIN_MAX_CRITSECT(OP,GOMP_FLAG) \ 1019 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 1020 MIN_MAX_CMPXCHG(TYPE,BITS,OP) \ 1023 MIN_MAX_CRITSECT(OP,LCK_ID) \ 1029 MIN_MAX_COMPXCHG( fixed1, max,
char, 8, <, 1i, 0, KMP_ARCH_X86 )
1030 MIN_MAX_COMPXCHG( fixed1, min,
char, 8, >, 1i, 0, KMP_ARCH_X86 )
1031 MIN_MAX_COMPXCHG( fixed2, max,
short, 16, <, 2i, 1, KMP_ARCH_X86 )
1032 MIN_MAX_COMPXCHG( fixed2, min,
short, 16, >, 2i, 1, KMP_ARCH_X86 )
1033 MIN_MAX_COMPXCHG( fixed4, max, kmp_int32, 32, <, 4i, 3, 0 )
1034 MIN_MAX_COMPXCHG( fixed4, min, kmp_int32, 32, >, 4i, 3, 0 )
1035 MIN_MAX_COMPXCHG( fixed8, max, kmp_int64, 64, <, 8i, 7, KMP_ARCH_X86 )
1036 MIN_MAX_COMPXCHG( fixed8, min, kmp_int64, 64, >, 8i, 7, KMP_ARCH_X86 )
1037 MIN_MAX_COMPXCHG( float4, max, kmp_real32, 32, <, 4r, 3, KMP_ARCH_X86 )
1038 MIN_MAX_COMPXCHG( float4, min, kmp_real32, 32, >, 4r, 3, KMP_ARCH_X86 )
1039 MIN_MAX_COMPXCHG( float8, max, kmp_real64, 64, <, 8r, 7, KMP_ARCH_X86 )
1040 MIN_MAX_COMPXCHG( float8, min, kmp_real64, 64, >, 8r, 7, KMP_ARCH_X86 )
1042 MIN_MAX_CRITICAL( float16, max, QUAD_LEGACY, <, 16r, 1 )
1043 MIN_MAX_CRITICAL( float16, min, QUAD_LEGACY, >, 16r, 1 )
1044 #if ( KMP_ARCH_X86 ) 1045 MIN_MAX_CRITICAL( float16, max_a16, Quad_a16_t, <, 16r, 1 )
1046 MIN_MAX_CRITICAL( float16, min_a16, Quad_a16_t, >, 16r, 1 )
1052 #define ATOMIC_CRIT_EQV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 1053 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1054 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \ 1055 OP_CRITICAL(^=~,LCK_ID) \ 1059 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1062 #define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 1063 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1064 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \ 1065 OP_CMPXCHG(TYPE,BITS,OP) \ 1071 #define ATOMIC_CMPX_EQV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,MASK,GOMP_FLAG) \ 1072 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1073 OP_GOMP_CRITICAL(^=~,GOMP_FLAG) \ 1074 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 1075 OP_CMPXCHG(TYPE,BITS,OP) \ 1078 OP_CRITICAL(^=~,LCK_ID) \ 1083 ATOMIC_CMPXCHG( fixed1, neqv, kmp_int8, 8, ^, 1i, 0, KMP_ARCH_X86 )
1084 ATOMIC_CMPXCHG( fixed2, neqv, kmp_int16, 16, ^, 2i, 1, KMP_ARCH_X86 )
1085 ATOMIC_CMPXCHG( fixed4, neqv, kmp_int32, 32, ^, 4i, 3, KMP_ARCH_X86 )
1086 ATOMIC_CMPXCHG( fixed8, neqv, kmp_int64, 64, ^, 8i, 7, KMP_ARCH_X86 )
1087 ATOMIC_CMPX_EQV( fixed1, eqv, kmp_int8, 8, ^~, 1i, 0, KMP_ARCH_X86 )
1088 ATOMIC_CMPX_EQV( fixed2, eqv, kmp_int16, 16, ^~, 2i, 1, KMP_ARCH_X86 )
1089 ATOMIC_CMPX_EQV( fixed4, eqv, kmp_int32, 32, ^~, 4i, 3, KMP_ARCH_X86 )
1090 ATOMIC_CMPX_EQV( fixed8, eqv, kmp_int64, 64, ^~, 8i, 7, KMP_ARCH_X86 )
1098 #define ATOMIC_CRITICAL(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 1099 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1100 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 1101 OP_CRITICAL(OP##=,LCK_ID) \ 1106 ATOMIC_CRITICAL( float10, add,
long double, +, 10r, 1 )
1107 ATOMIC_CRITICAL( float10, sub,
long double, -, 10r, 1 )
1108 ATOMIC_CRITICAL( float10, mul,
long double, *, 10r, 1 )
1109 ATOMIC_CRITICAL( float10, div,
long double, /, 10r, 1 )
1112 ATOMIC_CRITICAL( float16, add, QUAD_LEGACY, +, 16r, 1 )
1113 ATOMIC_CRITICAL( float16, sub, QUAD_LEGACY, -, 16r, 1 )
1114 ATOMIC_CRITICAL( float16, mul, QUAD_LEGACY, *, 16r, 1 )
1115 ATOMIC_CRITICAL( float16, div, QUAD_LEGACY, /, 16r, 1 )
1116 #if ( KMP_ARCH_X86 ) 1117 ATOMIC_CRITICAL( float16, add_a16, Quad_a16_t, +, 16r, 1 )
1118 ATOMIC_CRITICAL( float16, sub_a16, Quad_a16_t, -, 16r, 1 )
1119 ATOMIC_CRITICAL( float16, mul_a16, Quad_a16_t, *, 16r, 1 )
1120 ATOMIC_CRITICAL( float16, div_a16, Quad_a16_t, /, 16r, 1 )
1127 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, add, kmp_cmplx32, 64, +, 8c, 7, 1 )
1128 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7, 1 )
1129 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7, 1 )
1130 ATOMIC_CMPXCHG_WORKAROUND( cmplx4, div, kmp_cmplx32, 64, /, 8c, 7, 1 )
1133 ATOMIC_CRITICAL( cmplx4, add, kmp_cmplx32, +, 8c, 1 )
1134 ATOMIC_CRITICAL( cmplx4, sub, kmp_cmplx32, -, 8c, 1 )
1135 ATOMIC_CRITICAL( cmplx4, mul, kmp_cmplx32, *, 8c, 1 )
1136 ATOMIC_CRITICAL( cmplx4, div, kmp_cmplx32, /, 8c, 1 )
1137 #endif // USE_CMPXCHG_FIX 1139 ATOMIC_CRITICAL( cmplx8, add, kmp_cmplx64, +, 16c, 1 )
1140 ATOMIC_CRITICAL( cmplx8, sub, kmp_cmplx64, -, 16c, 1 )
1141 ATOMIC_CRITICAL( cmplx8, mul, kmp_cmplx64, *, 16c, 1 )
1142 ATOMIC_CRITICAL( cmplx8, div, kmp_cmplx64, /, 16c, 1 )
1143 ATOMIC_CRITICAL( cmplx10, add, kmp_cmplx80, +, 20c, 1 )
1144 ATOMIC_CRITICAL( cmplx10, sub, kmp_cmplx80, -, 20c, 1 )
1145 ATOMIC_CRITICAL( cmplx10, mul, kmp_cmplx80, *, 20c, 1 )
1146 ATOMIC_CRITICAL( cmplx10, div, kmp_cmplx80, /, 20c, 1 )
1148 ATOMIC_CRITICAL( cmplx16, add, CPLX128_LEG, +, 32c, 1 )
1149 ATOMIC_CRITICAL( cmplx16, sub, CPLX128_LEG, -, 32c, 1 )
1150 ATOMIC_CRITICAL( cmplx16, mul, CPLX128_LEG, *, 32c, 1 )
1151 ATOMIC_CRITICAL( cmplx16, div, CPLX128_LEG, /, 32c, 1 )
1152 #if ( KMP_ARCH_X86 ) 1153 ATOMIC_CRITICAL( cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c, 1 )
1154 ATOMIC_CRITICAL( cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c, 1 )
1155 ATOMIC_CRITICAL( cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c, 1 )
1156 ATOMIC_CRITICAL( cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c, 1 )
1164 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1172 #define OP_CRITICAL_REV(OP,LCK_ID) \ 1173 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 1175 (*lhs) = (rhs) OP (*lhs); \ 1177 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); 1179 #ifdef KMP_GOMP_COMPAT 1180 #define OP_GOMP_CRITICAL_REV(OP,FLAG) \ 1181 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 1183 OP_CRITICAL_REV( OP, 0 ); \ 1187 #define OP_GOMP_CRITICAL_REV(OP,FLAG) 1195 #define ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE, RET_TYPE) \ 1196 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \ 1198 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 1199 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid )); 1208 #define OP_CMPXCHG_REV(TYPE,BITS,OP) \ 1210 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1211 TYPE old_value, new_value; \ 1213 old_value = temp_val; \ 1214 new_value = rhs OP old_value; \ 1215 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 1216 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 1217 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \ 1222 old_value = temp_val; \ 1223 new_value = rhs OP old_value; \ 1228 #define ATOMIC_CMPXCHG_REV(TYPE_ID,OP_ID,TYPE,BITS,OP,LCK_ID,GOMP_FLAG) \ 1229 ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE,void) \ 1230 OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \ 1231 OP_CMPXCHG_REV(TYPE,BITS,OP) \ 1248 ATOMIC_CMPXCHG_REV( fixed1, div, kmp_int8, 8, /, 1i, KMP_ARCH_X86 )
1249 ATOMIC_CMPXCHG_REV( fixed1u, div, kmp_uint8, 8, /, 1i, KMP_ARCH_X86 )
1250 ATOMIC_CMPXCHG_REV( fixed1, shl, kmp_int8, 8, <<, 1i, KMP_ARCH_X86 )
1251 ATOMIC_CMPXCHG_REV( fixed1, shr, kmp_int8, 8, >>, 1i, KMP_ARCH_X86 )
1252 ATOMIC_CMPXCHG_REV( fixed1u, shr, kmp_uint8, 8, >>, 1i, KMP_ARCH_X86 )
1253 ATOMIC_CMPXCHG_REV( fixed1, sub, kmp_int8, 8, -, 1i, KMP_ARCH_X86 )
1255 ATOMIC_CMPXCHG_REV( fixed2, div, kmp_int16, 16, /, 2i, KMP_ARCH_X86 )
1256 ATOMIC_CMPXCHG_REV( fixed2u, div, kmp_uint16, 16, /, 2i, KMP_ARCH_X86 )
1257 ATOMIC_CMPXCHG_REV( fixed2, shl, kmp_int16, 16, <<, 2i, KMP_ARCH_X86 )
1258 ATOMIC_CMPXCHG_REV( fixed2, shr, kmp_int16, 16, >>, 2i, KMP_ARCH_X86 )
1259 ATOMIC_CMPXCHG_REV( fixed2u, shr, kmp_uint16, 16, >>, 2i, KMP_ARCH_X86 )
1260 ATOMIC_CMPXCHG_REV( fixed2, sub, kmp_int16, 16, -, 2i, KMP_ARCH_X86 )
1262 ATOMIC_CMPXCHG_REV( fixed4, div, kmp_int32, 32, /, 4i, KMP_ARCH_X86 )
1263 ATOMIC_CMPXCHG_REV( fixed4u, div, kmp_uint32, 32, /, 4i, KMP_ARCH_X86 )
1264 ATOMIC_CMPXCHG_REV( fixed4, shl, kmp_int32, 32, <<, 4i, KMP_ARCH_X86 )
1265 ATOMIC_CMPXCHG_REV( fixed4, shr, kmp_int32, 32, >>, 4i, KMP_ARCH_X86 )
1266 ATOMIC_CMPXCHG_REV( fixed4u, shr, kmp_uint32, 32, >>, 4i, KMP_ARCH_X86 )
1267 ATOMIC_CMPXCHG_REV( fixed4, sub, kmp_int32, 32, -, 4i, KMP_ARCH_X86 )
1269 ATOMIC_CMPXCHG_REV( fixed8, div, kmp_int64, 64, /, 8i, KMP_ARCH_X86 )
1270 ATOMIC_CMPXCHG_REV( fixed8u, div, kmp_uint64, 64, /, 8i, KMP_ARCH_X86 )
1271 ATOMIC_CMPXCHG_REV( fixed8, shl, kmp_int64, 64, <<, 8i, KMP_ARCH_X86 )
1272 ATOMIC_CMPXCHG_REV( fixed8, shr, kmp_int64, 64, >>, 8i, KMP_ARCH_X86 )
1273 ATOMIC_CMPXCHG_REV( fixed8u, shr, kmp_uint64, 64, >>, 8i, KMP_ARCH_X86 )
1274 ATOMIC_CMPXCHG_REV( fixed8, sub, kmp_int64, 64, -, 8i, KMP_ARCH_X86 )
1276 ATOMIC_CMPXCHG_REV( float4, div, kmp_real32, 32, /, 4r, KMP_ARCH_X86 )
1277 ATOMIC_CMPXCHG_REV( float4, sub, kmp_real32, 32, -, 4r, KMP_ARCH_X86 )
1279 ATOMIC_CMPXCHG_REV( float8, div, kmp_real64, 64, /, 8r, KMP_ARCH_X86 )
1280 ATOMIC_CMPXCHG_REV( float8, sub, kmp_real64, 64, -, 8r, KMP_ARCH_X86 )
1288 #define ATOMIC_CRITICAL_REV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 1289 ATOMIC_BEGIN_REV(TYPE_ID,OP_ID,TYPE,void) \ 1290 OP_GOMP_CRITICAL_REV(OP,GOMP_FLAG) \ 1291 OP_CRITICAL_REV(OP,LCK_ID) \ 1296 ATOMIC_CRITICAL_REV( float10, sub,
long double, -, 10r, 1 )
1297 ATOMIC_CRITICAL_REV( float10, div,
long double, /, 10r, 1 )
1300 ATOMIC_CRITICAL_REV( float16, sub, QUAD_LEGACY, -, 16r, 1 )
1301 ATOMIC_CRITICAL_REV( float16, div, QUAD_LEGACY, /, 16r, 1 )
1302 #if ( KMP_ARCH_X86 ) 1303 ATOMIC_CRITICAL_REV( float16, sub_a16, Quad_a16_t, -, 16r, 1 )
1304 ATOMIC_CRITICAL_REV( float16, div_a16, Quad_a16_t, /, 16r, 1 )
1309 ATOMIC_CRITICAL_REV( cmplx4, sub, kmp_cmplx32, -, 8c, 1 )
1310 ATOMIC_CRITICAL_REV( cmplx4, div, kmp_cmplx32, /, 8c, 1 )
1311 ATOMIC_CRITICAL_REV( cmplx8, sub, kmp_cmplx64, -, 16c, 1 )
1312 ATOMIC_CRITICAL_REV( cmplx8, div, kmp_cmplx64, /, 16c, 1 )
1313 ATOMIC_CRITICAL_REV( cmplx10, sub, kmp_cmplx80, -, 20c, 1 )
1314 ATOMIC_CRITICAL_REV( cmplx10, div, kmp_cmplx80, /, 20c, 1 )
1316 ATOMIC_CRITICAL_REV( cmplx16, sub, CPLX128_LEG, -, 32c, 1 )
1317 ATOMIC_CRITICAL_REV( cmplx16, div, CPLX128_LEG, /, 32c, 1 )
1318 #if ( KMP_ARCH_X86 ) 1319 ATOMIC_CRITICAL_REV( cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c, 1 )
1320 ATOMIC_CRITICAL_REV( cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c, 1 )
1325 #endif //KMP_ARCH_X86 || KMP_ARCH_X86_64 1328 #endif //OMP_40_ENABLED 1342 #define ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1343 void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( ident_t *id_ref, int gtid, TYPE * lhs, RTYPE rhs ) \ 1345 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 1346 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", gtid )); 1349 #define ATOMIC_CRITICAL_FP(TYPE_ID,TYPE,OP_ID,OP,RTYPE_ID,RTYPE,LCK_ID,GOMP_FLAG) \ 1350 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1351 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 1352 OP_CRITICAL(OP##=,LCK_ID) \ 1356 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1359 #define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \ 1360 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1361 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 1362 OP_CMPXCHG(TYPE,BITS,OP) \ 1368 #define ATOMIC_CMPXCHG_MIX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \ 1369 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1370 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 1371 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 1372 OP_CMPXCHG(TYPE,BITS,OP) \ 1375 OP_CRITICAL(OP##=,LCK_ID) \ 1381 ATOMIC_CMPXCHG_MIX( fixed1,
char, mul, 8, *, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 )
1382 ATOMIC_CMPXCHG_MIX( fixed1,
char, div, 8, /, float8, kmp_real64, 1i, 0, KMP_ARCH_X86 )
1383 ATOMIC_CMPXCHG_MIX( fixed2,
short, mul, 16, *, float8, kmp_real64, 2i, 1, KMP_ARCH_X86 )
1384 ATOMIC_CMPXCHG_MIX( fixed2,
short, div, 16, /, float8, kmp_real64, 2i, 1, KMP_ARCH_X86 )
1385 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3, 0 )
1386 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3, 0 )
1387 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7, KMP_ARCH_X86 )
1388 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7, KMP_ARCH_X86 )
1389 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1390 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1391 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1392 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3, KMP_ARCH_X86 )
1396 ATOMIC_CMPXCHG_MIX( fixed1,
char, add, 8, +, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1397 ATOMIC_CMPXCHG_MIX( fixed1,
char, sub, 8, -, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1398 ATOMIC_CMPXCHG_MIX( fixed1,
char, mul, 8, *, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1399 ATOMIC_CMPXCHG_MIX( fixed1,
char, div, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1400 ATOMIC_CMPXCHG_MIX( fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0, KMP_ARCH_X86 )
1402 ATOMIC_CMPXCHG_MIX( fixed2,
short, add, 16, +, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1403 ATOMIC_CMPXCHG_MIX( fixed2,
short, sub, 16, -, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1404 ATOMIC_CMPXCHG_MIX( fixed2,
short, mul, 16, *, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1405 ATOMIC_CMPXCHG_MIX( fixed2,
short, div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1406 ATOMIC_CMPXCHG_MIX( fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1, KMP_ARCH_X86 )
1408 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3, 0 )
1409 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3, 0 )
1410 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3, 0 )
1411 ATOMIC_CMPXCHG_MIX( fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3, 0 )
1412 ATOMIC_CMPXCHG_MIX( fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3, 0 )
1414 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1415 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1416 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1417 ATOMIC_CMPXCHG_MIX( fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1418 ATOMIC_CMPXCHG_MIX( fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7, KMP_ARCH_X86 )
1420 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1421 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1422 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1423 ATOMIC_CMPXCHG_MIX( float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3, KMP_ARCH_X86 )
1425 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1426 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1427 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1428 ATOMIC_CMPXCHG_MIX( float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7, KMP_ARCH_X86 )
1430 ATOMIC_CRITICAL_FP( float10,
long double, add, +, fp, _Quad, 10r, 1 )
1431 ATOMIC_CRITICAL_FP( float10,
long double, sub, -, fp, _Quad, 10r, 1 )
1432 ATOMIC_CRITICAL_FP( float10,
long double, mul, *, fp, _Quad, 10r, 1 )
1433 ATOMIC_CRITICAL_FP( float10,
long double, div, /, fp, _Quad, 10r, 1 )
1436 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1441 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \ 1442 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1443 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 1444 OP_CMPXCHG_WORKAROUND(TYPE,BITS,OP) \ 1448 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \ 1449 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1450 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 1451 OP_CMPXCHG(TYPE,BITS,OP) \ 1453 #endif // USE_CMPXCHG_FIX 1457 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID,TYPE,OP_ID,BITS,OP,RTYPE_ID,RTYPE,LCK_ID,MASK,GOMP_FLAG) \ 1458 ATOMIC_BEGIN_MIX(TYPE_ID,TYPE,OP_ID,RTYPE_ID,RTYPE) \ 1459 OP_GOMP_CRITICAL(OP##=,GOMP_FLAG) \ 1460 if ( ! ( (kmp_uintptr_t) lhs & 0x##MASK) ) { \ 1461 OP_CMPXCHG(TYPE,BITS,OP) \ 1464 OP_CRITICAL(OP##=,LCK_ID) \ 1469 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1470 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1471 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1472 ATOMIC_CMPXCHG_CMPLX( cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c, 7, KMP_ARCH_X86 )
1475 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1487 #define ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE, RET_TYPE) \ 1488 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * loc ) \ 1490 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 1491 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid )); 1503 #define OP_CMPXCHG_READ(TYPE,BITS,OP) \ 1505 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1508 kmp_int##BITS i_val; \ 1510 union f_i_union old_value; \ 1512 old_value.f_val = temp_val; \ 1513 old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( (kmp_int##BITS *) loc, \ 1514 *VOLATILE_CAST(kmp_int##BITS *) &old_value.i_val, \ 1515 *VOLATILE_CAST(kmp_int##BITS *) &old_value.i_val ); \ 1516 new_value = old_value.f_val; \ 1526 #define OP_CRITICAL_READ(OP,LCK_ID) \ 1527 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 1529 new_value = (*loc); \ 1531 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); 1534 #ifdef KMP_GOMP_COMPAT 1535 #define OP_GOMP_CRITICAL_READ(OP,FLAG) \ 1536 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 1538 OP_CRITICAL_READ( OP, 0 ); \ 1542 #define OP_GOMP_CRITICAL_READ(OP,FLAG) 1546 #define ATOMIC_FIXED_READ(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 1547 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \ 1549 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \ 1550 new_value = KMP_TEST_THEN_ADD##BITS( loc, OP 0 ); \ 1554 #define ATOMIC_CMPXCHG_READ(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 1555 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \ 1557 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \ 1558 OP_CMPXCHG_READ(TYPE,BITS,OP) \ 1565 #define ATOMIC_CRITICAL_READ(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 1566 ATOMIC_BEGIN_READ(TYPE_ID,OP_ID,TYPE,TYPE) \ 1568 OP_GOMP_CRITICAL_READ(OP##=,GOMP_FLAG) \ 1569 OP_CRITICAL_READ(OP,LCK_ID) \ 1577 #if ( KMP_OS_WINDOWS ) 1579 #define OP_CRITICAL_READ_WRK(OP,LCK_ID) \ 1580 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 1584 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); 1586 #ifdef KMP_GOMP_COMPAT 1587 #define OP_GOMP_CRITICAL_READ_WRK(OP,FLAG) \ 1588 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 1590 OP_CRITICAL_READ_WRK( OP, 0 ); \ 1593 #define OP_GOMP_CRITICAL_READ_WRK(OP,FLAG) 1596 #define ATOMIC_BEGIN_READ_WRK(TYPE_ID,OP_ID,TYPE) \ 1597 void __kmpc_atomic_##TYPE_ID##_##OP_ID( TYPE * out, ident_t *id_ref, int gtid, TYPE * loc ) \ 1599 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 1600 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid )); 1603 #define ATOMIC_CRITICAL_READ_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 1604 ATOMIC_BEGIN_READ_WRK(TYPE_ID,OP_ID,TYPE) \ 1605 OP_GOMP_CRITICAL_READ_WRK(OP##=,GOMP_FLAG) \ 1606 OP_CRITICAL_READ_WRK(OP,LCK_ID) \ 1609 #endif // KMP_OS_WINDOWS 1613 ATOMIC_FIXED_READ( fixed4, rd, kmp_int32, 32, +, 0 )
1614 ATOMIC_FIXED_READ( fixed8, rd, kmp_int64, 64, +, KMP_ARCH_X86 )
1615 ATOMIC_CMPXCHG_READ( float4, rd, kmp_real32, 32, +, KMP_ARCH_X86 )
1616 ATOMIC_CMPXCHG_READ( float8, rd, kmp_real64, 64, +, KMP_ARCH_X86 )
1619 ATOMIC_CMPXCHG_READ( fixed1, rd, kmp_int8, 8, +, KMP_ARCH_X86 )
1620 ATOMIC_CMPXCHG_READ( fixed2, rd, kmp_int16, 16, +, KMP_ARCH_X86 )
1622 ATOMIC_CRITICAL_READ( float10, rd,
long double, +, 10r, 1 )
1624 ATOMIC_CRITICAL_READ( float16, rd, QUAD_LEGACY, +, 16r, 1 )
1625 #endif // KMP_HAVE_QUAD 1628 #if ( KMP_OS_WINDOWS ) 1629 ATOMIC_CRITICAL_READ_WRK( cmplx4, rd, kmp_cmplx32, +, 8c, 1 )
1631 ATOMIC_CRITICAL_READ( cmplx4, rd, kmp_cmplx32, +, 8c, 1 )
1633 ATOMIC_CRITICAL_READ( cmplx8, rd, kmp_cmplx64, +, 16c, 1 )
1634 ATOMIC_CRITICAL_READ( cmplx10, rd, kmp_cmplx80, +, 20c, 1 )
1636 ATOMIC_CRITICAL_READ( cmplx16, rd, CPLX128_LEG, +, 32c, 1 )
1637 #if ( KMP_ARCH_X86 ) 1638 ATOMIC_CRITICAL_READ( float16, a16_rd, Quad_a16_t, +, 16r, 1 )
1639 ATOMIC_CRITICAL_READ( cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c, 1 )
1648 #define ATOMIC_XCHG_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 1649 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1650 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \ 1651 KMP_XCHG_FIXED##BITS( lhs, rhs ); \ 1654 #define ATOMIC_XCHG_FLOAT_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 1655 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1656 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \ 1657 KMP_XCHG_REAL##BITS( lhs, rhs ); \ 1668 #define OP_CMPXCHG_WR(TYPE,BITS,OP) \ 1670 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1671 TYPE old_value, new_value; \ 1673 old_value = temp_val; \ 1675 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 1676 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 1677 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \ 1682 old_value = temp_val; \ 1688 #define ATOMIC_CMPXCHG_WR(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 1689 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1690 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \ 1691 OP_CMPXCHG_WR(TYPE,BITS,OP) \ 1699 #define ATOMIC_CRITICAL_WR(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 1700 ATOMIC_BEGIN(TYPE_ID,OP_ID,TYPE,void) \ 1701 OP_GOMP_CRITICAL(OP,GOMP_FLAG) \ 1702 OP_CRITICAL(OP,LCK_ID) \ 1706 ATOMIC_XCHG_WR( fixed1, wr, kmp_int8, 8, =, KMP_ARCH_X86 )
1707 ATOMIC_XCHG_WR( fixed2, wr, kmp_int16, 16, =, KMP_ARCH_X86 )
1708 ATOMIC_XCHG_WR( fixed4, wr, kmp_int32, 32, =, KMP_ARCH_X86 )
1709 #if ( KMP_ARCH_X86 ) 1710 ATOMIC_CMPXCHG_WR( fixed8, wr, kmp_int64, 64, =, KMP_ARCH_X86 )
1712 ATOMIC_XCHG_WR( fixed8, wr, kmp_int64, 64, =, KMP_ARCH_X86 )
1715 ATOMIC_XCHG_FLOAT_WR( float4, wr, kmp_real32, 32, =, KMP_ARCH_X86 )
1716 #if ( KMP_ARCH_X86 ) 1717 ATOMIC_CMPXCHG_WR( float8, wr, kmp_real64, 64, =, KMP_ARCH_X86 )
1719 ATOMIC_XCHG_FLOAT_WR( float8, wr, kmp_real64, 64, =, KMP_ARCH_X86 )
1722 ATOMIC_CRITICAL_WR( float10, wr,
long double, =, 10r, 1 )
1724 ATOMIC_CRITICAL_WR( float16, wr, QUAD_LEGACY, =, 16r, 1 )
1726 ATOMIC_CRITICAL_WR( cmplx4, wr, kmp_cmplx32, =, 8c, 1 )
1727 ATOMIC_CRITICAL_WR( cmplx8, wr, kmp_cmplx64, =, 16c, 1 )
1728 ATOMIC_CRITICAL_WR( cmplx10, wr, kmp_cmplx80, =, 20c, 1 )
1730 ATOMIC_CRITICAL_WR( cmplx16, wr, CPLX128_LEG, =, 32c, 1 )
1731 #if ( KMP_ARCH_X86 ) 1732 ATOMIC_CRITICAL_WR( float16, a16_wr, Quad_a16_t, =, 16r, 1 )
1733 ATOMIC_CRITICAL_WR( cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c, 1 )
1746 #define ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,RET_TYPE) \ 1747 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, int flag ) \ 1749 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 1750 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid )); 1758 #define OP_CRITICAL_CPT(OP,LCK_ID) \ 1759 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 1763 new_value = (*lhs); \ 1765 new_value = (*lhs); \ 1769 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 1773 #ifdef KMP_GOMP_COMPAT 1774 #define OP_GOMP_CRITICAL_CPT(OP,FLAG) \ 1775 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 1777 OP_CRITICAL_CPT( OP##=, 0 ); \ 1780 #define OP_GOMP_CRITICAL_CPT(OP,FLAG) 1790 #define OP_CMPXCHG_CPT(TYPE,BITS,OP) \ 1792 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1793 TYPE old_value, new_value; \ 1795 old_value = temp_val; \ 1796 new_value = old_value OP rhs; \ 1797 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 1798 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 1799 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \ 1804 old_value = temp_val; \ 1805 new_value = old_value OP rhs; \ 1814 #define ATOMIC_CMPXCHG_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 1815 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 1817 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \ 1818 OP_CMPXCHG_CPT(TYPE,BITS,OP) \ 1822 #define ATOMIC_FIXED_ADD_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 1823 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 1824 TYPE old_value, new_value; \ 1825 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \ 1827 old_value = KMP_TEST_THEN_ADD##BITS( lhs, OP rhs ); \ 1829 return old_value OP rhs; \ 1835 ATOMIC_FIXED_ADD_CPT( fixed4, add_cpt, kmp_int32, 32, +, 0 )
1836 ATOMIC_FIXED_ADD_CPT( fixed4, sub_cpt, kmp_int32, 32, -, 0 )
1837 ATOMIC_FIXED_ADD_CPT( fixed8, add_cpt, kmp_int64, 64, +, KMP_ARCH_X86 )
1838 ATOMIC_FIXED_ADD_CPT( fixed8, sub_cpt, kmp_int64, 64, -, KMP_ARCH_X86 )
1840 ATOMIC_CMPXCHG_CPT( float4, add_cpt, kmp_real32, 32, +, KMP_ARCH_X86 )
1841 ATOMIC_CMPXCHG_CPT( float4, sub_cpt, kmp_real32, 32, -, KMP_ARCH_X86 )
1842 ATOMIC_CMPXCHG_CPT( float8, add_cpt, kmp_real64, 64, +, KMP_ARCH_X86 )
1843 ATOMIC_CMPXCHG_CPT( float8, sub_cpt, kmp_real64, 64, -, KMP_ARCH_X86 )
1857 ATOMIC_CMPXCHG_CPT( fixed1, add_cpt, kmp_int8, 8, +, KMP_ARCH_X86 )
1858 ATOMIC_CMPXCHG_CPT( fixed1, andb_cpt, kmp_int8, 8, &, 0 )
1859 ATOMIC_CMPXCHG_CPT( fixed1, div_cpt, kmp_int8, 8, /, KMP_ARCH_X86 )
1860 ATOMIC_CMPXCHG_CPT( fixed1u, div_cpt, kmp_uint8, 8, /, KMP_ARCH_X86 )
1861 ATOMIC_CMPXCHG_CPT( fixed1, mul_cpt, kmp_int8, 8, *, KMP_ARCH_X86 )
1862 ATOMIC_CMPXCHG_CPT( fixed1, orb_cpt, kmp_int8, 8, |, 0 )
1863 ATOMIC_CMPXCHG_CPT( fixed1, shl_cpt, kmp_int8, 8, <<, KMP_ARCH_X86 )
1864 ATOMIC_CMPXCHG_CPT( fixed1, shr_cpt, kmp_int8, 8, >>, KMP_ARCH_X86 )
1865 ATOMIC_CMPXCHG_CPT( fixed1u, shr_cpt, kmp_uint8, 8, >>, KMP_ARCH_X86 )
1866 ATOMIC_CMPXCHG_CPT( fixed1, sub_cpt, kmp_int8, 8, -, KMP_ARCH_X86 )
1867 ATOMIC_CMPXCHG_CPT( fixed1, xor_cpt, kmp_int8, 8, ^, 0 )
1868 ATOMIC_CMPXCHG_CPT( fixed2, add_cpt, kmp_int16, 16, +, KMP_ARCH_X86 )
1869 ATOMIC_CMPXCHG_CPT( fixed2, andb_cpt, kmp_int16, 16, &, 0 )
1870 ATOMIC_CMPXCHG_CPT( fixed2, div_cpt, kmp_int16, 16, /, KMP_ARCH_X86 )
1871 ATOMIC_CMPXCHG_CPT( fixed2u, div_cpt, kmp_uint16, 16, /, KMP_ARCH_X86 )
1872 ATOMIC_CMPXCHG_CPT( fixed2, mul_cpt, kmp_int16, 16, *, KMP_ARCH_X86 )
1873 ATOMIC_CMPXCHG_CPT( fixed2, orb_cpt, kmp_int16, 16, |, 0 )
1874 ATOMIC_CMPXCHG_CPT( fixed2, shl_cpt, kmp_int16, 16, <<, KMP_ARCH_X86 )
1875 ATOMIC_CMPXCHG_CPT( fixed2, shr_cpt, kmp_int16, 16, >>, KMP_ARCH_X86 )
1876 ATOMIC_CMPXCHG_CPT( fixed2u, shr_cpt, kmp_uint16, 16, >>, KMP_ARCH_X86 )
1877 ATOMIC_CMPXCHG_CPT( fixed2, sub_cpt, kmp_int16, 16, -, KMP_ARCH_X86 )
1878 ATOMIC_CMPXCHG_CPT( fixed2, xor_cpt, kmp_int16, 16, ^, 0 )
1879 ATOMIC_CMPXCHG_CPT( fixed4, andb_cpt, kmp_int32, 32, &, 0 )
1880 ATOMIC_CMPXCHG_CPT( fixed4, div_cpt, kmp_int32, 32, /, KMP_ARCH_X86 )
1881 ATOMIC_CMPXCHG_CPT( fixed4u, div_cpt, kmp_uint32, 32, /, KMP_ARCH_X86 )
1882 ATOMIC_CMPXCHG_CPT( fixed4, mul_cpt, kmp_int32, 32, *, KMP_ARCH_X86 )
1883 ATOMIC_CMPXCHG_CPT( fixed4, orb_cpt, kmp_int32, 32, |, 0 )
1884 ATOMIC_CMPXCHG_CPT( fixed4, shl_cpt, kmp_int32, 32, <<, KMP_ARCH_X86 )
1885 ATOMIC_CMPXCHG_CPT( fixed4, shr_cpt, kmp_int32, 32, >>, KMP_ARCH_X86 )
1886 ATOMIC_CMPXCHG_CPT( fixed4u, shr_cpt, kmp_uint32, 32, >>, KMP_ARCH_X86 )
1887 ATOMIC_CMPXCHG_CPT( fixed4, xor_cpt, kmp_int32, 32, ^, 0 )
1888 ATOMIC_CMPXCHG_CPT( fixed8, andb_cpt, kmp_int64, 64, &, KMP_ARCH_X86 )
1889 ATOMIC_CMPXCHG_CPT( fixed8, div_cpt, kmp_int64, 64, /, KMP_ARCH_X86 )
1890 ATOMIC_CMPXCHG_CPT( fixed8u, div_cpt, kmp_uint64, 64, /, KMP_ARCH_X86 )
1891 ATOMIC_CMPXCHG_CPT( fixed8, mul_cpt, kmp_int64, 64, *, KMP_ARCH_X86 )
1892 ATOMIC_CMPXCHG_CPT( fixed8, orb_cpt, kmp_int64, 64, |, KMP_ARCH_X86 )
1893 ATOMIC_CMPXCHG_CPT( fixed8, shl_cpt, kmp_int64, 64, <<, KMP_ARCH_X86 )
1894 ATOMIC_CMPXCHG_CPT( fixed8, shr_cpt, kmp_int64, 64, >>, KMP_ARCH_X86 )
1895 ATOMIC_CMPXCHG_CPT( fixed8u, shr_cpt, kmp_uint64, 64, >>, KMP_ARCH_X86 )
1896 ATOMIC_CMPXCHG_CPT( fixed8, xor_cpt, kmp_int64, 64, ^, KMP_ARCH_X86 )
1897 ATOMIC_CMPXCHG_CPT( float4, div_cpt, kmp_real32, 32, /, KMP_ARCH_X86 )
1898 ATOMIC_CMPXCHG_CPT( float4, mul_cpt, kmp_real32, 32, *, KMP_ARCH_X86 )
1899 ATOMIC_CMPXCHG_CPT( float8, div_cpt, kmp_real64, 64, /, KMP_ARCH_X86 )
1900 ATOMIC_CMPXCHG_CPT( float8, mul_cpt, kmp_real64, 64, *, KMP_ARCH_X86 )
1913 #define OP_CRITICAL_L_CPT(OP,LCK_ID) \ 1914 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 1919 new_value = (*lhs); \ 1921 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); 1924 #ifdef KMP_GOMP_COMPAT 1925 #define OP_GOMP_CRITICAL_L_CPT(OP,FLAG) \ 1926 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 1928 OP_CRITICAL_L_CPT( OP, 0 ); \ 1932 #define OP_GOMP_CRITICAL_L_CPT(OP,FLAG) 1937 #define ATOMIC_CMPX_L_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 1938 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 1940 OP_GOMP_CRITICAL_L_CPT( = *lhs OP, GOMP_FLAG ) \ 1941 OP_CMPXCHG_CPT(TYPE,BITS,OP) \ 1944 ATOMIC_CMPX_L_CPT( fixed1, andl_cpt,
char, 8, &&, KMP_ARCH_X86 )
1945 ATOMIC_CMPX_L_CPT( fixed1, orl_cpt,
char, 8, ||, KMP_ARCH_X86 )
1946 ATOMIC_CMPX_L_CPT( fixed2, andl_cpt,
short, 16, &&, KMP_ARCH_X86 )
1947 ATOMIC_CMPX_L_CPT( fixed2, orl_cpt,
short, 16, ||, KMP_ARCH_X86 )
1948 ATOMIC_CMPX_L_CPT( fixed4, andl_cpt, kmp_int32, 32, &&, 0 )
1949 ATOMIC_CMPX_L_CPT( fixed4, orl_cpt, kmp_int32, 32, ||, 0 )
1950 ATOMIC_CMPX_L_CPT( fixed8, andl_cpt, kmp_int64, 64, &&, KMP_ARCH_X86 )
1951 ATOMIC_CMPX_L_CPT( fixed8, orl_cpt, kmp_int64, 64, ||, KMP_ARCH_X86 )
1964 #define MIN_MAX_CRITSECT_CPT(OP,LCK_ID) \ 1965 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 1967 if ( *lhs OP rhs ) { \ 1973 new_value = old_value; \ 1975 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 1979 #ifdef KMP_GOMP_COMPAT 1980 #define GOMP_MIN_MAX_CRITSECT_CPT(OP,FLAG) \ 1981 if (( FLAG ) && ( __kmp_atomic_mode == 2 )) { \ 1983 MIN_MAX_CRITSECT_CPT( OP, 0 ); \ 1986 #define GOMP_MIN_MAX_CRITSECT_CPT(OP,FLAG) 1990 #define MIN_MAX_CMPXCHG_CPT(TYPE,BITS,OP) \ 1992 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 1995 old_value = temp_val; \ 1996 while ( old_value OP rhs && \ 1997 ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 1998 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 1999 *VOLATILE_CAST(kmp_int##BITS *) &rhs ) ) \ 2003 old_value = temp_val; \ 2013 #define MIN_MAX_CRITICAL_CPT(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 2014 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 2015 TYPE new_value, old_value; \ 2016 if ( *lhs OP rhs ) { \ 2017 GOMP_MIN_MAX_CRITSECT_CPT(OP,GOMP_FLAG) \ 2018 MIN_MAX_CRITSECT_CPT(OP,LCK_ID) \ 2023 #define MIN_MAX_COMPXCHG_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 2024 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 2025 TYPE new_value, old_value; \ 2026 if ( *lhs OP rhs ) { \ 2027 GOMP_MIN_MAX_CRITSECT_CPT(OP,GOMP_FLAG) \ 2028 MIN_MAX_CMPXCHG_CPT(TYPE,BITS,OP) \ 2034 MIN_MAX_COMPXCHG_CPT( fixed1, max_cpt,
char, 8, <, KMP_ARCH_X86 )
2035 MIN_MAX_COMPXCHG_CPT( fixed1, min_cpt,
char, 8, >, KMP_ARCH_X86 )
2036 MIN_MAX_COMPXCHG_CPT( fixed2, max_cpt,
short, 16, <, KMP_ARCH_X86 )
2037 MIN_MAX_COMPXCHG_CPT( fixed2, min_cpt,
short, 16, >, KMP_ARCH_X86 )
2038 MIN_MAX_COMPXCHG_CPT( fixed4, max_cpt, kmp_int32, 32, <, 0 )
2039 MIN_MAX_COMPXCHG_CPT( fixed4, min_cpt, kmp_int32, 32, >, 0 )
2040 MIN_MAX_COMPXCHG_CPT( fixed8, max_cpt, kmp_int64, 64, <, KMP_ARCH_X86 )
2041 MIN_MAX_COMPXCHG_CPT( fixed8, min_cpt, kmp_int64, 64, >, KMP_ARCH_X86 )
2042 MIN_MAX_COMPXCHG_CPT( float4, max_cpt, kmp_real32, 32, <, KMP_ARCH_X86 )
2043 MIN_MAX_COMPXCHG_CPT( float4, min_cpt, kmp_real32, 32, >, KMP_ARCH_X86 )
2044 MIN_MAX_COMPXCHG_CPT( float8, max_cpt, kmp_real64, 64, <, KMP_ARCH_X86 )
2045 MIN_MAX_COMPXCHG_CPT( float8, min_cpt, kmp_real64, 64, >, KMP_ARCH_X86 )
2047 MIN_MAX_CRITICAL_CPT( float16, max_cpt, QUAD_LEGACY, <, 16r, 1 )
2048 MIN_MAX_CRITICAL_CPT( float16, min_cpt, QUAD_LEGACY, >, 16r, 1 )
2049 #if ( KMP_ARCH_X86 ) 2050 MIN_MAX_CRITICAL_CPT( float16, max_a16_cpt, Quad_a16_t, <, 16r, 1 )
2051 MIN_MAX_CRITICAL_CPT( float16, min_a16_cpt, Quad_a16_t, >, 16r, 1 )
2056 #ifdef KMP_GOMP_COMPAT 2057 #define OP_GOMP_CRITICAL_EQV_CPT(OP,FLAG) \ 2058 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 2060 OP_CRITICAL_CPT( OP, 0 ); \ 2063 #define OP_GOMP_CRITICAL_EQV_CPT(OP,FLAG) 2066 #define ATOMIC_CMPX_EQV_CPT(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 2067 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 2069 OP_GOMP_CRITICAL_EQV_CPT(^=~,GOMP_FLAG) \ 2070 OP_CMPXCHG_CPT(TYPE,BITS,OP) \ 2075 ATOMIC_CMPXCHG_CPT( fixed1, neqv_cpt, kmp_int8, 8, ^, KMP_ARCH_X86 )
2076 ATOMIC_CMPXCHG_CPT( fixed2, neqv_cpt, kmp_int16, 16, ^, KMP_ARCH_X86 )
2077 ATOMIC_CMPXCHG_CPT( fixed4, neqv_cpt, kmp_int32, 32, ^, KMP_ARCH_X86 )
2078 ATOMIC_CMPXCHG_CPT( fixed8, neqv_cpt, kmp_int64, 64, ^, KMP_ARCH_X86 )
2079 ATOMIC_CMPX_EQV_CPT( fixed1, eqv_cpt, kmp_int8, 8, ^~, KMP_ARCH_X86 )
2080 ATOMIC_CMPX_EQV_CPT( fixed2, eqv_cpt, kmp_int16, 16, ^~, KMP_ARCH_X86 )
2081 ATOMIC_CMPX_EQV_CPT( fixed4, eqv_cpt, kmp_int32, 32, ^~, KMP_ARCH_X86 )
2082 ATOMIC_CMPX_EQV_CPT( fixed8, eqv_cpt, kmp_int64, 64, ^~, KMP_ARCH_X86 )
2089 #define ATOMIC_CRITICAL_CPT(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 2090 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 2092 OP_GOMP_CRITICAL_CPT(OP,GOMP_FLAG) \ 2093 OP_CRITICAL_CPT(OP##=,LCK_ID) \ 2100 #define OP_CRITICAL_CPT_WRK(OP,LCK_ID) \ 2101 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2111 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2115 #ifdef KMP_GOMP_COMPAT 2116 #define OP_GOMP_CRITICAL_CPT_WRK(OP,FLAG) \ 2117 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 2119 OP_CRITICAL_CPT_WRK( OP##=, 0 ); \ 2122 #define OP_GOMP_CRITICAL_CPT_WRK(OP,FLAG) 2126 #define ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \ 2127 void __kmpc_atomic_##TYPE_ID##_##OP_ID( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, TYPE * out, int flag ) \ 2129 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 2130 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid )); 2133 #define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 2134 ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \ 2135 OP_GOMP_CRITICAL_CPT_WRK(OP,GOMP_FLAG) \ 2136 OP_CRITICAL_CPT_WRK(OP##=,LCK_ID) \ 2142 ATOMIC_CRITICAL_CPT( float10, add_cpt,
long double, +, 10r, 1 )
2143 ATOMIC_CRITICAL_CPT( float10, sub_cpt,
long double, -, 10r, 1 )
2144 ATOMIC_CRITICAL_CPT( float10, mul_cpt,
long double, *, 10r, 1 )
2145 ATOMIC_CRITICAL_CPT( float10, div_cpt,
long double, /, 10r, 1 )
2148 ATOMIC_CRITICAL_CPT( float16, add_cpt, QUAD_LEGACY, +, 16r, 1 )
2149 ATOMIC_CRITICAL_CPT( float16, sub_cpt, QUAD_LEGACY, -, 16r, 1 )
2150 ATOMIC_CRITICAL_CPT( float16, mul_cpt, QUAD_LEGACY, *, 16r, 1 )
2151 ATOMIC_CRITICAL_CPT( float16, div_cpt, QUAD_LEGACY, /, 16r, 1 )
2152 #if ( KMP_ARCH_X86 ) 2153 ATOMIC_CRITICAL_CPT( float16, add_a16_cpt, Quad_a16_t, +, 16r, 1 )
2154 ATOMIC_CRITICAL_CPT( float16, sub_a16_cpt, Quad_a16_t, -, 16r, 1 )
2155 ATOMIC_CRITICAL_CPT( float16, mul_a16_cpt, Quad_a16_t, *, 16r, 1 )
2156 ATOMIC_CRITICAL_CPT( float16, div_a16_cpt, Quad_a16_t, /, 16r, 1 )
2163 ATOMIC_CRITICAL_CPT_WRK( cmplx4, add_cpt, kmp_cmplx32, +, 8c, 1 )
2164 ATOMIC_CRITICAL_CPT_WRK( cmplx4, sub_cpt, kmp_cmplx32, -, 8c, 1 )
2165 ATOMIC_CRITICAL_CPT_WRK( cmplx4, mul_cpt, kmp_cmplx32, *, 8c, 1 )
2166 ATOMIC_CRITICAL_CPT_WRK( cmplx4, div_cpt, kmp_cmplx32, /, 8c, 1 )
2168 ATOMIC_CRITICAL_CPT( cmplx8, add_cpt, kmp_cmplx64, +, 16c, 1 )
2169 ATOMIC_CRITICAL_CPT( cmplx8, sub_cpt, kmp_cmplx64, -, 16c, 1 )
2170 ATOMIC_CRITICAL_CPT( cmplx8, mul_cpt, kmp_cmplx64, *, 16c, 1 )
2171 ATOMIC_CRITICAL_CPT( cmplx8, div_cpt, kmp_cmplx64, /, 16c, 1 )
2172 ATOMIC_CRITICAL_CPT( cmplx10, add_cpt, kmp_cmplx80, +, 20c, 1 )
2173 ATOMIC_CRITICAL_CPT( cmplx10, sub_cpt, kmp_cmplx80, -, 20c, 1 )
2174 ATOMIC_CRITICAL_CPT( cmplx10, mul_cpt, kmp_cmplx80, *, 20c, 1 )
2175 ATOMIC_CRITICAL_CPT( cmplx10, div_cpt, kmp_cmplx80, /, 20c, 1 )
2177 ATOMIC_CRITICAL_CPT( cmplx16, add_cpt, CPLX128_LEG, +, 32c, 1 )
2178 ATOMIC_CRITICAL_CPT( cmplx16, sub_cpt, CPLX128_LEG, -, 32c, 1 )
2179 ATOMIC_CRITICAL_CPT( cmplx16, mul_cpt, CPLX128_LEG, *, 32c, 1 )
2180 ATOMIC_CRITICAL_CPT( cmplx16, div_cpt, CPLX128_LEG, /, 32c, 1 )
2181 #if ( KMP_ARCH_X86 ) 2182 ATOMIC_CRITICAL_CPT( cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c, 1 )
2183 ATOMIC_CRITICAL_CPT( cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c, 1 )
2184 ATOMIC_CRITICAL_CPT( cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c, 1 )
2185 ATOMIC_CRITICAL_CPT( cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c, 1 )
2200 #define OP_CRITICAL_CPT_REV(OP,LCK_ID) \ 2201 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2205 (*lhs) = (rhs) OP (*lhs); \ 2206 new_value = (*lhs); \ 2208 new_value = (*lhs);\ 2209 (*lhs) = (rhs) OP (*lhs); \ 2211 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2215 #ifdef KMP_GOMP_COMPAT 2216 #define OP_GOMP_CRITICAL_CPT_REV(OP,FLAG) \ 2217 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 2219 OP_CRITICAL_CPT_REV( OP, 0 ); \ 2222 #define OP_GOMP_CRITICAL_CPT_REV(OP,FLAG) 2232 #define OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \ 2234 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2235 TYPE old_value, new_value; \ 2237 old_value = temp_val; \ 2238 new_value = rhs OP old_value; \ 2239 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 2240 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 2241 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \ 2246 old_value = temp_val; \ 2247 new_value = rhs OP old_value; \ 2256 #define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID,OP_ID,TYPE,BITS,OP,GOMP_FLAG) \ 2257 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 2259 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2260 OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \ 2261 OP_CMPXCHG_CPT_REV(TYPE,BITS,OP) \ 2265 ATOMIC_CMPXCHG_CPT_REV( fixed1, div_cpt_rev, kmp_int8, 8, /, KMP_ARCH_X86 )
2266 ATOMIC_CMPXCHG_CPT_REV( fixed1u, div_cpt_rev, kmp_uint8, 8, /, KMP_ARCH_X86 )
2267 ATOMIC_CMPXCHG_CPT_REV( fixed1, shl_cpt_rev, kmp_int8, 8, <<, KMP_ARCH_X86 )
2268 ATOMIC_CMPXCHG_CPT_REV( fixed1, shr_cpt_rev, kmp_int8, 8, >>, KMP_ARCH_X86 )
2269 ATOMIC_CMPXCHG_CPT_REV( fixed1u, shr_cpt_rev, kmp_uint8, 8, >>, KMP_ARCH_X86 )
2270 ATOMIC_CMPXCHG_CPT_REV( fixed1, sub_cpt_rev, kmp_int8, 8, -, KMP_ARCH_X86 )
2271 ATOMIC_CMPXCHG_CPT_REV( fixed2, div_cpt_rev, kmp_int16, 16, /, KMP_ARCH_X86 )
2272 ATOMIC_CMPXCHG_CPT_REV( fixed2u, div_cpt_rev, kmp_uint16, 16, /, KMP_ARCH_X86 )
2273 ATOMIC_CMPXCHG_CPT_REV( fixed2, shl_cpt_rev, kmp_int16, 16, <<, KMP_ARCH_X86 )
2274 ATOMIC_CMPXCHG_CPT_REV( fixed2, shr_cpt_rev, kmp_int16, 16, >>, KMP_ARCH_X86 )
2275 ATOMIC_CMPXCHG_CPT_REV( fixed2u, shr_cpt_rev, kmp_uint16, 16, >>, KMP_ARCH_X86 )
2276 ATOMIC_CMPXCHG_CPT_REV( fixed2, sub_cpt_rev, kmp_int16, 16, -, KMP_ARCH_X86 )
2277 ATOMIC_CMPXCHG_CPT_REV( fixed4, div_cpt_rev, kmp_int32, 32, /, KMP_ARCH_X86 )
2278 ATOMIC_CMPXCHG_CPT_REV( fixed4u, div_cpt_rev, kmp_uint32, 32, /, KMP_ARCH_X86 )
2279 ATOMIC_CMPXCHG_CPT_REV( fixed4, shl_cpt_rev, kmp_int32, 32, <<, KMP_ARCH_X86 )
2280 ATOMIC_CMPXCHG_CPT_REV( fixed4, shr_cpt_rev, kmp_int32, 32, >>, KMP_ARCH_X86 )
2281 ATOMIC_CMPXCHG_CPT_REV( fixed4u, shr_cpt_rev, kmp_uint32, 32, >>, KMP_ARCH_X86 )
2282 ATOMIC_CMPXCHG_CPT_REV( fixed4, sub_cpt_rev, kmp_int32, 32, -, KMP_ARCH_X86 )
2283 ATOMIC_CMPXCHG_CPT_REV( fixed8, div_cpt_rev, kmp_int64, 64, /, KMP_ARCH_X86 )
2284 ATOMIC_CMPXCHG_CPT_REV( fixed8u, div_cpt_rev, kmp_uint64, 64, /, KMP_ARCH_X86 )
2285 ATOMIC_CMPXCHG_CPT_REV( fixed8, shl_cpt_rev, kmp_int64, 64, <<, KMP_ARCH_X86 )
2286 ATOMIC_CMPXCHG_CPT_REV( fixed8, shr_cpt_rev, kmp_int64, 64, >>, KMP_ARCH_X86 )
2287 ATOMIC_CMPXCHG_CPT_REV( fixed8u, shr_cpt_rev, kmp_uint64, 64, >>, KMP_ARCH_X86 )
2288 ATOMIC_CMPXCHG_CPT_REV( fixed8, sub_cpt_rev, kmp_int64, 64, -, KMP_ARCH_X86 )
2289 ATOMIC_CMPXCHG_CPT_REV( float4, div_cpt_rev, kmp_real32, 32, /, KMP_ARCH_X86 )
2290 ATOMIC_CMPXCHG_CPT_REV( float4, sub_cpt_rev, kmp_real32, 32, -, KMP_ARCH_X86 )
2291 ATOMIC_CMPXCHG_CPT_REV( float8, div_cpt_rev, kmp_real64, 64, /, KMP_ARCH_X86 )
2292 ATOMIC_CMPXCHG_CPT_REV( float8, sub_cpt_rev, kmp_real64, 64, -, KMP_ARCH_X86 )
2301 #define ATOMIC_CRITICAL_CPT_REV(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 2302 ATOMIC_BEGIN_CPT(TYPE_ID,OP_ID,TYPE,TYPE) \ 2304 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2306 OP_GOMP_CRITICAL_CPT_REV(OP,GOMP_FLAG) \ 2307 OP_CRITICAL_CPT_REV(OP,LCK_ID) \ 2313 ATOMIC_CRITICAL_CPT_REV( float10, sub_cpt_rev,
long double, -, 10r, 1 )
2314 ATOMIC_CRITICAL_CPT_REV( float10, div_cpt_rev,
long double, /, 10r, 1 )
2317 ATOMIC_CRITICAL_CPT_REV( float16, sub_cpt_rev, QUAD_LEGACY, -, 16r, 1 )
2318 ATOMIC_CRITICAL_CPT_REV( float16, div_cpt_rev, QUAD_LEGACY, /, 16r, 1 )
2319 #if ( KMP_ARCH_X86 ) 2320 ATOMIC_CRITICAL_CPT_REV( float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r, 1 )
2321 ATOMIC_CRITICAL_CPT_REV( float16, div_a16_cpt_rev, Quad_a16_t, /, 16r, 1 )
2331 #define OP_CRITICAL_CPT_REV_WRK(OP,LCK_ID) \ 2332 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2335 (*lhs) = (rhs) OP (*lhs); \ 2339 (*lhs) = (rhs) OP (*lhs); \ 2342 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2346 #ifdef KMP_GOMP_COMPAT 2347 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP,FLAG) \ 2348 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 2350 OP_CRITICAL_CPT_REV_WRK( OP, 0 ); \ 2353 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP,FLAG) 2357 #define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID,OP_ID,TYPE,OP,LCK_ID,GOMP_FLAG) \ 2358 ATOMIC_BEGIN_WRK(TYPE_ID,OP_ID,TYPE) \ 2359 OP_GOMP_CRITICAL_CPT_REV_WRK(OP,GOMP_FLAG) \ 2360 OP_CRITICAL_CPT_REV_WRK(OP,LCK_ID) \ 2367 ATOMIC_CRITICAL_CPT_REV_WRK( cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c, 1 )
2368 ATOMIC_CRITICAL_CPT_REV_WRK( cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c, 1 )
2370 ATOMIC_CRITICAL_CPT_REV( cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c, 1 )
2371 ATOMIC_CRITICAL_CPT_REV( cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c, 1 )
2372 ATOMIC_CRITICAL_CPT_REV( cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c, 1 )
2373 ATOMIC_CRITICAL_CPT_REV( cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c, 1 )
2375 ATOMIC_CRITICAL_CPT_REV( cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c, 1 )
2376 ATOMIC_CRITICAL_CPT_REV( cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c, 1 )
2377 #if ( KMP_ARCH_X86 ) 2378 ATOMIC_CRITICAL_CPT_REV( cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c, 1 )
2379 ATOMIC_CRITICAL_CPT_REV( cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c, 1 )
2385 #define ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \ 2386 TYPE __kmpc_atomic_##TYPE_ID##_swp( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs ) \ 2388 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 2389 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid )); 2391 #define CRITICAL_SWP(LCK_ID) \ 2392 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2394 old_value = (*lhs); \ 2397 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2401 #ifdef KMP_GOMP_COMPAT 2402 #define GOMP_CRITICAL_SWP(FLAG) \ 2403 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 2405 CRITICAL_SWP( 0 ); \ 2408 #define GOMP_CRITICAL_SWP(FLAG) 2412 #define ATOMIC_XCHG_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \ 2413 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \ 2415 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 2416 old_value = KMP_XCHG_FIXED##BITS( lhs, rhs ); \ 2420 #define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \ 2421 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \ 2423 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 2424 old_value = KMP_XCHG_REAL##BITS( lhs, rhs ); \ 2429 #define CMPXCHG_SWP(TYPE,BITS) \ 2431 TYPE KMP_ATOMIC_VOLATILE temp_val; \ 2432 TYPE old_value, new_value; \ 2434 old_value = temp_val; \ 2436 while ( ! KMP_COMPARE_AND_STORE_ACQ##BITS( (kmp_int##BITS *) lhs, \ 2437 *VOLATILE_CAST(kmp_int##BITS *) &old_value, \ 2438 *VOLATILE_CAST(kmp_int##BITS *) &new_value ) ) \ 2443 old_value = temp_val; \ 2450 #define ATOMIC_CMPXCHG_SWP(TYPE_ID,TYPE,BITS,GOMP_FLAG) \ 2451 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \ 2453 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 2454 CMPXCHG_SWP(TYPE,BITS) \ 2457 ATOMIC_XCHG_SWP( fixed1, kmp_int8, 8, KMP_ARCH_X86 )
2458 ATOMIC_XCHG_SWP( fixed2, kmp_int16, 16, KMP_ARCH_X86 )
2459 ATOMIC_XCHG_SWP( fixed4, kmp_int32, 32, KMP_ARCH_X86 )
2461 ATOMIC_XCHG_FLOAT_SWP( float4, kmp_real32, 32, KMP_ARCH_X86 )
2463 #if ( KMP_ARCH_X86 ) 2464 ATOMIC_CMPXCHG_SWP( fixed8, kmp_int64, 64, KMP_ARCH_X86 )
2465 ATOMIC_CMPXCHG_SWP( float8, kmp_real64, 64, KMP_ARCH_X86 )
2467 ATOMIC_XCHG_SWP( fixed8, kmp_int64, 64, KMP_ARCH_X86 )
2468 ATOMIC_XCHG_FLOAT_SWP( float8, kmp_real64, 64, KMP_ARCH_X86 )
2473 #define ATOMIC_CRITICAL_SWP(TYPE_ID,TYPE,LCK_ID,GOMP_FLAG) \ 2474 ATOMIC_BEGIN_SWP(TYPE_ID,TYPE) \ 2476 GOMP_CRITICAL_SWP(GOMP_FLAG) \ 2477 CRITICAL_SWP(LCK_ID) \ 2486 #define ATOMIC_BEGIN_SWP_WRK(TYPE_ID,TYPE) \ 2487 void __kmpc_atomic_##TYPE_ID##_swp( ident_t *id_ref, int gtid, TYPE * lhs, TYPE rhs, TYPE * out ) \ 2489 KMP_DEBUG_ASSERT( __kmp_init_serial ); \ 2490 KA_TRACE(100,("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid )); 2493 #define CRITICAL_SWP_WRK(LCK_ID) \ 2494 __kmp_acquire_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2499 __kmp_release_atomic_lock( & ATOMIC_LOCK##LCK_ID, gtid ); \ 2504 #ifdef KMP_GOMP_COMPAT 2505 #define GOMP_CRITICAL_SWP_WRK(FLAG) \ 2506 if ( (FLAG) && (__kmp_atomic_mode == 2) ) { \ 2508 CRITICAL_SWP_WRK( 0 ); \ 2511 #define GOMP_CRITICAL_SWP_WRK(FLAG) 2515 #define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE,LCK_ID,GOMP_FLAG) \ 2516 ATOMIC_BEGIN_SWP_WRK(TYPE_ID,TYPE) \ 2518 GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \ 2519 CRITICAL_SWP_WRK(LCK_ID) \ 2524 ATOMIC_CRITICAL_SWP( float10,
long double, 10r, 1 )
2526 ATOMIC_CRITICAL_SWP( float16, QUAD_LEGACY, 16r, 1 )
2529 ATOMIC_CRITICAL_SWP_WRK( cmplx4, kmp_cmplx32, 8c, 1 )
2534 ATOMIC_CRITICAL_SWP( cmplx8, kmp_cmplx64, 16c, 1 )
2535 ATOMIC_CRITICAL_SWP( cmplx10, kmp_cmplx80, 20c, 1 )
2537 ATOMIC_CRITICAL_SWP( cmplx16, CPLX128_LEG, 32c, 1 )
2538 #if ( KMP_ARCH_X86 ) 2539 ATOMIC_CRITICAL_SWP( float16_a16, Quad_a16_t, 16r, 1 )
2540 ATOMIC_CRITICAL_SWP( cmplx16_a16, kmp_cmplx128_a16_t, 32c, 1 )
2547 #endif //OMP_40_ENABLED 2549 #endif //KMP_ARCH_X86 || KMP_ARCH_X86_64 2559 __kmpc_atomic_1(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2561 KMP_DEBUG_ASSERT( __kmp_init_serial );
2564 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2571 kmp_int8 old_value, new_value;
2573 old_value = *(kmp_int8 *) lhs;
2574 (*f)( &new_value, &old_value, rhs );
2577 while ( ! KMP_COMPARE_AND_STORE_ACQ8 ( (kmp_int8 *) lhs,
2578 *(kmp_int8 *) &old_value, *(kmp_int8 *) &new_value ) )
2582 old_value = *(kmp_int8 *) lhs;
2583 (*f)( &new_value, &old_value, rhs );
2593 #ifdef KMP_GOMP_COMPAT 2594 if ( __kmp_atomic_mode == 2 ) {
2595 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2599 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_1i, gtid );
2601 (*f)( lhs, lhs, rhs );
2603 #ifdef KMP_GOMP_COMPAT 2604 if ( __kmp_atomic_mode == 2 ) {
2605 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2609 __kmp_release_atomic_lock( & __kmp_atomic_lock_1i, gtid );
2614 __kmpc_atomic_2(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2617 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2619 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
2622 ! ( (kmp_uintptr_t) lhs & 0x1)
2626 kmp_int16 old_value, new_value;
2628 old_value = *(kmp_int16 *) lhs;
2629 (*f)( &new_value, &old_value, rhs );
2632 while ( ! KMP_COMPARE_AND_STORE_ACQ16 ( (kmp_int16 *) lhs,
2633 *(kmp_int16 *) &old_value, *(kmp_int16 *) &new_value ) )
2637 old_value = *(kmp_int16 *) lhs;
2638 (*f)( &new_value, &old_value, rhs );
2648 #ifdef KMP_GOMP_COMPAT 2649 if ( __kmp_atomic_mode == 2 ) {
2650 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2654 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_2i, gtid );
2656 (*f)( lhs, lhs, rhs );
2658 #ifdef KMP_GOMP_COMPAT 2659 if ( __kmp_atomic_mode == 2 ) {
2660 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2664 __kmp_release_atomic_lock( & __kmp_atomic_lock_2i, gtid );
2669 __kmpc_atomic_4(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2671 KMP_DEBUG_ASSERT( __kmp_init_serial );
2678 #
if KMP_ARCH_X86 || KMP_ARCH_X86_64
2681 ! ( (kmp_uintptr_t) lhs & 0x3)
2685 kmp_int32 old_value, new_value;
2687 old_value = *(kmp_int32 *) lhs;
2688 (*f)( &new_value, &old_value, rhs );
2691 while ( ! KMP_COMPARE_AND_STORE_ACQ32 ( (kmp_int32 *) lhs,
2692 *(kmp_int32 *) &old_value, *(kmp_int32 *) &new_value ) )
2696 old_value = *(kmp_int32 *) lhs;
2697 (*f)( &new_value, &old_value, rhs );
2708 #ifdef KMP_GOMP_COMPAT 2709 if ( __kmp_atomic_mode == 2 ) {
2710 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2714 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_4i, gtid );
2716 (*f)( lhs, lhs, rhs );
2718 #ifdef KMP_GOMP_COMPAT 2719 if ( __kmp_atomic_mode == 2 ) {
2720 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2724 __kmp_release_atomic_lock( & __kmp_atomic_lock_4i, gtid );
2729 __kmpc_atomic_8(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2731 KMP_DEBUG_ASSERT( __kmp_init_serial );
2734 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
2736 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
2739 ! ( (kmp_uintptr_t) lhs & 0x7)
2743 kmp_int64 old_value, new_value;
2745 old_value = *(kmp_int64 *) lhs;
2746 (*f)( &new_value, &old_value, rhs );
2748 while ( ! KMP_COMPARE_AND_STORE_ACQ64 ( (kmp_int64 *) lhs,
2749 *(kmp_int64 *) &old_value,
2750 *(kmp_int64 *) &new_value ) )
2754 old_value = *(kmp_int64 *) lhs;
2755 (*f)( &new_value, &old_value, rhs );
2765 #ifdef KMP_GOMP_COMPAT 2766 if ( __kmp_atomic_mode == 2 ) {
2767 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2771 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_8i, gtid );
2773 (*f)( lhs, lhs, rhs );
2775 #ifdef KMP_GOMP_COMPAT 2776 if ( __kmp_atomic_mode == 2 ) {
2777 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2781 __kmp_release_atomic_lock( & __kmp_atomic_lock_8i, gtid );
2786 __kmpc_atomic_10(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2788 KMP_DEBUG_ASSERT( __kmp_init_serial );
2790 #ifdef KMP_GOMP_COMPAT 2791 if ( __kmp_atomic_mode == 2 ) {
2792 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2796 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_10r, gtid );
2798 (*f)( lhs, lhs, rhs );
2800 #ifdef KMP_GOMP_COMPAT 2801 if ( __kmp_atomic_mode == 2 ) {
2802 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2806 __kmp_release_atomic_lock( & __kmp_atomic_lock_10r, gtid );
2810 __kmpc_atomic_16(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2812 KMP_DEBUG_ASSERT( __kmp_init_serial );
2814 #ifdef KMP_GOMP_COMPAT 2815 if ( __kmp_atomic_mode == 2 ) {
2816 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2820 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_16c, gtid );
2822 (*f)( lhs, lhs, rhs );
2824 #ifdef KMP_GOMP_COMPAT 2825 if ( __kmp_atomic_mode == 2 ) {
2826 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2830 __kmp_release_atomic_lock( & __kmp_atomic_lock_16c, gtid );
2834 __kmpc_atomic_20(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2836 KMP_DEBUG_ASSERT( __kmp_init_serial );
2838 #ifdef KMP_GOMP_COMPAT 2839 if ( __kmp_atomic_mode == 2 ) {
2840 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2844 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_20c, gtid );
2846 (*f)( lhs, lhs, rhs );
2848 #ifdef KMP_GOMP_COMPAT 2849 if ( __kmp_atomic_mode == 2 ) {
2850 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2854 __kmp_release_atomic_lock( & __kmp_atomic_lock_20c, gtid );
2858 __kmpc_atomic_32(
ident_t *id_ref,
int gtid,
void* lhs,
void* rhs,
void (*f)(
void *,
void *,
void * ) )
2860 KMP_DEBUG_ASSERT( __kmp_init_serial );
2862 #ifdef KMP_GOMP_COMPAT 2863 if ( __kmp_atomic_mode == 2 ) {
2864 __kmp_acquire_atomic_lock( & __kmp_atomic_lock, gtid );
2868 __kmp_acquire_atomic_lock( & __kmp_atomic_lock_32c, gtid );
2870 (*f)( lhs, lhs, rhs );
2872 #ifdef KMP_GOMP_COMPAT 2873 if ( __kmp_atomic_mode == 2 ) {
2874 __kmp_release_atomic_lock( & __kmp_atomic_lock, gtid );
2878 __kmp_release_atomic_lock( & __kmp_atomic_lock_32c, gtid );
2885 __kmpc_atomic_start(
void)
2887 int gtid = __kmp_entry_gtid();
2888 KA_TRACE(20, (
"__kmpc_atomic_start: T#%d\n", gtid));
2889 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
2894 __kmpc_atomic_end(
void)
2896 int gtid = __kmp_get_gtid();
2897 KA_TRACE(20, (
"__kmpc_atomic_end: T#%d\n", gtid));
2898 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);