17 #include "kmp_wrapper_getpid.h" 23 #include "kmp_stats.h" 24 #include "kmp_wait_release.h" 26 #if !KMP_OS_FREEBSD && !KMP_OS_NETBSD 32 #include <sys/times.h> 33 #include <sys/resource.h> 34 #include <sys/syscall.h> 36 #if KMP_OS_LINUX && !KMP_OS_CNK 37 # include <sys/sysinfo.h> 52 # include <sys/sysctl.h> 53 # include <mach/mach.h> 55 # include <pthread_np.h> 65 struct kmp_sys_timer {
66 struct timespec start;
70 #define TS2NS(timespec) (((timespec).tv_sec * 1e9) + (timespec).tv_nsec) 72 static struct kmp_sys_timer __kmp_sys_timer_data;
74 #if KMP_HANDLE_SIGNALS 75 typedef void (* sig_func_t )( int );
76 STATIC_EFI2_WORKAROUND
struct sigaction __kmp_sighldrs[ NSIG ];
77 static sigset_t __kmp_sigset;
80 static int __kmp_init_runtime = FALSE;
82 static int __kmp_fork_count = 0;
84 static pthread_condattr_t __kmp_suspend_cond_attr;
85 static pthread_mutexattr_t __kmp_suspend_mutex_attr;
87 static kmp_cond_align_t __kmp_wait_cv;
88 static kmp_mutex_align_t __kmp_wait_mx;
95 __kmp_print_cond(
char *buffer, kmp_cond_align_t *cond )
97 KMP_SNPRINTF( buffer, 128,
"(cond (lock (%ld, %d)), (descr (%p)))",
98 cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
99 cond->c_cond.__c_waiting );
106 #if ( KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED) 119 # if KMP_ARCH_X86 || KMP_ARCH_ARM 120 # ifndef __NR_sched_setaffinity 121 # define __NR_sched_setaffinity 241 122 # elif __NR_sched_setaffinity != 241 123 # error Wrong code for setaffinity system call. 125 # ifndef __NR_sched_getaffinity 126 # define __NR_sched_getaffinity 242 127 # elif __NR_sched_getaffinity != 242 128 # error Wrong code for getaffinity system call. 131 # elif KMP_ARCH_AARCH64 132 # ifndef __NR_sched_setaffinity 133 # define __NR_sched_setaffinity 122 134 # elif __NR_sched_setaffinity != 122 135 # error Wrong code for setaffinity system call. 137 # ifndef __NR_sched_getaffinity 138 # define __NR_sched_getaffinity 123 139 # elif __NR_sched_getaffinity != 123 140 # error Wrong code for getaffinity system call. 143 # elif KMP_ARCH_X86_64 144 # ifndef __NR_sched_setaffinity 145 # define __NR_sched_setaffinity 203 146 # elif __NR_sched_setaffinity != 203 147 # error Wrong code for setaffinity system call. 149 # ifndef __NR_sched_getaffinity 150 # define __NR_sched_getaffinity 204 151 # elif __NR_sched_getaffinity != 204 152 # error Wrong code for getaffinity system call. 155 # elif KMP_ARCH_PPC64 156 # ifndef __NR_sched_setaffinity 157 # define __NR_sched_setaffinity 222 158 # elif __NR_sched_setaffinity != 222 159 # error Wrong code for setaffinity system call. 161 # ifndef __NR_sched_getaffinity 162 # define __NR_sched_getaffinity 223 163 # elif __NR_sched_getaffinity != 223 164 # error Wrong code for getaffinity system call. 168 # ifndef __NR_sched_setaffinity 169 # define __NR_sched_setaffinity 4239 170 # elif __NR_sched_setaffinity != 4239 171 # error Wrong code for setaffinity system call. 173 # ifndef __NR_sched_getaffinity 174 # define __NR_sched_getaffinity 4240 175 # elif __NR_sched_getaffinity != 4240 176 # error Wrong code for getaffinity system call. 179 # elif KMP_ARCH_MIPS64 180 # ifndef __NR_sched_setaffinity 181 # define __NR_sched_setaffinity 5195 182 # elif __NR_sched_setaffinity != 5195 183 # error Wrong code for setaffinity system call. 185 # ifndef __NR_sched_getaffinity 186 # define __NR_sched_getaffinity 5196 187 # elif __NR_sched_getaffinity != 5196 188 # error Wrong code for getaffinity system call. 192 # error Unknown or unsupported architecture 197 __kmp_set_system_affinity( kmp_affin_mask_t
const *mask,
int abort_on_error )
199 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
200 "Illegal set affinity operation when not capable");
202 int retval = hwloc_set_cpubind(__kmp_hwloc_topology, (hwloc_cpuset_t)mask, HWLOC_CPUBIND_THREAD);
204 int retval = syscall( __NR_sched_setaffinity, 0, __kmp_affin_mask_size, mask );
210 if (abort_on_error) {
213 KMP_MSG( FatalSysError ),
222 __kmp_get_system_affinity( kmp_affin_mask_t *mask,
int abort_on_error )
224 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
225 "Illegal get affinity operation when not capable");
228 int retval = hwloc_get_cpubind(__kmp_hwloc_topology, (hwloc_cpuset_t)mask, HWLOC_CPUBIND_THREAD);
230 int retval = syscall( __NR_sched_getaffinity, 0, __kmp_affin_mask_size, mask );
236 if (abort_on_error) {
239 KMP_MSG( FatalSysError ),
248 __kmp_affinity_bind_thread(
int which )
250 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
251 "Illegal set affinity operation when not capable");
253 kmp_affin_mask_t *mask;
254 KMP_CPU_ALLOC_ON_STACK(mask);
256 KMP_CPU_SET(which, mask);
257 __kmp_set_system_affinity(mask, TRUE);
258 KMP_CPU_FREE_FROM_STACK(mask);
267 __kmp_affinity_determine_capable(
const char *env_var)
273 # define KMP_CPU_SET_SIZE_LIMIT (1024*1024) 277 kmp_affin_mask_t *buf;
278 buf = ( kmp_affin_mask_t * ) KMP_INTERNAL_MALLOC( KMP_CPU_SET_SIZE_LIMIT );
283 gCode = syscall( __NR_sched_getaffinity, 0, KMP_CPU_SET_SIZE_LIMIT, buf );
284 KA_TRACE(30, (
"__kmp_affinity_determine_capable: " 285 "initial getaffinity call returned %d errno = %d\n",
293 if (__kmp_affinity_verbose || (__kmp_affinity_warnings
294 && (__kmp_affinity_type != affinity_none)
295 && (__kmp_affinity_type != affinity_default)
296 && (__kmp_affinity_type != affinity_disabled))) {
300 KMP_MSG( GetAffSysCallNotSupported, env_var ),
305 KMP_AFFINITY_DISABLE();
306 KMP_INTERNAL_FREE(buf);
315 sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
316 KA_TRACE(30, (
"__kmp_affinity_determine_capable: " 317 "setaffinity for mask size %d returned %d errno = %d\n",
318 gCode, sCode, errno));
320 if (errno == ENOSYS) {
321 if (__kmp_affinity_verbose || (__kmp_affinity_warnings
322 && (__kmp_affinity_type != affinity_none)
323 && (__kmp_affinity_type != affinity_default)
324 && (__kmp_affinity_type != affinity_disabled))) {
328 KMP_MSG( SetAffSysCallNotSupported, env_var ),
333 KMP_AFFINITY_DISABLE();
334 KMP_INTERNAL_FREE(buf);
336 if (errno == EFAULT) {
337 KMP_AFFINITY_ENABLE(gCode);
338 KA_TRACE(10, (
"__kmp_affinity_determine_capable: " 339 "affinity supported (mask size %d)\n",
340 (
int)__kmp_affin_mask_size));
341 KMP_INTERNAL_FREE(buf);
351 KA_TRACE(30, (
"__kmp_affinity_determine_capable: " 352 "searching for proper set size\n"));
354 for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
355 gCode = syscall( __NR_sched_getaffinity, 0, size, buf );
356 KA_TRACE(30, (
"__kmp_affinity_determine_capable: " 357 "getaffinity for mask size %d returned %d errno = %d\n", size,
361 if ( errno == ENOSYS )
366 KA_TRACE(30, (
"__kmp_affinity_determine_capable: " 367 "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
369 if (__kmp_affinity_verbose || (__kmp_affinity_warnings
370 && (__kmp_affinity_type != affinity_none)
371 && (__kmp_affinity_type != affinity_default)
372 && (__kmp_affinity_type != affinity_disabled))) {
376 KMP_MSG( GetAffSysCallNotSupported, env_var ),
381 KMP_AFFINITY_DISABLE();
382 KMP_INTERNAL_FREE(buf);
388 sCode = syscall( __NR_sched_setaffinity, 0, gCode, NULL );
389 KA_TRACE(30, (
"__kmp_affinity_determine_capable: " 390 "setaffinity for mask size %d returned %d errno = %d\n",
391 gCode, sCode, errno));
393 if (errno == ENOSYS) {
397 KA_TRACE(30, (
"__kmp_affinity_determine_capable: " 398 "inconsistent OS call behavior: errno == ENOSYS for mask size %d\n",
400 if (__kmp_affinity_verbose || (__kmp_affinity_warnings
401 && (__kmp_affinity_type != affinity_none)
402 && (__kmp_affinity_type != affinity_default)
403 && (__kmp_affinity_type != affinity_disabled))) {
407 KMP_MSG( SetAffSysCallNotSupported, env_var ),
412 KMP_AFFINITY_DISABLE();
413 KMP_INTERNAL_FREE(buf);
416 if (errno == EFAULT) {
417 KMP_AFFINITY_ENABLE(gCode);
418 KA_TRACE(10, (
"__kmp_affinity_determine_capable: " 419 "affinity supported (mask size %d)\n",
420 (
int)__kmp_affin_mask_size));
421 KMP_INTERNAL_FREE(buf);
427 KMP_INTERNAL_FREE(buf);
433 KMP_AFFINITY_DISABLE();
434 KA_TRACE(10, (
"__kmp_affinity_determine_capable: " 435 "cannot determine mask size - affinity not supported\n"));
436 if (__kmp_affinity_verbose || (__kmp_affinity_warnings
437 && (__kmp_affinity_type != affinity_none)
438 && (__kmp_affinity_type != affinity_default)
439 && (__kmp_affinity_type != affinity_disabled))) {
440 KMP_WARNING( AffCantGetMaskSize, env_var );
444 #endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 452 __kmp_futex_determine_capable()
455 int rc = syscall( __NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0 );
456 int retval = ( rc == 0 ) || ( errno != ENOSYS );
458 KA_TRACE(10, (
"__kmp_futex_determine_capable: rc = %d errno = %d\n", rc,
460 KA_TRACE(10, (
"__kmp_futex_determine_capable: futex syscall%s supported\n",
461 retval ?
"" :
" not" ) );
466 #endif // KMP_USE_FUTEX 471 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) 478 __kmp_test_then_or8(
volatile kmp_int8 *p, kmp_int8 d )
480 kmp_int8 old_value, new_value;
482 old_value = TCR_1( *p );
483 new_value = old_value | d;
485 while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
488 old_value = TCR_1( *p );
489 new_value = old_value | d;
495 __kmp_test_then_and8(
volatile kmp_int8 *p, kmp_int8 d )
497 kmp_int8 old_value, new_value;
499 old_value = TCR_1( *p );
500 new_value = old_value & d;
502 while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
505 old_value = TCR_1( *p );
506 new_value = old_value & d;
512 __kmp_test_then_or32(
volatile kmp_int32 *p, kmp_int32 d )
514 kmp_int32 old_value, new_value;
516 old_value = TCR_4( *p );
517 new_value = old_value | d;
519 while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
522 old_value = TCR_4( *p );
523 new_value = old_value | d;
529 __kmp_test_then_and32(
volatile kmp_int32 *p, kmp_int32 d )
531 kmp_int32 old_value, new_value;
533 old_value = TCR_4( *p );
534 new_value = old_value & d;
536 while ( ! KMP_COMPARE_AND_STORE_REL32 ( p, old_value, new_value ) )
539 old_value = TCR_4( *p );
540 new_value = old_value & d;
545 # if KMP_ARCH_X86 || KMP_ARCH_PPC64 || (KMP_OS_LINUX && KMP_ARCH_AARCH64) 547 __kmp_test_then_add8(
volatile kmp_int8 *p, kmp_int8 d )
549 kmp_int8 old_value, new_value;
551 old_value = TCR_1( *p );
552 new_value = old_value + d;
554 while ( ! KMP_COMPARE_AND_STORE_REL8 ( p, old_value, new_value ) )
557 old_value = TCR_1( *p );
558 new_value = old_value + d;
564 __kmp_test_then_add64(
volatile kmp_int64 *p, kmp_int64 d )
566 kmp_int64 old_value, new_value;
568 old_value = TCR_8( *p );
569 new_value = old_value + d;
571 while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
574 old_value = TCR_8( *p );
575 new_value = old_value + d;
582 __kmp_test_then_or64(
volatile kmp_int64 *p, kmp_int64 d )
584 kmp_int64 old_value, new_value;
586 old_value = TCR_8( *p );
587 new_value = old_value | d;
588 while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
591 old_value = TCR_8( *p );
592 new_value = old_value | d;
598 __kmp_test_then_and64(
volatile kmp_int64 *p, kmp_int64 d )
600 kmp_int64 old_value, new_value;
602 old_value = TCR_8( *p );
603 new_value = old_value & d;
604 while ( ! KMP_COMPARE_AND_STORE_REL64 ( p, old_value, new_value ) )
607 old_value = TCR_8( *p );
608 new_value = old_value & d;
616 __kmp_terminate_thread(
int gtid )
619 kmp_info_t *th = __kmp_threads[ gtid ];
623 #ifdef KMP_CANCEL_THREADS 624 KA_TRACE( 10, (
"__kmp_terminate_thread: kill (%d)\n", gtid ) );
625 status = pthread_cancel( th->th.th_info.ds.ds_thread );
626 if ( status != 0 && status != ESRCH ) {
629 KMP_MSG( CantTerminateWorkerThread ),
653 __kmp_set_stack_info(
int gtid, kmp_info_t *th )
656 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD 666 if ( ! KMP_UBER_GTID(gtid) ) {
669 status = pthread_attr_init( &attr );
670 KMP_CHECK_SYSFAIL(
"pthread_attr_init", status );
671 #if KMP_OS_FREEBSD || KMP_OS_NETBSD 672 status = pthread_attr_get_np( pthread_self(), &attr );
673 KMP_CHECK_SYSFAIL(
"pthread_attr_get_np", status );
675 status = pthread_getattr_np( pthread_self(), &attr );
676 KMP_CHECK_SYSFAIL(
"pthread_getattr_np", status );
678 status = pthread_attr_getstack( &attr, &addr, &size );
679 KMP_CHECK_SYSFAIL(
"pthread_attr_getstack", status );
680 KA_TRACE( 60, (
"__kmp_set_stack_info: T#%d pthread_attr_getstack returned size: %lu, " 684 status = pthread_attr_destroy( &attr );
685 KMP_CHECK_SYSFAIL(
"pthread_attr_destroy", status );
688 if ( size != 0 && addr != 0 ) {
690 TCW_PTR(th->th.th_info.ds.ds_stackbase, (((
char *)addr) + size));
691 TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
692 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
697 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
698 TCW_PTR(th -> th.th_info.ds.ds_stackbase, &stack_data);
699 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
704 __kmp_launch_worker(
void *thr )
706 int status, old_type, old_state;
707 #ifdef KMP_BLOCK_SIGNALS 708 sigset_t new_set, old_set;
711 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD 712 void *
volatile padding = 0;
716 gtid = ((kmp_info_t*)thr) -> th.th_info.ds.ds_gtid;
717 __kmp_gtid_set_specific( gtid );
718 #ifdef KMP_TDATA_GTID 721 #if KMP_STATS_ENABLED 723 __kmp_stats_thread_ptr = ((kmp_info_t*)thr)->th.th_stats;
725 KMP_SET_THREAD_STATE(IDLE);
730 __kmp_itt_thread_name( gtid );
733 #if KMP_AFFINITY_SUPPORTED 734 __kmp_affinity_set_init_mask( gtid, FALSE );
737 #ifdef KMP_CANCEL_THREADS 738 status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
739 KMP_CHECK_SYSFAIL(
"pthread_setcanceltype", status );
741 status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
742 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status );
745 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 750 __kmp_clear_x87_fpu_status_word();
751 __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
752 __kmp_load_mxcsr( &__kmp_init_mxcsr );
755 #ifdef KMP_BLOCK_SIGNALS 756 status = sigfillset( & new_set );
757 KMP_CHECK_SYSFAIL_ERRNO(
"sigfillset", status );
758 status = pthread_sigmask( SIG_BLOCK, & new_set, & old_set );
759 KMP_CHECK_SYSFAIL(
"pthread_sigmask", status );
762 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD 763 if ( __kmp_stkoffset > 0 && gtid > 0 ) {
764 padding = KMP_ALLOCA( gtid * __kmp_stkoffset );
769 __kmp_set_stack_info( gtid, (kmp_info_t*)thr );
771 __kmp_check_stack_overlap( (kmp_info_t*)thr );
773 exit_val = __kmp_launch_thread( (kmp_info_t *) thr );
775 #ifdef KMP_BLOCK_SIGNALS 776 status = pthread_sigmask( SIG_SETMASK, & old_set, NULL );
777 KMP_CHECK_SYSFAIL(
"pthread_sigmask", status );
786 __kmp_launch_monitor(
void *thr )
788 int status, old_type, old_state;
789 #ifdef KMP_BLOCK_SIGNALS 792 struct timespec interval;
794 int yield_cycles = 0;
798 KA_TRACE( 10, (
"__kmp_launch_monitor: #1 launched\n" ) );
801 __kmp_gtid_set_specific( KMP_GTID_MONITOR );
802 #ifdef KMP_TDATA_GTID 803 __kmp_gtid = KMP_GTID_MONITOR;
809 __kmp_itt_thread_ignore();
812 __kmp_set_stack_info( ((kmp_info_t*)thr)->th.th_info.ds.ds_gtid, (kmp_info_t*)thr );
814 __kmp_check_stack_overlap( (kmp_info_t*)thr );
816 #ifdef KMP_CANCEL_THREADS 817 status = pthread_setcanceltype( PTHREAD_CANCEL_ASYNCHRONOUS, & old_type );
818 KMP_CHECK_SYSFAIL(
"pthread_setcanceltype", status );
820 status = pthread_setcancelstate( PTHREAD_CANCEL_ENABLE, & old_state );
821 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status );
824 #if KMP_REAL_TIME_FIX 828 int sched = sched_getscheduler( 0 );
829 if ( sched == SCHED_FIFO || sched == SCHED_RR ) {
832 struct sched_param param;
833 int max_priority = sched_get_priority_max( sched );
835 KMP_WARNING( RealTimeSchedNotSupported );
836 sched_getparam( 0, & param );
837 if ( param.sched_priority < max_priority ) {
838 param.sched_priority += 1;
839 rc = sched_setscheduler( 0, sched, & param );
844 KMP_MSG( CantChangeMonitorPriority ),
846 KMP_MSG( MonitorWillStarve ),
855 KMP_MSG( RunningAtMaxPriority ),
856 KMP_MSG( MonitorWillStarve ),
857 KMP_HNT( RunningAtMaxPriority ),
862 TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
864 #endif // KMP_REAL_TIME_FIX 868 if ( __kmp_monitor_wakeups == 1 ) {
870 interval.tv_nsec = 0;
873 interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
876 KA_TRACE( 10, (
"__kmp_launch_monitor: #2 monitor\n" ) );
878 if (__kmp_yield_cycle) {
879 __kmp_yielding_on = 0;
880 yield_count = __kmp_yield_off_count;
882 __kmp_yielding_on = 1;
885 while( ! TCR_4( __kmp_global.g.g_done ) ) {
891 KA_TRACE( 15, (
"__kmp_launch_monitor: update\n" ) );
893 status = gettimeofday( &tval, NULL );
894 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status );
895 TIMEVAL_TO_TIMESPEC( &tval, &now );
897 now.tv_sec += interval.tv_sec;
898 now.tv_nsec += interval.tv_nsec;
900 if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
902 now.tv_nsec -= KMP_NSEC_PER_SEC;
905 status = pthread_mutex_lock( & __kmp_wait_mx.m_mutex );
906 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status );
908 if ( !TCR_4(__kmp_global.g.g_done) ) {
909 status = pthread_cond_timedwait( &__kmp_wait_cv.c_cond, &__kmp_wait_mx.m_mutex, &now );
911 if ( status != ETIMEDOUT && status != EINTR ) {
912 KMP_SYSFAIL(
"pthread_cond_timedwait", status );
916 status = pthread_mutex_unlock( & __kmp_wait_mx.m_mutex );
917 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
919 if (__kmp_yield_cycle) {
921 if ( (yield_cycles % yield_count) == 0 ) {
922 if (__kmp_yielding_on) {
923 __kmp_yielding_on = 0;
924 yield_count = __kmp_yield_off_count;
926 __kmp_yielding_on = 1;
927 yield_count = __kmp_yield_on_count;
932 __kmp_yielding_on = 1;
935 TCW_4( __kmp_global.g.g_time.dt.t_value,
936 TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
941 KA_TRACE( 10, (
"__kmp_launch_monitor: #3 cleanup\n" ) );
943 #ifdef KMP_BLOCK_SIGNALS 944 status = sigfillset( & new_set );
945 KMP_CHECK_SYSFAIL_ERRNO(
"sigfillset", status );
946 status = pthread_sigmask( SIG_UNBLOCK, & new_set, NULL );
947 KMP_CHECK_SYSFAIL(
"pthread_sigmask", status );
950 KA_TRACE( 10, (
"__kmp_launch_monitor: #4 finished\n" ) );
952 if( __kmp_global.g.g_abort != 0 ) {
958 KA_TRACE( 10, (
"__kmp_launch_monitor: #5 terminate sig=%d\n", __kmp_global.g.g_abort ) );
963 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
964 __kmp_terminate_thread( gtid );
968 KA_TRACE( 10, (
"__kmp_launch_monitor: #6 raise sig=%d\n", __kmp_global.g.g_abort ) );
970 if (__kmp_global.g.g_abort > 0)
971 raise( __kmp_global.g.g_abort );
975 KA_TRACE( 10, (
"__kmp_launch_monitor: #7 exit\n" ) );
981 __kmp_create_worker(
int gtid, kmp_info_t *th,
size_t stack_size )
984 pthread_attr_t thread_attr;
988 th->th.th_info.ds.ds_gtid = gtid;
990 #if KMP_STATS_ENABLED 992 __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
997 th->th.th_stats = __kmp_stats_list.push_back(gtid);
998 if(KMP_UBER_GTID(gtid)) {
999 __kmp_stats_start_time = tsc_tick_count::now();
1000 __kmp_stats_thread_ptr = th->th.th_stats;
1003 KMP_SET_THREAD_STATE(SERIAL_REGION);
1006 __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
1008 #endif // KMP_STATS_ENABLED 1010 if ( KMP_UBER_GTID(gtid) ) {
1011 KA_TRACE( 10, (
"__kmp_create_worker: uber thread (%d)\n", gtid ) );
1012 th -> th.th_info.ds.ds_thread = pthread_self();
1013 __kmp_set_stack_info( gtid, th );
1014 __kmp_check_stack_overlap( th );
1018 KA_TRACE( 10, (
"__kmp_create_worker: try to create thread (%d)\n", gtid ) );
1022 #ifdef KMP_THREAD_ATTR 1023 status = pthread_attr_init( &thread_attr );
1024 if ( status != 0 ) {
1025 __kmp_msg(kmp_ms_fatal, KMP_MSG( CantInitThreadAttrs ), KMP_ERR( status ), __kmp_msg_null);
1027 status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1028 if ( status != 0 ) {
1029 __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerState ), KMP_ERR( status ), __kmp_msg_null);
1038 stack_size += gtid * __kmp_stkoffset * 2;
1040 KA_TRACE( 10, (
"__kmp_create_worker: T#%d, default stacksize = %lu bytes, " 1041 "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
1042 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size ) );
1044 # ifdef _POSIX_THREAD_ATTR_STACKSIZE 1045 status = pthread_attr_setstacksize( & thread_attr, stack_size );
1046 # ifdef KMP_BACKUP_STKSIZE 1047 if ( status != 0 ) {
1048 if ( ! __kmp_env_stksize ) {
1049 stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
1050 __kmp_stksize = KMP_BACKUP_STKSIZE;
1051 KA_TRACE( 10, (
"__kmp_create_worker: T#%d, default stacksize = %lu bytes, " 1052 "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu " 1054 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size )
1056 status = pthread_attr_setstacksize( &thread_attr, stack_size );
1060 if ( status != 0 ) {
1061 __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerStackSize, stack_size ), KMP_ERR( status ),
1062 KMP_HNT( ChangeWorkerStackSize ), __kmp_msg_null);
1068 status = pthread_create( & handle, & thread_attr, __kmp_launch_worker, (
void *) th );
1069 if ( status != 0 || ! handle ) {
1070 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 1071 if ( status == EINVAL ) {
1072 __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerStackSize, stack_size ), KMP_ERR( status ),
1073 KMP_HNT( IncreaseWorkerStackSize ), __kmp_msg_null);
1075 if ( status == ENOMEM ) {
1076 __kmp_msg(kmp_ms_fatal, KMP_MSG( CantSetWorkerStackSize, stack_size ), KMP_ERR( status ),
1077 KMP_HNT( DecreaseWorkerStackSize ), __kmp_msg_null);
1080 if ( status == EAGAIN ) {
1081 __kmp_msg(kmp_ms_fatal, KMP_MSG( NoResourcesForWorkerThread ), KMP_ERR( status ),
1082 KMP_HNT( Decrease_NUM_THREADS ), __kmp_msg_null);
1084 KMP_SYSFAIL(
"pthread_create", status );
1087 th->th.th_info.ds.ds_thread = handle;
1089 #ifdef KMP_THREAD_ATTR 1090 status = pthread_attr_destroy( & thread_attr );
1092 __kmp_msg(kmp_ms_warning, KMP_MSG( CantDestroyThreadAttrs ), KMP_ERR( status ), __kmp_msg_null);
1098 KA_TRACE( 10, (
"__kmp_create_worker: done creating thread (%d)\n", gtid ) );
1104 __kmp_create_monitor( kmp_info_t *th )
1107 pthread_attr_t thread_attr;
1110 int auto_adj_size = FALSE;
1112 if( __kmp_dflt_blocktime == KMP_MAX_BLOCKTIME ) {
1114 KA_TRACE( 10, (
"__kmp_create_monitor: skipping monitor thread because of MAX blocktime\n" ) );
1115 th->th.th_info.ds.ds_tid = 0;
1116 th->th.th_info.ds.ds_gtid = 0;
1119 KA_TRACE( 10, (
"__kmp_create_monitor: try to create monitor\n" ) );
1123 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1124 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1125 #if KMP_REAL_TIME_FIX 1126 TCW_4( __kmp_global.g.g_time.dt.t_value, -1 );
1128 TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );
1129 #endif // KMP_REAL_TIME_FIX 1131 #ifdef KMP_THREAD_ATTR 1132 if ( __kmp_monitor_stksize == 0 ) {
1133 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1134 auto_adj_size = TRUE;
1136 status = pthread_attr_init( &thread_attr );
1137 if ( status != 0 ) {
1140 KMP_MSG( CantInitThreadAttrs ),
1145 status = pthread_attr_setdetachstate( & thread_attr, PTHREAD_CREATE_JOINABLE );
1146 if ( status != 0 ) {
1149 KMP_MSG( CantSetMonitorState ),
1155 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 1156 status = pthread_attr_getstacksize( & thread_attr, & size );
1157 KMP_CHECK_SYSFAIL(
"pthread_attr_getstacksize", status );
1159 size = __kmp_sys_min_stksize;
1163 if ( __kmp_monitor_stksize == 0 ) {
1164 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1166 if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
1167 __kmp_monitor_stksize = __kmp_sys_min_stksize;
1170 KA_TRACE( 10, (
"__kmp_create_monitor: default stacksize = %lu bytes," 1171 "requested stacksize = %lu bytes\n",
1172 size, __kmp_monitor_stksize ) );
1178 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 1179 KA_TRACE( 10, (
"__kmp_create_monitor: setting stacksize = %lu bytes,",
1180 __kmp_monitor_stksize ) );
1181 status = pthread_attr_setstacksize( & thread_attr, __kmp_monitor_stksize );
1182 if ( status != 0 ) {
1183 if ( auto_adj_size ) {
1184 __kmp_monitor_stksize *= 2;
1189 KMP_MSG( CantSetMonitorStackSize, (
long int) __kmp_monitor_stksize ),
1191 KMP_HNT( ChangeMonitorStackSize ),
1197 status = pthread_create( &handle, & thread_attr, __kmp_launch_monitor, (
void *) th );
1199 if ( status != 0 ) {
1200 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 1201 if ( status == EINVAL ) {
1202 if ( auto_adj_size && ( __kmp_monitor_stksize < (
size_t)0x40000000 ) ) {
1203 __kmp_monitor_stksize *= 2;
1208 KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1210 KMP_HNT( IncreaseMonitorStackSize ),
1214 if ( status == ENOMEM ) {
1217 KMP_MSG( CantSetMonitorStackSize, __kmp_monitor_stksize ),
1219 KMP_HNT( DecreaseMonitorStackSize ),
1224 if ( status == EAGAIN ) {
1227 KMP_MSG( NoResourcesForMonitorThread ),
1229 KMP_HNT( DecreaseNumberOfThreadsInUse ),
1233 KMP_SYSFAIL(
"pthread_create", status );
1236 th->th.th_info.ds.ds_thread = handle;
1238 #if KMP_REAL_TIME_FIX 1240 KMP_DEBUG_ASSERT(
sizeof( kmp_uint32 ) ==
sizeof( __kmp_global.g.g_time.dt.t_value ) );
1242 (kmp_uint32
volatile *) & __kmp_global.g.g_time.dt.t_value, -1, & __kmp_neq_4, NULL
1244 #endif // KMP_REAL_TIME_FIX 1246 #ifdef KMP_THREAD_ATTR 1247 status = pthread_attr_destroy( & thread_attr );
1248 if ( status != 0 ) {
1251 KMP_MSG( CantDestroyThreadAttrs ),
1260 KA_TRACE( 10, (
"__kmp_create_monitor: monitor created %#.8lx\n", th->th.th_info.ds.ds_thread ) );
1268 pthread_exit( (
void *)(intptr_t) exit_status );
1271 void __kmp_resume_monitor();
1274 __kmp_reap_monitor( kmp_info_t *th )
1279 KA_TRACE( 10, (
"__kmp_reap_monitor: try to reap monitor thread with handle %#.8lx\n",
1280 th->th.th_info.ds.ds_thread ) );
1285 KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
1286 if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
1287 KA_TRACE( 10, (
"__kmp_reap_monitor: monitor did not start, returning\n") );
1298 status = pthread_kill( th->th.th_info.ds.ds_thread, 0 );
1299 if (status != ESRCH) {
1300 __kmp_resume_monitor();
1302 KA_TRACE( 10, (
"__kmp_reap_monitor: try to join with monitor\n") );
1303 status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1304 if (exit_val != th) {
1307 KMP_MSG( ReapMonitorError ),
1313 th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1314 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1316 KA_TRACE( 10, (
"__kmp_reap_monitor: done reaping monitor thread with handle %#.8lx\n",
1317 th->th.th_info.ds.ds_thread ) );
1324 __kmp_reap_worker( kmp_info_t *th )
1331 KA_TRACE( 10, (
"__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid ) );
1333 status = pthread_join( th->th.th_info.ds.ds_thread, & exit_val);
1336 if ( status != 0 ) {
1337 __kmp_msg(kmp_ms_fatal, KMP_MSG( ReapWorkerError ), KMP_ERR( status ), __kmp_msg_null);
1339 if ( exit_val != th ) {
1340 KA_TRACE( 10, (
"__kmp_reap_worker: worker T#%d did not reap properly, exit_val = %p\n",
1341 th->th.th_info.ds.ds_gtid, exit_val ) );
1345 KA_TRACE( 10, (
"__kmp_reap_worker: done reaping T#%d\n", th->th.th_info.ds.ds_gtid ) );
1354 #if KMP_HANDLE_SIGNALS 1358 __kmp_null_handler(
int signo )
1365 __kmp_team_handler(
int signo )
1367 if ( __kmp_global.g.g_abort == 0 ) {
1370 __kmp_debug_printf(
"__kmp_team_handler: caught signal = %d\n", signo );
1385 if ( __kmp_debug_buf ) {
1386 __kmp_dump_debug_buffer( );
1389 TCW_4( __kmp_global.g.g_abort, signo );
1391 TCW_4( __kmp_global.g.g_done, TRUE );
1396 __kmp_debug_printf(
"__kmp_team_handler: unknown signal type" );
1405 void __kmp_sigaction(
int signum,
const struct sigaction * act,
struct sigaction * oldact ) {
1406 int rc = sigaction( signum, act, oldact );
1407 KMP_CHECK_SYSFAIL_ERRNO(
"sigaction", rc );
1412 __kmp_install_one_handler(
int sig, sig_func_t handler_func,
int parallel_init )
1415 KB_TRACE( 60, (
"__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init ) );
1416 if ( parallel_init ) {
1417 struct sigaction new_action;
1418 struct sigaction old_action;
1419 new_action.sa_handler = handler_func;
1420 new_action.sa_flags = 0;
1421 sigfillset( & new_action.sa_mask );
1422 __kmp_sigaction( sig, & new_action, & old_action );
1423 if ( old_action.sa_handler == __kmp_sighldrs[ sig ].sa_handler ) {
1424 sigaddset( & __kmp_sigset, sig );
1427 __kmp_sigaction( sig, & old_action, NULL );
1431 __kmp_sigaction( sig, NULL, & __kmp_sighldrs[ sig ] );
1438 __kmp_remove_one_handler(
int sig )
1440 KB_TRACE( 60, (
"__kmp_remove_one_handler( %d )\n", sig ) );
1441 if ( sigismember( & __kmp_sigset, sig ) ) {
1442 struct sigaction old;
1444 __kmp_sigaction( sig, & __kmp_sighldrs[ sig ], & old );
1445 if ( ( old.sa_handler != __kmp_team_handler ) && ( old.sa_handler != __kmp_null_handler ) ) {
1447 KB_TRACE( 10, (
"__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
1448 __kmp_sigaction( sig, & old, NULL );
1450 sigdelset( & __kmp_sigset, sig );
1457 __kmp_install_signals(
int parallel_init )
1459 KB_TRACE( 10, (
"__kmp_install_signals( %d )\n", parallel_init ) );
1460 if ( __kmp_handle_signals || ! parallel_init ) {
1463 sigemptyset( & __kmp_sigset );
1464 __kmp_install_one_handler( SIGHUP, __kmp_team_handler, parallel_init );
1465 __kmp_install_one_handler( SIGINT, __kmp_team_handler, parallel_init );
1466 __kmp_install_one_handler( SIGQUIT, __kmp_team_handler, parallel_init );
1467 __kmp_install_one_handler( SIGILL, __kmp_team_handler, parallel_init );
1468 __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
1469 __kmp_install_one_handler( SIGFPE, __kmp_team_handler, parallel_init );
1470 __kmp_install_one_handler( SIGBUS, __kmp_team_handler, parallel_init );
1471 __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
1473 __kmp_install_one_handler( SIGSYS, __kmp_team_handler, parallel_init );
1475 __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
1477 __kmp_install_one_handler( SIGPIPE, __kmp_team_handler, parallel_init );
1484 __kmp_remove_signals(
void )
1487 KB_TRACE( 10, (
"__kmp_remove_signals()\n" ) );
1488 for ( sig = 1; sig < NSIG; ++ sig ) {
1489 __kmp_remove_one_handler( sig );
1494 #endif // KMP_HANDLE_SIGNALS 1500 __kmp_enable(
int new_state )
1502 #ifdef KMP_CANCEL_THREADS 1503 int status, old_state;
1504 status = pthread_setcancelstate( new_state, & old_state );
1505 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status );
1506 KMP_DEBUG_ASSERT( old_state == PTHREAD_CANCEL_DISABLE );
1511 __kmp_disable(
int * old_state )
1513 #ifdef KMP_CANCEL_THREADS 1515 status = pthread_setcancelstate( PTHREAD_CANCEL_DISABLE, old_state );
1516 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status );
1524 __kmp_atfork_prepare (
void)
1530 __kmp_atfork_parent (
void)
1541 __kmp_atfork_child (
void)
1549 __kmp_init_runtime = FALSE;
1550 __kmp_init_monitor = 0;
1551 __kmp_init_parallel = FALSE;
1552 __kmp_init_middle = FALSE;
1553 __kmp_init_serial = FALSE;
1554 TCW_4(__kmp_init_gtid, FALSE);
1555 __kmp_init_common = FALSE;
1557 TCW_4(__kmp_init_user_locks, FALSE);
1558 #if ! KMP_USE_DYNAMIC_LOCK 1559 __kmp_user_lock_table.used = 1;
1560 __kmp_user_lock_table.allocated = 0;
1561 __kmp_user_lock_table.table = NULL;
1562 __kmp_lock_blocks = NULL;
1566 TCW_4(__kmp_nth, 0);
1570 KA_TRACE( 10, (
"__kmp_atfork_child: checking cache address list %p\n",
1571 __kmp_threadpriv_cache_list ) );
1573 while ( __kmp_threadpriv_cache_list != NULL ) {
1575 if ( *__kmp_threadpriv_cache_list -> addr != NULL ) {
1576 KC_TRACE( 50, (
"__kmp_atfork_child: zeroing cache at address %p\n",
1577 &(*__kmp_threadpriv_cache_list -> addr) ) );
1579 *__kmp_threadpriv_cache_list -> addr = NULL;
1581 __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list -> next;
1584 __kmp_init_runtime = FALSE;
1587 __kmp_init_bootstrap_lock( &__kmp_initz_lock );
1588 __kmp_init_bootstrap_lock( &__kmp_stdio_lock );
1589 __kmp_init_bootstrap_lock( &__kmp_console_lock );
1604 __kmp_register_atfork(
void) {
1605 if ( __kmp_need_register_atfork ) {
1606 int status = pthread_atfork( __kmp_atfork_prepare, __kmp_atfork_parent, __kmp_atfork_child );
1607 KMP_CHECK_SYSFAIL(
"pthread_atfork", status );
1608 __kmp_need_register_atfork = FALSE;
1613 __kmp_suspend_initialize(
void )
1616 status = pthread_mutexattr_init( &__kmp_suspend_mutex_attr );
1617 KMP_CHECK_SYSFAIL(
"pthread_mutexattr_init", status );
1618 status = pthread_condattr_init( &__kmp_suspend_cond_attr );
1619 KMP_CHECK_SYSFAIL(
"pthread_condattr_init", status );
1623 __kmp_suspend_initialize_thread( kmp_info_t *th )
1625 if ( th->th.th_suspend_init_count <= __kmp_fork_count ) {
1629 status = pthread_cond_init( &th->th.th_suspend_cv.c_cond, &__kmp_suspend_cond_attr );
1630 KMP_CHECK_SYSFAIL(
"pthread_cond_init", status );
1631 status = pthread_mutex_init( &th->th.th_suspend_mx.m_mutex, & __kmp_suspend_mutex_attr );
1632 KMP_CHECK_SYSFAIL(
"pthread_mutex_init", status );
1633 *(
volatile int*)&th->th.th_suspend_init_count = __kmp_fork_count + 1;
1638 __kmp_suspend_uninitialize_thread( kmp_info_t *th )
1640 if(th->th.th_suspend_init_count > __kmp_fork_count) {
1645 status = pthread_cond_destroy( &th->th.th_suspend_cv.c_cond );
1646 if ( status != 0 && status != EBUSY ) {
1647 KMP_SYSFAIL(
"pthread_cond_destroy", status );
1649 status = pthread_mutex_destroy( &th->th.th_suspend_mx.m_mutex );
1650 if ( status != 0 && status != EBUSY ) {
1651 KMP_SYSFAIL(
"pthread_mutex_destroy", status );
1653 --th->th.th_suspend_init_count;
1654 KMP_DEBUG_ASSERT(th->th.th_suspend_init_count == __kmp_fork_count);
1662 static inline void __kmp_suspend_template(
int th_gtid, C *flag )
1664 KMP_TIME_DEVELOPER_BLOCK(USER_suspend);
1665 kmp_info_t *th = __kmp_threads[th_gtid];
1667 typename C::flag_t old_spin;
1669 KF_TRACE( 30, (
"__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid, flag->get() ) );
1671 __kmp_suspend_initialize_thread( th );
1673 status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1674 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status );
1676 KF_TRACE( 10, (
"__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1677 th_gtid, flag->get() ) );
1682 old_spin = flag->set_sleeping();
1684 KF_TRACE( 5, (
"__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x, was %x\n",
1685 th_gtid, flag->get(), *(flag->get()), old_spin ) );
1687 if ( flag->done_check_val(old_spin) ) {
1688 old_spin = flag->unset_sleeping();
1689 KF_TRACE( 5, (
"__kmp_suspend_template: T#%d false alarm, reset sleep bit for spin(%p)\n",
1690 th_gtid, flag->get()) );
1696 int deactivated = FALSE;
1697 TCW_PTR(th->th.th_sleep_loc, (
void *)flag);
1698 while ( flag->is_sleeping() ) {
1699 #ifdef DEBUG_SUSPEND 1701 __kmp_suspend_count++;
1702 __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1703 __kmp_printf(
"__kmp_suspend_template: suspending T#%d: %s\n", th_gtid, buffer );
1706 if ( ! deactivated ) {
1707 th->th.th_active = FALSE;
1708 if ( th->th.th_active_in_pool ) {
1709 th->th.th_active_in_pool = FALSE;
1710 KMP_TEST_THEN_DEC32(
1711 (kmp_int32 *) &__kmp_thread_pool_active_nth );
1712 KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
1717 #if USE_SUSPEND_TIMEOUT 1718 struct timespec now;
1719 struct timeval tval;
1722 status = gettimeofday( &tval, NULL );
1723 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status );
1724 TIMEVAL_TO_TIMESPEC( &tval, &now );
1726 msecs = (4*__kmp_dflt_blocktime) + 200;
1727 now.tv_sec += msecs / 1000;
1728 now.tv_nsec += (msecs % 1000)*1000;
1730 KF_TRACE( 15, (
"__kmp_suspend_template: T#%d about to perform pthread_cond_timedwait\n",
1732 status = pthread_cond_timedwait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex, & now );
1734 KF_TRACE( 15, (
"__kmp_suspend_template: T#%d about to perform pthread_cond_wait\n",
1736 status = pthread_cond_wait( &th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex );
1739 if ( (status != 0) && (status != EINTR) && (status != ETIMEDOUT) ) {
1740 KMP_SYSFAIL(
"pthread_cond_wait", status );
1743 if (status == ETIMEDOUT) {
1744 if ( flag->is_sleeping() ) {
1745 KF_TRACE( 100, (
"__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid ) );
1747 KF_TRACE( 2, (
"__kmp_suspend_template: T#%d timeout wakeup, sleep bit not set!\n",
1750 }
else if ( flag->is_sleeping() ) {
1751 KF_TRACE( 100, (
"__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid ) );
1757 if ( deactivated ) {
1758 th->th.th_active = TRUE;
1759 if ( TCR_4(th->th.th_in_pool) ) {
1760 KMP_TEST_THEN_INC32( (kmp_int32 *) &__kmp_thread_pool_active_nth );
1761 th->th.th_active_in_pool = TRUE;
1766 #ifdef DEBUG_SUSPEND 1769 __kmp_print_cond( buffer, &th->th.th_suspend_cv);
1770 __kmp_printf(
"__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid, buffer );
1774 status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1775 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
1777 KF_TRACE( 30, (
"__kmp_suspend_template: T#%d exit\n", th_gtid ) );
1780 void __kmp_suspend_32(
int th_gtid, kmp_flag_32 *flag) {
1781 __kmp_suspend_template(th_gtid, flag);
1783 void __kmp_suspend_64(
int th_gtid, kmp_flag_64 *flag) {
1784 __kmp_suspend_template(th_gtid, flag);
1786 void __kmp_suspend_oncore(
int th_gtid, kmp_flag_oncore *flag) {
1787 __kmp_suspend_template(th_gtid, flag);
1796 static inline void __kmp_resume_template(
int target_gtid, C *flag )
1798 KMP_TIME_DEVELOPER_BLOCK(USER_resume);
1799 kmp_info_t *th = __kmp_threads[target_gtid];
1803 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1806 KF_TRACE( 30, (
"__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );
1807 KMP_DEBUG_ASSERT( gtid != target_gtid );
1809 __kmp_suspend_initialize_thread( th );
1811 status = pthread_mutex_lock( &th->th.th_suspend_mx.m_mutex );
1812 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status );
1815 flag = (C *)th->th.th_sleep_loc;
1819 if (!flag || flag->get_type() != flag->get_ptr_type()) {
1820 KF_TRACE( 5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p)\n",
1821 gtid, target_gtid, NULL ) );
1822 status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1823 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
1827 typename C::flag_t old_spin = flag->unset_sleeping();
1828 if ( ! flag->is_sleeping_val(old_spin) ) {
1829 KF_TRACE( 5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag(%p): " 1831 gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1832 status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1833 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
1836 KF_TRACE( 5, (
"__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p): " 1838 gtid, target_gtid, flag->get(), old_spin, *flag->get() ) );
1840 TCW_PTR(th->th.th_sleep_loc, NULL);
1843 #ifdef DEBUG_SUSPEND 1846 __kmp_print_cond( buffer, &th->th.th_suspend_cv );
1847 __kmp_printf(
"__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid, target_gtid, buffer );
1851 status = pthread_cond_signal( &th->th.th_suspend_cv.c_cond );
1852 KMP_CHECK_SYSFAIL(
"pthread_cond_signal", status );
1853 status = pthread_mutex_unlock( &th->th.th_suspend_mx.m_mutex );
1854 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
1855 KF_TRACE( 30, (
"__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n",
1856 gtid, target_gtid ) );
1859 void __kmp_resume_32(
int target_gtid, kmp_flag_32 *flag) {
1860 __kmp_resume_template(target_gtid, flag);
1862 void __kmp_resume_64(
int target_gtid, kmp_flag_64 *flag) {
1863 __kmp_resume_template(target_gtid, flag);
1865 void __kmp_resume_oncore(
int target_gtid, kmp_flag_oncore *flag) {
1866 __kmp_resume_template(target_gtid, flag);
1870 __kmp_resume_monitor()
1872 KMP_TIME_DEVELOPER_BLOCK(USER_resume);
1875 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1876 KF_TRACE( 30, (
"__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n",
1877 gtid, KMP_GTID_MONITOR ) );
1878 KMP_DEBUG_ASSERT( gtid != KMP_GTID_MONITOR );
1880 status = pthread_mutex_lock( &__kmp_wait_mx.m_mutex );
1881 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status );
1882 #ifdef DEBUG_SUSPEND 1885 __kmp_print_cond( buffer, &__kmp_wait_cv.c_cond );
1886 __kmp_printf(
"__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid, KMP_GTID_MONITOR, buffer );
1889 status = pthread_cond_signal( &__kmp_wait_cv.c_cond );
1890 KMP_CHECK_SYSFAIL(
"pthread_cond_signal", status );
1891 status = pthread_mutex_unlock( &__kmp_wait_mx.m_mutex );
1892 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status );
1893 KF_TRACE( 30, (
"__kmp_resume_monitor: T#%d exiting after signaling wake up for T#%d\n",
1894 gtid, KMP_GTID_MONITOR ) );
1901 __kmp_yield(
int cond )
1903 if (cond && __kmp_yielding_on) {
1912 __kmp_gtid_set_specific(
int gtid )
1914 if( __kmp_init_gtid ) {
1916 status = pthread_setspecific( __kmp_gtid_threadprivate_key, (
void*)(intptr_t)(gtid+1) );
1917 KMP_CHECK_SYSFAIL(
"pthread_setspecific", status );
1919 KA_TRACE( 50, (
"__kmp_gtid_set_specific: runtime shutdown, returning\n" ) );
1924 __kmp_gtid_get_specific()
1927 if ( !__kmp_init_gtid ) {
1928 KA_TRACE( 50, (
"__kmp_gtid_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
1929 return KMP_GTID_SHUTDOWN;
1931 gtid = (int)(
size_t)pthread_getspecific( __kmp_gtid_threadprivate_key );
1933 gtid = KMP_GTID_DNE;
1938 KA_TRACE( 50, (
"__kmp_gtid_get_specific: key:%d gtid:%d\n",
1939 __kmp_gtid_threadprivate_key, gtid ));
1947 __kmp_read_cpu_time(
void )
1954 return (buffer.tms_utime + buffer.tms_cutime) / (double) CLOCKS_PER_SEC;
1958 __kmp_read_system_info(
struct kmp_sys_info *info )
1961 struct rusage r_usage;
1963 memset( info, 0,
sizeof( *info ) );
1965 status = getrusage( RUSAGE_SELF, &r_usage);
1966 KMP_CHECK_SYSFAIL_ERRNO(
"getrusage", status );
1968 info->maxrss = r_usage.ru_maxrss;
1969 info->minflt = r_usage.ru_minflt;
1970 info->majflt = r_usage.ru_majflt;
1971 info->nswap = r_usage.ru_nswap;
1972 info->inblock = r_usage.ru_inblock;
1973 info->oublock = r_usage.ru_oublock;
1974 info->nvcsw = r_usage.ru_nvcsw;
1975 info->nivcsw = r_usage.ru_nivcsw;
1977 return (status != 0);
1984 __kmp_read_system_time(
double *delta )
1987 struct timeval tval;
1988 struct timespec stop;
1991 status = gettimeofday( &tval, NULL );
1992 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status );
1993 TIMEVAL_TO_TIMESPEC( &tval, &stop );
1994 t_ns = TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start);
1995 *delta = (t_ns * 1e-9);
1999 __kmp_clear_system_time(
void )
2001 struct timeval tval;
2003 status = gettimeofday( &tval, NULL );
2004 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status );
2005 TIMEVAL_TO_TIMESPEC( &tval, &__kmp_sys_timer_data.start );
2014 __kmp_tv_threadprivate_store( kmp_info_t *th,
void *global_addr,
void *thread_addr )
2018 p = (
struct tv_data *) __kmp_allocate(
sizeof( *p ) );
2020 p->u.tp.global_addr = global_addr;
2021 p->u.tp.thread_addr = thread_addr;
2023 p->type = (
void *) 1;
2025 p->next = th->th.th_local.tv_data;
2026 th->th.th_local.tv_data = p;
2028 if ( p->next == 0 ) {
2029 int rc = pthread_setspecific( __kmp_tv_key, p );
2030 KMP_CHECK_SYSFAIL(
"pthread_setspecific", rc );
2040 __kmp_get_xproc(
void ) {
2044 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD 2046 r = sysconf( _SC_NPROCESSORS_ONLN );
2054 host_basic_info_data_t info;
2055 mach_msg_type_number_t num = HOST_BASIC_INFO_COUNT;
2056 rc = host_info( mach_host_self(), HOST_BASIC_INFO, (host_info_t) & info, & num );
2057 if ( rc == 0 && num == HOST_BASIC_INFO_COUNT ) {
2060 r = info.avail_cpus;
2062 KMP_WARNING( CantGetNumAvailCPU );
2063 KMP_INFORM( AssumedNumCPU );
2068 #error "Unknown or unsupported OS." 2072 return r > 0 ? r : 2;
2077 __kmp_read_from_file(
char const *path,
char const *format, ... )
2082 va_start(args, format);
2083 FILE *f = fopen(path,
"rb");
2086 result = vfscanf(f, format, args);
2093 __kmp_runtime_initialize(
void )
2096 pthread_mutexattr_t mutex_attr;
2097 pthread_condattr_t cond_attr;
2099 if ( __kmp_init_runtime ) {
2103 #if ( KMP_ARCH_X86 || KMP_ARCH_X86_64 ) 2104 if ( ! __kmp_cpuinfo.initialized ) {
2105 __kmp_query_cpuid( &__kmp_cpuinfo );
2109 __kmp_xproc = __kmp_get_xproc();
2111 if ( sysconf( _SC_THREADS ) ) {
2114 __kmp_sys_max_nth = sysconf( _SC_THREAD_THREADS_MAX );
2115 if ( __kmp_sys_max_nth == -1 ) {
2117 __kmp_sys_max_nth = INT_MAX;
2119 else if ( __kmp_sys_max_nth <= 1 ) {
2121 __kmp_sys_max_nth = KMP_MAX_NTH;
2125 __kmp_sys_min_stksize = sysconf( _SC_THREAD_STACK_MIN );
2126 if ( __kmp_sys_min_stksize <= 1 ) {
2127 __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
2132 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
2136 int rc = pthread_key_create( & __kmp_tv_key, 0 );
2137 KMP_CHECK_SYSFAIL(
"pthread_key_create", rc );
2141 status = pthread_key_create( &__kmp_gtid_threadprivate_key, __kmp_internal_end_dest );
2142 KMP_CHECK_SYSFAIL(
"pthread_key_create", status );
2143 status = pthread_mutexattr_init( & mutex_attr );
2144 KMP_CHECK_SYSFAIL(
"pthread_mutexattr_init", status );
2145 status = pthread_mutex_init( & __kmp_wait_mx.m_mutex, & mutex_attr );
2146 KMP_CHECK_SYSFAIL(
"pthread_mutex_init", status );
2147 status = pthread_condattr_init( & cond_attr );
2148 KMP_CHECK_SYSFAIL(
"pthread_condattr_init", status );
2149 status = pthread_cond_init( & __kmp_wait_cv.c_cond, & cond_attr );
2150 KMP_CHECK_SYSFAIL(
"pthread_cond_init", status );
2152 __kmp_itt_initialize();
2155 __kmp_init_runtime = TRUE;
2159 __kmp_runtime_destroy(
void )
2163 if ( ! __kmp_init_runtime ) {
2168 __kmp_itt_destroy();
2171 status = pthread_key_delete( __kmp_gtid_threadprivate_key );
2172 KMP_CHECK_SYSFAIL(
"pthread_key_delete", status );
2174 status = pthread_key_delete( __kmp_tv_key );
2175 KMP_CHECK_SYSFAIL(
"pthread_key_delete", status );
2178 status = pthread_mutex_destroy( & __kmp_wait_mx.m_mutex );
2179 if ( status != 0 && status != EBUSY ) {
2180 KMP_SYSFAIL(
"pthread_mutex_destroy", status );
2182 status = pthread_cond_destroy( & __kmp_wait_cv.c_cond );
2183 if ( status != 0 && status != EBUSY ) {
2184 KMP_SYSFAIL(
"pthread_cond_destroy", status );
2186 #if KMP_AFFINITY_SUPPORTED 2187 __kmp_affinity_uninitialize();
2190 __kmp_init_runtime = FALSE;
2197 __kmp_thread_sleep(
int millis )
2199 sleep( ( millis + 500 ) / 1000 );
2204 __kmp_elapsed(
double *t )
2207 # ifdef FIX_SGI_CLOCK 2210 status = clock_gettime( CLOCK_PROCESS_CPUTIME_ID, &ts );
2211 KMP_CHECK_SYSFAIL_ERRNO(
"clock_gettime", status );
2212 *t = (double) ts.tv_nsec * (1.0 / (
double) KMP_NSEC_PER_SEC) +
2217 status = gettimeofday( & tv, NULL );
2218 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status );
2219 *t = (double) tv.tv_usec * (1.0 / (
double) KMP_USEC_PER_SEC) +
2226 __kmp_elapsed_tick(
double *t )
2228 *t = 1 / (double) CLOCKS_PER_SEC;
2236 gettimeofday(&t, NULL);
2237 return KMP_NSEC_PER_SEC*t.tv_sec + 1000*t.tv_usec;
2245 __kmp_is_address_mapped(
void * addr ) {
2250 #if KMP_OS_LINUX || KMP_OS_FREEBSD 2257 char * name = __kmp_str_format(
"/proc/%d/maps", getpid() );
2260 file = fopen( name,
"r" );
2261 KMP_ASSERT( file != NULL );
2265 void * beginning = NULL;
2266 void * ending = NULL;
2269 rc = fscanf( file,
"%p-%p %4s %*[^\n]\n", & beginning, & ending, perms );
2273 KMP_ASSERT( rc == 3 && KMP_STRLEN( perms ) == 4 );
2276 if ( ( addr >= beginning ) && ( addr < ending ) ) {
2278 if ( strcmp( perms,
"rw" ) == 0 ) {
2289 KMP_INTERNAL_FREE( name );
2303 (vm_address_t)( addr ),
2305 (vm_address_t)( & buffer ),
2313 #elif KMP_OS_FREEBSD || KMP_OS_NETBSD 2320 #error "Unknown or unsupported OS" 2328 #ifdef USE_LOAD_BALANCE 2339 __kmp_get_load_balance(
int max )
2344 int res = getloadavg( averages, 3 );
2349 if ( __kmp_load_balance_interval < 180 && ( res >= 1 ) ) {
2350 ret_avg = averages[0];
2351 }
else if ( ( __kmp_load_balance_interval >= 180
2352 && __kmp_load_balance_interval < 600 ) && ( res >= 2 ) ) {
2353 ret_avg = averages[1];
2354 }
else if ( ( __kmp_load_balance_interval >= 600 ) && ( res == 3 ) ) {
2355 ret_avg = averages[2];
2369 __kmp_get_load_balance(
int max )
2371 static int permanent_error = 0;
2373 static int glb_running_threads = 0;
2374 static double glb_call_time = 0;
2376 int running_threads = 0;
2378 DIR * proc_dir = NULL;
2379 struct dirent * proc_entry = NULL;
2381 kmp_str_buf_t task_path;
2382 DIR * task_dir = NULL;
2383 struct dirent * task_entry = NULL;
2384 int task_path_fixed_len;
2386 kmp_str_buf_t stat_path;
2388 int stat_path_fixed_len;
2390 int total_processes = 0;
2391 int total_threads = 0;
2393 double call_time = 0.0;
2395 __kmp_str_buf_init( & task_path );
2396 __kmp_str_buf_init( & stat_path );
2398 __kmp_elapsed( & call_time );
2400 if ( glb_call_time &&
2401 ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
2402 running_threads = glb_running_threads;
2406 glb_call_time = call_time;
2409 if ( permanent_error ) {
2410 running_threads = -1;
2419 proc_dir = opendir(
"/proc" );
2420 if ( proc_dir == NULL ) {
2423 running_threads = -1;
2424 permanent_error = 1;
2429 __kmp_str_buf_cat( & task_path,
"/proc/", 6 );
2430 task_path_fixed_len = task_path.used;
2432 proc_entry = readdir( proc_dir );
2433 while ( proc_entry != NULL ) {
2436 if ( proc_entry->d_type == DT_DIR && isdigit( proc_entry->d_name[ 0 ] ) ) {
2444 KMP_DEBUG_ASSERT( total_processes != 1 || strcmp( proc_entry->d_name,
"1" ) == 0 );
2447 task_path.used = task_path_fixed_len;
2448 __kmp_str_buf_cat( & task_path, proc_entry->d_name, KMP_STRLEN( proc_entry->d_name ) );
2449 __kmp_str_buf_cat( & task_path,
"/task", 5 );
2451 task_dir = opendir( task_path.str );
2452 if ( task_dir == NULL ) {
2460 if ( strcmp( proc_entry->d_name,
"1" ) == 0 ) {
2461 running_threads = -1;
2462 permanent_error = 1;
2467 __kmp_str_buf_clear( & stat_path );
2468 __kmp_str_buf_cat( & stat_path, task_path.str, task_path.used );
2469 __kmp_str_buf_cat( & stat_path,
"/", 1 );
2470 stat_path_fixed_len = stat_path.used;
2472 task_entry = readdir( task_dir );
2473 while ( task_entry != NULL ) {
2475 if ( proc_entry->d_type == DT_DIR && isdigit( task_entry->d_name[ 0 ] ) ) {
2482 stat_path.used = stat_path_fixed_len;
2483 __kmp_str_buf_cat( & stat_path, task_entry->d_name, KMP_STRLEN( task_entry->d_name ) );
2484 __kmp_str_buf_cat( & stat_path,
"/stat", 5 );
2488 stat_file = open( stat_path.str, O_RDONLY );
2489 if ( stat_file == -1 ) {
2522 len = read( stat_file, buffer,
sizeof( buffer ) - 1 );
2529 char * close_parent = strstr( buffer,
") " );
2530 if ( close_parent != NULL ) {
2531 char state = * ( close_parent + 2 );
2532 if ( state ==
'R' ) {
2534 if ( running_threads >= max ) {
2544 task_entry = readdir( task_dir );
2546 closedir( task_dir );
2550 proc_entry = readdir( proc_dir );
2558 KMP_DEBUG_ASSERT( running_threads > 0 );
2559 if ( running_threads <= 0 ) {
2560 running_threads = 1;
2564 if ( proc_dir != NULL ) {
2565 closedir( proc_dir );
2567 __kmp_str_buf_free( & task_path );
2568 if ( task_dir != NULL ) {
2569 closedir( task_dir );
2571 __kmp_str_buf_free( & stat_path );
2572 if ( stat_file != -1 ) {
2576 glb_running_threads = running_threads;
2578 return running_threads;
2582 # endif // KMP_OS_DARWIN 2584 #endif // USE_LOAD_BALANCE 2586 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || (KMP_OS_LINUX && KMP_ARCH_AARCH64) || KMP_ARCH_PPC64) 2591 __kmp_invoke_microtask( microtask_t pkfn,
2593 int argc,
void *p_argv[]
2595 ,
void **exit_frame_ptr
2600 *exit_frame_ptr = __builtin_frame_address(0);
2605 fprintf(stderr,
"Too many args to microtask: %d!\n", argc);
2609 (*pkfn)(>id, &tid);
2612 (*pkfn)(>id, &tid, p_argv[0]);
2615 (*pkfn)(>id, &tid, p_argv[0], p_argv[1]);
2618 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2]);
2621 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3]);
2624 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4]);
2627 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2631 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2632 p_argv[5], p_argv[6]);
2635 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2636 p_argv[5], p_argv[6], p_argv[7]);
2639 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2640 p_argv[5], p_argv[6], p_argv[7], p_argv[8]);
2643 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2644 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9]);
2647 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2648 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2651 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2652 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2656 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2657 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2658 p_argv[11], p_argv[12]);
2661 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2662 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2663 p_argv[11], p_argv[12], p_argv[13]);
2666 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2667 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2668 p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2673 *exit_frame_ptr = 0;
#define KMP_START_EXPLICIT_TIMER(name)
"Starts" an explicit timer which will need a corresponding KMP_STOP_EXPLICIT_TIMER() macro...
#define KMP_INIT_PARTITIONED_TIMERS(name)
Initializes the paritioned timers to begin with name.