openmp/runtime/src/z_Windows_NT_util.c

/*
 * z_Windows_NT_util.c -- platform specific routines.
 */


//===----------------------------------------------------------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is dual licensed under the MIT and the University of Illinois Open
// Source Licenses. See LICENSE.txt for details.
//
//===----------------------------------------------------------------------===//


#include "kmp.h"
#include "kmp_itt.h"
#include "kmp_i18n.h"
#include "kmp_io.h"
#include "kmp_wait_release.h"

/* This code is related to NtQuerySystemInformation() function. This function
   is used in the Load balance algorithm for OMP_DYNAMIC=true to find the
   number of running threads in the system. */

#include <ntstatus.h>
#include <ntsecapi.h>   // UNICODE_STRING

enum SYSTEM_INFORMATION_CLASS {
    SystemProcessInformation = 5
}; // SYSTEM_INFORMATION_CLASS

struct CLIENT_ID {
    HANDLE UniqueProcess;
    HANDLE UniqueThread;
}; // struct CLIENT_ID

enum THREAD_STATE {
    StateInitialized,
    StateReady,
    StateRunning,
    StateStandby,
    StateTerminated,
    StateWait,
    StateTransition,
    StateUnknown
}; // enum THREAD_STATE

struct VM_COUNTERS {
    SIZE_T        PeakVirtualSize;
    SIZE_T        VirtualSize;
    ULONG         PageFaultCount;
    SIZE_T        PeakWorkingSetSize;
    SIZE_T        WorkingSetSize;
    SIZE_T        QuotaPeakPagedPoolUsage;
    SIZE_T        QuotaPagedPoolUsage;
    SIZE_T        QuotaPeakNonPagedPoolUsage;
    SIZE_T        QuotaNonPagedPoolUsage;
    SIZE_T        PagefileUsage;
    SIZE_T        PeakPagefileUsage;
    SIZE_T        PrivatePageCount;
}; // struct VM_COUNTERS

struct SYSTEM_THREAD {
  LARGE_INTEGER   KernelTime;
  LARGE_INTEGER   UserTime;
  LARGE_INTEGER   CreateTime;
  ULONG           WaitTime;
  LPVOID          StartAddress;
  CLIENT_ID       ClientId;
  DWORD           Priority;
  LONG            BasePriority;
  ULONG           ContextSwitchCount;
  THREAD_STATE    State;
  ULONG           WaitReason;
}; // SYSTEM_THREAD

KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, KernelTime ) == 0 );
#if KMP_ARCH_X86
    KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 28 );
    KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State        ) == 52 );
#else
    KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, StartAddress ) == 32 );
    KMP_BUILD_ASSERT( offsetof( SYSTEM_THREAD, State        ) == 68 );
#endif

struct SYSTEM_PROCESS_INFORMATION {
  ULONG           NextEntryOffset;
  ULONG           NumberOfThreads;
  LARGE_INTEGER   Reserved[ 3 ];
  LARGE_INTEGER   CreateTime;
  LARGE_INTEGER   UserTime;
  LARGE_INTEGER   KernelTime;
  UNICODE_STRING  ImageName;
  DWORD           BasePriority;
  HANDLE          ProcessId;
  HANDLE          ParentProcessId;
  ULONG           HandleCount;
  ULONG           Reserved2[ 2 ];
  VM_COUNTERS     VMCounters;
  IO_COUNTERS     IOCounters;
  SYSTEM_THREAD   Threads[ 1 ];
}; // SYSTEM_PROCESS_INFORMATION
typedef SYSTEM_PROCESS_INFORMATION * PSYSTEM_PROCESS_INFORMATION;

KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, NextEntryOffset ) ==  0 );
KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, CreateTime      ) == 32 );
KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ImageName       ) == 56 );
#if KMP_ARCH_X86
    KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId       ) ==  68 );
    KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount     ) ==  76 );
    KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters      ) ==  88 );
    KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters      ) == 136 );
    KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads         ) == 184 );
#else
    KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, ProcessId       ) ==  80 );
    KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, HandleCount     ) ==  96 );
    KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, VMCounters      ) == 112 );
    KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, IOCounters      ) == 208 );
    KMP_BUILD_ASSERT( offsetof( SYSTEM_PROCESS_INFORMATION, Threads         ) == 256 );
#endif

typedef NTSTATUS (NTAPI *NtQuerySystemInformation_t)( SYSTEM_INFORMATION_CLASS, PVOID, ULONG, PULONG );
NtQuerySystemInformation_t NtQuerySystemInformation = NULL;

HMODULE ntdll = NULL;

/* End of NtQuerySystemInformation()-related code */

#if KMP_GROUP_AFFINITY
static HMODULE kernel32 = NULL;
#endif /* KMP_GROUP_AFFINITY */

/* ----------------------------------------------------------------------------------- */
/* ----------------------------------------------------------------------------------- */

#if KMP_HANDLE_SIGNALS
    typedef void    (* sig_func_t )( int );
    static sig_func_t  __kmp_sighldrs[ NSIG ];
    static int         __kmp_siginstalled[ NSIG ];
#endif

static HANDLE   __kmp_monitor_ev;
static kmp_int64 __kmp_win32_time;
double __kmp_win32_tick;

int __kmp_init_runtime = FALSE;
CRITICAL_SECTION __kmp_win32_section;

void
__kmp_win32_mutex_init( kmp_win32_mutex_t *mx )
{
    InitializeCriticalSection( & mx->cs );
#if USE_ITT_BUILD
    __kmp_itt_system_object_created( & mx->cs, "Critical Section" );
#endif /* USE_ITT_BUILD */
}

void
__kmp_win32_mutex_destroy( kmp_win32_mutex_t *mx )
{
    DeleteCriticalSection( & mx->cs );
}

void
__kmp_win32_mutex_lock( kmp_win32_mutex_t *mx )
{
    EnterCriticalSection( & mx->cs );
}

void
__kmp_win32_mutex_unlock( kmp_win32_mutex_t *mx )
{
    LeaveCriticalSection( & mx->cs );
}

void
__kmp_win32_cond_init( kmp_win32_cond_t *cv )
{
    cv->waiters_count_         = 0;
    cv->wait_generation_count_ = 0;
    cv->release_count_         = 0;

    /* Initialize the critical section */
    __kmp_win32_mutex_init( & cv->waiters_count_lock_ );

    /* Create a manual-reset event. */
    cv->event_ = CreateEvent( NULL,     // no security
                              TRUE,     // manual-reset
                              FALSE,    // non-signaled initially
                              NULL );   // unnamed
#if USE_ITT_BUILD
    __kmp_itt_system_object_created( cv->event_, "Event" );
#endif /* USE_ITT_BUILD */
}

void
__kmp_win32_cond_destroy( kmp_win32_cond_t *cv )
{
    __kmp_win32_mutex_destroy( & cv->waiters_count_lock_ );
    __kmp_free_handle( cv->event_ );
    memset( cv, '\0', sizeof( *cv ) );
}

/* TODO associate cv with a team instead of a thread so as to optimize
 * the case where we wake up a whole team */

void
__kmp_win32_cond_wait( kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx, kmp_info_t *th, int need_decrease_load )
{
    int my_generation;
    int last_waiter;

    /* Avoid race conditions */
    __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );

    /* Increment count of waiters */
    cv->waiters_count_++;

    /* Store current generation in our activation record. */
    my_generation = cv->wait_generation_count_;

    __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
    __kmp_win32_mutex_unlock( mx );

    for (;;) {
        int wait_done;

        /* Wait until the event is signaled */
        WaitForSingleObject( cv->event_, INFINITE );

        __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );

        /* Exit the loop when the <cv->event_> is signaled and
         * there are still waiting threads from this <wait_generation>
         * that haven't been released from this wait yet.              */
        wait_done = ( cv->release_count_ > 0 ) &&
                    ( cv->wait_generation_count_ != my_generation );

        __kmp_win32_mutex_unlock( &cv->waiters_count_lock_);

        /* there used to be a semicolon after the if statement,
         * it looked like a bug, so i removed it */
        if( wait_done )
            break;
    }

    __kmp_win32_mutex_lock( mx );
    __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );

    cv->waiters_count_--;
    cv->release_count_--;

    last_waiter =  ( cv->release_count_ == 0 );

    __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );

    if( last_waiter ) {
        /* We're the last waiter to be notified, so reset the manual event. */
        ResetEvent( cv->event_ );
    }
}

void
__kmp_win32_cond_broadcast( kmp_win32_cond_t *cv )
{
    __kmp_win32_mutex_lock( &cv->waiters_count_lock_ );

    if( cv->waiters_count_ > 0 ) {
        SetEvent( cv->event_ );
        /* Release all the threads in this generation. */

        cv->release_count_ = cv->waiters_count_;

        /* Start a new generation. */
        cv->wait_generation_count_++;
    }

    __kmp_win32_mutex_unlock( &cv->waiters_count_lock_ );
}

void
__kmp_win32_cond_signal( kmp_win32_cond_t *cv )
{
    __kmp_win32_cond_broadcast( cv );
}

/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */

void
__kmp_enable( int new_state )
{
    if (__kmp_init_runtime)
        LeaveCriticalSection( & __kmp_win32_section );
}

void
__kmp_disable( int *old_state )
{
    *old_state = 0;

    if (__kmp_init_runtime)
        EnterCriticalSection( & __kmp_win32_section );
}

void
__kmp_suspend_initialize( void )
{
    /* do nothing */
}

static void
__kmp_suspend_initialize_thread( kmp_info_t *th )
{
    if ( ! TCR_4( th->th.th_suspend_init ) ) {
      /* this means we haven't initialized the suspension pthread objects for this thread
         in this instance of the process */
        __kmp_win32_cond_init(  &th->th.th_suspend_cv );
        __kmp_win32_mutex_init( &th->th.th_suspend_mx );
        TCW_4( th->th.th_suspend_init, TRUE );
    }
}

void
__kmp_suspend_uninitialize_thread( kmp_info_t *th )
{
    if ( TCR_4( th->th.th_suspend_init ) ) {
      /* this means we have initialize the suspension pthread objects for this thread
         in this instance of the process */
      __kmp_win32_cond_destroy( & th->th.th_suspend_cv );
      __kmp_win32_mutex_destroy( & th->th.th_suspend_mx );
      TCW_4( th->th.th_suspend_init, FALSE );
    }
}

/* This routine puts the calling thread to sleep after setting the
 * sleep bit for the indicated flag variable to true.
 */
template <class C>
static inline void __kmp_suspend_template( int th_gtid, C *flag )
{
    kmp_info_t *th = __kmp_threads[th_gtid];
    int status;
    typename C::flag_t old_spin;

    KF_TRACE( 30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n", th_gtid, flag->get() ) );

    __kmp_suspend_initialize_thread( th );
    __kmp_win32_mutex_lock( &th->th.th_suspend_mx );

    KF_TRACE( 10, ( "__kmp_suspend_template: T#%d setting sleep bit for flag's loc(%p)\n",
                    th_gtid, flag->get() ) );

    /* TODO: shouldn't this use release semantics to ensure that __kmp_suspend_initialize_thread
       gets called first?
    */
    old_spin = flag->set_sleeping();

    KF_TRACE( 5, ( "__kmp_suspend_template: T#%d set sleep bit for flag's loc(%p)==%d\n",
                   th_gtid, flag->get(), *(flag->get()) ) );

    if ( flag->done_check_val(old_spin) ) {
        old_spin = flag->unset_sleeping();
        KF_TRACE( 5, ( "__kmp_suspend_template: T#%d false alarm, reset sleep bit for flag's loc(%p)\n",
                       th_gtid, flag->get()) );
    } else {
#ifdef DEBUG_SUSPEND
        __kmp_suspend_count++;
#endif
        /* Encapsulate in a loop as the documentation states that this may
         * "with low probability" return when the condition variable has
         * not been signaled or broadcast
         */
        int deactivated = FALSE;
        TCW_PTR(th->th.th_sleep_loc, (void *)flag);
        while ( flag->is_sleeping() ) {
            KF_TRACE( 15, ("__kmp_suspend_template: T#%d about to perform kmp_win32_cond_wait()\n",
                     th_gtid ) );
            // Mark the thread as no longer active (only in the first iteration of the loop).
            if ( ! deactivated ) {
                th->th.th_active = FALSE;
                if ( th->th.th_active_in_pool ) {
                    th->th.th_active_in_pool = FALSE;
                    KMP_TEST_THEN_DEC32(
                      (kmp_int32 *) &__kmp_thread_pool_active_nth );
                    KMP_DEBUG_ASSERT( TCR_4(__kmp_thread_pool_active_nth) >= 0 );
                }
                deactivated = TRUE;

                __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
            }
            else {
                __kmp_win32_cond_wait( &th->th.th_suspend_cv, &th->th.th_suspend_mx, 0, 0 );
            }

#ifdef KMP_DEBUG
            if( flag->is_sleeping() ) {
                KF_TRACE( 100, ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid ));
            }
#endif /* KMP_DEBUG */

        } // while

        // Mark the thread as active again (if it was previous marked as inactive)
        if ( deactivated ) {
            th->th.th_active = TRUE;
            if ( TCR_4(th->th.th_in_pool) ) {
                KMP_TEST_THEN_INC32(
                  (kmp_int32 *) &__kmp_thread_pool_active_nth );
                th->th.th_active_in_pool = TRUE;
            }
        }
    }

    __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );

    KF_TRACE( 30, ("__kmp_suspend_template: T#%d exit\n", th_gtid ) );
}

void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
    __kmp_suspend_template(th_gtid, flag);
}
void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
    __kmp_suspend_template(th_gtid, flag);
}
void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
    __kmp_suspend_template(th_gtid, flag);
}


/* This routine signals the thread specified by target_gtid to wake up
 * after setting the sleep bit indicated by the flag argument to FALSE
 */
template <class C>
static inline void __kmp_resume_template( int target_gtid, C *flag )
{
    kmp_info_t *th = __kmp_threads[target_gtid];
    int status;

#ifdef KMP_DEBUG
    int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
#endif

    KF_TRACE( 30, ( "__kmp_resume_template: T#%d wants to wakeup T#%d enter\n", gtid, target_gtid ) );

    __kmp_suspend_initialize_thread( th );
    __kmp_win32_mutex_lock( &th->th.th_suspend_mx );

    if (!flag) { // coming from __kmp_null_resume_wrapper
        flag = (C *)th->th.th_sleep_loc;
    }

    // First, check if the flag is null or its type has changed. If so, someone else woke it up.
    if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type simply shows what flag was cast to
        KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p)\n",
                       gtid, target_gtid, NULL ) );
        __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
        return;
    }
    else {
        typename C::flag_t old_spin = flag->unset_sleeping();
        if ( !flag->is_sleeping_val(old_spin) ) {
            KF_TRACE( 5, ( "__kmp_resume_template: T#%d exiting, thread T#%d already awake: flag's loc(%p): "
                           "%u => %u\n",
                           gtid, target_gtid, flag->get(), old_spin, *(flag->get()) ) );
            __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );
            return;
        }
    }
    TCW_PTR(th->th.th_sleep_loc, NULL);

    KF_TRACE( 5, ( "__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep bit for flag's loc(%p)\n",
                   gtid, target_gtid, flag->get() ) );

    __kmp_win32_cond_signal(  &th->th.th_suspend_cv );
    __kmp_win32_mutex_unlock( &th->th.th_suspend_mx );

    KF_TRACE( 30, ( "__kmp_resume_template: T#%d exiting after signaling wake up for T#%d\n",
                    gtid, target_gtid ) );
}

void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
    __kmp_resume_template(target_gtid, flag);
}
void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
    __kmp_resume_template(target_gtid, flag);
}
void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
    __kmp_resume_template(target_gtid, flag);
}


/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */

void
__kmp_yield( int cond )
{
    if (cond)
        Sleep(0);
}

/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */

void
__kmp_gtid_set_specific( int gtid )
{
    KA_TRACE( 50, ("__kmp_gtid_set_specific: T#%d key:%d\n",
                gtid, __kmp_gtid_threadprivate_key ));
    KMP_ASSERT( __kmp_init_runtime );
    if( ! TlsSetValue( __kmp_gtid_threadprivate_key, (LPVOID)(gtid+1)) )
        KMP_FATAL( TLSSetValueFailed );
}

int
__kmp_gtid_get_specific()
{
    int gtid;
    if( !__kmp_init_runtime ) {
        KA_TRACE( 50, ("__kmp_get_specific: runtime shutdown, returning KMP_GTID_SHUTDOWN\n" ) );
        return KMP_GTID_SHUTDOWN;
    }
    gtid = (int)(kmp_intptr_t)TlsGetValue( __kmp_gtid_threadprivate_key );
    if ( gtid == 0 ) {
        gtid = KMP_GTID_DNE;
    }
    else {
        gtid--;
    }
    KA_TRACE( 50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
                __kmp_gtid_threadprivate_key, gtid ));
    return gtid;
}

/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */

#if KMP_GROUP_AFFINITY

//
// Only 1 DWORD in the mask should have any procs set.
// Return the appropriate index, or -1 for an invalid mask.
//
int
__kmp_get_proc_group( kmp_affin_mask_t const *mask )
{
    int i;
    int group = -1;
    for (i = 0; i < __kmp_num_proc_groups; i++) {
        if (mask[i] == 0) {
            continue;
        }
        if (group >= 0) {
            return -1;
        }
        group = i;
    }
    return group;
}

#endif /* KMP_GROUP_AFFINITY */

int
__kmp_set_system_affinity( kmp_affin_mask_t const *mask, int abort_on_error )
{

#if KMP_GROUP_AFFINITY

    if (__kmp_num_proc_groups > 1) {
        //
        // Check for a valid mask.
        //
        GROUP_AFFINITY ga;
        int group = __kmp_get_proc_group( mask );
        if (group < 0) {
            if (abort_on_error) {
                KMP_FATAL(AffinityInvalidMask, "kmp_set_affinity");
            }
            return -1;
        }

        //
        // Transform the bit vector into a GROUP_AFFINITY struct
        // and make the system call to set affinity.
        //
        ga.Group = group;
        ga.Mask = mask[group];
        ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;

        KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
        if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
            DWORD error = GetLastError();
            if (abort_on_error) {
                __kmp_msg(
                    kmp_ms_fatal,
                    KMP_MSG( CantSetThreadAffMask ),
                    KMP_ERR( error ),
                    __kmp_msg_null
                );
            }
            return error;
        }
    }
    else

#endif /* KMP_GROUP_AFFINITY */

    {
        if (!SetThreadAffinityMask( GetCurrentThread(), *mask )) {
            DWORD error = GetLastError();
            if (abort_on_error) {
                __kmp_msg(
                    kmp_ms_fatal,
                    KMP_MSG( CantSetThreadAffMask ),
                    KMP_ERR( error ),
                    __kmp_msg_null
                );
            }
            return error;
        }
    }
    return 0;
}

int
__kmp_get_system_affinity( kmp_affin_mask_t *mask, int abort_on_error )
{

#if KMP_GROUP_AFFINITY

    if (__kmp_num_proc_groups > 1) {
        KMP_CPU_ZERO(mask);
        GROUP_AFFINITY ga;
        KMP_DEBUG_ASSERT(__kmp_GetThreadGroupAffinity != NULL);

        if (__kmp_GetThreadGroupAffinity(GetCurrentThread(), &ga) == 0) {
            DWORD error = GetLastError();
            if (abort_on_error) {
                __kmp_msg(
                    kmp_ms_fatal,
                    KMP_MSG(FunctionError, "GetThreadGroupAffinity()"),
                    KMP_ERR(error),
                    __kmp_msg_null
                );
            }
            return error;
        }

        if ((ga.Group < 0) || (ga.Group > __kmp_num_proc_groups)
          || (ga.Mask == 0)) {
            return -1;
        }

        mask[ga.Group] = ga.Mask;
    }
    else

#endif /* KMP_GROUP_AFFINITY */

    {
        kmp_affin_mask_t newMask, sysMask, retval;

        if (!GetProcessAffinityMask(GetCurrentProcess(), &newMask, &sysMask)) {
            DWORD error = GetLastError();
            if (abort_on_error) {
                __kmp_msg(
                    kmp_ms_fatal,
                    KMP_MSG(FunctionError, "GetProcessAffinityMask()"),
                    KMP_ERR(error),
                    __kmp_msg_null
                );
            }
            return error;
        }
        retval = SetThreadAffinityMask(GetCurrentThread(), newMask);
        if (! retval) {
            DWORD error = GetLastError();
            if (abort_on_error) {
                __kmp_msg(
                    kmp_ms_fatal,
                    KMP_MSG(FunctionError, "SetThreadAffinityMask()"),
                    KMP_ERR(error),
                    __kmp_msg_null
                );
            }
            return error;
        }
        newMask = SetThreadAffinityMask(GetCurrentThread(), retval);
        if (! newMask) {
            DWORD error = GetLastError();
            if (abort_on_error) {
                __kmp_msg(
                    kmp_ms_fatal,
                    KMP_MSG(FunctionError, "SetThreadAffinityMask()"),
                    KMP_ERR(error),
                    __kmp_msg_null
                );
            }
        }
        *mask = retval;
    }
    return 0;
}

void
__kmp_affinity_bind_thread( int proc )
{

#if KMP_GROUP_AFFINITY

    if (__kmp_num_proc_groups > 1) {
        //
        // Form the GROUP_AFFINITY struct directly, rather than filling
        // out a bit vector and calling __kmp_set_system_affinity().
        //
        GROUP_AFFINITY ga;
        KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups
           * CHAR_BIT * sizeof(DWORD_PTR))));
        ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR));
        ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR)));
        ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;

        KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
        if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
            DWORD error = GetLastError();
            if (__kmp_affinity_verbose) { // AC: continue silently if not verbose
                __kmp_msg(
                    kmp_ms_warning,
                    KMP_MSG( CantSetThreadAffMask ),
                    KMP_ERR( error ),
                    __kmp_msg_null
                );
            }
        }
    }
    else

#endif /* KMP_GROUP_AFFINITY */

    {
        kmp_affin_mask_t mask;
        KMP_CPU_ZERO(&mask);
        KMP_CPU_SET(proc, &mask);
        __kmp_set_system_affinity(&mask, TRUE);
    }
}

void
__kmp_affinity_determine_capable( const char *env_var )
{
    //
    // All versions of Windows* OS (since Win '95) support SetThreadAffinityMask().
    //

#if KMP_GROUP_AFFINITY
    KMP_AFFINITY_ENABLE(__kmp_num_proc_groups*sizeof(kmp_affin_mask_t));
#else
    KMP_AFFINITY_ENABLE(sizeof(kmp_affin_mask_t));
#endif

    KA_TRACE( 10, (
        "__kmp_affinity_determine_capable: "
            "Windows* OS affinity interface functional (mask size = %" KMP_SIZE_T_SPEC ").\n",
        __kmp_affin_mask_size
    ) );
}

double
__kmp_read_cpu_time( void )
{
    FILETIME    CreationTime, ExitTime, KernelTime, UserTime;
    int         status;
    double      cpu_time;

    cpu_time = 0;

    status = GetProcessTimes( GetCurrentProcess(), &CreationTime,
                              &ExitTime, &KernelTime, &UserTime );

    if (status) {
        double  sec = 0;

        sec += KernelTime.dwHighDateTime;
        sec += UserTime.dwHighDateTime;

        /* Shift left by 32 bits */
        sec *= (double) (1 << 16) * (double) (1 << 16);

        sec += KernelTime.dwLowDateTime;
        sec += UserTime.dwLowDateTime;

        cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
    }

    return cpu_time;
}

int
__kmp_read_system_info( struct kmp_sys_info *info )
{
    info->maxrss  = 0;                   /* the maximum resident set size utilized (in kilobytes)     */
    info->minflt  = 0;                   /* the number of page faults serviced without any I/O        */
    info->majflt  = 0;                   /* the number of page faults serviced that required I/O      */
    info->nswap   = 0;                   /* the number of times a process was "swapped" out of memory */
    info->inblock = 0;                   /* the number of times the file system had to perform input  */
    info->oublock = 0;                   /* the number of times the file system had to perform output */
    info->nvcsw   = 0;                   /* the number of times a context switch was voluntarily      */
    info->nivcsw  = 0;                   /* the number of times a context switch was forced           */

    return 1;
}

/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */


void
__kmp_runtime_initialize( void )
{
    SYSTEM_INFO info;
    kmp_str_buf_t path;
    UINT path_size;

    if ( __kmp_init_runtime ) {
        return;
    };

#if KMP_DYNAMIC_LIB
    /* Pin dynamic library for the lifetime of application */
    {
        // First, turn off error message boxes
        UINT err_mode = SetErrorMode (SEM_FAILCRITICALERRORS);
        HMODULE h;
        BOOL ret = GetModuleHandleEx( GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS
                                     |GET_MODULE_HANDLE_EX_FLAG_PIN,
                                     (LPCTSTR)&__kmp_serial_initialize, &h);
        KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded");
        SetErrorMode (err_mode);   // Restore error mode
        KA_TRACE( 10, ("__kmp_runtime_initialize: dynamic library pinned\n") );
    }
#endif

    InitializeCriticalSection( & __kmp_win32_section );
#if USE_ITT_BUILD
    __kmp_itt_system_object_created( & __kmp_win32_section, "Critical Section" );
#endif /* USE_ITT_BUILD */
    __kmp_initialize_system_tick();

    #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
        if ( ! __kmp_cpuinfo.initialized ) {
            __kmp_query_cpuid( & __kmp_cpuinfo );
        }; // if
    #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */

    /* Set up minimum number of threads to switch to TLS gtid */
    #if KMP_OS_WINDOWS && ! defined KMP_DYNAMIC_LIB
        // Windows* OS, static library.
        /*
            New thread may use stack space previously used by another thread, currently terminated.
            On Windows* OS, in case of static linking, we do not know the moment of thread termination,
            and our structures (__kmp_threads and __kmp_root arrays) are still keep info about dead
            threads. This leads to problem in __kmp_get_global_thread_id() function: it wrongly
            finds gtid (by searching through stack addresses of all known threads) for unregistered
            foreign tread.

            Setting __kmp_tls_gtid_min to 0 workarounds this problem: __kmp_get_global_thread_id()
            does not search through stacks, but get gtid from TLS immediately.

            --ln
        */
        __kmp_tls_gtid_min = 0;
    #else
        __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
    #endif

    /* for the static library */
    if ( !__kmp_gtid_threadprivate_key ) {
        __kmp_gtid_threadprivate_key = TlsAlloc();
        if( __kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES ) {
            KMP_FATAL( TLSOutOfIndexes );
        }
    }


    //
    // Load ntdll.dll.
    //
    /*
        Simple
            GetModuleHandle( "ntdll.dl" )
        is not suitable due to security issue (see
        http://www.microsoft.com/technet/security/advisory/2269637.mspx). We have to specify full
        path to the library.
    */
    __kmp_str_buf_init( & path );
    path_size = GetSystemDirectory( path.str, path.size );
    KMP_DEBUG_ASSERT( path_size > 0 );
    if ( path_size >= path.size ) {
        //
        // Buffer is too short.  Expand the buffer and try again.
        //
        __kmp_str_buf_reserve( & path, path_size );
        path_size = GetSystemDirectory( path.str, path.size );
        KMP_DEBUG_ASSERT( path_size > 0 );
    }; // if
    if ( path_size > 0 && path_size < path.size ) {
        //
        // Now we have system directory name in the buffer.
        // Append backslash and name of dll to form full path,
        //
        path.used = path_size;
        __kmp_str_buf_print( & path, "\\%s", "ntdll.dll" );

        //
        // Now load ntdll using full path.
        //
        ntdll = GetModuleHandle( path.str );
    }

    KMP_DEBUG_ASSERT( ntdll != NULL );
    if ( ntdll != NULL ) {
        NtQuerySystemInformation = (NtQuerySystemInformation_t) GetProcAddress( ntdll, "NtQuerySystemInformation" );
    }
    KMP_DEBUG_ASSERT( NtQuerySystemInformation != NULL );

#if KMP_GROUP_AFFINITY
    //
    // Load kernel32.dll.
    // Same caveat - must use full system path name.
    //
    if ( path_size > 0 && path_size < path.size ) {
        //
        // Truncate the buffer back to just the system path length,
        // discarding "\\ntdll.dll", and replacing it with "kernel32.dll".
        //
        path.used = path_size;
        __kmp_str_buf_print( & path, "\\%s", "kernel32.dll" );

        //
        // Load kernel32.dll using full path.
        //
        kernel32 = GetModuleHandle( path.str );
        KA_TRACE( 10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str ) );

        //
        // Load the function pointers to kernel32.dll routines
        // that may or may not exist on this system.
        //
        if ( kernel32 != NULL ) {
            __kmp_GetActiveProcessorCount = (kmp_GetActiveProcessorCount_t) GetProcAddress( kernel32, "GetActiveProcessorCount" );
            __kmp_GetActiveProcessorGroupCount = (kmp_GetActiveProcessorGroupCount_t) GetProcAddress( kernel32, "GetActiveProcessorGroupCount" );
            __kmp_GetThreadGroupAffinity = (kmp_GetThreadGroupAffinity_t) GetProcAddress( kernel32, "GetThreadGroupAffinity" );
            __kmp_SetThreadGroupAffinity = (kmp_SetThreadGroupAffinity_t) GetProcAddress( kernel32, "SetThreadGroupAffinity" );

            KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount = %p\n", __kmp_GetActiveProcessorCount ) );
            KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorGroupCount = %p\n", __kmp_GetActiveProcessorGroupCount ) );
            KA_TRACE( 10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity = %p\n", __kmp_GetThreadGroupAffinity ) );
            KA_TRACE( 10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity = %p\n", __kmp_SetThreadGroupAffinity ) );
            KA_TRACE( 10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n", sizeof(kmp_affin_mask_t) ) );

            //
            // See if group affinity is supported on this system.
            // If so, calculate the #groups and #procs.
            //
            // Group affinity was introduced with Windows* 7 OS and
            // Windows* Server 2008 R2 OS.
            //
            if ( ( __kmp_GetActiveProcessorCount != NULL )
              && ( __kmp_GetActiveProcessorGroupCount != NULL )
              && ( __kmp_GetThreadGroupAffinity != NULL )
              && ( __kmp_SetThreadGroupAffinity != NULL )
              && ( ( __kmp_num_proc_groups
              = __kmp_GetActiveProcessorGroupCount() ) > 1 ) ) {
                //
                // Calculate the total number of active OS procs.
                //
                int i;

                KA_TRACE( 10, ("__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) );

                __kmp_xproc = 0;

                for ( i = 0; i < __kmp_num_proc_groups; i++ ) {
                    DWORD size = __kmp_GetActiveProcessorCount( i );
                    __kmp_xproc += size;
                    KA_TRACE( 10, ("__kmp_runtime_initialize: proc group %d size = %d\n", i, size ) );
                }
                }
            else {
                KA_TRACE( 10, ("__kmp_runtime_initialize: %d processor groups detected\n", __kmp_num_proc_groups ) );
            }
        }
    }
    if ( __kmp_num_proc_groups <= 1 ) {
        GetSystemInfo( & info );
        __kmp_xproc = info.dwNumberOfProcessors;
    }
#else
    GetSystemInfo( & info );
    __kmp_xproc = info.dwNumberOfProcessors;
#endif /* KMP_GROUP_AFFINITY */

    //
    // If the OS said there were 0 procs, take a guess and use a value of 2.
    // This is done for Linux* OS, also.  Do we need error / warning?
    //
    if ( __kmp_xproc <= 0 ) {
        __kmp_xproc = 2;
    }

    KA_TRACE( 5, ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc) );

    __kmp_str_buf_free( & path );

#if USE_ITT_BUILD
    __kmp_itt_initialize();
#endif /* USE_ITT_BUILD */

    __kmp_init_runtime = TRUE;
} // __kmp_runtime_initialize

void
__kmp_runtime_destroy( void )
{
    if ( ! __kmp_init_runtime ) {
        return;
    }

#if USE_ITT_BUILD
    __kmp_itt_destroy();
#endif /* USE_ITT_BUILD */

    /* we can't DeleteCriticalsection( & __kmp_win32_section ); */
    /* due to the KX_TRACE() commands */
    KA_TRACE( 40, ("__kmp_runtime_destroy\n" ));

    if( __kmp_gtid_threadprivate_key ) {
        TlsFree( __kmp_gtid_threadprivate_key );
        __kmp_gtid_threadprivate_key = 0;
    }

    __kmp_affinity_uninitialize();
    DeleteCriticalSection( & __kmp_win32_section );

    ntdll = NULL;
    NtQuerySystemInformation = NULL;

#if KMP_ARCH_X86_64
    kernel32 = NULL;
    __kmp_GetActiveProcessorCount = NULL;
    __kmp_GetActiveProcessorGroupCount = NULL;
    __kmp_GetThreadGroupAffinity = NULL;
    __kmp_SetThreadGroupAffinity = NULL;
#endif // KMP_ARCH_X86_64

    __kmp_init_runtime = FALSE;
}


void
__kmp_terminate_thread( int gtid )
{
    kmp_info_t  *th = __kmp_threads[ gtid ];

    if( !th ) return;

    KA_TRACE( 10, ("__kmp_terminate_thread: kill (%d)\n", gtid ) );

    if (TerminateThread( th->th.th_info.ds.ds_thread, (DWORD) -1) == FALSE) {
        /* It's OK, the thread may have exited already */
    }
    __kmp_free_handle( th->th.th_info.ds.ds_thread );
}

/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */

void
__kmp_clear_system_time( void )
{
    BOOL status;
    LARGE_INTEGER time;
    status = QueryPerformanceCounter( & time );
    __kmp_win32_time = (kmp_int64) time.QuadPart;
}

void
__kmp_initialize_system_tick( void )
{
    {
  BOOL status;
  LARGE_INTEGER freq;

  status = QueryPerformanceFrequency( & freq );
  if (! status) {
        DWORD error = GetLastError();
        __kmp_msg(
            kmp_ms_fatal,
            KMP_MSG( FunctionError, "QueryPerformanceFrequency()" ),
            KMP_ERR( error ),
            __kmp_msg_null
        );

  }
  else {
      __kmp_win32_tick = ((double) 1.0) / (double) freq.QuadPart;
  }
    }
}

/* Calculate the elapsed wall clock time for the user */

void
__kmp_elapsed( double *t )
{
    BOOL status;
    LARGE_INTEGER now;
    status = QueryPerformanceCounter( & now );
    *t = ((double) now.QuadPart) * __kmp_win32_tick;
}

/* Calculate the elapsed wall clock tick for the user */

void
__kmp_elapsed_tick( double *t )
{
    *t = __kmp_win32_tick;
}

void
__kmp_read_system_time( double *delta )
{
    if (delta != NULL) {
        BOOL status;
        LARGE_INTEGER now;

        status = QueryPerformanceCounter( & now );

        *delta = ((double) (((kmp_int64) now.QuadPart) - __kmp_win32_time))
    * __kmp_win32_tick;
    }
}

/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */

void * __stdcall
__kmp_launch_worker( void *arg )
{
    volatile void *stack_data;
    void *exit_val;
    void *padding = 0;
    kmp_info_t *this_thr = (kmp_info_t *) arg;
    int gtid;

    gtid = this_thr->th.th_info.ds.ds_gtid;
    __kmp_gtid_set_specific( gtid );
#ifdef KMP_TDATA_GTID
    #error "This define causes problems with LoadLibrary() + declspec(thread) " \
        "on Windows* OS.  See CQ50564, tests kmp_load_library*.c and this MSDN " \
        "reference: http://support.microsoft.com/kb/118816"
    //__kmp_gtid = gtid;
#endif

#if USE_ITT_BUILD
    __kmp_itt_thread_name( gtid );
#endif /* USE_ITT_BUILD */

    __kmp_affinity_set_init_mask( gtid, FALSE );

#if KMP_ARCH_X86 || KMP_ARCH_X86_64
    //
    // Set the FP control regs to be a copy of
    // the parallel initialization thread's.
    //
    __kmp_clear_x87_fpu_status_word();
    __kmp_load_x87_fpu_control_word( &__kmp_init_x87_fpu_control_word );
    __kmp_load_mxcsr( &__kmp_init_mxcsr );
#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */

    if ( __kmp_stkoffset > 0 && gtid > 0 ) {
        padding = KMP_ALLOCA( gtid * __kmp_stkoffset );
    }

    KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
    this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
    TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );

    if ( TCR_4(__kmp_gtid_mode) < 2 ) { // check stack only if it is used to get gtid
        TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
        KMP_ASSERT( this_thr -> th.th_info.ds.ds_stackgrow == FALSE );
        __kmp_check_stack_overlap( this_thr );
    }
    KMP_MB();
    exit_val = __kmp_launch_thread( this_thr );
    KMP_FSYNC_RELEASING( &this_thr -> th.th_info.ds.ds_alive );
    TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );
    KMP_MB();
    return exit_val;
}

/* The monitor thread controls all of the threads in the complex */

void * __stdcall
__kmp_launch_monitor( void *arg )
{
    DWORD        wait_status;
    kmp_thread_t monitor;
    int          status;
    int          interval;
    kmp_info_t *this_thr = (kmp_info_t *) arg;

    KMP_DEBUG_ASSERT(__kmp_init_monitor);
    TCW_4( __kmp_init_monitor, 2 );    // AC: Signal the library that monitor has started
                                       // TODO: hide "2" in enum (like {true,false,started})
    this_thr -> th.th_info.ds.ds_thread_id = GetCurrentThreadId();
    TCW_4( this_thr -> th.th_info.ds.ds_alive, TRUE );

    KMP_MB();       /* Flush all pending memory write invalidates.  */
    KA_TRACE( 10, ("__kmp_launch_monitor: launched\n" ) );

    monitor = GetCurrentThread();

    /* set thread priority */
    status = SetThreadPriority( monitor, THREAD_PRIORITY_HIGHEST );
    if (! status) {
        DWORD error = GetLastError();
        __kmp_msg(
            kmp_ms_fatal,
            KMP_MSG( CantSetThreadPriority ),
            KMP_ERR( error ),
            __kmp_msg_null
        );
    }

    /* register us as monitor */
    __kmp_gtid_set_specific( KMP_GTID_MONITOR );
#ifdef KMP_TDATA_GTID
    #error "This define causes problems with LoadLibrary() + declspec(thread) " \
        "on Windows* OS.  See CQ50564, tests kmp_load_library*.c and this MSDN " \
        "reference: http://support.microsoft.com/kb/118816"
    //__kmp_gtid = KMP_GTID_MONITOR;
#endif

#if USE_ITT_BUILD
    __kmp_itt_thread_ignore();    // Instruct Intel(R) Threading Tools to ignore monitor thread.
#endif /* USE_ITT_BUILD */

    KMP_MB();       /* Flush all pending memory write invalidates.  */

    interval = ( 1000 / __kmp_monitor_wakeups ); /* in milliseconds */

    while (! TCR_4(__kmp_global.g.g_done)) {
        /*  This thread monitors the state of the system */

        KA_TRACE( 15, ( "__kmp_launch_monitor: update\n" ) );

        wait_status = WaitForSingleObject( __kmp_monitor_ev, interval );

        if (wait_status == WAIT_TIMEOUT) {
            TCW_4( __kmp_global.g.g_time.dt.t_value,
              TCR_4( __kmp_global.g.g_time.dt.t_value ) + 1 );
        }

        KMP_MB();       /* Flush all pending memory write invalidates.  */
    }

    KA_TRACE( 10, ("__kmp_launch_monitor: finished\n" ) );

    status = SetThreadPriority( monitor, THREAD_PRIORITY_NORMAL );
    if (! status) {
        DWORD error = GetLastError();
        __kmp_msg(
            kmp_ms_fatal,
            KMP_MSG( CantSetThreadPriority ),
            KMP_ERR( error ),
            __kmp_msg_null
        );
    }

    if (__kmp_global.g.g_abort != 0) {
        /* now we need to terminate the worker threads   */
        /* the value of t_abort is the signal we caught */

        int gtid;

        KA_TRACE( 10, ("__kmp_launch_monitor: terminate sig=%d\n", (__kmp_global.g.g_abort) ) );

        /* terminate the OpenMP worker threads */
        /* TODO this is not valid for sibling threads!!
         * the uber master might not be 0 anymore.. */
        for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
            __kmp_terminate_thread( gtid );

        __kmp_cleanup();

        Sleep( 0 );

        KA_TRACE( 10, ("__kmp_launch_monitor: raise sig=%d\n", (__kmp_global.g.g_abort) ) );

        if (__kmp_global.g.g_abort > 0) {
            raise( __kmp_global.g.g_abort );
        }
    }

    TCW_4( this_thr -> th.th_info.ds.ds_alive, FALSE );

    KMP_MB();
    return arg;
}

void
__kmp_create_worker( int gtid, kmp_info_t *th, size_t stack_size )
{
    kmp_thread_t   handle;
    DWORD          idThread;

    KA_TRACE( 10, ("__kmp_create_worker: try to create thread (%d)\n", gtid ) );

    th->th.th_info.ds.ds_gtid = gtid;

    if ( KMP_UBER_GTID(gtid) ) {
        int     stack_data;

        /* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for other threads to use.
           Is it appropriate to just use GetCurrentThread?  When should we close this handle?  When
           unregistering the root?
        */
        {
            BOOL rc;
            rc = DuplicateHandle(
                                 GetCurrentProcess(),
                                 GetCurrentThread(),
                                 GetCurrentProcess(),
                                 &th->th.th_info.ds.ds_thread,
                                 0,
                                 FALSE,
                                 DUPLICATE_SAME_ACCESS
                                 );
            KMP_ASSERT( rc );
            KA_TRACE( 10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, handle = %" KMP_UINTPTR_SPEC "\n",
                           (LPVOID)th,
                           th->th.th_info.ds.ds_thread ) );
            th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
        }
        if ( TCR_4(__kmp_gtid_mode) < 2 ) { // check stack only if it is used to get gtid
            /* we will dynamically update the stack range if gtid_mode == 1 */
            TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
            TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
            TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
            __kmp_check_stack_overlap( th );
        }
    }
    else {
        KMP_MB();       /* Flush all pending memory write invalidates.  */

        /* Set stack size for this thread now. */
        KA_TRACE( 10, ( "__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC
                        " bytes\n", stack_size ) );

        stack_size += gtid * __kmp_stkoffset;

        TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
        TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);

        KA_TRACE( 10, ( "__kmp_create_worker: (before) stack_size = %"
                        KMP_SIZE_T_SPEC
                        " bytes, &__kmp_launch_worker = %p, th = %p, "
                        "&idThread = %p\n",
                        (SIZE_T) stack_size,
                        (LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
                        (LPVOID) th, &idThread ) );

        handle = CreateThread( NULL, (SIZE_T) stack_size,
                               (LPTHREAD_START_ROUTINE) __kmp_launch_worker,
                               (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );

        KA_TRACE( 10, ( "__kmp_create_worker: (after) stack_size = %"
                        KMP_SIZE_T_SPEC
                        " bytes, &__kmp_launch_worker = %p, th = %p, "
                        "idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n",
                        (SIZE_T) stack_size,
                        (LPTHREAD_START_ROUTINE) & __kmp_launch_worker,
                        (LPVOID) th, idThread, handle ) );

        if ( handle == 0 ) {
            DWORD error = GetLastError();
            __kmp_msg(kmp_ms_fatal, KMP_MSG( CantCreateThread ), KMP_ERR( error ), __kmp_msg_null);
        } else {
            th->th.th_info.ds.ds_thread = handle;
        }

        KMP_MB();       /* Flush all pending memory write invalidates.  */
    }

    KA_TRACE( 10, ("__kmp_create_worker: done creating thread (%d)\n", gtid ) );
}

int
__kmp_still_running(kmp_info_t *th) {
    return (WAIT_TIMEOUT == WaitForSingleObject( th->th.th_info.ds.ds_thread, 0));
}

void
__kmp_create_monitor( kmp_info_t *th )
{
    kmp_thread_t        handle;
    DWORD               idThread;
    int                 ideal, new_ideal;

    if( __kmp_dflt_blocktime == KMP_MAX_BLOCKTIME ) {
        // We don't need monitor thread in case of MAX_BLOCKTIME
        KA_TRACE( 10, ("__kmp_create_monitor: skipping monitor thread because of MAX blocktime\n" ) );
        th->th.th_info.ds.ds_tid  = 0; // this makes reap_monitor no-op
        th->th.th_info.ds.ds_gtid = 0;
        TCW_4( __kmp_init_monitor, 2 ); // Signal to stop waiting for monitor creation
        return;
    }
    KA_TRACE( 10, ("__kmp_create_monitor: try to create monitor\n" ) );

    KMP_MB();       /* Flush all pending memory write invalidates.  */

    __kmp_monitor_ev = CreateEvent( NULL, TRUE, FALSE, NULL );
    if ( __kmp_monitor_ev == NULL ) {
        DWORD error = GetLastError();
        __kmp_msg(
            kmp_ms_fatal,
            KMP_MSG( CantCreateEvent ),
            KMP_ERR( error ),
            __kmp_msg_null
        );
    }; // if
#if USE_ITT_BUILD
    __kmp_itt_system_object_created( __kmp_monitor_ev, "Event" );
#endif /* USE_ITT_BUILD */

    th->th.th_info.ds.ds_tid  = KMP_GTID_MONITOR;
    th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;

    // FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how
    // to automatically expand stacksize based on CreateThread error code.
    if ( __kmp_monitor_stksize == 0 ) {
        __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
    }
    if ( __kmp_monitor_stksize < __kmp_sys_min_stksize ) {
        __kmp_monitor_stksize = __kmp_sys_min_stksize;
    }

    KA_TRACE( 10, ("__kmp_create_monitor: requested stacksize = %d bytes\n",
                   (int) __kmp_monitor_stksize ) );

    TCW_4( __kmp_global.g.g_time.dt.t_value, 0 );

    handle = CreateThread( NULL, (SIZE_T) __kmp_monitor_stksize,
                           (LPTHREAD_START_ROUTINE) __kmp_launch_monitor,
                           (LPVOID) th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread );
    if (handle == 0) {
        DWORD error = GetLastError();
        __kmp_msg(
            kmp_ms_fatal,
            KMP_MSG( CantCreateThread ),
            KMP_ERR( error ),
            __kmp_msg_null
        );
    }
    else
        th->th.th_info.ds.ds_thread = handle;

    KMP_MB();       /* Flush all pending memory write invalidates.  */

    KA_TRACE( 10, ("__kmp_create_monitor: monitor created %p\n",
                   (void *) th->th.th_info.ds.ds_thread ) );
}

/*
  Check to see if thread is still alive.

  NOTE:  The ExitProcess(code) system call causes all threads to Terminate
         with a exit_val = code.  Because of this we can not rely on
         exit_val having any particular value.  So this routine may
         return STILL_ALIVE in exit_val even after the thread is dead.
*/

int
__kmp_is_thread_alive( kmp_info_t * th, DWORD *exit_val )
{
    DWORD rc;
    rc = GetExitCodeThread( th->th.th_info.ds.ds_thread, exit_val );
    if ( rc == 0 ) {
        DWORD error = GetLastError();
        __kmp_msg(
            kmp_ms_fatal,
            KMP_MSG( FunctionError, "GetExitCodeThread()" ),
            KMP_ERR( error ),
            __kmp_msg_null
        );
    }; // if
    return ( *exit_val == STILL_ACTIVE );
}


void
__kmp_exit_thread(
    int exit_status
) {
    ExitThread( exit_status );
} // __kmp_exit_thread

/*
    This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor().
*/
static void
__kmp_reap_common( kmp_info_t * th )
{
    DWORD exit_val;

    KMP_MB();       /* Flush all pending memory write invalidates.  */

    KA_TRACE( 10, ( "__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid ) );

    /*
        2006-10-19:

        There are two opposite situations:

            1. Windows* OS keep thread alive after it resets ds_alive flag and exits from thread
               function. (For example, see C70770/Q394281 "unloading of dll based on OMP is very
               slow".)
            2. Windows* OS may kill thread before it resets ds_alive flag.

        Right solution seems to be waiting for *either* thread termination *or* ds_alive resetting.

    */

    {
        // TODO: This code is very similar to KMP_WAIT_YIELD. Need to generalize KMP_WAIT_YIELD to
        // cover this usage also.
        void * obj = NULL;
        register kmp_uint32 spins;
#if USE_ITT_BUILD
        KMP_FSYNC_SPIN_INIT( obj, (void*) & th->th.th_info.ds.ds_alive );
#endif /* USE_ITT_BUILD */
        KMP_INIT_YIELD( spins );
        do {
#if USE_ITT_BUILD
            KMP_FSYNC_SPIN_PREPARE( obj );
#endif /* USE_ITT_BUILD */
            __kmp_is_thread_alive( th, &exit_val );
            KMP_YIELD( TCR_4(__kmp_nth) > __kmp_avail_proc );
            KMP_YIELD_SPIN( spins );
        } while ( exit_val == STILL_ACTIVE && TCR_4( th->th.th_info.ds.ds_alive ) );
#if USE_ITT_BUILD
        if ( exit_val == STILL_ACTIVE ) {
            KMP_FSYNC_CANCEL( obj );
        } else {
            KMP_FSYNC_SPIN_ACQUIRED( obj );
        }; // if
#endif /* USE_ITT_BUILD */
    }

    __kmp_free_handle( th->th.th_info.ds.ds_thread );

    /*
     * NOTE:  The ExitProcess(code) system call causes all threads to Terminate
     *        with a exit_val = code.  Because of this we can not rely on
     *        exit_val having any particular value.
     */
    if ( exit_val == STILL_ACTIVE ) {
        KA_TRACE( 1, ( "__kmp_reap_common: thread still active.\n" ) );
    } else if ( (void *) exit_val != (void *) th) {
        KA_TRACE( 1, ( "__kmp_reap_common: ExitProcess / TerminateThread used?\n" ) );
    }; // if

    KA_TRACE( 10,
        (
            "__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC "\n",
            th->th.th_info.ds.ds_gtid,
            th->th.th_info.ds.ds_thread
        )
    );

    th->th.th_info.ds.ds_thread    = 0;
    th->th.th_info.ds.ds_tid       = KMP_GTID_DNE;
    th->th.th_info.ds.ds_gtid      = KMP_GTID_DNE;
    th->th.th_info.ds.ds_thread_id = 0;

    KMP_MB();       /* Flush all pending memory write invalidates.  */
}

void
__kmp_reap_monitor( kmp_info_t *th )
{
    int status;

    KA_TRACE( 10, ("__kmp_reap_monitor: try to reap %p\n",
                   (void *) th->th.th_info.ds.ds_thread ) );

    // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
    // If both tid and gtid are 0, it means the monitor did not ever start.
    // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
    KMP_DEBUG_ASSERT( th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid );
    if ( th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR ) {
        KA_TRACE( 10, ("__kmp_reap_monitor: monitor did not start, returning\n") );
        return;
    }; // if

    KMP_MB();       /* Flush all pending memory write invalidates.  */

    status = SetEvent( __kmp_monitor_ev );
    if ( status == FALSE ) {
        DWORD error = GetLastError();
        __kmp_msg(
            kmp_ms_fatal,
            KMP_MSG( CantSetEvent ),
            KMP_ERR( error ),
            __kmp_msg_null
        );
    }
    KA_TRACE( 10, ( "__kmp_reap_monitor: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
    __kmp_reap_common( th );

    __kmp_free_handle( __kmp_monitor_ev );

    KMP_MB();       /* Flush all pending memory write invalidates.  */
}

void
__kmp_reap_worker( kmp_info_t * th )
{
    KA_TRACE( 10, ( "__kmp_reap_worker: reaping thread (%d)\n", th->th.th_info.ds.ds_gtid ) );
    __kmp_reap_common( th );
}

/* ------------------------------------------------------------------------ */
/* ------------------------------------------------------------------------ */

#if KMP_HANDLE_SIGNALS


static void
__kmp_team_handler( int signo )
{
    if ( __kmp_global.g.g_abort == 0 ) {
        // Stage 1 signal handler, let's shut down all of the threads.
        if ( __kmp_debug_buf ) {
            __kmp_dump_debug_buffer();
        }; // if
        KMP_MB();       // Flush all pending memory write invalidates.
        TCW_4( __kmp_global.g.g_abort, signo );
        KMP_MB();       // Flush all pending memory write invalidates.
        TCW_4( __kmp_global.g.g_done, TRUE );
        KMP_MB();       // Flush all pending memory write invalidates.
    }
} // __kmp_team_handler



static
sig_func_t __kmp_signal( int signum, sig_func_t handler ) {
    sig_func_t old = signal( signum, handler );
    if ( old == SIG_ERR ) {
        int error = errno;
        __kmp_msg( kmp_ms_fatal, KMP_MSG( FunctionError, "signal" ), KMP_ERR( error ), __kmp_msg_null );
    }; // if
    return old;
}

static void
__kmp_install_one_handler(
    int           sig,
    sig_func_t    handler,
    int           parallel_init
) {
    sig_func_t old;
    KMP_MB();       /* Flush all pending memory write invalidates.  */
    KB_TRACE( 60, ("__kmp_install_one_handler: called: sig=%d\n", sig ) );
    if ( parallel_init ) {
        old = __kmp_signal( sig, handler );
        // SIG_DFL on Windows* OS in NULL or 0.
        if ( old == __kmp_sighldrs[ sig ] ) {
            __kmp_siginstalled[ sig ] = 1;
        } else {
            // Restore/keep user's handler if one previously installed.
            old = __kmp_signal( sig, old );
        }; // if
    } else {
        // Save initial/system signal handlers to see if user handlers installed.
        // 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals called once with
        // parallel_init == TRUE.
        old = __kmp_signal( sig, SIG_DFL );
        __kmp_sighldrs[ sig ] = old;
        __kmp_signal( sig, old );
    }; // if
    KMP_MB();       /* Flush all pending memory write invalidates.  */
} // __kmp_install_one_handler

static void
__kmp_remove_one_handler( int sig ) {
    if ( __kmp_siginstalled[ sig ] ) {
        sig_func_t old;
        KMP_MB();       // Flush all pending memory write invalidates.
        KB_TRACE( 60, ( "__kmp_remove_one_handler: called: sig=%d\n", sig ) );
        old = __kmp_signal( sig, __kmp_sighldrs[ sig ] );
        if ( old != __kmp_team_handler ) {
            KB_TRACE( 10, ( "__kmp_remove_one_handler: oops, not our handler, restoring: sig=%d\n", sig ) );
            old = __kmp_signal( sig, old );
        }; // if
        __kmp_sighldrs[ sig ] = NULL;
        __kmp_siginstalled[ sig ] = 0;
        KMP_MB();       // Flush all pending memory write invalidates.
    }; // if
} // __kmp_remove_one_handler


void
__kmp_install_signals( int parallel_init )
{
    KB_TRACE( 10, ( "__kmp_install_signals: called\n" ) );
    if ( ! __kmp_handle_signals ) {
        KB_TRACE( 10, ( "__kmp_install_signals: KMP_HANDLE_SIGNALS is false - handlers not installed\n" ) );
        return;
    }; // if
    __kmp_install_one_handler( SIGINT,  __kmp_team_handler, parallel_init );
    __kmp_install_one_handler( SIGILL,  __kmp_team_handler, parallel_init );
    __kmp_install_one_handler( SIGABRT, __kmp_team_handler, parallel_init );
    __kmp_install_one_handler( SIGFPE,  __kmp_team_handler, parallel_init );
    __kmp_install_one_handler( SIGSEGV, __kmp_team_handler, parallel_init );
    __kmp_install_one_handler( SIGTERM, __kmp_team_handler, parallel_init );
} // __kmp_install_signals


void
__kmp_remove_signals( void )
{
    int sig;
    KB_TRACE( 10, ("__kmp_remove_signals: called\n" ) );
    for ( sig = 1; sig < NSIG; ++ sig ) {
        __kmp_remove_one_handler( sig );
    }; // for sig
} // __kmp_remove_signals


#endif // KMP_HANDLE_SIGNALS

/* Put the thread to sleep for a time period */
void
__kmp_thread_sleep( int millis )
{
    DWORD status;

    status = SleepEx( (DWORD) millis, FALSE );
    if ( status ) {
        DWORD error = GetLastError();
        __kmp_msg(
            kmp_ms_fatal,
            KMP_MSG( FunctionError, "SleepEx()" ),
            KMP_ERR( error ),
            __kmp_msg_null
        );
    }
}

/* Determine whether the given address is mapped into the current address space. */
int
__kmp_is_address_mapped( void * addr )
{
    DWORD status;
    MEMORY_BASIC_INFORMATION lpBuffer;
    SIZE_T dwLength;

    dwLength = sizeof(MEMORY_BASIC_INFORMATION);

    status = VirtualQuery( addr, &lpBuffer, dwLength );

    return !((( lpBuffer.State == MEM_RESERVE) || ( lpBuffer.State == MEM_FREE )) ||
       (( lpBuffer.Protect == PAGE_NOACCESS ) || ( lpBuffer.Protect == PAGE_EXECUTE )));
}

kmp_uint64
__kmp_hardware_timestamp(void)
{
    kmp_uint64 r = 0;

    QueryPerformanceCounter((LARGE_INTEGER*) &r);
    return r;
}

/* Free handle and check the error code */
void
__kmp_free_handle( kmp_thread_t tHandle )
{
/* called with parameter type HANDLE also, thus suppose kmp_thread_t defined as HANDLE */
    BOOL rc;
    rc = CloseHandle( tHandle );
    if ( !rc ) {
        DWORD error = GetLastError();
        __kmp_msg(
            kmp_ms_fatal,
            KMP_MSG( CantCloseHandle ),
            KMP_ERR( error ),
            __kmp_msg_null
        );
    }
}

int
__kmp_get_load_balance( int max ) {

    static ULONG glb_buff_size = 100 * 1024;

    static int     glb_running_threads  = 0;  /* Saved count of the running threads for the thread balance algortihm */
    static double  glb_call_time        = 0;  /* Thread balance algorithm call time */

    int running_threads = 0;              // Number of running threads in the system.
    NTSTATUS  status        = 0;
    ULONG     buff_size     = 0;
    ULONG     info_size     = 0;
    void *    buffer        = NULL;
    PSYSTEM_PROCESS_INFORMATION spi = NULL;
    int first_time          = 1;

    double call_time = 0.0; //start, finish;

    __kmp_elapsed( & call_time );

    if ( glb_call_time &&
            ( call_time - glb_call_time < __kmp_load_balance_interval ) ) {
        running_threads = glb_running_threads;
        goto finish;
    }
    glb_call_time = call_time;

    // Do not spend time on running algorithm if we have a permanent error.
    if ( NtQuerySystemInformation == NULL ) {
        running_threads = -1;
        goto finish;
    }; // if

    if ( max <= 0 ) {
        max = INT_MAX;
    }; // if

    do {

        if ( first_time ) {
            buff_size = glb_buff_size;
        } else {
            buff_size = 2 * buff_size;
        }

        buffer = KMP_INTERNAL_REALLOC( buffer, buff_size );
        if ( buffer == NULL ) {
            running_threads = -1;
            goto finish;
        }; // if
        status = NtQuerySystemInformation( SystemProcessInformation, buffer, buff_size, & info_size );
        first_time = 0;

    } while ( status == STATUS_INFO_LENGTH_MISMATCH );
    glb_buff_size = buff_size;

    #define CHECK( cond )                       \
        {                                       \
            KMP_DEBUG_ASSERT( cond );           \
            if ( ! ( cond ) ) {                 \
                running_threads = -1;           \
                goto finish;                    \
            }                                   \
        }

    CHECK( buff_size >= info_size );
    spi = PSYSTEM_PROCESS_INFORMATION( buffer );
    for ( ; ; ) {
        ptrdiff_t offset = uintptr_t( spi ) - uintptr_t( buffer );
        CHECK( 0 <= offset && offset + sizeof( SYSTEM_PROCESS_INFORMATION ) < info_size );
        HANDLE pid = spi->ProcessId;
        ULONG num = spi->NumberOfThreads;
        CHECK( num >= 1 );
        size_t spi_size = sizeof( SYSTEM_PROCESS_INFORMATION ) + sizeof( SYSTEM_THREAD ) * ( num - 1 );
        CHECK( offset + spi_size < info_size );          // Make sure process info record fits the buffer.
        if ( spi->NextEntryOffset != 0 ) {
            CHECK( spi_size <= spi->NextEntryOffset );   // And do not overlap with the next record.
        }; // if
        // pid == 0 corresponds to the System Idle Process. It always has running threads
        // on all cores. So, we don't consider the running threads of this process.
        if ( pid != 0 ) {
            for ( int i = 0; i < num; ++ i ) {
                THREAD_STATE state = spi->Threads[ i ].State;
                // Count threads that have Ready or Running state.
                // !!! TODO: Why comment does not match the code???
                if ( state == StateRunning ) {
                    ++ running_threads;
                    // Stop counting running threads if the number is already greater than
                    // the number of available cores
                    if ( running_threads >= max ) {
                        goto finish;
                    }
                } // if
            }; // for i
        } // if
        if ( spi->NextEntryOffset == 0 ) {
            break;
        }; // if
        spi = PSYSTEM_PROCESS_INFORMATION( uintptr_t( spi ) + spi->NextEntryOffset );
    }; // forever

    #undef CHECK

    finish: // Clean up and exit.

        if ( buffer != NULL ) {
            KMP_INTERNAL_FREE( buffer );
        }; // if

        glb_running_threads = running_threads;

        return running_threads;

} //__kmp_get_load_balance()