blob: ae639ba1d55bfe13b3af6229161fc5015372d1ee [file] [log] [blame]
Jim Cownie5e8470a2013-09-27 10:38:44 +00001/*
Jonathan Peytonde4749b2016-12-14 23:01:24 +00002 * kmp_runtime.cpp -- KPTS runtime support library
Jim Cownie5e8470a2013-09-27 10:38:44 +00003 */
4
5
6//===----------------------------------------------------------------------===//
7//
8// The LLVM Compiler Infrastructure
9//
10// This file is dual licensed under the MIT and the University of Illinois Open
11// Source Licenses. See LICENSE.txt for details.
12//
13//===----------------------------------------------------------------------===//
14
15
16#include "kmp.h"
Jonathan Peyton30419822017-05-12 18:01:32 +000017#include "kmp_affinity.h"
Jim Cownie5e8470a2013-09-27 10:38:44 +000018#include "kmp_atomic.h"
Jim Cownie5e8470a2013-09-27 10:38:44 +000019#include "kmp_environment.h"
Jonathan Peyton30419822017-05-12 18:01:32 +000020#include "kmp_error.h"
Jim Cownie5e8470a2013-09-27 10:38:44 +000021#include "kmp_i18n.h"
22#include "kmp_io.h"
Jonathan Peyton30419822017-05-12 18:01:32 +000023#include "kmp_itt.h"
24#include "kmp_settings.h"
Jim Cownie4cc4bb42014-10-07 16:25:50 +000025#include "kmp_stats.h"
Jonathan Peyton30419822017-05-12 18:01:32 +000026#include "kmp_str.h"
Jim Cownie4cc4bb42014-10-07 16:25:50 +000027#include "kmp_wait_release.h"
Jonathan Peyton30419822017-05-12 18:01:32 +000028#include "kmp_wrapper_getpid.h"
Jim Cownie5e8470a2013-09-27 10:38:44 +000029
Andrey Churbanovd7d088f2015-04-29 16:42:24 +000030#if OMPT_SUPPORT
31#include "ompt-specific.h"
32#endif
33
Jim Cownie5e8470a2013-09-27 10:38:44 +000034/* these are temporary issues to be dealt with */
35#define KMP_USE_PRCTL 0
Jim Cownie5e8470a2013-09-27 10:38:44 +000036
Jim Cownie5e8470a2013-09-27 10:38:44 +000037#if KMP_OS_WINDOWS
38#include <process.h>
39#endif
40
Jonas Hahnfeld50fed042016-11-07 15:58:36 +000041#include "tsan_annotations.h"
Jim Cownie5e8470a2013-09-27 10:38:44 +000042
43#if defined(KMP_GOMP_COMPAT)
Jonathan Peyton30419822017-05-12 18:01:32 +000044char const __kmp_version_alt_comp[] =
45 KMP_VERSION_PREFIX "alternative compiler support: yes";
Jim Cownie5e8470a2013-09-27 10:38:44 +000046#endif /* defined(KMP_GOMP_COMPAT) */
47
48char const __kmp_version_omp_api[] = KMP_VERSION_PREFIX "API version: "
Jonathan Peytone844a542017-03-06 22:07:40 +000049#if OMP_50_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +000050 "5.0 (201611)";
Jonathan Peytone844a542017-03-06 22:07:40 +000051#elif OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +000052 "4.5 (201511)";
Jonathan Peyton74f3ffc2016-09-30 15:50:14 +000053#elif OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +000054 "4.0 (201307)";
Jim Cownie5e8470a2013-09-27 10:38:44 +000055#else
Jonathan Peyton30419822017-05-12 18:01:32 +000056 "3.1 (201107)";
Jim Cownie5e8470a2013-09-27 10:38:44 +000057#endif
58
59#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +000060char const __kmp_version_lock[] =
61 KMP_VERSION_PREFIX "lock type: run time selectable";
Jim Cownie5e8470a2013-09-27 10:38:44 +000062#endif /* KMP_DEBUG */
63
Jonathan Peyton30419822017-05-12 18:01:32 +000064#define KMP_MIN(x, y) ((x) < (y) ? (x) : (y))
Jim Cownie181b4bb2013-12-23 17:28:57 +000065
Jim Cownie5e8470a2013-09-27 10:38:44 +000066/* ------------------------------------------------------------------------ */
Jim Cownie5e8470a2013-09-27 10:38:44 +000067
68kmp_info_t __kmp_monitor;
69
Jim Cownie5e8470a2013-09-27 10:38:44 +000070/* Forward declarations */
71
Jonathan Peyton30419822017-05-12 18:01:32 +000072void __kmp_cleanup(void);
Jim Cownie5e8470a2013-09-27 10:38:44 +000073
Jonathan Peyton30419822017-05-12 18:01:32 +000074static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *, int tid,
75 int gtid);
76static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
77 kmp_internal_control_t *new_icvs,
78 ident_t *loc);
Jonathan Peyton2321d572015-06-08 19:25:25 +000079#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +000080static void __kmp_partition_places(kmp_team_t *team,
81 int update_master_only = 0);
Jonathan Peyton2321d572015-06-08 19:25:25 +000082#endif
Jonathan Peyton30419822017-05-12 18:01:32 +000083static void __kmp_do_serial_initialize(void);
84void __kmp_fork_barrier(int gtid, int tid);
85void __kmp_join_barrier(int gtid);
86void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc,
87 kmp_internal_control_t *new_icvs, ident_t *loc);
Jim Cownie5e8470a2013-09-27 10:38:44 +000088
Jim Cownie5e8470a2013-09-27 10:38:44 +000089#ifdef USE_LOAD_BALANCE
Jonathan Peyton30419822017-05-12 18:01:32 +000090static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc);
Jim Cownie5e8470a2013-09-27 10:38:44 +000091#endif
92
93static int __kmp_expand_threads(int nWish, int nNeed);
Jonathan Peyton2321d572015-06-08 19:25:25 +000094#if KMP_OS_WINDOWS
Jonathan Peyton30419822017-05-12 18:01:32 +000095static int __kmp_unregister_root_other_thread(int gtid);
Jonathan Peyton2321d572015-06-08 19:25:25 +000096#endif
Jonathan Peyton30419822017-05-12 18:01:32 +000097static void __kmp_unregister_library(void); // called by __kmp_internal_end()
98static void __kmp_reap_thread(kmp_info_t *thread, int is_root);
Jim Cownie5e8470a2013-09-27 10:38:44 +000099static kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
100
Jim Cownie5e8470a2013-09-27 10:38:44 +0000101/* Calculate the identifier of the current thread */
Jonathan Peyton30419822017-05-12 18:01:32 +0000102/* fast (and somewhat portable) way to get unique identifier of executing
103 thread. Returns KMP_GTID_DNE if we haven't been assigned a gtid. */
Jonathan Peyton30419822017-05-12 18:01:32 +0000104int __kmp_get_global_thread_id() {
105 int i;
106 kmp_info_t **other_threads;
107 size_t stack_data;
108 char *stack_addr;
109 size_t stack_size;
110 char *stack_base;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000111
Jonathan Peyton30419822017-05-12 18:01:32 +0000112 KA_TRACE(
113 1000,
114 ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n",
115 __kmp_nth, __kmp_all_nth));
Jim Cownie5e8470a2013-09-27 10:38:44 +0000116
Jonathan Peyton30419822017-05-12 18:01:32 +0000117 /* JPH - to handle the case where __kmpc_end(0) is called immediately prior to
118 a parallel region, made it return KMP_GTID_DNE to force serial_initialize
119 by caller. Had to handle KMP_GTID_DNE at all call-sites, or else guarantee
120 __kmp_init_gtid for this to work. */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000121
Jonathan Peyton30419822017-05-12 18:01:32 +0000122 if (!TCR_4(__kmp_init_gtid))
123 return KMP_GTID_DNE;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000124
125#ifdef KMP_TDATA_GTID
Jonathan Peyton30419822017-05-12 18:01:32 +0000126 if (TCR_4(__kmp_gtid_mode) >= 3) {
127 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using TDATA\n"));
128 return __kmp_gtid;
129 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000130#endif
Jonathan Peyton30419822017-05-12 18:01:32 +0000131 if (TCR_4(__kmp_gtid_mode) >= 2) {
132 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using keyed TLS\n"));
133 return __kmp_gtid_get_specific();
134 }
135 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using internal alg.\n"));
136
137 stack_addr = (char *)&stack_data;
138 other_threads = __kmp_threads;
139
140 /* ATT: The code below is a source of potential bugs due to unsynchronized
141 access to __kmp_threads array. For example:
142 1. Current thread loads other_threads[i] to thr and checks it, it is
143 non-NULL.
144 2. Current thread is suspended by OS.
145 3. Another thread unregisters and finishes (debug versions of free()
146 may fill memory with something like 0xEF).
147 4. Current thread is resumed.
148 5. Current thread reads junk from *thr.
149 TODO: Fix it. --ln */
150
151 for (i = 0; i < __kmp_threads_capacity; i++) {
152
153 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i]);
154 if (!thr)
155 continue;
156
157 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize);
158 stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase);
159
160 /* stack grows down -- search through all of the active threads */
161
162 if (stack_addr <= stack_base) {
163 size_t stack_diff = stack_base - stack_addr;
164
165 if (stack_diff <= stack_size) {
166 /* The only way we can be closer than the allocated */
167 /* stack size is if we are running on this thread. */
168 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i);
169 return i;
170 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000171 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000172 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000173
Jonathan Peyton30419822017-05-12 18:01:32 +0000174 /* get specific to try and determine our gtid */
175 KA_TRACE(1000,
176 ("*** __kmp_get_global_thread_id: internal alg. failed to find "
177 "thread, using TLS\n"));
178 i = __kmp_gtid_get_specific();
Jim Cownie5e8470a2013-09-27 10:38:44 +0000179
Jonathan Peyton30419822017-05-12 18:01:32 +0000180 /*fprintf( stderr, "=== %d\n", i ); */ /* GROO */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000181
Jonathan Peyton30419822017-05-12 18:01:32 +0000182 /* if we havn't been assigned a gtid, then return code */
183 if (i < 0)
Jim Cownie5e8470a2013-09-27 10:38:44 +0000184 return i;
Jonathan Peyton30419822017-05-12 18:01:32 +0000185
186 /* dynamically updated stack window for uber threads to avoid get_specific
187 call */
188 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) {
189 KMP_FATAL(StackOverflow, i);
190 }
191
192 stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
193 if (stack_addr > stack_base) {
194 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr);
195 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
196 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr -
197 stack_base);
198 } else {
199 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
200 stack_base - stack_addr);
201 }
202
203 /* Reprint stack bounds for ubermaster since they have been refined */
204 if (__kmp_storage_map) {
205 char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
206 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize;
207 __kmp_print_storage_map_gtid(i, stack_beg, stack_end,
208 other_threads[i]->th.th_info.ds.ds_stacksize,
209 "th_%d stack (refinement)", i);
210 }
211 return i;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000212}
213
Jonathan Peyton30419822017-05-12 18:01:32 +0000214int __kmp_get_global_thread_id_reg() {
215 int gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000216
Jonathan Peyton30419822017-05-12 18:01:32 +0000217 if (!__kmp_init_serial) {
218 gtid = KMP_GTID_DNE;
219 } else
Jim Cownie5e8470a2013-09-27 10:38:44 +0000220#ifdef KMP_TDATA_GTID
Jonathan Peyton30419822017-05-12 18:01:32 +0000221 if (TCR_4(__kmp_gtid_mode) >= 3) {
222 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using TDATA\n"));
223 gtid = __kmp_gtid;
224 } else
Jim Cownie5e8470a2013-09-27 10:38:44 +0000225#endif
Jonathan Peyton30419822017-05-12 18:01:32 +0000226 if (TCR_4(__kmp_gtid_mode) >= 2) {
227 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n"));
228 gtid = __kmp_gtid_get_specific();
229 } else {
230 KA_TRACE(1000,
231 ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"));
232 gtid = __kmp_get_global_thread_id();
233 }
234
235 /* we must be a new uber master sibling thread */
236 if (gtid == KMP_GTID_DNE) {
237 KA_TRACE(10,
238 ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
239 "Registering a new gtid.\n"));
240 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
241 if (!__kmp_init_serial) {
242 __kmp_do_serial_initialize();
243 gtid = __kmp_gtid_get_specific();
Jim Cownie5e8470a2013-09-27 10:38:44 +0000244 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +0000245 gtid = __kmp_register_root(FALSE);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000246 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000247 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
248 /*__kmp_printf( "+++ %d\n", gtid ); */ /* GROO */
249 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000250
Jonathan Peyton30419822017-05-12 18:01:32 +0000251 KMP_DEBUG_ASSERT(gtid >= 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000252
Jonathan Peyton30419822017-05-12 18:01:32 +0000253 return gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000254}
255
256/* caller must hold forkjoin_lock */
Jonathan Peyton30419822017-05-12 18:01:32 +0000257void __kmp_check_stack_overlap(kmp_info_t *th) {
258 int f;
259 char *stack_beg = NULL;
260 char *stack_end = NULL;
261 int gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000262
Jonathan Peyton30419822017-05-12 18:01:32 +0000263 KA_TRACE(10, ("__kmp_check_stack_overlap: called\n"));
264 if (__kmp_storage_map) {
265 stack_end = (char *)th->th.th_info.ds.ds_stackbase;
266 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000267
Jonathan Peyton30419822017-05-12 18:01:32 +0000268 gtid = __kmp_gtid_from_thread(th);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000269
Jonathan Peyton30419822017-05-12 18:01:32 +0000270 if (gtid == KMP_GTID_MONITOR) {
271 __kmp_print_storage_map_gtid(
272 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
273 "th_%s stack (%s)", "mon",
274 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
Jim Cownie5e8470a2013-09-27 10:38:44 +0000275 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +0000276 __kmp_print_storage_map_gtid(
277 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
278 "th_%d stack (%s)", gtid,
279 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
280 }
281 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000282
Jonathan Peyton30419822017-05-12 18:01:32 +0000283 /* No point in checking ubermaster threads since they use refinement and
284 * cannot overlap */
285 gtid = __kmp_gtid_from_thread(th);
286 if (__kmp_env_checks == TRUE && !KMP_UBER_GTID(gtid)) {
287 KA_TRACE(10,
288 ("__kmp_check_stack_overlap: performing extensive checking\n"));
289 if (stack_beg == NULL) {
290 stack_end = (char *)th->th.th_info.ds.ds_stackbase;
291 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
292 }
293
294 for (f = 0; f < __kmp_threads_capacity; f++) {
295 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f]);
296
297 if (f_th && f_th != th) {
298 char *other_stack_end =
299 (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase);
300 char *other_stack_beg =
301 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize);
302 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) ||
303 (stack_end > other_stack_beg && stack_end < other_stack_end)) {
304
305 /* Print the other stack values before the abort */
306 if (__kmp_storage_map)
307 __kmp_print_storage_map_gtid(
308 -1, other_stack_beg, other_stack_end,
309 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize),
310 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th));
311
Jonathan Peyton6a393f72017-09-05 15:43:58 +0000312 __kmp_fatal(KMP_MSG(StackOverlap), KMP_HNT(ChangeStackLimit),
313 __kmp_msg_null);
Jonathan Peyton30419822017-05-12 18:01:32 +0000314 }
315 }
316 }
317 }
318 KA_TRACE(10, ("__kmp_check_stack_overlap: returning\n"));
319}
320
321/* ------------------------------------------------------------------------ */
322
323void __kmp_infinite_loop(void) {
324 static int done = FALSE;
325
326 while (!done) {
327 KMP_YIELD(1);
328 }
329}
330
331#define MAX_MESSAGE 512
332
333void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2, size_t size,
334 char const *format, ...) {
335 char buffer[MAX_MESSAGE];
336 va_list ap;
337
338 va_start(ap, format);
339 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP storage map: %p %p%8lu %s\n", p1,
340 p2, (unsigned long)size, format);
341 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
342 __kmp_vprintf(kmp_err, buffer, ap);
343#if KMP_PRINT_DATA_PLACEMENT
344 int node;
345 if (gtid >= 0) {
346 if (p1 <= p2 && (char *)p2 - (char *)p1 == size) {
347 if (__kmp_storage_map_verbose) {
348 node = __kmp_get_host_node(p1);
349 if (node < 0) /* doesn't work, so don't try this next time */
350 __kmp_storage_map_verbose = FALSE;
351 else {
352 char *last;
353 int lastNode;
354 int localProc = __kmp_get_cpu_from_gtid(gtid);
355
356 const int page_size = KMP_GET_PAGE_SIZE();
357
358 p1 = (void *)((size_t)p1 & ~((size_t)page_size - 1));
359 p2 = (void *)(((size_t)p2 - 1) & ~((size_t)page_size - 1));
360 if (localProc >= 0)
361 __kmp_printf_no_lock(" GTID %d localNode %d\n", gtid,
362 localProc >> 1);
363 else
364 __kmp_printf_no_lock(" GTID %d\n", gtid);
365#if KMP_USE_PRCTL
366 /* The more elaborate format is disabled for now because of the prctl
367 * hanging bug. */
368 do {
369 last = p1;
370 lastNode = node;
371 /* This loop collates adjacent pages with the same host node. */
372 do {
373 (char *)p1 += page_size;
374 } while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode);
375 __kmp_printf_no_lock(" %p-%p memNode %d\n", last, (char *)p1 - 1,
376 lastNode);
377 } while (p1 <= p2);
378#else
379 __kmp_printf_no_lock(" %p-%p memNode %d\n", p1,
380 (char *)p1 + (page_size - 1),
381 __kmp_get_host_node(p1));
382 if (p1 < p2) {
383 __kmp_printf_no_lock(" %p-%p memNode %d\n", p2,
384 (char *)p2 + (page_size - 1),
385 __kmp_get_host_node(p2));
386 }
387#endif
388 }
389 }
390 } else
391 __kmp_printf_no_lock(" %s\n", KMP_I18N_STR(StorageMapWarning));
392 }
393#endif /* KMP_PRINT_DATA_PLACEMENT */
394 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
395}
396
397void __kmp_warn(char const *format, ...) {
398 char buffer[MAX_MESSAGE];
399 va_list ap;
400
401 if (__kmp_generate_warnings == kmp_warnings_off) {
402 return;
403 }
404
405 va_start(ap, format);
406
407 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP warning: %s\n", format);
408 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
409 __kmp_vprintf(kmp_err, buffer, ap);
410 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
411
412 va_end(ap);
413}
414
415void __kmp_abort_process() {
416 // Later threads may stall here, but that's ok because abort() will kill them.
417 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock);
418
419 if (__kmp_debug_buf) {
420 __kmp_dump_debug_buffer();
Jonathan Peytonbd3a7632017-09-27 20:36:27 +0000421 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000422
423 if (KMP_OS_WINDOWS) {
424 // Let other threads know of abnormal termination and prevent deadlock
425 // if abort happened during library initialization or shutdown
426 __kmp_global.g.g_abort = SIGABRT;
427
428 /* On Windows* OS by default abort() causes pop-up error box, which stalls
429 nightly testing. Unfortunately, we cannot reliably suppress pop-up error
430 boxes. _set_abort_behavior() works well, but this function is not
431 available in VS7 (this is not problem for DLL, but it is a problem for
432 static OpenMP RTL). SetErrorMode (and so, timelimit utility) does not
433 help, at least in some versions of MS C RTL.
434
435 It seems following sequence is the only way to simulate abort() and
436 avoid pop-up error box. */
437 raise(SIGABRT);
438 _exit(3); // Just in case, if signal ignored, exit anyway.
439 } else {
440 abort();
Jonathan Peytonbd3a7632017-09-27 20:36:27 +0000441 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000442
443 __kmp_infinite_loop();
444 __kmp_release_bootstrap_lock(&__kmp_exit_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000445
446} // __kmp_abort_process
447
Jonathan Peyton30419822017-05-12 18:01:32 +0000448void __kmp_abort_thread(void) {
449 // TODO: Eliminate g_abort global variable and this function.
450 // In case of abort just call abort(), it will kill all the threads.
451 __kmp_infinite_loop();
Jim Cownie5e8470a2013-09-27 10:38:44 +0000452} // __kmp_abort_thread
453
Jonathan Peyton30419822017-05-12 18:01:32 +0000454/* Print out the storage map for the major kmp_info_t thread data structures
455 that are allocated together. */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000456
Jonathan Peyton30419822017-05-12 18:01:32 +0000457static void __kmp_print_thread_storage_map(kmp_info_t *thr, int gtid) {
458 __kmp_print_storage_map_gtid(gtid, thr, thr + 1, sizeof(kmp_info_t), "th_%d",
459 gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000460
Jonathan Peyton30419822017-05-12 18:01:32 +0000461 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team,
462 sizeof(kmp_desc_t), "th_%d.th_info", gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000463
Jonathan Peyton30419822017-05-12 18:01:32 +0000464 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head,
465 sizeof(kmp_local_t), "th_%d.th_local", gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000466
Jonathan Peyton30419822017-05-12 18:01:32 +0000467 __kmp_print_storage_map_gtid(
468 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier],
469 sizeof(kmp_balign_t) * bs_last_barrier, "th_%d.th_bar", gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000470
Jonathan Peyton30419822017-05-12 18:01:32 +0000471 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier],
472 &thr->th.th_bar[bs_plain_barrier + 1],
473 sizeof(kmp_balign_t), "th_%d.th_bar[plain]",
474 gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000475
Jonathan Peyton30419822017-05-12 18:01:32 +0000476 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier],
477 &thr->th.th_bar[bs_forkjoin_barrier + 1],
478 sizeof(kmp_balign_t), "th_%d.th_bar[forkjoin]",
479 gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000480
Jonathan Peyton30419822017-05-12 18:01:32 +0000481#if KMP_FAST_REDUCTION_BARRIER
482 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier],
483 &thr->th.th_bar[bs_reduction_barrier + 1],
484 sizeof(kmp_balign_t), "th_%d.th_bar[reduction]",
485 gtid);
486#endif // KMP_FAST_REDUCTION_BARRIER
Jim Cownie5e8470a2013-09-27 10:38:44 +0000487}
488
Jonathan Peyton30419822017-05-12 18:01:32 +0000489/* Print out the storage map for the major kmp_team_t team data structures
490 that are allocated together. */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000491
Jonathan Peyton30419822017-05-12 18:01:32 +0000492static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team,
493 int team_id, int num_thr) {
494 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;
495 __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d",
496 header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000497
Jonathan Peyton30419822017-05-12 18:01:32 +0000498 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0],
499 &team->t.t_bar[bs_last_barrier],
500 sizeof(kmp_balign_team_t) * bs_last_barrier,
501 "%s_%d.t_bar", header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000502
Jonathan Peyton30419822017-05-12 18:01:32 +0000503 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier],
504 &team->t.t_bar[bs_plain_barrier + 1],
505 sizeof(kmp_balign_team_t), "%s_%d.t_bar[plain]",
506 header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000507
Jonathan Peyton30419822017-05-12 18:01:32 +0000508 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier],
509 &team->t.t_bar[bs_forkjoin_barrier + 1],
510 sizeof(kmp_balign_team_t),
511 "%s_%d.t_bar[forkjoin]", header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000512
Jonathan Peyton30419822017-05-12 18:01:32 +0000513#if KMP_FAST_REDUCTION_BARRIER
514 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier],
515 &team->t.t_bar[bs_reduction_barrier + 1],
516 sizeof(kmp_balign_team_t),
517 "%s_%d.t_bar[reduction]", header, team_id);
518#endif // KMP_FAST_REDUCTION_BARRIER
Jim Cownie5e8470a2013-09-27 10:38:44 +0000519
Jonathan Peyton30419822017-05-12 18:01:32 +0000520 __kmp_print_storage_map_gtid(
521 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr],
522 sizeof(kmp_disp_t) * num_thr, "%s_%d.t_dispatch", header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000523
Jonathan Peyton30419822017-05-12 18:01:32 +0000524 __kmp_print_storage_map_gtid(
525 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr],
526 sizeof(kmp_info_t *) * num_thr, "%s_%d.t_threads", header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000527
Jonathan Peyton30419822017-05-12 18:01:32 +0000528 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0],
529 &team->t.t_disp_buffer[num_disp_buff],
530 sizeof(dispatch_shared_info_t) * num_disp_buff,
531 "%s_%d.t_disp_buffer", header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000532
Jonathan Peyton30419822017-05-12 18:01:32 +0000533 __kmp_print_storage_map_gtid(-1, &team->t.t_taskq, &team->t.t_copypriv_data,
534 sizeof(kmp_taskq_t), "%s_%d.t_taskq", header,
535 team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000536}
537
538static void __kmp_init_allocator() {}
539static void __kmp_fini_allocator() {}
Jim Cownie5e8470a2013-09-27 10:38:44 +0000540
541/* ------------------------------------------------------------------------ */
542
Jonathan Peyton99016992015-05-26 17:32:53 +0000543#ifdef KMP_DYNAMIC_LIB
Jonathan Peyton30419822017-05-12 18:01:32 +0000544#if KMP_OS_WINDOWS
Jim Cownie5e8470a2013-09-27 10:38:44 +0000545
Jonathan Peyton30419822017-05-12 18:01:32 +0000546static void __kmp_reset_lock(kmp_bootstrap_lock_t *lck) {
547 // TODO: Change to __kmp_break_bootstrap_lock().
548 __kmp_init_bootstrap_lock(lck); // make the lock released
Jim Cownie5e8470a2013-09-27 10:38:44 +0000549}
550
Jonathan Peyton30419822017-05-12 18:01:32 +0000551static void __kmp_reset_locks_on_process_detach(int gtid_req) {
552 int i;
553 int thread_count;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000554
Jonathan Peyton30419822017-05-12 18:01:32 +0000555 // PROCESS_DETACH is expected to be called by a thread that executes
556 // ProcessExit() or FreeLibrary(). OS terminates other threads (except the one
557 // calling ProcessExit or FreeLibrary). So, it might be safe to access the
558 // __kmp_threads[] without taking the forkjoin_lock. However, in fact, some
559 // threads can be still alive here, although being about to be terminated. The
560 // threads in the array with ds_thread==0 are most suspicious. Actually, it
561 // can be not safe to access the __kmp_threads[].
Jim Cownie5e8470a2013-09-27 10:38:44 +0000562
Jonathan Peyton30419822017-05-12 18:01:32 +0000563 // TODO: does it make sense to check __kmp_roots[] ?
Jim Cownie5e8470a2013-09-27 10:38:44 +0000564
Jonathan Peyton30419822017-05-12 18:01:32 +0000565 // Let's check that there are no other alive threads registered with the OMP
566 // lib.
567 while (1) {
568 thread_count = 0;
569 for (i = 0; i < __kmp_threads_capacity; ++i) {
570 if (!__kmp_threads)
571 continue;
572 kmp_info_t *th = __kmp_threads[i];
573 if (th == NULL)
574 continue;
575 int gtid = th->th.th_info.ds.ds_gtid;
576 if (gtid == gtid_req)
577 continue;
578 if (gtid < 0)
579 continue;
580 DWORD exit_val;
581 int alive = __kmp_is_thread_alive(th, &exit_val);
582 if (alive) {
583 ++thread_count;
584 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000585 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000586 if (thread_count == 0)
587 break; // success
588 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000589
Jonathan Peyton30419822017-05-12 18:01:32 +0000590 // Assume that I'm alone. Now it might be safe to check and reset locks.
591 // __kmp_forkjoin_lock and __kmp_stdio_lock are expected to be reset.
592 __kmp_reset_lock(&__kmp_forkjoin_lock);
593#ifdef KMP_DEBUG
594 __kmp_reset_lock(&__kmp_stdio_lock);
595#endif // KMP_DEBUG
Jim Cownie5e8470a2013-09-27 10:38:44 +0000596}
597
Jonathan Peyton30419822017-05-12 18:01:32 +0000598BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) {
599 //__kmp_acquire_bootstrap_lock( &__kmp_initz_lock );
Jim Cownie5e8470a2013-09-27 10:38:44 +0000600
Jonathan Peyton30419822017-05-12 18:01:32 +0000601 switch (fdwReason) {
Jim Cownie5e8470a2013-09-27 10:38:44 +0000602
Jonathan Peyton30419822017-05-12 18:01:32 +0000603 case DLL_PROCESS_ATTACH:
604 KA_TRACE(10, ("DllMain: PROCESS_ATTACH\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +0000605
606 return TRUE;
Jonathan Peyton30419822017-05-12 18:01:32 +0000607
608 case DLL_PROCESS_DETACH:
609 KA_TRACE(10, ("DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific()));
610
611 if (lpReserved != NULL) {
612 // lpReserved is used for telling the difference:
613 // lpReserved == NULL when FreeLibrary() was called,
614 // lpReserved != NULL when the process terminates.
615 // When FreeLibrary() is called, worker threads remain alive. So they will
616 // release the forkjoin lock by themselves. When the process terminates,
617 // worker threads disappear triggering the problem of unreleased forkjoin
618 // lock as described below.
619
620 // A worker thread can take the forkjoin lock. The problem comes up if
621 // that worker thread becomes dead before it releases the forkjoin lock.
622 // The forkjoin lock remains taken, while the thread executing
623 // DllMain()->PROCESS_DETACH->__kmp_internal_end_library() below will try
624 // to take the forkjoin lock and will always fail, so that the application
625 // will never finish [normally]. This scenario is possible if
626 // __kmpc_end() has not been executed. It looks like it's not a corner
627 // case, but common cases:
628 // - the main function was compiled by an alternative compiler;
629 // - the main function was compiled by icl but without /Qopenmp
630 // (application with plugins);
631 // - application terminates by calling C exit(), Fortran CALL EXIT() or
632 // Fortran STOP.
633 // - alive foreign thread prevented __kmpc_end from doing cleanup.
634 //
635 // This is a hack to work around the problem.
636 // TODO: !!! figure out something better.
637 __kmp_reset_locks_on_process_detach(__kmp_gtid_get_specific());
638 }
639
640 __kmp_internal_end_library(__kmp_gtid_get_specific());
641
642 return TRUE;
643
644 case DLL_THREAD_ATTACH:
645 KA_TRACE(10, ("DllMain: THREAD_ATTACH\n"));
646
647 /* if we want to register new siblings all the time here call
648 * __kmp_get_gtid(); */
649 return TRUE;
650
651 case DLL_THREAD_DETACH:
652 KA_TRACE(10, ("DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific()));
653
654 __kmp_internal_end_thread(__kmp_gtid_get_specific());
655 return TRUE;
656 }
657
658 return TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000659}
660
Jonathan Peyton30419822017-05-12 18:01:32 +0000661#endif /* KMP_OS_WINDOWS */
Jonathan Peyton99016992015-05-26 17:32:53 +0000662#endif /* KMP_DYNAMIC_LIB */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000663
Jim Cownie5e8470a2013-09-27 10:38:44 +0000664/* Change the library type to "status" and return the old type */
665/* called from within initialization routines where __kmp_initz_lock is held */
Jonathan Peyton30419822017-05-12 18:01:32 +0000666int __kmp_change_library(int status) {
667 int old_status;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000668
Jonathan Peyton30419822017-05-12 18:01:32 +0000669 old_status = __kmp_yield_init &
670 1; // check whether KMP_LIBRARY=throughput (even init count)
Jim Cownie5e8470a2013-09-27 10:38:44 +0000671
Jonathan Peyton30419822017-05-12 18:01:32 +0000672 if (status) {
673 __kmp_yield_init |= 1; // throughput => turnaround (odd init count)
674 } else {
675 __kmp_yield_init &= ~1; // turnaround => throughput (even init count)
676 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000677
Jonathan Peyton30419822017-05-12 18:01:32 +0000678 return old_status; // return previous setting of whether
679 // KMP_LIBRARY=throughput
Jim Cownie5e8470a2013-09-27 10:38:44 +0000680}
681
Jonathan Peyton30419822017-05-12 18:01:32 +0000682/* __kmp_parallel_deo -- Wait until it's our turn. */
683void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
684 int gtid = *gtid_ref;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000685#ifdef BUILD_PARALLEL_ORDERED
Jonathan Peyton30419822017-05-12 18:01:32 +0000686 kmp_team_t *team = __kmp_team_from_gtid(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000687#endif /* BUILD_PARALLEL_ORDERED */
688
Jonathan Peyton30419822017-05-12 18:01:32 +0000689 if (__kmp_env_consistency_check) {
690 if (__kmp_threads[gtid]->th.th_root->r.r_active)
Andrey Churbanov5c56fb52015-02-20 18:05:17 +0000691#if KMP_USE_DYNAMIC_LOCK
Jonathan Peyton30419822017-05-12 18:01:32 +0000692 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL, 0);
Andrey Churbanov5c56fb52015-02-20 18:05:17 +0000693#else
Jonathan Peyton30419822017-05-12 18:01:32 +0000694 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL);
Andrey Churbanov5c56fb52015-02-20 18:05:17 +0000695#endif
Jonathan Peyton30419822017-05-12 18:01:32 +0000696 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000697#ifdef BUILD_PARALLEL_ORDERED
Jonathan Peyton30419822017-05-12 18:01:32 +0000698 if (!team->t.t_serialized) {
699 KMP_MB();
700 KMP_WAIT_YIELD(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid),
701 KMP_EQ, NULL);
702 KMP_MB();
703 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000704#endif /* BUILD_PARALLEL_ORDERED */
705}
706
Jonathan Peyton30419822017-05-12 18:01:32 +0000707/* __kmp_parallel_dxo -- Signal the next task. */
708void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
709 int gtid = *gtid_ref;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000710#ifdef BUILD_PARALLEL_ORDERED
Jonathan Peyton30419822017-05-12 18:01:32 +0000711 int tid = __kmp_tid_from_gtid(gtid);
712 kmp_team_t *team = __kmp_team_from_gtid(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000713#endif /* BUILD_PARALLEL_ORDERED */
714
Jonathan Peyton30419822017-05-12 18:01:32 +0000715 if (__kmp_env_consistency_check) {
716 if (__kmp_threads[gtid]->th.th_root->r.r_active)
717 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref);
718 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000719#ifdef BUILD_PARALLEL_ORDERED
Jonathan Peyton30419822017-05-12 18:01:32 +0000720 if (!team->t.t_serialized) {
721 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000722
Jonathan Peyton30419822017-05-12 18:01:32 +0000723 /* use the tid of the next thread in this team */
724 /* TODO replace with general release procedure */
725 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000726
Andrey Churbanovd7d088f2015-04-29 16:42:24 +0000727#if OMPT_SUPPORT && OMPT_BLAME
Jonathan Peyton30419822017-05-12 18:01:32 +0000728 if (ompt_enabled &&
729 ompt_callbacks.ompt_callback(ompt_event_release_ordered)) {
730 /* accept blame for "ordered" waiting */
731 kmp_info_t *this_thread = __kmp_threads[gtid];
732 ompt_callbacks.ompt_callback(ompt_event_release_ordered)(
733 this_thread->th.ompt_thread_info.wait_id);
734 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +0000735#endif
736
Jonathan Peyton30419822017-05-12 18:01:32 +0000737 KMP_MB(); /* Flush all pending memory write invalidates. */
738 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000739#endif /* BUILD_PARALLEL_ORDERED */
740}
741
742/* ------------------------------------------------------------------------ */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000743/* The BARRIER for a SINGLE process section is always explicit */
744
Jonathan Peyton30419822017-05-12 18:01:32 +0000745int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws) {
746 int status;
747 kmp_info_t *th;
748 kmp_team_t *team;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000749
Jonathan Peyton30419822017-05-12 18:01:32 +0000750 if (!TCR_4(__kmp_init_parallel))
751 __kmp_parallel_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +0000752
Jonathan Peyton30419822017-05-12 18:01:32 +0000753 th = __kmp_threads[gtid];
754 team = th->th.th_team;
755 status = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000756
Jonathan Peyton30419822017-05-12 18:01:32 +0000757 th->th.th_ident = id_ref;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000758
Jonathan Peyton30419822017-05-12 18:01:32 +0000759 if (team->t.t_serialized) {
760 status = 1;
761 } else {
762 kmp_int32 old_this = th->th.th_local.this_construct;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000763
Jonathan Peyton30419822017-05-12 18:01:32 +0000764 ++th->th.th_local.this_construct;
765 /* try to set team count to thread count--success means thread got the
766 single block */
767 /* TODO: Should this be acquire or release? */
768 if (team->t.t_construct == old_this) {
769 status = KMP_COMPARE_AND_STORE_ACQ32(&team->t.t_construct, old_this,
770 th->th.th_local.this_construct);
771 }
Andrey Churbanov51aecb82015-05-06 19:22:36 +0000772#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +0000773 if (__itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 &&
774 KMP_MASTER_GTID(gtid) &&
Andrey Churbanov51aecb82015-05-06 19:22:36 +0000775#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +0000776 th->th.th_teams_microtask == NULL &&
Andrey Churbanov51aecb82015-05-06 19:22:36 +0000777#endif
Jonathan Peyton30419822017-05-12 18:01:32 +0000778 team->t.t_active_level ==
779 1) { // Only report metadata by master of active team at level 1
780 __kmp_itt_metadata_single(id_ref);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000781 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000782#endif /* USE_ITT_BUILD */
783 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000784
Jonathan Peyton30419822017-05-12 18:01:32 +0000785 if (__kmp_env_consistency_check) {
786 if (status && push_ws) {
787 __kmp_push_workshare(gtid, ct_psingle, id_ref);
788 } else {
789 __kmp_check_workshare(gtid, ct_psingle, id_ref);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000790 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000791 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000792#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +0000793 if (status) {
794 __kmp_itt_single_start(gtid);
795 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000796#endif /* USE_ITT_BUILD */
Jonathan Peyton30419822017-05-12 18:01:32 +0000797 return status;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000798}
799
Jonathan Peyton30419822017-05-12 18:01:32 +0000800void __kmp_exit_single(int gtid) {
Jim Cownie5e8470a2013-09-27 10:38:44 +0000801#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +0000802 __kmp_itt_single_end(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000803#endif /* USE_ITT_BUILD */
Jonathan Peyton30419822017-05-12 18:01:32 +0000804 if (__kmp_env_consistency_check)
805 __kmp_pop_workshare(gtid, ct_psingle, NULL);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000806}
807
Jonathan Peyton30419822017-05-12 18:01:32 +0000808/* determine if we can go parallel or must use a serialized parallel region and
Jim Cownie5e8470a2013-09-27 10:38:44 +0000809 * how many threads we can use
810 * set_nproc is the number of threads requested for the team
811 * returns 0 if we should serialize or only use one thread,
812 * otherwise the number of threads to use
Jonathan Peyton30419822017-05-12 18:01:32 +0000813 * The forkjoin lock is held by the caller. */
814static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team,
815 int master_tid, int set_nthreads
Jim Cownie5e8470a2013-09-27 10:38:44 +0000816#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +0000817 ,
818 int enter_teams
Jim Cownie5e8470a2013-09-27 10:38:44 +0000819#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +0000820 ) {
821 int capacity;
822 int new_nthreads;
823 KMP_DEBUG_ASSERT(__kmp_init_serial);
824 KMP_DEBUG_ASSERT(root && parent_team);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000825
Jonathan Peyton30419822017-05-12 18:01:32 +0000826 // If dyn-var is set, dynamically adjust the number of desired threads,
827 // according to the method specified by dynamic_mode.
828 new_nthreads = set_nthreads;
829 if (!get__dynamic_2(parent_team, master_tid)) {
830 ;
831 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000832#ifdef USE_LOAD_BALANCE
Jonathan Peyton30419822017-05-12 18:01:32 +0000833 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) {
834 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads);
835 if (new_nthreads == 1) {
836 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "
837 "reservation to 1 thread\n",
838 master_tid));
839 return 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000840 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000841 if (new_nthreads < set_nthreads) {
842 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "
843 "reservation to %d threads\n",
844 master_tid, new_nthreads));
845 }
846 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000847#endif /* USE_LOAD_BALANCE */
Jonathan Peyton30419822017-05-12 18:01:32 +0000848 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) {
849 new_nthreads = __kmp_avail_proc - __kmp_nth +
850 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
851 if (new_nthreads <= 1) {
852 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "
853 "reservation to 1 thread\n",
854 master_tid));
855 return 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000856 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000857 if (new_nthreads < set_nthreads) {
858 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "
859 "reservation to %d threads\n",
860 master_tid, new_nthreads));
861 } else {
862 new_nthreads = set_nthreads;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000863 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000864 } else if (__kmp_global.g.g_dynamic_mode == dynamic_random) {
865 if (set_nthreads > 2) {
866 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]);
867 new_nthreads = (new_nthreads % set_nthreads) + 1;
868 if (new_nthreads == 1) {
869 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "
870 "reservation to 1 thread\n",
871 master_tid));
Jim Cownie5e8470a2013-09-27 10:38:44 +0000872 return 1;
Jonathan Peyton30419822017-05-12 18:01:32 +0000873 }
874 if (new_nthreads < set_nthreads) {
875 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "
876 "reservation to %d threads\n",
877 master_tid, new_nthreads));
878 }
879 }
880 } else {
881 KMP_ASSERT(0);
882 }
883
Jonathan Peytonf4392462017-07-27 20:58:41 +0000884 // Respect KMP_ALL_THREADS/KMP_DEVICE_THREAD_LIMIT.
Jonathan Peyton30419822017-05-12 18:01:32 +0000885 if (__kmp_nth + new_nthreads -
886 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
887 __kmp_max_nth) {
888 int tl_nthreads = __kmp_max_nth - __kmp_nth +
889 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
890 if (tl_nthreads <= 0) {
891 tl_nthreads = 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000892 }
893
Jonathan Peyton30419822017-05-12 18:01:32 +0000894 // If dyn-var is false, emit a 1-time warning.
895 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
896 __kmp_reserve_warn = 1;
897 __kmp_msg(kmp_ms_warning,
898 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
899 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
900 }
901 if (tl_nthreads == 1) {
Jonathan Peytonf4392462017-07-27 20:58:41 +0000902 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
903 "reduced reservation to 1 thread\n",
Jonathan Peyton30419822017-05-12 18:01:32 +0000904 master_tid));
905 return 1;
906 }
Jonathan Peytonf4392462017-07-27 20:58:41 +0000907 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
908 "reservation to %d threads\n",
909 master_tid, tl_nthreads));
910 new_nthreads = tl_nthreads;
911 }
912
913 // Respect OMP_THREAD_LIMIT
914 if (root->r.r_cg_nthreads + new_nthreads -
915 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
916 __kmp_cg_max_nth) {
917 int tl_nthreads = __kmp_cg_max_nth - root->r.r_cg_nthreads +
918 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
919 if (tl_nthreads <= 0) {
920 tl_nthreads = 1;
921 }
922
923 // If dyn-var is false, emit a 1-time warning.
924 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
925 __kmp_reserve_warn = 1;
926 __kmp_msg(kmp_ms_warning,
927 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
928 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
929 }
930 if (tl_nthreads == 1) {
931 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
932 "reduced reservation to 1 thread\n",
933 master_tid));
934 return 1;
935 }
936 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
Jonathan Peyton30419822017-05-12 18:01:32 +0000937 "reservation to %d threads\n",
938 master_tid, tl_nthreads));
939 new_nthreads = tl_nthreads;
940 }
941
942 // Check if the threads array is large enough, or needs expanding.
Jonathan Peyton30419822017-05-12 18:01:32 +0000943 // See comment in __kmp_register_root() about the adjustment if
944 // __kmp_threads[0] == NULL.
945 capacity = __kmp_threads_capacity;
946 if (TCR_PTR(__kmp_threads[0]) == NULL) {
947 --capacity;
948 }
949 if (__kmp_nth + new_nthreads -
950 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
951 capacity) {
952 // Expand the threads array.
953 int slotsRequired = __kmp_nth + new_nthreads -
954 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) -
955 capacity;
956 int slotsAdded = __kmp_expand_threads(slotsRequired, slotsRequired);
957 if (slotsAdded < slotsRequired) {
958 // The threads array was not expanded enough.
959 new_nthreads -= (slotsRequired - slotsAdded);
960 KMP_ASSERT(new_nthreads >= 1);
961
962 // If dyn-var is false, emit a 1-time warning.
963 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
964 __kmp_reserve_warn = 1;
965 if (__kmp_tp_cached) {
966 __kmp_msg(kmp_ms_warning,
967 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
968 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
969 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
970 } else {
971 __kmp_msg(kmp_ms_warning,
972 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
973 KMP_HNT(SystemLimitOnThreads), __kmp_msg_null);
974 }
975 }
976 }
977 }
978
Jonathan Peyton642688b2017-06-01 16:46:36 +0000979#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +0000980 if (new_nthreads == 1) {
981 KC_TRACE(10,
982 ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
983 "dead roots and rechecking; requested %d threads\n",
984 __kmp_get_gtid(), set_nthreads));
Jonathan Peyton642688b2017-06-01 16:46:36 +0000985 } else {
986 KC_TRACE(10, ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
987 " %d threads\n",
988 __kmp_get_gtid(), new_nthreads, set_nthreads));
Jonathan Peyton30419822017-05-12 18:01:32 +0000989 }
Jonathan Peyton642688b2017-06-01 16:46:36 +0000990#endif // KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +0000991 return new_nthreads;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000992}
993
Jonathan Peyton30419822017-05-12 18:01:32 +0000994/* Allocate threads from the thread pool and assign them to the new team. We are
995 assured that there are enough threads available, because we checked on that
996 earlier within critical section forkjoin */
997static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
998 kmp_info_t *master_th, int master_gtid) {
999 int i;
1000 int use_hot_team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001001
Jonathan Peyton30419822017-05-12 18:01:32 +00001002 KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc));
1003 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid());
1004 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00001005
Jonathan Peyton30419822017-05-12 18:01:32 +00001006 /* first, let's setup the master thread */
1007 master_th->th.th_info.ds.ds_tid = 0;
1008 master_th->th.th_team = team;
1009 master_th->th.th_team_nproc = team->t.t_nproc;
1010 master_th->th.th_team_master = master_th;
1011 master_th->th.th_team_serialized = FALSE;
1012 master_th->th.th_dispatch = &team->t.t_dispatch[0];
Jim Cownie5e8470a2013-09-27 10:38:44 +00001013
Jonathan Peyton30419822017-05-12 18:01:32 +00001014/* make sure we are not the optimized hot team */
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001015#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00001016 use_hot_team = 0;
1017 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams;
1018 if (hot_teams) { // hot teams array is not allocated if
1019 // KMP_HOT_TEAMS_MAX_LEVEL=0
1020 int level = team->t.t_active_level - 1; // index in array of hot teams
1021 if (master_th->th.th_teams_microtask) { // are we inside the teams?
1022 if (master_th->th.th_teams_size.nteams > 1) {
1023 ++level; // level was not increased in teams construct for
1024 // team_of_masters
1025 }
1026 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
1027 master_th->th.th_teams_level == team->t.t_level) {
1028 ++level; // level was not increased in teams construct for
1029 // team_of_workers before the parallel
1030 } // team->t.t_level will be increased inside parallel
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001031 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001032 if (level < __kmp_hot_teams_max_level) {
1033 if (hot_teams[level].hot_team) {
1034 // hot team has already been allocated for given level
1035 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team);
1036 use_hot_team = 1; // the team is ready to use
1037 } else {
1038 use_hot_team = 0; // AC: threads are not allocated yet
1039 hot_teams[level].hot_team = team; // remember new hot team
1040 hot_teams[level].hot_team_nth = team->t.t_nproc;
1041 }
1042 } else {
1043 use_hot_team = 0;
1044 }
1045 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001046#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001047 use_hot_team = team == root->r.r_hot_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001048#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001049 if (!use_hot_team) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00001050
Jonathan Peyton30419822017-05-12 18:01:32 +00001051 /* install the master thread */
1052 team->t.t_threads[0] = master_th;
1053 __kmp_initialize_info(master_th, team, 0, master_gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001054
Jonathan Peyton30419822017-05-12 18:01:32 +00001055 /* now, install the worker threads */
1056 for (i = 1; i < team->t.t_nproc; i++) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00001057
Jonathan Peyton30419822017-05-12 18:01:32 +00001058 /* fork or reallocate a new thread and install it in team */
1059 kmp_info_t *thr = __kmp_allocate_thread(root, team, i);
1060 team->t.t_threads[i] = thr;
1061 KMP_DEBUG_ASSERT(thr);
1062 KMP_DEBUG_ASSERT(thr->th.th_team == team);
1063 /* align team and thread arrived states */
1064 KA_TRACE(20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
1065 "T#%d(%d:%d) join =%llu, plain=%llu\n",
1066 __kmp_gtid_from_tid(0, team), team->t.t_id, 0,
1067 __kmp_gtid_from_tid(i, team), team->t.t_id, i,
1068 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
1069 team->t.t_bar[bs_plain_barrier].b_arrived));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001070#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001071 thr->th.th_teams_microtask = master_th->th.th_teams_microtask;
1072 thr->th.th_teams_level = master_th->th.th_teams_level;
1073 thr->th.th_teams_size = master_th->th.th_teams_size;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001074#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001075 { // Initialize threads' barrier data.
1076 int b;
1077 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar;
1078 for (b = 0; b < bs_last_barrier; ++b) {
1079 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
1080 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00001081#if USE_DEBUGGER
Jonathan Peyton30419822017-05-12 18:01:32 +00001082 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00001083#endif
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00001084 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001085 }
1086 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001087
Alp Toker98758b02014-03-02 04:12:06 +00001088#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00001089 __kmp_partition_places(team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001090#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001091 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001092
Jonathan Peyton30419822017-05-12 18:01:32 +00001093 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00001094}
1095
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001096#if KMP_ARCH_X86 || KMP_ARCH_X86_64
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001097// Propagate any changes to the floating point control registers out to the team
Jonathan Peyton30419822017-05-12 18:01:32 +00001098// We try to avoid unnecessary writes to the relevant cache line in the team
1099// structure, so we don't make changes unless they are needed.
1100inline static void propagateFPControl(kmp_team_t *team) {
1101 if (__kmp_inherit_fp_control) {
1102 kmp_int16 x87_fpu_control_word;
1103 kmp_uint32 mxcsr;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001104
Jonathan Peyton30419822017-05-12 18:01:32 +00001105 // Get master values of FPU control flags (both X87 and vector)
1106 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1107 __kmp_store_mxcsr(&mxcsr);
1108 mxcsr &= KMP_X86_MXCSR_MASK;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001109
Jonathan Peyton30419822017-05-12 18:01:32 +00001110// There is no point looking at t_fp_control_saved here.
1111// If it is TRUE, we still have to update the values if they are different from
1112// those we now have.
1113// If it is FALSE we didn't save anything yet, but our objective is the same. We
1114// have to ensure that the values in the team are the same as those we have.
1115// So, this code achieves what we need whether or not t_fp_control_saved is
1116// true. By checking whether the value needs updating we avoid unnecessary
1117// writes that would put the cache-line into a written state, causing all
1118// threads in the team to have to read it again.
1119 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word);
1120 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr);
1121 // Although we don't use this value, other code in the runtime wants to know
1122 // whether it should restore them. So we must ensure it is correct.
1123 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE);
1124 } else {
1125 // Similarly here. Don't write to this cache-line in the team structure
1126 // unless we have to.
1127 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE);
1128 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001129}
1130
Jonathan Peyton30419822017-05-12 18:01:32 +00001131// Do the opposite, setting the hardware registers to the updated values from
1132// the team.
1133inline static void updateHWFPControl(kmp_team_t *team) {
1134 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) {
1135 // Only reset the fp control regs if they have been changed in the team.
1136 // the parallel region that we are exiting.
1137 kmp_int16 x87_fpu_control_word;
1138 kmp_uint32 mxcsr;
1139 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1140 __kmp_store_mxcsr(&mxcsr);
1141 mxcsr &= KMP_X86_MXCSR_MASK;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001142
Jonathan Peyton30419822017-05-12 18:01:32 +00001143 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) {
1144 __kmp_clear_x87_fpu_status_word();
1145 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001146 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001147
1148 if (team->t.t_mxcsr != mxcsr) {
1149 __kmp_load_mxcsr(&team->t.t_mxcsr);
1150 }
1151 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001152}
1153#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001154#define propagateFPControl(x) ((void)0)
1155#define updateHWFPControl(x) ((void)0)
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001156#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1157
Jonathan Peyton30419822017-05-12 18:01:32 +00001158static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team,
1159 int realloc); // forward declaration
Jim Cownie5e8470a2013-09-27 10:38:44 +00001160
Jonathan Peyton30419822017-05-12 18:01:32 +00001161/* Run a parallel region that has been serialized, so runs only in a team of the
1162 single master thread. */
1163void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
1164 kmp_info_t *this_thr;
1165 kmp_team_t *serial_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001166
Jonathan Peyton30419822017-05-12 18:01:32 +00001167 KC_TRACE(10, ("__kmpc_serialized_parallel: called by T#%d\n", global_tid));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001168
Jonathan Peyton30419822017-05-12 18:01:32 +00001169 /* Skip all this code for autopar serialized loops since it results in
1170 unacceptable overhead */
1171 if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR))
1172 return;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001173
Jonathan Peyton30419822017-05-12 18:01:32 +00001174 if (!TCR_4(__kmp_init_parallel))
1175 __kmp_parallel_initialize();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001176
Jonathan Peyton30419822017-05-12 18:01:32 +00001177 this_thr = __kmp_threads[global_tid];
1178 serial_team = this_thr->th.th_serial_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001179
Jonathan Peyton30419822017-05-12 18:01:32 +00001180 /* utilize the serialized team held by this thread */
1181 KMP_DEBUG_ASSERT(serial_team);
1182 KMP_MB();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001183
Jonathan Peyton30419822017-05-12 18:01:32 +00001184 if (__kmp_tasking_mode != tskm_immediate_exec) {
1185 KMP_DEBUG_ASSERT(
1186 this_thr->th.th_task_team ==
1187 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]);
1188 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] ==
1189 NULL);
1190 KA_TRACE(20, ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
1191 "team %p, new task_team = NULL\n",
1192 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
1193 this_thr->th.th_task_team = NULL;
1194 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001195
1196#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001197 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
1198 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1199 proc_bind = proc_bind_false;
1200 } else if (proc_bind == proc_bind_default) {
1201 // No proc_bind clause was specified, so use the current value
1202 // of proc-bind-var for this parallel region.
1203 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
1204 }
1205 // Reset for next parallel region
1206 this_thr->th.th_set_proc_bind = proc_bind_default;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001207#endif /* OMP_40_ENABLED */
1208
Jonathan Peyton30419822017-05-12 18:01:32 +00001209 if (this_thr->th.th_team != serial_team) {
1210 // Nested level will be an index in the nested nthreads array
1211 int level = this_thr->th.th_team->t.t_level;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001212
Jonathan Peyton30419822017-05-12 18:01:32 +00001213 if (serial_team->t.t_serialized) {
1214 /* this serial team was already used
1215 TODO increase performance by making this locks more specific */
1216 kmp_team_t *new_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001217
Jonathan Peyton30419822017-05-12 18:01:32 +00001218 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001219
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001220#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001221 ompt_parallel_id_t ompt_parallel_id = __ompt_parallel_id_new(global_tid);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001222#endif
1223
Jonathan Peyton30419822017-05-12 18:01:32 +00001224 new_team = __kmp_allocate_team(this_thr->th.th_root, 1, 1,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001225#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001226 ompt_parallel_id,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001227#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001228#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001229 proc_bind,
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001230#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001231 &this_thr->th.th_current_task->td_icvs,
1232 0 USE_NESTED_HOT_ARG(NULL));
1233 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1234 KMP_ASSERT(new_team);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001235
Jonathan Peyton30419822017-05-12 18:01:32 +00001236 /* setup new serialized team and install it */
1237 new_team->t.t_threads[0] = this_thr;
1238 new_team->t.t_parent = this_thr->th.th_team;
1239 serial_team = new_team;
1240 this_thr->th.th_serial_team = serial_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001241
Jonathan Peyton30419822017-05-12 18:01:32 +00001242 KF_TRACE(
1243 10,
1244 ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",
1245 global_tid, serial_team));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001246
Jonathan Peyton30419822017-05-12 18:01:32 +00001247 /* TODO the above breaks the requirement that if we run out of resources,
1248 then we can still guarantee that serialized teams are ok, since we may
1249 need to allocate a new one */
1250 } else {
1251 KF_TRACE(
1252 10,
1253 ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",
1254 global_tid, serial_team));
1255 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001256
Jonathan Peyton30419822017-05-12 18:01:32 +00001257 /* we have to initialize this serial team */
1258 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1259 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1260 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team);
1261 serial_team->t.t_ident = loc;
1262 serial_team->t.t_serialized = 1;
1263 serial_team->t.t_nproc = 1;
1264 serial_team->t.t_parent = this_thr->th.th_team;
1265 serial_team->t.t_sched = this_thr->th.th_team->t.t_sched;
1266 this_thr->th.th_team = serial_team;
1267 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001268
Jonathan Peyton30419822017-05-12 18:01:32 +00001269 KF_TRACE(10, ("__kmpc_serialized_parallel: T#d curtask=%p\n", global_tid,
1270 this_thr->th.th_current_task));
1271 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1);
1272 this_thr->th.th_current_task->td_flags.executing = 0;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001273
Jonathan Peyton30419822017-05-12 18:01:32 +00001274 __kmp_push_current_task_to_thread(this_thr, serial_team, 0);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001275
Jonathan Peyton30419822017-05-12 18:01:32 +00001276 /* TODO: GEH: do ICVs work for nested serialized teams? Don't we need an
1277 implicit task for each serialized task represented by
1278 team->t.t_serialized? */
1279 copy_icvs(&this_thr->th.th_current_task->td_icvs,
1280 &this_thr->th.th_current_task->td_parent->td_icvs);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001281
Jonathan Peyton30419822017-05-12 18:01:32 +00001282 // Thread value exists in the nested nthreads array for the next nested
1283 // level
1284 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1285 this_thr->th.th_current_task->td_icvs.nproc =
1286 __kmp_nested_nth.nth[level + 1];
1287 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001288
1289#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001290 if (__kmp_nested_proc_bind.used &&
1291 (level + 1 < __kmp_nested_proc_bind.used)) {
1292 this_thr->th.th_current_task->td_icvs.proc_bind =
1293 __kmp_nested_proc_bind.bind_types[level + 1];
1294 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001295#endif /* OMP_40_ENABLED */
1296
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00001297#if USE_DEBUGGER
Jonathan Peyton30419822017-05-12 18:01:32 +00001298 serial_team->t.t_pkfn = (microtask_t)(~0); // For the debugger.
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00001299#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001300 this_thr->th.th_info.ds.ds_tid = 0;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001301
Jonathan Peyton30419822017-05-12 18:01:32 +00001302 /* set thread cache values */
1303 this_thr->th.th_team_nproc = 1;
1304 this_thr->th.th_team_master = this_thr;
1305 this_thr->th.th_team_serialized = 1;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001306
Jonathan Peyton30419822017-05-12 18:01:32 +00001307 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1;
1308 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001309
Jonathan Peyton30419822017-05-12 18:01:32 +00001310 propagateFPControl(serial_team);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001311
Jonathan Peyton30419822017-05-12 18:01:32 +00001312 /* check if we need to allocate dispatch buffers stack */
1313 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1314 if (!serial_team->t.t_dispatch->th_disp_buffer) {
1315 serial_team->t.t_dispatch->th_disp_buffer =
1316 (dispatch_private_info_t *)__kmp_allocate(
1317 sizeof(dispatch_private_info_t));
1318 }
1319 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001320
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001321#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001322 ompt_parallel_id_t ompt_parallel_id = __ompt_parallel_id_new(global_tid);
1323 __ompt_team_assign_id(serial_team, ompt_parallel_id);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001324#endif
1325
Jonathan Peyton30419822017-05-12 18:01:32 +00001326 KMP_MB();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001327
Jonathan Peyton30419822017-05-12 18:01:32 +00001328 } else {
1329 /* this serialized team is already being used,
1330 * that's fine, just add another nested level */
1331 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
1332 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1333 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1334 ++serial_team->t.t_serialized;
1335 this_thr->th.th_team_serialized = serial_team->t.t_serialized;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001336
Jonathan Peyton30419822017-05-12 18:01:32 +00001337 // Nested level will be an index in the nested nthreads array
1338 int level = this_thr->th.th_team->t.t_level;
1339 // Thread value exists in the nested nthreads array for the next nested
1340 // level
1341 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1342 this_thr->th.th_current_task->td_icvs.nproc =
1343 __kmp_nested_nth.nth[level + 1];
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001344 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001345 serial_team->t.t_level++;
1346 KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d increasing nesting level "
1347 "of serial team %p to %d\n",
1348 global_tid, serial_team, serial_team->t.t_level));
1349
1350 /* allocate/push dispatch buffers stack */
1351 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1352 {
1353 dispatch_private_info_t *disp_buffer =
1354 (dispatch_private_info_t *)__kmp_allocate(
1355 sizeof(dispatch_private_info_t));
1356 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;
1357 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;
1358 }
1359 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1360
1361 KMP_MB();
1362 }
Olga Malyshevadbdcfa12017-04-04 13:56:50 +00001363#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001364 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq);
Olga Malyshevadbdcfa12017-04-04 13:56:50 +00001365#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001366
Jonathan Peyton30419822017-05-12 18:01:32 +00001367 if (__kmp_env_consistency_check)
1368 __kmp_push_parallel(global_tid, NULL);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001369}
Jim Cownie181b4bb2013-12-23 17:28:57 +00001370
Jim Cownie5e8470a2013-09-27 10:38:44 +00001371/* most of the work for a fork */
1372/* return true if we really went parallel, false if serialized */
Jonathan Peyton30419822017-05-12 18:01:32 +00001373int __kmp_fork_call(ident_t *loc, int gtid,
1374 enum fork_context_e call_context, // Intel, GNU, ...
1375 kmp_int32 argc,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001376#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001377 void *unwrapped_task,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001378#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001379 microtask_t microtask, launch_t invoker,
Jim Cownie5e8470a2013-09-27 10:38:44 +00001380/* TODO: revert workaround for Intel(R) 64 tracker #96 */
Andrey Churbanovcbda8682015-01-13 14:43:35 +00001381#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
Jonathan Peyton30419822017-05-12 18:01:32 +00001382 va_list *ap
Jim Cownie5e8470a2013-09-27 10:38:44 +00001383#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001384 va_list ap
Jim Cownie5e8470a2013-09-27 10:38:44 +00001385#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001386 ) {
1387 void **argv;
1388 int i;
1389 int master_tid;
1390 int master_this_cons;
1391 kmp_team_t *team;
1392 kmp_team_t *parent_team;
1393 kmp_info_t *master_th;
1394 kmp_root_t *root;
1395 int nthreads;
1396 int master_active;
1397 int master_set_numthreads;
1398 int level;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001399#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001400 int active_level;
1401 int teams_level;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001402#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001403#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00001404 kmp_hot_team_ptr_t **p_hot_teams;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001405#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001406 { // KMP_TIME_BLOCK
Jonathan Peyton5375fe82016-11-14 21:13:44 +00001407 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call);
Jonathan Peyton45be4502015-08-11 21:36:41 +00001408 KMP_COUNT_VALUE(OMP_PARALLEL_args, argc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001409
Jonathan Peyton30419822017-05-12 18:01:32 +00001410 KA_TRACE(20, ("__kmp_fork_call: enter T#%d\n", gtid));
1411 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL) {
1412 /* Some systems prefer the stack for the root thread(s) to start with */
1413 /* some gap from the parent stack to prevent false sharing. */
1414 void *dummy = KMP_ALLOCA(__kmp_stkpadding);
1415 /* These 2 lines below are so this does not get optimized out */
1416 if (__kmp_stkpadding > KMP_MAX_STKPADDING)
1417 __kmp_stkpadding += (short)((kmp_int64)dummy);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001418 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001419
1420 /* initialize if needed */
Jonathan Peyton30419822017-05-12 18:01:32 +00001421 KMP_DEBUG_ASSERT(
1422 __kmp_init_serial); // AC: potentially unsafe, not in sync with shutdown
1423 if (!TCR_4(__kmp_init_parallel))
1424 __kmp_parallel_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00001425
1426 /* setup current data */
Jonathan Peyton30419822017-05-12 18:01:32 +00001427 master_th = __kmp_threads[gtid]; // AC: potentially unsafe, not in sync with
1428 // shutdown
1429 parent_team = master_th->th.th_team;
1430 master_tid = master_th->th.th_info.ds.ds_tid;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001431 master_this_cons = master_th->th.th_local.this_construct;
Jonathan Peyton30419822017-05-12 18:01:32 +00001432 root = master_th->th.th_root;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001433 master_active = root->r.r_active;
1434 master_set_numthreads = master_th->th.th_set_nproc;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001435
1436#if OMPT_SUPPORT
1437 ompt_parallel_id_t ompt_parallel_id;
1438 ompt_task_id_t ompt_task_id;
1439 ompt_frame_t *ompt_frame;
1440 ompt_task_id_t my_task_id;
1441 ompt_parallel_id_t my_parallel_id;
1442
Jonathan Peytonb68a85d2015-09-21 18:11:22 +00001443 if (ompt_enabled) {
Jonathan Peyton30419822017-05-12 18:01:32 +00001444 ompt_parallel_id = __ompt_parallel_id_new(gtid);
1445 ompt_task_id = __ompt_get_task_id_internal(0);
1446 ompt_frame = __ompt_get_task_frame_internal(0);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001447 }
1448#endif
1449
Jim Cownie5e8470a2013-09-27 10:38:44 +00001450 // Nested level will be an index in the nested nthreads array
Jonathan Peyton30419822017-05-12 18:01:32 +00001451 level = parent_team->t.t_level;
1452 // used to launch non-serial teams even if nested is not allowed
1453 active_level = parent_team->t.t_active_level;
Jonathan Peytonc76f9f02016-06-21 19:12:07 +00001454#if OMP_40_ENABLED
Jonathan Peyton642688b2017-06-01 16:46:36 +00001455 // needed to check nesting inside the teams
1456 teams_level = master_th->th.th_teams_level;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001457#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001458#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00001459 p_hot_teams = &master_th->th.th_hot_teams;
1460 if (*p_hot_teams == NULL && __kmp_hot_teams_max_level > 0) {
1461 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate(
1462 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level);
1463 (*p_hot_teams)[0].hot_team = root->r.r_hot_team;
Jonathan Peyton642688b2017-06-01 16:46:36 +00001464 // it is either actual or not needed (when active_level > 0)
1465 (*p_hot_teams)[0].hot_team_nth = 1;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001466 }
1467#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001468
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001469#if OMPT_SUPPORT
Jonathan Peytonb68a85d2015-09-21 18:11:22 +00001470 if (ompt_enabled &&
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001471 ompt_callbacks.ompt_callback(ompt_event_parallel_begin)) {
Jonathan Peyton30419822017-05-12 18:01:32 +00001472 int team_size = master_set_numthreads;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001473
Jonathan Peyton30419822017-05-12 18:01:32 +00001474 ompt_callbacks.ompt_callback(ompt_event_parallel_begin)(
1475 ompt_task_id, ompt_frame, ompt_parallel_id, team_size, unwrapped_task,
1476 OMPT_INVOKER(call_context));
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001477 }
1478#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001479
Jim Cownie5e8470a2013-09-27 10:38:44 +00001480 master_th->th.th_ident = loc;
1481
1482#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001483 if (master_th->th.th_teams_microtask && ap &&
1484 microtask != (microtask_t)__kmp_teams_master && level == teams_level) {
1485 // AC: This is start of parallel that is nested inside teams construct.
1486 // The team is actual (hot), all workers are ready at the fork barrier.
1487 // No lock needed to initialize the team a bit, then free workers.
1488 parent_team->t.t_ident = loc;
1489 __kmp_alloc_argv_entries(argc, parent_team, TRUE);
1490 parent_team->t.t_argc = argc;
1491 argv = (void **)parent_team->t.t_argv;
1492 for (i = argc - 1; i >= 0; --i)
Jim Cownie5e8470a2013-09-27 10:38:44 +00001493/* TODO: revert workaround for Intel(R) 64 tracker #96 */
Andrey Churbanovcbda8682015-01-13 14:43:35 +00001494#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
Jonathan Peyton30419822017-05-12 18:01:32 +00001495 *argv++ = va_arg(*ap, void *);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001496#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001497 *argv++ = va_arg(ap, void *);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001498#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001499 // Increment our nested depth levels, but not increase the serialization
1500 if (parent_team == master_th->th.th_serial_team) {
1501 // AC: we are in serialized parallel
1502 __kmpc_serialized_parallel(loc, gtid);
1503 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1);
1504 // AC: need this in order enquiry functions work
1505 // correctly, will restore at join time
1506 parent_team->t.t_serialized--;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001507#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001508 void *dummy;
1509 void **exit_runtime_p;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001510
Jonathan Peyton30419822017-05-12 18:01:32 +00001511 ompt_lw_taskteam_t lw_taskteam;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001512
Jonathan Peyton30419822017-05-12 18:01:32 +00001513 if (ompt_enabled) {
1514 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid, unwrapped_task,
1515 ompt_parallel_id);
1516 lw_taskteam.ompt_task_info.task_id = __ompt_task_id_new(gtid);
1517 exit_runtime_p =
1518 &(lw_taskteam.ompt_task_info.frame.exit_runtime_frame);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001519
Jonathan Peyton30419822017-05-12 18:01:32 +00001520 __ompt_lw_taskteam_link(&lw_taskteam, master_th);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001521
1522#if OMPT_TRACE
Jonathan Peyton30419822017-05-12 18:01:32 +00001523 /* OMPT implicit task begin */
1524 my_task_id = lw_taskteam.ompt_task_info.task_id;
1525 my_parallel_id = parent_team->t.ompt_team_info.parallel_id;
1526 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)) {
1527 ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)(
1528 my_parallel_id, my_task_id);
1529 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001530#endif
1531
Jonathan Peyton30419822017-05-12 18:01:32 +00001532 /* OMPT state */
1533 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1534 } else {
1535 exit_runtime_p = &dummy;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001536 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001537#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001538
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001539 {
Jonathan Peyton30419822017-05-12 18:01:32 +00001540 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1541 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1542 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv
1543#if OMPT_SUPPORT
1544 ,
1545 exit_runtime_p
1546#endif
1547 );
Jim Cownie5e8470a2013-09-27 10:38:44 +00001548 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001549
Jonathan Peyton30419822017-05-12 18:01:32 +00001550#if OMPT_SUPPORT
1551 *exit_runtime_p = NULL;
1552 if (ompt_enabled) {
1553#if OMPT_TRACE
1554 lw_taskteam.ompt_task_info.frame.exit_runtime_frame = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001555
Jonathan Peyton30419822017-05-12 18:01:32 +00001556 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
1557 ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
1558 ompt_parallel_id, ompt_task_id);
1559 }
1560
1561 __ompt_lw_taskteam_unlink(master_th);
1562 // reset clear the task id only after unlinking the task
1563 lw_taskteam.ompt_task_info.task_id = ompt_task_id_none;
1564#endif
1565
1566 if (ompt_callbacks.ompt_callback(ompt_event_parallel_end)) {
1567 ompt_callbacks.ompt_callback(ompt_event_parallel_end)(
1568 ompt_parallel_id, ompt_task_id, OMPT_INVOKER(call_context));
1569 }
1570 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1571 }
1572#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001573 return TRUE;
Jonathan Peyton30419822017-05-12 18:01:32 +00001574 }
1575
1576 parent_team->t.t_pkfn = microtask;
1577#if OMPT_SUPPORT
1578 parent_team->t.ompt_team_info.microtask = unwrapped_task;
1579#endif
1580 parent_team->t.t_invoke = invoker;
1581 KMP_TEST_THEN_INC32((kmp_int32 *)&root->r.r_in_parallel);
1582 parent_team->t.t_active_level++;
1583 parent_team->t.t_level++;
1584
1585 /* Change number of threads in the team if requested */
1586 if (master_set_numthreads) { // The parallel has num_threads clause
1587 if (master_set_numthreads < master_th->th.th_teams_size.nth) {
1588 // AC: only can reduce number of threads dynamically, can't increase
1589 kmp_info_t **other_threads = parent_team->t.t_threads;
1590 parent_team->t.t_nproc = master_set_numthreads;
1591 for (i = 0; i < master_set_numthreads; ++i) {
1592 other_threads[i]->th.th_team_nproc = master_set_numthreads;
1593 }
1594 // Keep extra threads hot in the team for possible next parallels
1595 }
1596 master_th->th.th_set_nproc = 0;
1597 }
1598
1599#if USE_DEBUGGER
1600 if (__kmp_debugging) { // Let debugger override number of threads.
1601 int nth = __kmp_omp_num_threads(loc);
Jonathan Peyton642688b2017-06-01 16:46:36 +00001602 if (nth > 0) { // 0 means debugger doesn't want to change num threads
Jonathan Peyton30419822017-05-12 18:01:32 +00001603 master_set_numthreads = nth;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00001604 }
1605 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001606#endif
1607
1608 KF_TRACE(10, ("__kmp_fork_call: before internal fork: root=%p, team=%p, "
1609 "master_th=%p, gtid=%d\n",
1610 root, parent_team, master_th, gtid));
1611 __kmp_internal_fork(loc, gtid, parent_team);
1612 KF_TRACE(10, ("__kmp_fork_call: after internal fork: root=%p, team=%p, "
1613 "master_th=%p, gtid=%d\n",
1614 root, parent_team, master_th, gtid));
1615
1616 /* Invoke microtask for MASTER thread */
1617 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
1618 parent_team->t.t_id, parent_team->t.t_pkfn));
1619
1620 {
1621 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1622 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1623 if (!parent_team->t.t_invoke(gtid)) {
1624 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread");
1625 }
1626 }
1627 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
1628 parent_team->t.t_id, parent_team->t.t_pkfn));
1629 KMP_MB(); /* Flush all pending memory write invalidates. */
1630
1631 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid));
1632
1633 return TRUE;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001634 } // Parallel closely nested in teams construct
Jim Cownie5e8470a2013-09-27 10:38:44 +00001635#endif /* OMP_40_ENABLED */
1636
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001637#if KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00001638 if (__kmp_tasking_mode != tskm_immediate_exec) {
1639 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
1640 parent_team->t.t_task_team[master_th->th.th_task_state]);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001641 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001642#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001643
Jonathan Peyton30419822017-05-12 18:01:32 +00001644 if (parent_team->t.t_active_level >=
1645 master_th->th.th_current_task->td_icvs.max_active_levels) {
1646 nthreads = 1;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001647 } else {
Andrey Churbanov92effc42015-08-18 10:08:27 +00001648#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001649 int enter_teams = ((ap == NULL && active_level == 0) ||
1650 (ap && teams_level > 0 && teams_level == level));
Andrey Churbanov92effc42015-08-18 10:08:27 +00001651#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001652 nthreads =
1653 master_set_numthreads
1654 ? master_set_numthreads
1655 : get__nproc_2(
1656 parent_team,
1657 master_tid); // TODO: get nproc directly from current task
Andrey Churbanov92effc42015-08-18 10:08:27 +00001658
Jonathan Peyton30419822017-05-12 18:01:32 +00001659 // Check if we need to take forkjoin lock? (no need for serialized
1660 // parallel out of teams construct). This code moved here from
1661 // __kmp_reserve_threads() to speedup nested serialized parallels.
1662 if (nthreads > 1) {
1663 if ((!get__nested(master_th) && (root->r.r_in_parallel
Andrey Churbanov92effc42015-08-18 10:08:27 +00001664#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001665 && !enter_teams
Andrey Churbanov92effc42015-08-18 10:08:27 +00001666#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00001667 )) ||
1668 (__kmp_library == library_serial)) {
Jonathan Peyton642688b2017-06-01 16:46:36 +00001669 KC_TRACE(10, ("__kmp_fork_call: T#%d serializing team; requested %d"
1670 " threads\n",
1671 gtid, nthreads));
Jonathan Peyton30419822017-05-12 18:01:32 +00001672 nthreads = 1;
Andrey Churbanov92effc42015-08-18 10:08:27 +00001673 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001674 }
1675 if (nthreads > 1) {
1676 /* determine how many new threads we can use */
1677 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
Jonathan Peyton30419822017-05-12 18:01:32 +00001678 nthreads = __kmp_reserve_threads(
1679 root, parent_team, master_tid, nthreads
Jim Cownie5e8470a2013-09-27 10:38:44 +00001680#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001681 /* AC: If we execute teams from parallel region (on host), then
1682 teams should be created but each can only have 1 thread if
1683 nesting is disabled. If teams called from serial region, then
1684 teams and their threads should be created regardless of the
1685 nesting setting. */
1686 ,
1687 enter_teams
Jim Cownie5e8470a2013-09-27 10:38:44 +00001688#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00001689 );
1690 if (nthreads == 1) {
1691 // Free lock for single thread execution here; for multi-thread
1692 // execution it will be freed later after team of threads created
1693 // and initialized
1694 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
Andrey Churbanov92effc42015-08-18 10:08:27 +00001695 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001696 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001697 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001698 KMP_DEBUG_ASSERT(nthreads > 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001699
Jonathan Peyton30419822017-05-12 18:01:32 +00001700 // If we temporarily changed the set number of threads then restore it now
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001701 master_th->th.th_set_nproc = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001702
Jim Cownie5e8470a2013-09-27 10:38:44 +00001703 /* create a serialized parallel region? */
Jonathan Peyton30419822017-05-12 18:01:32 +00001704 if (nthreads == 1) {
1705/* josh todo: hypothetical question: what do we do for OS X*? */
1706#if KMP_OS_LINUX && \
1707 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1708 void *args[argc];
Jim Cownie5e8470a2013-09-27 10:38:44 +00001709#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001710 void **args = (void **)KMP_ALLOCA(argc * sizeof(void *));
1711#endif /* KMP_OS_LINUX && ( KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || \
1712 KMP_ARCH_AARCH64) */
Jim Cownie5e8470a2013-09-27 10:38:44 +00001713
Jonathan Peyton30419822017-05-12 18:01:32 +00001714 KA_TRACE(20,
1715 ("__kmp_fork_call: T#%d serializing parallel region\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00001716
Jonathan Peyton30419822017-05-12 18:01:32 +00001717 __kmpc_serialized_parallel(loc, gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001718
Jonathan Peyton30419822017-05-12 18:01:32 +00001719 if (call_context == fork_context_intel) {
1720 /* TODO this sucks, use the compiler itself to pass args! :) */
1721 master_th->th.th_serial_team->t.t_ident = loc;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001722#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001723 if (!ap) {
1724 // revert change made in __kmpc_serialized_parallel()
1725 master_th->th.th_serial_team->t.t_level--;
1726// Get args from parent team for teams construct
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001727
1728#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001729 void *dummy;
1730 void **exit_runtime_p;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001731
Jonathan Peyton30419822017-05-12 18:01:32 +00001732 ompt_lw_taskteam_t lw_taskteam;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001733
Jonathan Peyton30419822017-05-12 18:01:32 +00001734 if (ompt_enabled) {
1735 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1736 unwrapped_task, ompt_parallel_id);
1737 lw_taskteam.ompt_task_info.task_id = __ompt_task_id_new(gtid);
1738 exit_runtime_p =
1739 &(lw_taskteam.ompt_task_info.frame.exit_runtime_frame);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001740
Jonathan Peyton30419822017-05-12 18:01:32 +00001741 __ompt_lw_taskteam_link(&lw_taskteam, master_th);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001742
1743#if OMPT_TRACE
Jonathan Peyton30419822017-05-12 18:01:32 +00001744 my_task_id = lw_taskteam.ompt_task_info.task_id;
1745 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)) {
1746 ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)(
1747 ompt_parallel_id, my_task_id);
1748 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001749#endif
1750
Jonathan Peyton30419822017-05-12 18:01:32 +00001751 /* OMPT state */
1752 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1753 } else {
1754 exit_runtime_p = &dummy;
1755 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001756#endif
1757
Jonathan Peyton30419822017-05-12 18:01:32 +00001758 {
1759 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1760 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1761 __kmp_invoke_microtask(microtask, gtid, 0, argc,
1762 parent_team->t.t_argv
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001763#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001764 ,
1765 exit_runtime_p
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001766#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001767 );
1768 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001769
1770#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001771 *exit_runtime_p = NULL;
1772 if (ompt_enabled) {
1773 lw_taskteam.ompt_task_info.frame.exit_runtime_frame = NULL;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001774
1775#if OMPT_TRACE
Jonathan Peyton30419822017-05-12 18:01:32 +00001776 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
1777 ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
1778 ompt_parallel_id, ompt_task_id);
1779 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001780#endif
1781
Jonathan Peyton30419822017-05-12 18:01:32 +00001782 __ompt_lw_taskteam_unlink(master_th);
1783 // reset clear the task id only after unlinking the task
1784 lw_taskteam.ompt_task_info.task_id = ompt_task_id_none;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001785
Jonathan Peyton30419822017-05-12 18:01:32 +00001786 if (ompt_callbacks.ompt_callback(ompt_event_parallel_end)) {
1787 ompt_callbacks.ompt_callback(ompt_event_parallel_end)(
1788 ompt_parallel_id, ompt_task_id, OMPT_INVOKER(call_context));
1789 }
1790 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1791 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001792#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001793 } else if (microtask == (microtask_t)__kmp_teams_master) {
1794 KMP_DEBUG_ASSERT(master_th->th.th_team ==
1795 master_th->th.th_serial_team);
1796 team = master_th->th.th_team;
1797 // team->t.t_pkfn = microtask;
1798 team->t.t_invoke = invoker;
1799 __kmp_alloc_argv_entries(argc, team, TRUE);
1800 team->t.t_argc = argc;
1801 argv = (void **)team->t.t_argv;
1802 if (ap) {
1803 for (i = argc - 1; i >= 0; --i)
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001804// TODO: revert workaround for Intel(R) 64 tracker #96
Andrey Churbanovcbda8682015-01-13 14:43:35 +00001805#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
Jonathan Peyton30419822017-05-12 18:01:32 +00001806 *argv++ = va_arg(*ap, void *);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001807#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001808 *argv++ = va_arg(ap, void *);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001809#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001810 } else {
1811 for (i = 0; i < argc; ++i)
1812 // Get args from parent team for teams construct
1813 argv[i] = parent_team->t.t_argv[i];
1814 }
1815 // AC: revert change made in __kmpc_serialized_parallel()
1816 // because initial code in teams should have level=0
1817 team->t.t_level--;
1818 // AC: call special invoker for outer "parallel" of teams construct
1819 {
1820 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1821 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1822 invoker(gtid);
1823 }
1824 } else {
1825#endif /* OMP_40_ENABLED */
1826 argv = args;
1827 for (i = argc - 1; i >= 0; --i)
1828// TODO: revert workaround for Intel(R) 64 tracker #96
1829#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
1830 *argv++ = va_arg(*ap, void *);
1831#else
1832 *argv++ = va_arg(ap, void *);
1833#endif
1834 KMP_MB();
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001835
1836#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001837 void *dummy;
1838 void **exit_runtime_p;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001839
Jonathan Peyton30419822017-05-12 18:01:32 +00001840 ompt_lw_taskteam_t lw_taskteam;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001841
Jonathan Peyton30419822017-05-12 18:01:32 +00001842 if (ompt_enabled) {
1843 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1844 unwrapped_task, ompt_parallel_id);
1845 lw_taskteam.ompt_task_info.task_id = __ompt_task_id_new(gtid);
1846 exit_runtime_p =
1847 &(lw_taskteam.ompt_task_info.frame.exit_runtime_frame);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001848
Jonathan Peyton30419822017-05-12 18:01:32 +00001849 __ompt_lw_taskteam_link(&lw_taskteam, master_th);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001850
1851#if OMPT_TRACE
Jonathan Peyton30419822017-05-12 18:01:32 +00001852 /* OMPT implicit task begin */
1853 my_task_id = lw_taskteam.ompt_task_info.task_id;
1854 my_parallel_id = ompt_parallel_id;
1855 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)) {
1856 ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)(
1857 my_parallel_id, my_task_id);
1858 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001859#endif
1860
Jonathan Peyton30419822017-05-12 18:01:32 +00001861 /* OMPT state */
1862 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1863 } else {
1864 exit_runtime_p = &dummy;
1865 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001866#endif
1867
Jonathan Peyton30419822017-05-12 18:01:32 +00001868 {
1869 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1870 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1871 __kmp_invoke_microtask(microtask, gtid, 0, argc, args
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001872#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001873 ,
1874 exit_runtime_p
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001875#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001876 );
1877 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001878
1879#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001880 *exit_runtime_p = NULL;
1881 if (ompt_enabled) {
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001882#if OMPT_TRACE
Jonathan Peyton30419822017-05-12 18:01:32 +00001883 lw_taskteam.ompt_task_info.frame.exit_runtime_frame = NULL;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001884
Jonathan Peyton30419822017-05-12 18:01:32 +00001885 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
1886 ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
1887 my_parallel_id, my_task_id);
1888 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001889#endif
1890
Jonathan Peyton30419822017-05-12 18:01:32 +00001891 __ompt_lw_taskteam_unlink(master_th);
1892 // reset clear the task id only after unlinking the task
1893 lw_taskteam.ompt_task_info.task_id = ompt_task_id_none;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001894
Jonathan Peyton30419822017-05-12 18:01:32 +00001895 if (ompt_callbacks.ompt_callback(ompt_event_parallel_end)) {
1896 ompt_callbacks.ompt_callback(ompt_event_parallel_end)(
1897 ompt_parallel_id, ompt_task_id, OMPT_INVOKER(call_context));
1898 }
1899 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1900 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001901#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001902#if OMP_40_ENABLED
Jim Cownie5e8470a2013-09-27 10:38:44 +00001903 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001904#endif /* OMP_40_ENABLED */
1905 } else if (call_context == fork_context_gnu) {
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001906#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001907 ompt_lw_taskteam_t *lwt =
1908 (ompt_lw_taskteam_t *)__kmp_allocate(sizeof(ompt_lw_taskteam_t));
1909 __ompt_lw_taskteam_init(lwt, master_th, gtid, unwrapped_task,
1910 ompt_parallel_id);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001911
Jonathan Peyton30419822017-05-12 18:01:32 +00001912 lwt->ompt_task_info.task_id = __ompt_task_id_new(gtid);
1913 lwt->ompt_task_info.frame.exit_runtime_frame = NULL;
1914 __ompt_lw_taskteam_link(lwt, master_th);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001915#endif
1916
Jonathan Peyton30419822017-05-12 18:01:32 +00001917 // we were called from GNU native code
1918 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00001919 return FALSE;
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00001920 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00001921 KMP_ASSERT2(call_context < fork_context_last,
1922 "__kmp_fork_call: unknown fork_context parameter");
1923 }
1924
1925 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid));
1926 KMP_MB();
1927 return FALSE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001928 }
1929
Jim Cownie5e8470a2013-09-27 10:38:44 +00001930 // GEH: only modify the executing flag in the case when not serialized
1931 // serialized case is handled in kmpc_serialized_parallel
Jonathan Peyton30419822017-05-12 18:01:32 +00001932 KF_TRACE(10, ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
1933 "curtask=%p, curtask_max_aclevel=%d\n",
1934 parent_team->t.t_active_level, master_th,
1935 master_th->th.th_current_task,
1936 master_th->th.th_current_task->td_icvs.max_active_levels));
1937 // TODO: GEH - cannot do this assertion because root thread not set up as
1938 // executing
Jim Cownie5e8470a2013-09-27 10:38:44 +00001939 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 1 );
1940 master_th->th.th_current_task->td_flags.executing = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001941
1942#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001943 if (!master_th->th.th_teams_microtask || level > teams_level)
Jim Cownie5e8470a2013-09-27 10:38:44 +00001944#endif /* OMP_40_ENABLED */
1945 {
Jonathan Peyton30419822017-05-12 18:01:32 +00001946 /* Increment our nested depth level */
1947 KMP_TEST_THEN_INC32((kmp_int32 *)&root->r.r_in_parallel);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001948 }
1949
Jim Cownie5e8470a2013-09-27 10:38:44 +00001950 // See if we need to make a copy of the ICVs.
Jim Cownie5e8470a2013-09-27 10:38:44 +00001951 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc;
Jonathan Peyton30419822017-05-12 18:01:32 +00001952 if ((level + 1 < __kmp_nested_nth.used) &&
1953 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) {
1954 nthreads_icv = __kmp_nested_nth.nth[level + 1];
1955 } else {
1956 nthreads_icv = 0; // don't update
Jim Cownie5e8470a2013-09-27 10:38:44 +00001957 }
1958
1959#if OMP_40_ENABLED
Jim Cownie5e8470a2013-09-27 10:38:44 +00001960 // Figure out the proc_bind_policy for the new team.
Jim Cownie5e8470a2013-09-27 10:38:44 +00001961 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;
Jonathan Peyton30419822017-05-12 18:01:32 +00001962 kmp_proc_bind_t proc_bind_icv =
1963 proc_bind_default; // proc_bind_default means don't update
1964 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1965 proc_bind = proc_bind_false;
1966 } else {
1967 if (proc_bind == proc_bind_default) {
1968 // No proc_bind clause specified; use current proc-bind-var for this
1969 // parallel region
1970 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;
1971 }
1972 /* else: The proc_bind policy was specified explicitly on parallel clause.
1973 This overrides proc-bind-var for this parallel region, but does not
1974 change proc-bind-var. */
1975 // Figure the value of proc-bind-var for the child threads.
1976 if ((level + 1 < __kmp_nested_proc_bind.used) &&
1977 (__kmp_nested_proc_bind.bind_types[level + 1] !=
1978 master_th->th.th_current_task->td_icvs.proc_bind)) {
1979 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1];
1980 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001981 }
1982
Jim Cownie5e8470a2013-09-27 10:38:44 +00001983 // Reset for next parallel region
Jim Cownie5e8470a2013-09-27 10:38:44 +00001984 master_th->th.th_set_proc_bind = proc_bind_default;
1985#endif /* OMP_40_ENABLED */
1986
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001987 if ((nthreads_icv > 0)
Jim Cownie5e8470a2013-09-27 10:38:44 +00001988#if OMP_40_ENABLED
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001989 || (proc_bind_icv != proc_bind_default)
Jim Cownie5e8470a2013-09-27 10:38:44 +00001990#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00001991 ) {
1992 kmp_internal_control_t new_icvs;
1993 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs);
1994 new_icvs.next = NULL;
1995 if (nthreads_icv > 0) {
1996 new_icvs.nproc = nthreads_icv;
1997 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001998
1999#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002000 if (proc_bind_icv != proc_bind_default) {
2001 new_icvs.proc_bind = proc_bind_icv;
2002 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002003#endif /* OMP_40_ENABLED */
2004
Jonathan Peyton30419822017-05-12 18:01:32 +00002005 /* allocate a new parallel team */
2006 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n"));
2007 team = __kmp_allocate_team(root, nthreads, nthreads,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002008#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00002009 ompt_parallel_id,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002010#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002011#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002012 proc_bind,
Jim Cownie5e8470a2013-09-27 10:38:44 +00002013#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002014 &new_icvs, argc USE_NESTED_HOT_ARG(master_th));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002015 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00002016 /* allocate a new parallel team */
2017 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n"));
2018 team = __kmp_allocate_team(root, nthreads, nthreads,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002019#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00002020 ompt_parallel_id,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002021#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002022#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002023 proc_bind,
Jim Cownie5e8470a2013-09-27 10:38:44 +00002024#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002025 &master_th->th.th_current_task->td_icvs,
2026 argc USE_NESTED_HOT_ARG(master_th));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002027 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002028 KF_TRACE(
2029 10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002030
2031 /* setup the new team */
Jonathan Peytonb044e4f2016-05-23 18:01:19 +00002032 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid);
2033 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons);
2034 KMP_CHECK_UPDATE(team->t.t_ident, loc);
2035 KMP_CHECK_UPDATE(team->t.t_parent, parent_team);
2036 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002037#if OMPT_SUPPORT
Jonathan Peytonb044e4f2016-05-23 18:01:19 +00002038 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.microtask, unwrapped_task);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002039#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002040 KMP_CHECK_UPDATE(team->t.t_invoke, invoker); // TODO move to root, maybe
2041// TODO: parent_team->t.t_level == INT_MAX ???
Jim Cownie5e8470a2013-09-27 10:38:44 +00002042#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002043 if (!master_th->th.th_teams_microtask || level > teams_level) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002044#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00002045 int new_level = parent_team->t.t_level + 1;
2046 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2047 new_level = parent_team->t.t_active_level + 1;
2048 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002049#if OMP_40_ENABLED
2050 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00002051 // AC: Do not increase parallel level at start of the teams construct
2052 int new_level = parent_team->t.t_level;
2053 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2054 new_level = parent_team->t.t_active_level;
2055 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002056 }
2057#endif /* OMP_40_ENABLED */
Jonathan Peytonb044e4f2016-05-23 18:01:19 +00002058 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid);
Jonathan Peyton30419822017-05-12 18:01:32 +00002059 if (team->t.t_sched.r_sched_type != new_sched.r_sched_type ||
2060 team->t.t_sched.chunk != new_sched.chunk)
2061 team->t.t_sched =
2062 new_sched; // set master's schedule as new run-time schedule
Jim Cownie5e8470a2013-09-27 10:38:44 +00002063
Jonathan Peyton45ca5da2015-10-19 19:33:38 +00002064#if OMP_40_ENABLED
Jonathan Peytonb044e4f2016-05-23 18:01:19 +00002065 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq);
Jonathan Peyton45ca5da2015-10-19 19:33:38 +00002066#endif
2067
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002068 // Update the floating point rounding in the team if required.
2069 propagateFPControl(team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002070
Jonathan Peyton30419822017-05-12 18:01:32 +00002071 if (__kmp_tasking_mode != tskm_immediate_exec) {
2072 // Set master's task team to team's task team. Unless this is hot team, it
2073 // should be NULL.
Jonathan Peyton30419822017-05-12 18:01:32 +00002074 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2075 parent_team->t.t_task_team[master_th->th.th_task_state]);
Jonathan Peyton30419822017-05-12 18:01:32 +00002076 KA_TRACE(20, ("__kmp_fork_call: Master T#%d pushing task_team %p / team "
2077 "%p, new task_team %p / team %p\n",
2078 __kmp_gtid_from_thread(master_th),
2079 master_th->th.th_task_team, parent_team,
2080 team->t.t_task_team[master_th->th.th_task_state], team));
Jonathan Peytond3f2b942016-02-09 22:32:41 +00002081
Jonathan Peyton30419822017-05-12 18:01:32 +00002082 if (active_level || master_th->th.th_task_team) {
2083 // Take a memo of master's task_state
2084 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2085 if (master_th->th.th_task_state_top >=
2086 master_th->th.th_task_state_stack_sz) { // increase size
2087 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz;
2088 kmp_uint8 *old_stack, *new_stack;
2089 kmp_uint32 i;
2090 new_stack = (kmp_uint8 *)__kmp_allocate(new_size);
2091 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) {
2092 new_stack[i] = master_th->th.th_task_state_memo_stack[i];
2093 }
2094 for (i = master_th->th.th_task_state_stack_sz; i < new_size;
2095 ++i) { // zero-init rest of stack
2096 new_stack[i] = 0;
2097 }
2098 old_stack = master_th->th.th_task_state_memo_stack;
2099 master_th->th.th_task_state_memo_stack = new_stack;
2100 master_th->th.th_task_state_stack_sz = new_size;
2101 __kmp_free(old_stack);
Andrey Churbanov6d224db2015-02-10 18:37:43 +00002102 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002103 // Store master's task_state on stack
2104 master_th->th
2105 .th_task_state_memo_stack[master_th->th.th_task_state_top] =
2106 master_th->th.th_task_state;
2107 master_th->th.th_task_state_top++;
2108#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton642688b2017-06-01 16:46:36 +00002109 if (team == master_th->th.th_hot_teams[active_level].hot_team) {
2110 // Restore master's nested state if nested hot team
Jonathan Peyton30419822017-05-12 18:01:32 +00002111 master_th->th.th_task_state =
2112 master_th->th
2113 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2114 } else {
2115#endif
2116 master_th->th.th_task_state = 0;
2117#if KMP_NESTED_HOT_TEAMS
2118 }
2119#endif
2120 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002121#if !KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00002122 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) ||
2123 (team == root->r.r_hot_team));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002124#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002125 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002126
Jonathan Peyton30419822017-05-12 18:01:32 +00002127 KA_TRACE(
2128 20,
2129 ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",
2130 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id,
2131 team->t.t_nproc));
2132 KMP_DEBUG_ASSERT(team != root->r.r_hot_team ||
2133 (team->t.t_master_tid == 0 &&
2134 (team->t.t_parent == root->r.r_root_team ||
2135 team->t.t_parent->t.t_serialized)));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002136 KMP_MB();
2137
2138 /* now, setup the arguments */
Jonathan Peyton30419822017-05-12 18:01:32 +00002139 argv = (void **)team->t.t_argv;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002140#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002141 if (ap) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002142#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00002143 for (i = argc - 1; i >= 0; --i) {
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002144// TODO: revert workaround for Intel(R) 64 tracker #96
Andrey Churbanovcbda8682015-01-13 14:43:35 +00002145#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
Jonathan Peyton30419822017-05-12 18:01:32 +00002146 void *new_argv = va_arg(*ap, void *);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002147#else
Jonathan Peyton30419822017-05-12 18:01:32 +00002148 void *new_argv = va_arg(ap, void *);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002149#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002150 KMP_CHECK_UPDATE(*argv, new_argv);
2151 argv++;
2152 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002153#if OMP_40_ENABLED
2154 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00002155 for (i = 0; i < argc; ++i) {
2156 // Get args from parent team for teams construct
2157 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]);
2158 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002159 }
2160#endif /* OMP_40_ENABLED */
2161
2162 /* now actually fork the threads */
Jonathan Peytonb044e4f2016-05-23 18:01:19 +00002163 KMP_CHECK_UPDATE(team->t.t_master_active, master_active);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002164 if (!root->r.r_active) // Only do assignment if it prevents cache ping-pong
Jonathan Peyton30419822017-05-12 18:01:32 +00002165 root->r.r_active = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002166
Jonathan Peyton30419822017-05-12 18:01:32 +00002167 __kmp_fork_team_threads(root, team, master_th, gtid);
2168 __kmp_setup_icv_copy(team, nthreads,
2169 &master_th->th.th_current_task->td_icvs, loc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002170
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002171#if OMPT_SUPPORT
2172 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
2173#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002174
Jonathan Peyton30419822017-05-12 18:01:32 +00002175 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002176
Jim Cownie5e8470a2013-09-27 10:38:44 +00002177#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +00002178 if (team->t.t_active_level == 1 // only report frames at level 1
2179#if OMP_40_ENABLED
Andrey Churbanov51aecb82015-05-06 19:22:36 +00002180 && !master_th->th.th_teams_microtask // not in teams construct
Jonathan Peyton30419822017-05-12 18:01:32 +00002181#endif /* OMP_40_ENABLED */
2182 ) {
Andrey Churbanov51aecb82015-05-06 19:22:36 +00002183#if USE_ITT_NOTIFY
Jonathan Peyton30419822017-05-12 18:01:32 +00002184 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2185 (__kmp_forkjoin_frames_mode == 3 ||
2186 __kmp_forkjoin_frames_mode == 1)) {
2187 kmp_uint64 tmp_time = 0;
2188 if (__itt_get_timestamp_ptr)
2189 tmp_time = __itt_get_timestamp();
2190 // Internal fork - report frame begin
2191 master_th->th.th_frame_time = tmp_time;
2192 if (__kmp_forkjoin_frames_mode == 3)
2193 team->t.t_region_time = tmp_time;
Jonathan Peyton642688b2017-06-01 16:46:36 +00002194 } else
2195// only one notification scheme (either "submit" or "forking/joined", not both)
Andrey Churbanov51aecb82015-05-06 19:22:36 +00002196#endif /* USE_ITT_NOTIFY */
Jonathan Peyton30419822017-05-12 18:01:32 +00002197 if ((__itt_frame_begin_v3_ptr || KMP_ITT_DEBUG) &&
2198 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) {
2199 // Mark start of "parallel" region for VTune.
2200 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
2201 }
Andrey Churbanovf6451d92015-01-16 15:58:03 +00002202 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002203#endif /* USE_ITT_BUILD */
2204
2205 /* now go on and do the work */
Jonathan Peyton30419822017-05-12 18:01:32 +00002206 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002207 KMP_MB();
Jonathan Peyton30419822017-05-12 18:01:32 +00002208 KF_TRACE(10,
2209 ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n",
2210 root, team, master_th, gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002211
2212#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +00002213 if (__itt_stack_caller_create_ptr) {
2214 team->t.t_stack_id =
2215 __kmp_itt_stack_caller_create(); // create new stack stitching id
2216 // before entering fork barrier
Jim Cownie5e8470a2013-09-27 10:38:44 +00002217 }
2218#endif /* USE_ITT_BUILD */
2219
2220#if OMP_40_ENABLED
Jonathan Peyton642688b2017-06-01 16:46:36 +00002221 // AC: skip __kmp_internal_fork at teams construct, let only master
2222 // threads execute
2223 if (ap)
Jim Cownie5e8470a2013-09-27 10:38:44 +00002224#endif /* OMP_40_ENABLED */
2225 {
Jonathan Peyton30419822017-05-12 18:01:32 +00002226 __kmp_internal_fork(loc, gtid, team);
2227 KF_TRACE(10, ("__kmp_internal_fork : after : root=%p, team=%p, "
2228 "master_th=%p, gtid=%d\n",
2229 root, team, master_th, gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002230 }
2231
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002232 if (call_context == fork_context_gnu) {
Jonathan Peyton30419822017-05-12 18:01:32 +00002233 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid));
2234 return TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002235 }
2236
2237 /* Invoke microtask for MASTER thread */
Jonathan Peyton30419822017-05-12 18:01:32 +00002238 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
2239 team->t.t_id, team->t.t_pkfn));
2240 } // END of timer KMP_fork_call block
Jim Cownie5e8470a2013-09-27 10:38:44 +00002241
Jonathan Peyton30419822017-05-12 18:01:32 +00002242 {
2243 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
2244 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
2245 if (!team->t.t_invoke(gtid)) {
2246 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread");
Jim Cownie5e8470a2013-09-27 10:38:44 +00002247 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002248 }
2249 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
2250 team->t.t_id, team->t.t_pkfn));
2251 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00002252
Jonathan Peyton30419822017-05-12 18:01:32 +00002253 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002254
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002255#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00002256 if (ompt_enabled) {
2257 master_th->th.ompt_thread_info.state = ompt_state_overhead;
2258 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002259#endif
2260
Jonathan Peyton30419822017-05-12 18:01:32 +00002261 return TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002262}
2263
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002264#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00002265static inline void __kmp_join_restore_state(kmp_info_t *thread,
2266 kmp_team_t *team) {
2267 // restore state outside the region
2268 thread->th.ompt_thread_info.state =
2269 ((team->t.t_serialized) ? ompt_state_work_serial
2270 : ompt_state_work_parallel);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002271}
2272
Jonathan Peyton30419822017-05-12 18:01:32 +00002273static inline void __kmp_join_ompt(kmp_info_t *thread, kmp_team_t *team,
2274 ompt_parallel_id_t parallel_id,
2275 fork_context_e fork_context) {
2276 ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
2277 if (ompt_callbacks.ompt_callback(ompt_event_parallel_end)) {
2278 ompt_callbacks.ompt_callback(ompt_event_parallel_end)(
2279 parallel_id, task_info->task_id, OMPT_INVOKER(fork_context));
2280 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002281
Jonathan Peyton30419822017-05-12 18:01:32 +00002282 task_info->frame.reenter_runtime_frame = NULL;
2283 __kmp_join_restore_state(thread, team);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002284}
2285#endif
2286
Jonathan Peyton30419822017-05-12 18:01:32 +00002287void __kmp_join_call(ident_t *loc, int gtid
Jonathan Peytonf89fbbb2015-08-31 18:15:00 +00002288#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00002289 ,
2290 enum fork_context_e fork_context
Jonathan Peytonf89fbbb2015-08-31 18:15:00 +00002291#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002292#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002293 ,
2294 int exit_teams
Jim Cownie5e8470a2013-09-27 10:38:44 +00002295#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00002296 ) {
2297 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call);
2298 kmp_team_t *team;
2299 kmp_team_t *parent_team;
2300 kmp_info_t *master_th;
2301 kmp_root_t *root;
2302 int master_active;
2303 int i;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002304
Jonathan Peyton30419822017-05-12 18:01:32 +00002305 KA_TRACE(20, ("__kmp_join_call: enter T#%d\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002306
Jonathan Peyton30419822017-05-12 18:01:32 +00002307 /* setup current data */
2308 master_th = __kmp_threads[gtid];
2309 root = master_th->th.th_root;
2310 team = master_th->th.th_team;
2311 parent_team = team->t.t_parent;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002312
Jonathan Peyton30419822017-05-12 18:01:32 +00002313 master_th->th.th_ident = loc;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002314
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002315#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00002316 if (ompt_enabled) {
2317 master_th->th.ompt_thread_info.state = ompt_state_overhead;
2318 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002319#endif
2320
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002321#if KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00002322 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) {
2323 KA_TRACE(20, ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
2324 "th_task_team = %p\n",
2325 __kmp_gtid_from_thread(master_th), team,
2326 team->t.t_task_team[master_th->th.th_task_state],
2327 master_th->th.th_task_team));
2328 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2329 team->t.t_task_team[master_th->th.th_task_state]);
2330 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002331#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002332
Jonathan Peyton30419822017-05-12 18:01:32 +00002333 if (team->t.t_serialized) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002334#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002335 if (master_th->th.th_teams_microtask) {
2336 // We are in teams construct
2337 int level = team->t.t_level;
2338 int tlevel = master_th->th.th_teams_level;
2339 if (level == tlevel) {
2340 // AC: we haven't incremented it earlier at start of teams construct,
2341 // so do it here - at the end of teams construct
2342 team->t.t_level++;
2343 } else if (level == tlevel + 1) {
2344 // AC: we are exiting parallel inside teams, need to increment
2345 // serialization in order to restore it in the next call to
2346 // __kmpc_end_serialized_parallel
2347 team->t.t_serialized++;
2348 }
Andrey Churbanov6d224db2015-02-10 18:37:43 +00002349 }
Jonathan Peyton441f3372015-09-21 17:24:46 +00002350#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00002351 __kmpc_end_serialized_parallel(loc, gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002352
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002353#if OMPT_SUPPORT
Jonathan Peytonb68a85d2015-09-21 18:11:22 +00002354 if (ompt_enabled) {
Jonathan Peyton30419822017-05-12 18:01:32 +00002355 __kmp_join_restore_state(master_th, parent_team);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002356 }
2357#endif
2358
Jonathan Peyton30419822017-05-12 18:01:32 +00002359 return;
2360 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002361
Jonathan Peyton30419822017-05-12 18:01:32 +00002362 master_active = team->t.t_master_active;
2363
2364#if OMP_40_ENABLED
2365 if (!exit_teams)
2366#endif /* OMP_40_ENABLED */
2367 {
2368 // AC: No barrier for internal teams at exit from teams construct.
2369 // But there is barrier for external team (league).
2370 __kmp_internal_join(loc, gtid, team);
2371 }
2372#if OMP_40_ENABLED
2373 else {
2374 master_th->th.th_task_state =
2375 0; // AC: no tasking in teams (out of any parallel)
2376 }
2377#endif /* OMP_40_ENABLED */
2378
2379 KMP_MB();
2380
2381#if OMPT_SUPPORT
2382 ompt_parallel_id_t parallel_id = team->t.ompt_team_info.parallel_id;
2383#endif
2384
2385#if USE_ITT_BUILD
2386 if (__itt_stack_caller_create_ptr) {
2387 __kmp_itt_stack_caller_destroy(
2388 (__itt_caller)team->t
2389 .t_stack_id); // destroy the stack stitching id after join barrier
2390 }
2391
2392 // Mark end of "parallel" region for VTune.
2393 if (team->t.t_active_level == 1
2394#if OMP_40_ENABLED
2395 && !master_th->th.th_teams_microtask /* not in teams construct */
2396#endif /* OMP_40_ENABLED */
2397 ) {
2398 master_th->th.th_ident = loc;
2399 // only one notification scheme (either "submit" or "forking/joined", not
2400 // both)
2401 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2402 __kmp_forkjoin_frames_mode == 3)
2403 __kmp_itt_frame_submit(gtid, team->t.t_region_time,
2404 master_th->th.th_frame_time, 0, loc,
2405 master_th->th.th_team_nproc, 1);
2406 else if ((__itt_frame_end_v3_ptr || KMP_ITT_DEBUG) &&
2407 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames)
2408 __kmp_itt_region_joined(gtid);
2409 } // active_level == 1
2410#endif /* USE_ITT_BUILD */
2411
2412#if OMP_40_ENABLED
2413 if (master_th->th.th_teams_microtask && !exit_teams &&
2414 team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
2415 team->t.t_level == master_th->th.th_teams_level + 1) {
2416 // AC: We need to leave the team structure intact at the end of parallel
2417 // inside the teams construct, so that at the next parallel same (hot) team
2418 // works, only adjust nesting levels
2419
2420 /* Decrement our nested depth level */
2421 team->t.t_level--;
2422 team->t.t_active_level--;
2423 KMP_TEST_THEN_DEC32((kmp_int32 *)&root->r.r_in_parallel);
2424
2425 /* Restore number of threads in the team if needed */
2426 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) {
2427 int old_num = master_th->th.th_team_nproc;
2428 int new_num = master_th->th.th_teams_size.nth;
2429 kmp_info_t **other_threads = team->t.t_threads;
2430 team->t.t_nproc = new_num;
2431 for (i = 0; i < old_num; ++i) {
2432 other_threads[i]->th.th_team_nproc = new_num;
2433 }
2434 // Adjust states of non-used threads of the team
2435 for (i = old_num; i < new_num; ++i) {
2436 // Re-initialize thread's barrier data.
2437 int b;
2438 kmp_balign_t *balign = other_threads[i]->th.th_bar;
2439 for (b = 0; b < bs_last_barrier; ++b) {
2440 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
2441 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
2442#if USE_DEBUGGER
2443 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
2444#endif
2445 }
2446 if (__kmp_tasking_mode != tskm_immediate_exec) {
2447 // Synchronize thread's task state
2448 other_threads[i]->th.th_task_state = master_th->th.th_task_state;
2449 }
2450 }
2451 }
2452
2453#if OMPT_SUPPORT
2454 if (ompt_enabled) {
2455 __kmp_join_ompt(master_th, parent_team, parallel_id, fork_context);
2456 }
2457#endif
2458
2459 return;
2460 }
2461#endif /* OMP_40_ENABLED */
2462
2463 /* do cleanup and restore the parent team */
2464 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid;
2465 master_th->th.th_local.this_construct = team->t.t_master_this_cons;
2466
2467 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid];
2468
2469 /* jc: The following lock has instructions with REL and ACQ semantics,
2470 separating the parallel user code called in this parallel region
2471 from the serial user code called after this function returns. */
2472 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2473
2474#if OMP_40_ENABLED
2475 if (!master_th->th.th_teams_microtask ||
2476 team->t.t_level > master_th->th.th_teams_level)
2477#endif /* OMP_40_ENABLED */
2478 {
2479 /* Decrement our nested depth level */
2480 KMP_TEST_THEN_DEC32((kmp_int32 *)&root->r.r_in_parallel);
2481 }
2482 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0);
2483
2484#if OMPT_SUPPORT && OMPT_TRACE
2485 if (ompt_enabled) {
2486 ompt_task_info_t *task_info = __ompt_get_taskinfo(0);
2487 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
2488 ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
2489 parallel_id, task_info->task_id);
2490 }
2491 task_info->frame.exit_runtime_frame = NULL;
2492 task_info->task_id = 0;
2493 }
2494#endif
2495
2496 KF_TRACE(10, ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0,
2497 master_th, team));
2498 __kmp_pop_current_task_from_thread(master_th);
2499
2500#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
2501 // Restore master thread's partition.
2502 master_th->th.th_first_place = team->t.t_first_place;
2503 master_th->th.th_last_place = team->t.t_last_place;
2504#endif /* OMP_40_ENABLED */
2505
2506 updateHWFPControl(team);
2507
2508 if (root->r.r_active != master_active)
2509 root->r.r_active = master_active;
2510
2511 __kmp_free_team(root, team USE_NESTED_HOT_ARG(
2512 master_th)); // this will free worker threads
2513
2514 /* this race was fun to find. make sure the following is in the critical
2515 region otherwise assertions may fail occasionally since the old team may be
2516 reallocated and the hierarchy appears inconsistent. it is actually safe to
2517 run and won't cause any bugs, but will cause those assertion failures. it's
2518 only one deref&assign so might as well put this in the critical region */
2519 master_th->th.th_team = parent_team;
2520 master_th->th.th_team_nproc = parent_team->t.t_nproc;
2521 master_th->th.th_team_master = parent_team->t.t_threads[0];
2522 master_th->th.th_team_serialized = parent_team->t.t_serialized;
2523
2524 /* restore serialized team, if need be */
2525 if (parent_team->t.t_serialized &&
2526 parent_team != master_th->th.th_serial_team &&
2527 parent_team != root->r.r_root_team) {
2528 __kmp_free_team(root,
2529 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL));
2530 master_th->th.th_serial_team = parent_team;
2531 }
2532
2533 if (__kmp_tasking_mode != tskm_immediate_exec) {
2534 if (master_th->th.th_task_state_top >
2535 0) { // Restore task state from memo stack
2536 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2537 // Remember master's state if we re-use this nested hot team
2538 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] =
2539 master_th->th.th_task_state;
2540 --master_th->th.th_task_state_top; // pop
2541 // Now restore state at this level
2542 master_th->th.th_task_state =
2543 master_th->th
2544 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2545 }
2546 // Copy the task team from the parent team to the master thread
2547 master_th->th.th_task_team =
2548 parent_team->t.t_task_team[master_th->th.th_task_state];
2549 KA_TRACE(20,
2550 ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n",
2551 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,
2552 parent_team));
2553 }
2554
2555 // TODO: GEH - cannot do this assertion because root thread not set up as
2556 // executing
2557 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 );
2558 master_th->th.th_current_task->td_flags.executing = 1;
2559
2560 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2561
2562#if OMPT_SUPPORT
2563 if (ompt_enabled) {
2564 __kmp_join_ompt(master_th, parent_team, parallel_id, fork_context);
2565 }
2566#endif
2567
2568 KMP_MB();
2569 KA_TRACE(20, ("__kmp_join_call: exit T#%d\n", gtid));
2570}
Jim Cownie5e8470a2013-09-27 10:38:44 +00002571
2572/* Check whether we should push an internal control record onto the
2573 serial team stack. If so, do it. */
Jonathan Peyton30419822017-05-12 18:01:32 +00002574void __kmp_save_internal_controls(kmp_info_t *thread) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002575
Jonathan Peyton30419822017-05-12 18:01:32 +00002576 if (thread->th.th_team != thread->th.th_serial_team) {
2577 return;
2578 }
2579 if (thread->th.th_team->t.t_serialized > 1) {
2580 int push = 0;
2581
2582 if (thread->th.th_team->t.t_control_stack_top == NULL) {
2583 push = 1;
2584 } else {
2585 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level !=
2586 thread->th.th_team->t.t_serialized) {
2587 push = 1;
2588 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002589 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002590 if (push) { /* push a record on the serial team's stack */
2591 kmp_internal_control_t *control =
2592 (kmp_internal_control_t *)__kmp_allocate(
2593 sizeof(kmp_internal_control_t));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002594
Jonathan Peyton30419822017-05-12 18:01:32 +00002595 copy_icvs(control, &thread->th.th_current_task->td_icvs);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002596
Jonathan Peyton30419822017-05-12 18:01:32 +00002597 control->serial_nesting_level = thread->th.th_team->t.t_serialized;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002598
Jonathan Peyton30419822017-05-12 18:01:32 +00002599 control->next = thread->th.th_team->t.t_control_stack_top;
2600 thread->th.th_team->t.t_control_stack_top = control;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002601 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002602 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002603}
2604
2605/* Changes set_nproc */
Jonathan Peyton30419822017-05-12 18:01:32 +00002606void __kmp_set_num_threads(int new_nth, int gtid) {
2607 kmp_info_t *thread;
2608 kmp_root_t *root;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002609
Jonathan Peyton30419822017-05-12 18:01:32 +00002610 KF_TRACE(10, ("__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth));
2611 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002612
Jonathan Peyton30419822017-05-12 18:01:32 +00002613 if (new_nth < 1)
2614 new_nth = 1;
2615 else if (new_nth > __kmp_max_nth)
2616 new_nth = __kmp_max_nth;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002617
Jonathan Peyton30419822017-05-12 18:01:32 +00002618 KMP_COUNT_VALUE(OMP_set_numthreads, new_nth);
2619 thread = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00002620
Jonathan Peyton30419822017-05-12 18:01:32 +00002621 __kmp_save_internal_controls(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002622
Jonathan Peyton30419822017-05-12 18:01:32 +00002623 set__nproc(thread, new_nth);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002624
Jonathan Peyton30419822017-05-12 18:01:32 +00002625 // If this omp_set_num_threads() call will cause the hot team size to be
2626 // reduced (in the absence of a num_threads clause), then reduce it now,
2627 // rather than waiting for the next parallel region.
2628 root = thread->th.th_root;
2629 if (__kmp_init_parallel && (!root->r.r_active) &&
2630 (root->r.r_hot_team->t.t_nproc > new_nth)
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002631#if KMP_NESTED_HOT_TEAMS
2632 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode
2633#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002634 ) {
2635 kmp_team_t *hot_team = root->r.r_hot_team;
2636 int f;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002637
Jonathan Peyton30419822017-05-12 18:01:32 +00002638 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002639
Jonathan Peyton30419822017-05-12 18:01:32 +00002640 // Release the extra threads we don't need any more.
2641 for (f = new_nth; f < hot_team->t.t_nproc; f++) {
2642 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2643 if (__kmp_tasking_mode != tskm_immediate_exec) {
2644 // When decreasing team size, threads no longer in the team should unref
2645 // task team.
2646 hot_team->t.t_threads[f]->th.th_task_team = NULL;
2647 }
2648 __kmp_free_thread(hot_team->t.t_threads[f]);
2649 hot_team->t.t_threads[f] = NULL;
2650 }
2651 hot_team->t.t_nproc = new_nth;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002652#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00002653 if (thread->th.th_hot_teams) {
2654 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team);
2655 thread->th.th_hot_teams[0].hot_team_nth = new_nth;
2656 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002657#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002658
Jonathan Peyton30419822017-05-12 18:01:32 +00002659 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002660
Jonathan Peyton30419822017-05-12 18:01:32 +00002661 // Update the t_nproc field in the threads that are still active.
2662 for (f = 0; f < new_nth; f++) {
2663 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2664 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002665 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002666 // Special flag in case omp_set_num_threads() call
2667 hot_team->t.t_size_changed = -1;
2668 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002669}
2670
Jim Cownie5e8470a2013-09-27 10:38:44 +00002671/* Changes max_active_levels */
Jonathan Peyton30419822017-05-12 18:01:32 +00002672void __kmp_set_max_active_levels(int gtid, int max_active_levels) {
2673 kmp_info_t *thread;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002674
Jonathan Peyton30419822017-05-12 18:01:32 +00002675 KF_TRACE(10, ("__kmp_set_max_active_levels: new max_active_levels for thread "
2676 "%d = (%d)\n",
2677 gtid, max_active_levels));
2678 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002679
Jonathan Peyton30419822017-05-12 18:01:32 +00002680 // validate max_active_levels
2681 if (max_active_levels < 0) {
2682 KMP_WARNING(ActiveLevelsNegative, max_active_levels);
2683 // We ignore this call if the user has specified a negative value.
2684 // The current setting won't be changed. The last valid setting will be
2685 // used. A warning will be issued (if warnings are allowed as controlled by
2686 // the KMP_WARNINGS env var).
2687 KF_TRACE(10, ("__kmp_set_max_active_levels: the call is ignored: new "
2688 "max_active_levels for thread %d = (%d)\n",
2689 gtid, max_active_levels));
2690 return;
2691 }
2692 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT) {
2693 // it's OK, the max_active_levels is within the valid range: [ 0;
2694 // KMP_MAX_ACTIVE_LEVELS_LIMIT ]
2695 // We allow a zero value. (implementation defined behavior)
2696 } else {
2697 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels,
2698 KMP_MAX_ACTIVE_LEVELS_LIMIT);
2699 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT;
2700 // Current upper limit is MAX_INT. (implementation defined behavior)
2701 // If the input exceeds the upper limit, we correct the input to be the
2702 // upper limit. (implementation defined behavior)
2703 // Actually, the flow should never get here until we use MAX_INT limit.
2704 }
2705 KF_TRACE(10, ("__kmp_set_max_active_levels: after validation: new "
2706 "max_active_levels for thread %d = (%d)\n",
2707 gtid, max_active_levels));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002708
Jonathan Peyton30419822017-05-12 18:01:32 +00002709 thread = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00002710
Jonathan Peyton30419822017-05-12 18:01:32 +00002711 __kmp_save_internal_controls(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002712
Jonathan Peyton30419822017-05-12 18:01:32 +00002713 set__max_active_levels(thread, max_active_levels);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002714}
2715
2716/* Gets max_active_levels */
Jonathan Peyton30419822017-05-12 18:01:32 +00002717int __kmp_get_max_active_levels(int gtid) {
2718 kmp_info_t *thread;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002719
Jonathan Peyton30419822017-05-12 18:01:32 +00002720 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d\n", gtid));
2721 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002722
Jonathan Peyton30419822017-05-12 18:01:32 +00002723 thread = __kmp_threads[gtid];
2724 KMP_DEBUG_ASSERT(thread->th.th_current_task);
2725 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
2726 "curtask_maxaclevel=%d\n",
2727 gtid, thread->th.th_current_task,
2728 thread->th.th_current_task->td_icvs.max_active_levels));
2729 return thread->th.th_current_task->td_icvs.max_active_levels;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002730}
2731
2732/* Changes def_sched_var ICV values (run-time schedule kind and chunk) */
Jonathan Peyton30419822017-05-12 18:01:32 +00002733void __kmp_set_schedule(int gtid, kmp_sched_t kind, int chunk) {
2734 kmp_info_t *thread;
2735 // kmp_team_t *team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002736
Jonathan Peyton30419822017-05-12 18:01:32 +00002737 KF_TRACE(10, ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n",
2738 gtid, (int)kind, chunk));
2739 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002740
Jonathan Peyton30419822017-05-12 18:01:32 +00002741 // Check if the kind parameter is valid, correct if needed.
2742 // Valid parameters should fit in one of two intervals - standard or extended:
2743 // <lower>, <valid>, <upper_std>, <lower_ext>, <valid>, <upper>
2744 // 2008-01-25: 0, 1 - 4, 5, 100, 101 - 102, 103
2745 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper ||
2746 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) {
2747 // TODO: Hint needs attention in case we change the default schedule.
2748 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind),
2749 KMP_HNT(DefaultScheduleKindUsed, "static, no chunk"),
2750 __kmp_msg_null);
2751 kind = kmp_sched_default;
2752 chunk = 0; // ignore chunk value in case of bad kind
2753 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002754
Jonathan Peyton30419822017-05-12 18:01:32 +00002755 thread = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00002756
Jonathan Peyton30419822017-05-12 18:01:32 +00002757 __kmp_save_internal_controls(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002758
Jonathan Peyton30419822017-05-12 18:01:32 +00002759 if (kind < kmp_sched_upper_std) {
2760 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK) {
2761 // differ static chunked vs. unchunked: chunk should be invalid to
2762 // indicate unchunked schedule (which is the default)
2763 thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002764 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00002765 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2766 __kmp_sch_map[kind - kmp_sched_lower - 1];
Jim Cownie5e8470a2013-09-27 10:38:44 +00002767 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002768 } else {
2769 // __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2770 // kmp_sched_lower - 2 ];
2771 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2772 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2773 kmp_sched_lower - 2];
2774 }
Andrey Churbanovd454c732017-06-05 17:17:33 +00002775 if (kind == kmp_sched_auto || chunk < 1) {
Jonathan Peyton30419822017-05-12 18:01:32 +00002776 // ignore parameter chunk for schedule auto
2777 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK;
2778 } else {
2779 thread->th.th_current_task->td_icvs.sched.chunk = chunk;
2780 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002781}
2782
2783/* Gets def_sched_var ICV values */
Jonathan Peyton30419822017-05-12 18:01:32 +00002784void __kmp_get_schedule(int gtid, kmp_sched_t *kind, int *chunk) {
2785 kmp_info_t *thread;
2786 enum sched_type th_type;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002787
Jonathan Peyton30419822017-05-12 18:01:32 +00002788 KF_TRACE(10, ("__kmp_get_schedule: thread %d\n", gtid));
2789 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002790
Jonathan Peyton30419822017-05-12 18:01:32 +00002791 thread = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00002792
Jonathan Peyton30419822017-05-12 18:01:32 +00002793 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002794
Jonathan Peyton30419822017-05-12 18:01:32 +00002795 switch (th_type) {
2796 case kmp_sch_static:
2797 case kmp_sch_static_greedy:
2798 case kmp_sch_static_balanced:
2799 *kind = kmp_sched_static;
2800 *chunk = 0; // chunk was not set, try to show this fact via zero value
2801 return;
2802 case kmp_sch_static_chunked:
2803 *kind = kmp_sched_static;
2804 break;
2805 case kmp_sch_dynamic_chunked:
2806 *kind = kmp_sched_dynamic;
2807 break;
2808 case kmp_sch_guided_chunked:
2809 case kmp_sch_guided_iterative_chunked:
2810 case kmp_sch_guided_analytical_chunked:
2811 *kind = kmp_sched_guided;
2812 break;
2813 case kmp_sch_auto:
2814 *kind = kmp_sched_auto;
2815 break;
2816 case kmp_sch_trapezoidal:
2817 *kind = kmp_sched_trapezoidal;
2818 break;
Jonathan Peytona1234cf2016-10-07 18:01:35 +00002819#if KMP_STATIC_STEAL_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002820 case kmp_sch_static_steal:
2821 *kind = kmp_sched_static_steal;
2822 break;
Jonathan Peytona1234cf2016-10-07 18:01:35 +00002823#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002824 default:
2825 KMP_FATAL(UnknownSchedulingType, th_type);
2826 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002827
Jonathan Peyton30419822017-05-12 18:01:32 +00002828 *chunk = thread->th.th_current_task->td_icvs.sched.chunk;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002829}
2830
Jonathan Peyton30419822017-05-12 18:01:32 +00002831int __kmp_get_ancestor_thread_num(int gtid, int level) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002832
Jonathan Peyton30419822017-05-12 18:01:32 +00002833 int ii, dd;
2834 kmp_team_t *team;
2835 kmp_info_t *thr;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002836
Jonathan Peyton30419822017-05-12 18:01:32 +00002837 KF_TRACE(10, ("__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level));
2838 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002839
Jonathan Peyton30419822017-05-12 18:01:32 +00002840 // validate level
2841 if (level == 0)
2842 return 0;
2843 if (level < 0)
2844 return -1;
2845 thr = __kmp_threads[gtid];
2846 team = thr->th.th_team;
2847 ii = team->t.t_level;
2848 if (level > ii)
2849 return -1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002850
2851#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002852 if (thr->th.th_teams_microtask) {
2853 // AC: we are in teams region where multiple nested teams have same level
2854 int tlevel = thr->th.th_teams_level; // the level of the teams construct
2855 if (level <=
2856 tlevel) { // otherwise usual algorithm works (will not touch the teams)
2857 KMP_DEBUG_ASSERT(ii >= tlevel);
2858 // AC: As we need to pass by the teams league, we need to artificially
2859 // increase ii
2860 if (ii == tlevel) {
2861 ii += 2; // three teams have same level
2862 } else {
2863 ii++; // two teams have same level
2864 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002865 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002866 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002867#endif
2868
Jonathan Peyton30419822017-05-12 18:01:32 +00002869 if (ii == level)
2870 return __kmp_tid_from_gtid(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002871
Jonathan Peyton30419822017-05-12 18:01:32 +00002872 dd = team->t.t_serialized;
2873 level++;
2874 while (ii > level) {
2875 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002876 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002877 if ((team->t.t_serialized) && (!dd)) {
2878 team = team->t.t_parent;
2879 continue;
2880 }
2881 if (ii > level) {
2882 team = team->t.t_parent;
2883 dd = team->t.t_serialized;
2884 ii--;
2885 }
2886 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002887
Jonathan Peyton30419822017-05-12 18:01:32 +00002888 return (dd > 1) ? (0) : (team->t.t_master_tid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002889}
2890
Jonathan Peyton30419822017-05-12 18:01:32 +00002891int __kmp_get_team_size(int gtid, int level) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002892
Jonathan Peyton30419822017-05-12 18:01:32 +00002893 int ii, dd;
2894 kmp_team_t *team;
2895 kmp_info_t *thr;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002896
Jonathan Peyton30419822017-05-12 18:01:32 +00002897 KF_TRACE(10, ("__kmp_get_team_size: thread %d %d\n", gtid, level));
2898 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002899
Jonathan Peyton30419822017-05-12 18:01:32 +00002900 // validate level
2901 if (level == 0)
2902 return 1;
2903 if (level < 0)
2904 return -1;
2905 thr = __kmp_threads[gtid];
2906 team = thr->th.th_team;
2907 ii = team->t.t_level;
2908 if (level > ii)
2909 return -1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002910
2911#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002912 if (thr->th.th_teams_microtask) {
2913 // AC: we are in teams region where multiple nested teams have same level
2914 int tlevel = thr->th.th_teams_level; // the level of the teams construct
2915 if (level <=
2916 tlevel) { // otherwise usual algorithm works (will not touch the teams)
2917 KMP_DEBUG_ASSERT(ii >= tlevel);
2918 // AC: As we need to pass by the teams league, we need to artificially
2919 // increase ii
2920 if (ii == tlevel) {
2921 ii += 2; // three teams have same level
2922 } else {
2923 ii++; // two teams have same level
2924 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002925 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002926 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002927#endif
2928
Jonathan Peyton30419822017-05-12 18:01:32 +00002929 while (ii > level) {
2930 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002931 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002932 if (team->t.t_serialized && (!dd)) {
2933 team = team->t.t_parent;
2934 continue;
2935 }
2936 if (ii > level) {
2937 team = team->t.t_parent;
2938 ii--;
2939 }
2940 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002941
Jonathan Peyton30419822017-05-12 18:01:32 +00002942 return team->t.t_nproc;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002943}
2944
Jonathan Peyton30419822017-05-12 18:01:32 +00002945kmp_r_sched_t __kmp_get_schedule_global() {
2946 // This routine created because pairs (__kmp_sched, __kmp_chunk) and
2947 // (__kmp_static, __kmp_guided) may be changed by kmp_set_defaults
2948 // independently. So one can get the updated schedule here.
Jim Cownie5e8470a2013-09-27 10:38:44 +00002949
Jonathan Peyton30419822017-05-12 18:01:32 +00002950 kmp_r_sched_t r_sched;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002951
Jonathan Peyton30419822017-05-12 18:01:32 +00002952 // create schedule from 4 globals: __kmp_sched, __kmp_chunk, __kmp_static,
2953 // __kmp_guided. __kmp_sched should keep original value, so that user can set
2954 // KMP_SCHEDULE multiple times, and thus have different run-time schedules in
2955 // different roots (even in OMP 2.5)
2956 if (__kmp_sched == kmp_sch_static) {
2957 r_sched.r_sched_type = __kmp_static; // replace STATIC with more detailed
2958 // schedule (balanced or greedy)
2959 } else if (__kmp_sched == kmp_sch_guided_chunked) {
2960 r_sched.r_sched_type = __kmp_guided; // replace GUIDED with more detailed
2961 // schedule (iterative or analytical)
2962 } else {
2963 r_sched.r_sched_type =
2964 __kmp_sched; // (STATIC_CHUNKED), or (DYNAMIC_CHUNKED), or other
2965 }
2966
2967 if (__kmp_chunk < KMP_DEFAULT_CHUNK) { // __kmp_chunk may be wrong here (if it
2968 // was not ever set)
2969 r_sched.chunk = KMP_DEFAULT_CHUNK;
2970 } else {
2971 r_sched.chunk = __kmp_chunk;
2972 }
2973
2974 return r_sched;
2975}
2976
2977/* Allocate (realloc == FALSE) * or reallocate (realloc == TRUE)
2978 at least argc number of *t_argv entries for the requested team. */
2979static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, int realloc) {
2980
2981 KMP_DEBUG_ASSERT(team);
2982 if (!realloc || argc > team->t.t_max_argc) {
2983
2984 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "
2985 "current entries=%d\n",
2986 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0));
2987 /* if previously allocated heap space for args, free them */
2988 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0])
2989 __kmp_free((void *)team->t.t_argv);
2990
2991 if (argc <= KMP_INLINE_ARGV_ENTRIES) {
2992 /* use unused space in the cache line for arguments */
2993 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES;
2994 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: inline allocate %d "
2995 "argv entries\n",
2996 team->t.t_id, team->t.t_max_argc));
2997 team->t.t_argv = &team->t.t_inline_argv[0];
2998 if (__kmp_storage_map) {
2999 __kmp_print_storage_map_gtid(
3000 -1, &team->t.t_inline_argv[0],
3001 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES],
3002 (sizeof(void *) * KMP_INLINE_ARGV_ENTRIES), "team_%d.t_inline_argv",
3003 team->t.t_id);
3004 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003005 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00003006 /* allocate space for arguments in the heap */
3007 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1))
3008 ? KMP_MIN_MALLOC_ARGV_ENTRIES
3009 : 2 * argc;
3010 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
3011 "argv entries\n",
3012 team->t.t_id, team->t.t_max_argc));
3013 team->t.t_argv =
3014 (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc);
3015 if (__kmp_storage_map) {
3016 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0],
3017 &team->t.t_argv[team->t.t_max_argc],
3018 sizeof(void *) * team->t.t_max_argc,
3019 "team_%d.t_argv", team->t.t_id);
3020 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003021 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003022 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003023}
3024
Jonathan Peyton30419822017-05-12 18:01:32 +00003025static void __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth) {
3026 int i;
3027 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2;
3028 team->t.t_threads =
3029 (kmp_info_t **)__kmp_allocate(sizeof(kmp_info_t *) * max_nth);
3030 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate(
3031 sizeof(dispatch_shared_info_t) * num_disp_buff);
3032 team->t.t_dispatch =
3033 (kmp_disp_t *)__kmp_allocate(sizeof(kmp_disp_t) * max_nth);
3034 team->t.t_implicit_task_taskdata =
3035 (kmp_taskdata_t *)__kmp_allocate(sizeof(kmp_taskdata_t) * max_nth);
3036 team->t.t_max_nproc = max_nth;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003037
Jonathan Peyton30419822017-05-12 18:01:32 +00003038 /* setup dispatch buffers */
3039 for (i = 0; i < num_disp_buff; ++i) {
3040 team->t.t_disp_buffer[i].buffer_index = i;
Jonathan Peytondf6818b2016-06-14 17:57:47 +00003041#if OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003042 team->t.t_disp_buffer[i].doacross_buf_idx = i;
Jonathan Peyton71909c52016-03-02 22:42:06 +00003043#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003044 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003045}
3046
Jonathan Peyton30419822017-05-12 18:01:32 +00003047static void __kmp_free_team_arrays(kmp_team_t *team) {
3048 /* Note: this does not free the threads in t_threads (__kmp_free_threads) */
3049 int i;
3050 for (i = 0; i < team->t.t_max_nproc; ++i) {
3051 if (team->t.t_dispatch[i].th_disp_buffer != NULL) {
3052 __kmp_free(team->t.t_dispatch[i].th_disp_buffer);
3053 team->t.t_dispatch[i].th_disp_buffer = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003054 }
3055 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003056 __kmp_free(team->t.t_threads);
3057 __kmp_free(team->t.t_disp_buffer);
3058 __kmp_free(team->t.t_dispatch);
3059 __kmp_free(team->t.t_implicit_task_taskdata);
3060 team->t.t_threads = NULL;
3061 team->t.t_disp_buffer = NULL;
3062 team->t.t_dispatch = NULL;
3063 team->t.t_implicit_task_taskdata = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003064}
3065
Jonathan Peyton30419822017-05-12 18:01:32 +00003066static void __kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) {
3067 kmp_info_t **oldThreads = team->t.t_threads;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003068
Jonathan Peyton30419822017-05-12 18:01:32 +00003069 __kmp_free(team->t.t_disp_buffer);
3070 __kmp_free(team->t.t_dispatch);
3071 __kmp_free(team->t.t_implicit_task_taskdata);
3072 __kmp_allocate_team_arrays(team, max_nth);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003073
Jonathan Peyton30419822017-05-12 18:01:32 +00003074 KMP_MEMCPY(team->t.t_threads, oldThreads,
3075 team->t.t_nproc * sizeof(kmp_info_t *));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003076
Jonathan Peyton30419822017-05-12 18:01:32 +00003077 __kmp_free(oldThreads);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003078}
3079
Jonathan Peyton30419822017-05-12 18:01:32 +00003080static kmp_internal_control_t __kmp_get_global_icvs(void) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003081
Jonathan Peyton30419822017-05-12 18:01:32 +00003082 kmp_r_sched_t r_sched =
3083 __kmp_get_schedule_global(); // get current state of scheduling globals
Jim Cownie5e8470a2013-09-27 10:38:44 +00003084
3085#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003086 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003087#endif /* OMP_40_ENABLED */
3088
Jonathan Peyton30419822017-05-12 18:01:32 +00003089 kmp_internal_control_t g_icvs = {
3090 0, // int serial_nesting_level; //corresponds to value of th_team_serialized
3091 (kmp_int8)__kmp_dflt_nested, // int nested; //internal control
3092 // for nested parallelism (per thread)
3093 (kmp_int8)__kmp_global.g.g_dynamic, // internal control for dynamic
3094 // adjustment of threads (per thread)
3095 (kmp_int8)__kmp_env_blocktime, // int bt_set; //internal control for
3096 // whether blocktime is explicitly set
3097 __kmp_dflt_blocktime, // int blocktime; //internal control for blocktime
Jonathan Peytone1c7c132016-10-07 18:12:19 +00003098#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00003099 __kmp_bt_intervals, // int bt_intervals; //internal control for blocktime
3100// intervals
Jonathan Peytone1c7c132016-10-07 18:12:19 +00003101#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003102 __kmp_dflt_team_nth, // int nproc; //internal control for # of threads for
3103 // next parallel region (per thread)
3104 // (use a max ub on value if __kmp_parallel_initialize not called yet)
3105 __kmp_dflt_max_active_levels, // int max_active_levels; //internal control
3106 // for max_active_levels
3107 r_sched, // kmp_r_sched_t sched; //internal control for runtime schedule
3108// {sched,chunk} pair
Jim Cownie5e8470a2013-09-27 10:38:44 +00003109#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003110 __kmp_nested_proc_bind.bind_types[0],
3111 __kmp_default_device,
Jim Cownie5e8470a2013-09-27 10:38:44 +00003112#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00003113 NULL // struct kmp_internal_control *next;
3114 };
Jim Cownie5e8470a2013-09-27 10:38:44 +00003115
Jonathan Peyton30419822017-05-12 18:01:32 +00003116 return g_icvs;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003117}
3118
Jonathan Peyton30419822017-05-12 18:01:32 +00003119static kmp_internal_control_t __kmp_get_x_global_icvs(const kmp_team_t *team) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003120
Jonathan Peyton30419822017-05-12 18:01:32 +00003121 kmp_internal_control_t gx_icvs;
3122 gx_icvs.serial_nesting_level =
3123 0; // probably =team->t.t_serial like in save_inter_controls
3124 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs);
3125 gx_icvs.next = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003126
Jonathan Peyton30419822017-05-12 18:01:32 +00003127 return gx_icvs;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003128}
3129
Jonathan Peyton30419822017-05-12 18:01:32 +00003130static void __kmp_initialize_root(kmp_root_t *root) {
3131 int f;
3132 kmp_team_t *root_team;
3133 kmp_team_t *hot_team;
3134 int hot_team_max_nth;
3135 kmp_r_sched_t r_sched =
3136 __kmp_get_schedule_global(); // get current state of scheduling globals
3137 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3138 KMP_DEBUG_ASSERT(root);
3139 KMP_ASSERT(!root->r.r_begin);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003140
Jonathan Peyton30419822017-05-12 18:01:32 +00003141 /* setup the root state structure */
3142 __kmp_init_lock(&root->r.r_begin_lock);
3143 root->r.r_begin = FALSE;
3144 root->r.r_active = FALSE;
3145 root->r.r_in_parallel = 0;
3146 root->r.r_blocktime = __kmp_dflt_blocktime;
3147 root->r.r_nested = __kmp_dflt_nested;
Jonathan Peytonf4392462017-07-27 20:58:41 +00003148 root->r.r_cg_nthreads = 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003149
Jonathan Peyton30419822017-05-12 18:01:32 +00003150 /* setup the root team for this task */
3151 /* allocate the root team structure */
3152 KF_TRACE(10, ("__kmp_initialize_root: before root_team\n"));
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003153
Jonathan Peyton30419822017-05-12 18:01:32 +00003154 root_team =
3155 __kmp_allocate_team(root,
3156 1, // new_nproc
3157 1, // max_nproc
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003158#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00003159 0, // root parallel id
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003160#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00003161#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003162 __kmp_nested_proc_bind.bind_types[0],
Jim Cownie5e8470a2013-09-27 10:38:44 +00003163#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003164 &r_icvs,
3165 0 // argc
3166 USE_NESTED_HOT_ARG(NULL) // master thread is unknown
3167 );
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00003168#if USE_DEBUGGER
Jonathan Peyton30419822017-05-12 18:01:32 +00003169 // Non-NULL value should be assigned to make the debugger display the root
3170 // team.
3171 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0));
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00003172#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00003173
Jonathan Peyton30419822017-05-12 18:01:32 +00003174 KF_TRACE(10, ("__kmp_initialize_root: after root_team = %p\n", root_team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003175
Jonathan Peyton30419822017-05-12 18:01:32 +00003176 root->r.r_root_team = root_team;
3177 root_team->t.t_control_stack_top = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003178
Jonathan Peyton30419822017-05-12 18:01:32 +00003179 /* initialize root team */
3180 root_team->t.t_threads[0] = NULL;
3181 root_team->t.t_nproc = 1;
3182 root_team->t.t_serialized = 1;
3183 // TODO???: root_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
3184 root_team->t.t_sched.r_sched_type = r_sched.r_sched_type;
3185 root_team->t.t_sched.chunk = r_sched.chunk;
3186 KA_TRACE(
3187 20,
3188 ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n",
3189 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003190
Jonathan Peyton30419822017-05-12 18:01:32 +00003191 /* setup the hot team for this task */
3192 /* allocate the hot team structure */
3193 KF_TRACE(10, ("__kmp_initialize_root: before hot_team\n"));
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003194
Jonathan Peyton30419822017-05-12 18:01:32 +00003195 hot_team =
3196 __kmp_allocate_team(root,
3197 1, // new_nproc
3198 __kmp_dflt_team_nth_ub * 2, // max_nproc
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003199#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00003200 0, // root parallel id
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003201#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00003202#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003203 __kmp_nested_proc_bind.bind_types[0],
Jim Cownie5e8470a2013-09-27 10:38:44 +00003204#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003205 &r_icvs,
3206 0 // argc
3207 USE_NESTED_HOT_ARG(NULL) // master thread is unknown
3208 );
3209 KF_TRACE(10, ("__kmp_initialize_root: after hot_team = %p\n", hot_team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003210
Jonathan Peyton30419822017-05-12 18:01:32 +00003211 root->r.r_hot_team = hot_team;
3212 root_team->t.t_control_stack_top = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003213
Jonathan Peyton30419822017-05-12 18:01:32 +00003214 /* first-time initialization */
3215 hot_team->t.t_parent = root_team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003216
Jonathan Peyton30419822017-05-12 18:01:32 +00003217 /* initialize hot team */
3218 hot_team_max_nth = hot_team->t.t_max_nproc;
3219 for (f = 0; f < hot_team_max_nth; ++f) {
3220 hot_team->t.t_threads[f] = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003221 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003222 hot_team->t.t_nproc = 1;
3223 // TODO???: hot_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
3224 hot_team->t.t_sched.r_sched_type = r_sched.r_sched_type;
3225 hot_team->t.t_sched.chunk = r_sched.chunk;
3226 hot_team->t.t_size_changed = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003227}
3228
3229#ifdef KMP_DEBUG
3230
Jim Cownie5e8470a2013-09-27 10:38:44 +00003231typedef struct kmp_team_list_item {
Jonathan Peyton30419822017-05-12 18:01:32 +00003232 kmp_team_p const *entry;
3233 struct kmp_team_list_item *next;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003234} kmp_team_list_item_t;
Jonathan Peyton30419822017-05-12 18:01:32 +00003235typedef kmp_team_list_item_t *kmp_team_list_t;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003236
Jonathan Peyton30419822017-05-12 18:01:32 +00003237static void __kmp_print_structure_team_accum( // Add team to list of teams.
3238 kmp_team_list_t list, // List of teams.
3239 kmp_team_p const *team // Team to add.
3240 ) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003241
Jonathan Peyton30419822017-05-12 18:01:32 +00003242 // List must terminate with item where both entry and next are NULL.
3243 // Team is added to the list only once.
3244 // List is sorted in ascending order by team id.
3245 // Team id is *not* a key.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003246
Jonathan Peyton30419822017-05-12 18:01:32 +00003247 kmp_team_list_t l;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003248
Jonathan Peyton30419822017-05-12 18:01:32 +00003249 KMP_DEBUG_ASSERT(list != NULL);
3250 if (team == NULL) {
3251 return;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003252 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003253
Jonathan Peyton30419822017-05-12 18:01:32 +00003254 __kmp_print_structure_team_accum(list, team->t.t_parent);
3255 __kmp_print_structure_team_accum(list, team->t.t_next_pool);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003256
Jonathan Peyton30419822017-05-12 18:01:32 +00003257 // Search list for the team.
3258 l = list;
3259 while (l->next != NULL && l->entry != team) {
3260 l = l->next;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003261 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003262 if (l->next != NULL) {
3263 return; // Team has been added before, exit.
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003264 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003265
Jonathan Peyton30419822017-05-12 18:01:32 +00003266 // Team is not found. Search list again for insertion point.
3267 l = list;
3268 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) {
3269 l = l->next;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003270 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003271
Jonathan Peyton30419822017-05-12 18:01:32 +00003272 // Insert team.
3273 {
3274 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(
3275 sizeof(kmp_team_list_item_t));
3276 *item = *l;
3277 l->entry = team;
3278 l->next = item;
3279 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003280}
3281
Jonathan Peyton30419822017-05-12 18:01:32 +00003282static void __kmp_print_structure_team(char const *title, kmp_team_p const *team
Jim Cownie5e8470a2013-09-27 10:38:44 +00003283
Jonathan Peyton30419822017-05-12 18:01:32 +00003284 ) {
3285 __kmp_printf("%s", title);
3286 if (team != NULL) {
3287 __kmp_printf("%2x %p\n", team->t.t_id, team);
3288 } else {
3289 __kmp_printf(" - (nil)\n");
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003290 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003291}
3292
Jonathan Peyton30419822017-05-12 18:01:32 +00003293static void __kmp_print_structure_thread(char const *title,
3294 kmp_info_p const *thread) {
3295 __kmp_printf("%s", title);
3296 if (thread != NULL) {
3297 __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread);
3298 } else {
3299 __kmp_printf(" - (nil)\n");
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003300 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003301}
3302
Jonathan Peyton30419822017-05-12 18:01:32 +00003303void __kmp_print_structure(void) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003304
Jonathan Peyton30419822017-05-12 18:01:32 +00003305 kmp_team_list_t list;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003306
Jonathan Peyton30419822017-05-12 18:01:32 +00003307 // Initialize list of teams.
3308 list =
3309 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(sizeof(kmp_team_list_item_t));
3310 list->entry = NULL;
3311 list->next = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003312
Jonathan Peyton30419822017-05-12 18:01:32 +00003313 __kmp_printf("\n------------------------------\nGlobal Thread "
3314 "Table\n------------------------------\n");
3315 {
3316 int gtid;
3317 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3318 __kmp_printf("%2d", gtid);
3319 if (__kmp_threads != NULL) {
3320 __kmp_printf(" %p", __kmp_threads[gtid]);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003321 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003322 if (__kmp_root != NULL) {
3323 __kmp_printf(" %p", __kmp_root[gtid]);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003324 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003325 __kmp_printf("\n");
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003326 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003327 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003328
Jonathan Peyton30419822017-05-12 18:01:32 +00003329 // Print out __kmp_threads array.
3330 __kmp_printf("\n------------------------------\nThreads\n--------------------"
3331 "----------\n");
3332 if (__kmp_threads != NULL) {
3333 int gtid;
3334 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3335 kmp_info_t const *thread = __kmp_threads[gtid];
3336 if (thread != NULL) {
3337 __kmp_printf("GTID %2d %p:\n", gtid, thread);
3338 __kmp_printf(" Our Root: %p\n", thread->th.th_root);
3339 __kmp_print_structure_team(" Our Team: ", thread->th.th_team);
3340 __kmp_print_structure_team(" Serial Team: ",
3341 thread->th.th_serial_team);
3342 __kmp_printf(" Threads: %2d\n", thread->th.th_team_nproc);
3343 __kmp_print_structure_thread(" Master: ",
3344 thread->th.th_team_master);
3345 __kmp_printf(" Serialized?: %2d\n", thread->th.th_team_serialized);
3346 __kmp_printf(" Set NProc: %2d\n", thread->th.th_set_nproc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003347#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003348 __kmp_printf(" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003349#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003350 __kmp_print_structure_thread(" Next in pool: ",
3351 thread->th.th_next_pool);
3352 __kmp_printf("\n");
3353 __kmp_print_structure_team_accum(list, thread->th.th_team);
3354 __kmp_print_structure_team_accum(list, thread->th.th_serial_team);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003355 }
3356 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003357 } else {
3358 __kmp_printf("Threads array is not allocated.\n");
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003359 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003360
Jonathan Peyton30419822017-05-12 18:01:32 +00003361 // Print out __kmp_root array.
3362 __kmp_printf("\n------------------------------\nUbers\n----------------------"
3363 "--------\n");
3364 if (__kmp_root != NULL) {
3365 int gtid;
3366 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3367 kmp_root_t const *root = __kmp_root[gtid];
3368 if (root != NULL) {
3369 __kmp_printf("GTID %2d %p:\n", gtid, root);
3370 __kmp_print_structure_team(" Root Team: ", root->r.r_root_team);
3371 __kmp_print_structure_team(" Hot Team: ", root->r.r_hot_team);
3372 __kmp_print_structure_thread(" Uber Thread: ",
3373 root->r.r_uber_thread);
3374 __kmp_printf(" Active?: %2d\n", root->r.r_active);
3375 __kmp_printf(" Nested?: %2d\n", root->r.r_nested);
3376 __kmp_printf(" In Parallel: %2d\n", root->r.r_in_parallel);
3377 __kmp_printf("\n");
3378 __kmp_print_structure_team_accum(list, root->r.r_root_team);
3379 __kmp_print_structure_team_accum(list, root->r.r_hot_team);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003380 }
3381 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003382 } else {
3383 __kmp_printf("Ubers array is not allocated.\n");
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003384 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003385
Jonathan Peyton30419822017-05-12 18:01:32 +00003386 __kmp_printf("\n------------------------------\nTeams\n----------------------"
3387 "--------\n");
3388 while (list->next != NULL) {
3389 kmp_team_p const *team = list->entry;
3390 int i;
3391 __kmp_printf("Team %2x %p:\n", team->t.t_id, team);
3392 __kmp_print_structure_team(" Parent Team: ", team->t.t_parent);
3393 __kmp_printf(" Master TID: %2d\n", team->t.t_master_tid);
3394 __kmp_printf(" Max threads: %2d\n", team->t.t_max_nproc);
3395 __kmp_printf(" Levels of serial: %2d\n", team->t.t_serialized);
3396 __kmp_printf(" Number threads: %2d\n", team->t.t_nproc);
3397 for (i = 0; i < team->t.t_nproc; ++i) {
3398 __kmp_printf(" Thread %2d: ", i);
3399 __kmp_print_structure_thread("", team->t.t_threads[i]);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003400 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003401 __kmp_print_structure_team(" Next in pool: ", team->t.t_next_pool);
3402 __kmp_printf("\n");
3403 list = list->next;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003404 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003405
Jonathan Peyton30419822017-05-12 18:01:32 +00003406 // Print out __kmp_thread_pool and __kmp_team_pool.
3407 __kmp_printf("\n------------------------------\nPools\n----------------------"
3408 "--------\n");
3409 __kmp_print_structure_thread("Thread pool: ",
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00003410 CCAST(kmp_info_t *, __kmp_thread_pool));
Jonathan Peyton30419822017-05-12 18:01:32 +00003411 __kmp_print_structure_team("Team pool: ",
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00003412 CCAST(kmp_team_t *, __kmp_team_pool));
Jonathan Peyton30419822017-05-12 18:01:32 +00003413 __kmp_printf("\n");
Jim Cownie5e8470a2013-09-27 10:38:44 +00003414
Jonathan Peyton30419822017-05-12 18:01:32 +00003415 // Free team list.
3416 while (list != NULL) {
3417 kmp_team_list_item_t *item = list;
3418 list = list->next;
3419 KMP_INTERNAL_FREE(item);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003420 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003421}
3422
3423#endif
3424
Jim Cownie5e8470a2013-09-27 10:38:44 +00003425//---------------------------------------------------------------------------
3426// Stuff for per-thread fast random number generator
3427// Table of primes
Jim Cownie5e8470a2013-09-27 10:38:44 +00003428static const unsigned __kmp_primes[] = {
Jonathan Peyton30419822017-05-12 18:01:32 +00003429 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877,
3430 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231,
3431 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201,
3432 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3,
3433 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7,
3434 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9,
3435 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45,
3436 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7,
3437 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363,
3438 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3,
3439 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f};
Jim Cownie5e8470a2013-09-27 10:38:44 +00003440
3441//---------------------------------------------------------------------------
3442// __kmp_get_random: Get a random number using a linear congruential method.
Jonathan Peyton30419822017-05-12 18:01:32 +00003443unsigned short __kmp_get_random(kmp_info_t *thread) {
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003444 unsigned x = thread->th.th_x;
Jonathan Peyton30419822017-05-12 18:01:32 +00003445 unsigned short r = x >> 16;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003446
Jonathan Peyton30419822017-05-12 18:01:32 +00003447 thread->th.th_x = x * thread->th.th_a + 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003448
3449 KA_TRACE(30, ("__kmp_get_random: THREAD: %d, RETURN: %u\n",
Jonathan Peyton30419822017-05-12 18:01:32 +00003450 thread->th.th_info.ds.ds_tid, r));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003451
3452 return r;
3453}
3454//--------------------------------------------------------
3455// __kmp_init_random: Initialize a random number generator
Jonathan Peyton30419822017-05-12 18:01:32 +00003456void __kmp_init_random(kmp_info_t *thread) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003457 unsigned seed = thread->th.th_info.ds.ds_tid;
3458
Jonathan Peyton30419822017-05-12 18:01:32 +00003459 thread->th.th_a =
3460 __kmp_primes[seed % (sizeof(__kmp_primes) / sizeof(__kmp_primes[0]))];
3461 thread->th.th_x = (seed + 1) * thread->th.th_a + 1;
3462 KA_TRACE(30,
3463 ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003464}
3465
Jim Cownie5e8470a2013-09-27 10:38:44 +00003466#if KMP_OS_WINDOWS
Jonathan Peyton30419822017-05-12 18:01:32 +00003467/* reclaim array entries for root threads that are already dead, returns number
3468 * reclaimed */
3469static int __kmp_reclaim_dead_roots(void) {
3470 int i, r = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003471
Jonathan Peyton30419822017-05-12 18:01:32 +00003472 for (i = 0; i < __kmp_threads_capacity; ++i) {
3473 if (KMP_UBER_GTID(i) &&
3474 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])) &&
3475 !__kmp_root[i]
3476 ->r.r_active) { // AC: reclaim only roots died in non-active state
3477 r += __kmp_unregister_root_other_thread(i);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003478 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003479 }
3480 return r;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003481}
3482#endif
3483
Jonathan Peyton30419822017-05-12 18:01:32 +00003484/* This function attempts to create free entries in __kmp_threads and
3485 __kmp_root, and returns the number of free entries generated.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003486
Jonathan Peyton30419822017-05-12 18:01:32 +00003487 For Windows* OS static library, the first mechanism used is to reclaim array
3488 entries for root threads that are already dead.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003489
Jonathan Peyton30419822017-05-12 18:01:32 +00003490 On all platforms, expansion is attempted on the arrays __kmp_threads_ and
3491 __kmp_root, with appropriate update to __kmp_threads_capacity. Array
3492 capacity is increased by doubling with clipping to __kmp_tp_capacity, if
3493 threadprivate cache array has been created. Synchronization with
3494 __kmpc_threadprivate_cached is done using __kmp_tp_cached_lock.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003495
Jonathan Peyton30419822017-05-12 18:01:32 +00003496 After any dead root reclamation, if the clipping value allows array expansion
3497 to result in the generation of a total of nWish free slots, the function does
3498 that expansion. If not, but the clipping value allows array expansion to
3499 result in the generation of a total of nNeed free slots, the function does
3500 that expansion. Otherwise, nothing is done beyond the possible initial root
3501 thread reclamation. However, if nNeed is zero, a best-effort attempt is made
3502 to fulfil nWish as far as possible, i.e. the function will attempt to create
Jim Cownie5e8470a2013-09-27 10:38:44 +00003503 as many free slots as possible up to nWish.
3504
Jonathan Peyton30419822017-05-12 18:01:32 +00003505 If any argument is negative, the behavior is undefined. */
3506static int __kmp_expand_threads(int nWish, int nNeed) {
3507 int added = 0;
3508 int old_tp_cached;
3509 int __kmp_actual_max_nth;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003510
Jonathan Peyton30419822017-05-12 18:01:32 +00003511 if (nNeed > nWish) /* normalize the arguments */
3512 nWish = nNeed;
Jonathan Peyton99016992015-05-26 17:32:53 +00003513#if KMP_OS_WINDOWS && !defined KMP_DYNAMIC_LIB
Jonathan Peyton30419822017-05-12 18:01:32 +00003514 /* only for Windows static library */
3515 /* reclaim array entries for root threads that are already dead */
3516 added = __kmp_reclaim_dead_roots();
Jim Cownie5e8470a2013-09-27 10:38:44 +00003517
Jonathan Peyton30419822017-05-12 18:01:32 +00003518 if (nNeed) {
3519 nNeed -= added;
3520 if (nNeed < 0)
3521 nNeed = 0;
3522 }
3523 if (nWish) {
3524 nWish -= added;
3525 if (nWish < 0)
3526 nWish = 0;
3527 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003528#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003529 if (nWish <= 0)
Jim Cownie5e8470a2013-09-27 10:38:44 +00003530 return added;
Jonathan Peyton30419822017-05-12 18:01:32 +00003531
3532 while (1) {
3533 int nTarget;
3534 int minimumRequiredCapacity;
3535 int newCapacity;
3536 kmp_info_t **newThreads;
3537 kmp_root_t **newRoot;
3538
3539 // Note that __kmp_threads_capacity is not bounded by __kmp_max_nth. If
3540 // __kmp_max_nth is set to some value less than __kmp_sys_max_nth by the
Jonathan Peytonf4392462017-07-27 20:58:41 +00003541 // user via KMP_DEVICE_THREAD_LIMIT, then __kmp_threads_capacity may become
Jonathan Peyton30419822017-05-12 18:01:32 +00003542 // > __kmp_max_nth in one of two ways:
3543 //
3544 // 1) The initialization thread (gtid = 0) exits. __kmp_threads[0]
3545 // may not be resused by another thread, so we may need to increase
Jonathan Peyton09244f32017-07-26 20:07:58 +00003546 // __kmp_threads_capacity to __kmp_max_nth + 1.
Jonathan Peyton30419822017-05-12 18:01:32 +00003547 //
3548 // 2) New foreign root(s) are encountered. We always register new foreign
3549 // roots. This may cause a smaller # of threads to be allocated at
3550 // subsequent parallel regions, but the worker threads hang around (and
3551 // eventually go to sleep) and need slots in the __kmp_threads[] array.
3552 //
3553 // Anyway, that is the reason for moving the check to see if
Jonathan Peyton09244f32017-07-26 20:07:58 +00003554 // __kmp_max_nth was exceeded into __kmp_reserve_threads()
Jonathan Peyton30419822017-05-12 18:01:32 +00003555 // instead of having it performed here. -BB
3556 old_tp_cached = __kmp_tp_cached;
3557 __kmp_actual_max_nth =
3558 old_tp_cached ? __kmp_tp_capacity : __kmp_sys_max_nth;
3559 KMP_DEBUG_ASSERT(__kmp_actual_max_nth >= __kmp_threads_capacity);
3560
3561 /* compute expansion headroom to check if we can expand and whether to aim
3562 for nWish or nNeed */
3563 nTarget = nWish;
3564 if (__kmp_actual_max_nth - __kmp_threads_capacity < nTarget) {
3565 /* can't fulfil nWish, so try nNeed */
3566 if (nNeed) {
3567 nTarget = nNeed;
3568 if (__kmp_actual_max_nth - __kmp_threads_capacity < nTarget) {
3569 /* possible expansion too small -- give up */
3570 break;
3571 }
3572 } else {
3573 /* best-effort */
3574 nTarget = __kmp_actual_max_nth - __kmp_threads_capacity;
3575 if (!nTarget) {
3576 /* can expand at all -- give up */
3577 break;
3578 }
3579 }
3580 }
3581 minimumRequiredCapacity = __kmp_threads_capacity + nTarget;
3582
3583 newCapacity = __kmp_threads_capacity;
3584 do {
3585 newCapacity = newCapacity <= (__kmp_actual_max_nth >> 1)
3586 ? (newCapacity << 1)
3587 : __kmp_actual_max_nth;
3588 } while (newCapacity < minimumRequiredCapacity);
3589 newThreads = (kmp_info_t **)__kmp_allocate(
3590 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * newCapacity +
3591 CACHE_LINE);
3592 newRoot = (kmp_root_t **)((char *)newThreads +
3593 sizeof(kmp_info_t *) * newCapacity);
3594 KMP_MEMCPY(newThreads, __kmp_threads,
3595 __kmp_threads_capacity * sizeof(kmp_info_t *));
3596 KMP_MEMCPY(newRoot, __kmp_root,
3597 __kmp_threads_capacity * sizeof(kmp_root_t *));
3598 memset(newThreads + __kmp_threads_capacity, 0,
3599 (newCapacity - __kmp_threads_capacity) * sizeof(kmp_info_t *));
3600 memset(newRoot + __kmp_threads_capacity, 0,
3601 (newCapacity - __kmp_threads_capacity) * sizeof(kmp_root_t *));
3602
3603 if (!old_tp_cached && __kmp_tp_cached && newCapacity > __kmp_tp_capacity) {
3604 /* __kmp_tp_cached has changed, i.e. __kmpc_threadprivate_cached has
3605 allocated a threadprivate cache while we were allocating the expanded
3606 array, and our new capacity is larger than the threadprivate cache
3607 capacity, so we should deallocate the expanded arrays and try again.
3608 This is the first check of a double-check pair. */
3609 __kmp_free(newThreads);
3610 continue; /* start over and try again */
3611 }
3612 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
3613 if (!old_tp_cached && __kmp_tp_cached && newCapacity > __kmp_tp_capacity) {
3614 /* Same check as above, but this time with the lock so we can be sure if
3615 we can succeed. */
3616 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
3617 __kmp_free(newThreads);
3618 continue; /* start over and try again */
3619 } else {
3620 /* success */
3621 // __kmp_free( __kmp_threads ); // ATT: It leads to crash. Need to be
3622 // investigated.
3623 *(kmp_info_t * *volatile *)&__kmp_threads = newThreads;
3624 *(kmp_root_t * *volatile *)&__kmp_root = newRoot;
3625 added += newCapacity - __kmp_threads_capacity;
3626 *(volatile int *)&__kmp_threads_capacity = newCapacity;
3627 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
3628 break; /* succeeded, so we can exit the loop */
3629 }
3630 }
3631 return added;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003632}
3633
Jonathan Peyton30419822017-05-12 18:01:32 +00003634/* Register the current thread as a root thread and obtain our gtid. We must
3635 have the __kmp_initz_lock held at this point. Argument TRUE only if are the
3636 thread that calls from __kmp_do_serial_initialize() */
3637int __kmp_register_root(int initial_thread) {
3638 kmp_info_t *root_thread;
3639 kmp_root_t *root;
3640 int gtid;
3641 int capacity;
3642 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3643 KA_TRACE(20, ("__kmp_register_root: entered\n"));
3644 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00003645
Jonathan Peyton30419822017-05-12 18:01:32 +00003646 /* 2007-03-02:
3647 If initial thread did not invoke OpenMP RTL yet, and this thread is not an
3648 initial one, "__kmp_all_nth >= __kmp_threads_capacity" condition does not
3649 work as expected -- it may return false (that means there is at least one
3650 empty slot in __kmp_threads array), but it is possible the only free slot
3651 is #0, which is reserved for initial thread and so cannot be used for this
3652 one. Following code workarounds this bug.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003653
Jonathan Peyton30419822017-05-12 18:01:32 +00003654 However, right solution seems to be not reserving slot #0 for initial
3655 thread because:
3656 (1) there is no magic in slot #0,
3657 (2) we cannot detect initial thread reliably (the first thread which does
3658 serial initialization may be not a real initial thread).
3659 */
3660 capacity = __kmp_threads_capacity;
3661 if (!initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) {
3662 --capacity;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003663 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003664
Jonathan Peyton30419822017-05-12 18:01:32 +00003665 /* see if there are too many threads */
3666 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1, 1)) {
3667 if (__kmp_tp_cached) {
Jonathan Peyton6a393f72017-09-05 15:43:58 +00003668 __kmp_fatal(KMP_MSG(CantRegisterNewThread),
3669 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
3670 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
Jonathan Peyton30419822017-05-12 18:01:32 +00003671 } else {
Jonathan Peyton6a393f72017-09-05 15:43:58 +00003672 __kmp_fatal(KMP_MSG(CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads),
3673 __kmp_msg_null);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003674 }
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003675 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003676
3677 /* find an available thread slot */
3678 /* Don't reassign the zero slot since we need that to only be used by initial
3679 thread */
3680 for (gtid = (initial_thread ? 0 : 1); TCR_PTR(__kmp_threads[gtid]) != NULL;
3681 gtid++)
3682 ;
3683 KA_TRACE(1,
3684 ("__kmp_register_root: found slot in threads array: T#%d\n", gtid));
3685 KMP_ASSERT(gtid < __kmp_threads_capacity);
3686
3687 /* update global accounting */
3688 __kmp_all_nth++;
3689 TCW_4(__kmp_nth, __kmp_nth + 1);
3690
3691 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low
3692 // numbers of procs, and method #2 (keyed API call) for higher numbers.
3693 if (__kmp_adjust_gtid_mode) {
3694 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
3695 if (TCR_4(__kmp_gtid_mode) != 2) {
3696 TCW_4(__kmp_gtid_mode, 2);
3697 }
3698 } else {
3699 if (TCR_4(__kmp_gtid_mode) != 1) {
3700 TCW_4(__kmp_gtid_mode, 1);
3701 }
3702 }
3703 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003704
3705#ifdef KMP_ADJUST_BLOCKTIME
Jonathan Peyton30419822017-05-12 18:01:32 +00003706 /* Adjust blocktime to zero if necessary */
3707 /* Middle initialization might not have occurred yet */
3708 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
3709 if (__kmp_nth > __kmp_avail_proc) {
3710 __kmp_zero_bt = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003711 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003712 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003713#endif /* KMP_ADJUST_BLOCKTIME */
3714
Jonathan Peyton30419822017-05-12 18:01:32 +00003715 /* setup this new hierarchy */
3716 if (!(root = __kmp_root[gtid])) {
3717 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(sizeof(kmp_root_t));
3718 KMP_DEBUG_ASSERT(!root->r.r_root_team);
3719 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003720
Jonathan Peyton5375fe82016-11-14 21:13:44 +00003721#if KMP_STATS_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003722 // Initialize stats as soon as possible (right after gtid assignment).
3723 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid);
3724 KMP_START_EXPLICIT_TIMER(OMP_worker_thread_life);
3725 KMP_SET_THREAD_STATE(SERIAL_REGION);
3726 KMP_INIT_PARTITIONED_TIMERS(OMP_serial);
Jonathan Peyton5375fe82016-11-14 21:13:44 +00003727#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003728 __kmp_initialize_root(root);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003729
Jonathan Peyton30419822017-05-12 18:01:32 +00003730 /* setup new root thread structure */
3731 if (root->r.r_uber_thread) {
3732 root_thread = root->r.r_uber_thread;
3733 } else {
3734 root_thread = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t));
3735 if (__kmp_storage_map) {
3736 __kmp_print_thread_storage_map(root_thread, gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003737 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003738 root_thread->th.th_info.ds.ds_gtid = gtid;
3739 root_thread->th.th_root = root;
3740 if (__kmp_env_consistency_check) {
3741 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid);
3742 }
3743#if USE_FAST_MEMORY
3744 __kmp_initialize_fast_memory(root_thread);
3745#endif /* USE_FAST_MEMORY */
Jim Cownie5e8470a2013-09-27 10:38:44 +00003746
Jonathan Peyton30419822017-05-12 18:01:32 +00003747#if KMP_USE_BGET
3748 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL);
3749 __kmp_initialize_bget(root_thread);
3750#endif
3751 __kmp_init_random(root_thread); // Initialize random number generator
3752 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003753
Jonathan Peyton30419822017-05-12 18:01:32 +00003754 /* setup the serial team held in reserve by the root thread */
3755 if (!root_thread->th.th_serial_team) {
3756 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3757 KF_TRACE(10, ("__kmp_register_root: before serial_team\n"));
3758 root_thread->th.th_serial_team =
3759 __kmp_allocate_team(root, 1, 1,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003760#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00003761 0, // root parallel id
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003762#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00003763#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003764 proc_bind_default,
Jim Cownie5e8470a2013-09-27 10:38:44 +00003765#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003766 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL));
3767 }
3768 KMP_ASSERT(root_thread->th.th_serial_team);
3769 KF_TRACE(10, ("__kmp_register_root: after serial_team = %p\n",
3770 root_thread->th.th_serial_team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003771
Jonathan Peyton30419822017-05-12 18:01:32 +00003772 /* drop root_thread into place */
3773 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003774
Jonathan Peyton30419822017-05-12 18:01:32 +00003775 root->r.r_root_team->t.t_threads[0] = root_thread;
3776 root->r.r_hot_team->t.t_threads[0] = root_thread;
3777 root_thread->th.th_serial_team->t.t_threads[0] = root_thread;
3778 // AC: the team created in reserve, not for execution (it is unused for now).
3779 root_thread->th.th_serial_team->t.t_serialized = 0;
3780 root->r.r_uber_thread = root_thread;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003781
Jonathan Peyton30419822017-05-12 18:01:32 +00003782 /* initialize the thread, get it ready to go */
3783 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid);
3784 TCW_4(__kmp_init_gtid, TRUE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003785
Jonathan Peyton30419822017-05-12 18:01:32 +00003786 /* prepare the master thread for get_gtid() */
3787 __kmp_gtid_set_specific(gtid);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003788
Jonathan Peyton7abf9d52016-05-26 18:19:10 +00003789#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +00003790 __kmp_itt_thread_name(gtid);
Jonathan Peyton7abf9d52016-05-26 18:19:10 +00003791#endif /* USE_ITT_BUILD */
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003792
Jonathan Peyton30419822017-05-12 18:01:32 +00003793#ifdef KMP_TDATA_GTID
3794 __kmp_gtid = gtid;
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00003795#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003796 __kmp_create_worker(gtid, root_thread, __kmp_stksize);
3797 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid);
3798
3799 KA_TRACE(20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
3800 "plain=%u\n",
3801 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team),
3802 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,
3803 KMP_INIT_BARRIER_STATE));
3804 { // Initialize barrier data.
3805 int b;
3806 for (b = 0; b < bs_last_barrier; ++b) {
3807 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE;
3808#if USE_DEBUGGER
3809 root_thread->th.th_bar[b].bb.b_worker_arrived = 0;
3810#endif
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003811 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003812 }
3813 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived ==
3814 KMP_INIT_BARRIER_STATE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003815
Alp Toker763b9392014-02-28 09:42:41 +00003816#if KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00003817#if OMP_40_ENABLED
3818 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED;
3819 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED;
3820 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED;
3821 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED;
3822#endif
Jonathan Peyton2f7c0772016-02-25 18:49:52 +00003823
Jonathan Peyton30419822017-05-12 18:01:32 +00003824 if (TCR_4(__kmp_init_middle)) {
3825 __kmp_affinity_set_init_mask(gtid, TRUE);
3826 }
Alp Toker763b9392014-02-28 09:42:41 +00003827#endif /* KMP_AFFINITY_SUPPORTED */
Jim Cownie5e8470a2013-09-27 10:38:44 +00003828
Jonathan Peyton30419822017-05-12 18:01:32 +00003829 __kmp_root_counter++;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003830
Jonathan Peyton30419822017-05-12 18:01:32 +00003831 KMP_MB();
3832 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003833
Jonathan Peyton30419822017-05-12 18:01:32 +00003834 return gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003835}
3836
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003837#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00003838static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr, int level,
3839 const int max_level) {
3840 int i, n, nth;
3841 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams;
3842 if (!hot_teams || !hot_teams[level].hot_team) {
3843 return 0;
3844 }
3845 KMP_DEBUG_ASSERT(level < max_level);
3846 kmp_team_t *team = hot_teams[level].hot_team;
3847 nth = hot_teams[level].hot_team_nth;
3848 n = nth - 1; // master is not freed
3849 if (level < max_level - 1) {
3850 for (i = 0; i < nth; ++i) {
3851 kmp_info_t *th = team->t.t_threads[i];
3852 n += __kmp_free_hot_teams(root, th, level + 1, max_level);
3853 if (i > 0 && th->th.th_hot_teams) {
3854 __kmp_free(th->th.th_hot_teams);
3855 th->th.th_hot_teams = NULL;
3856 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003857 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003858 }
3859 __kmp_free_team(root, team, NULL);
3860 return n;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003861}
3862#endif
3863
Jonathan Peyton30419822017-05-12 18:01:32 +00003864// Resets a root thread and clear its root and hot teams.
3865// Returns the number of __kmp_threads entries directly and indirectly freed.
3866static int __kmp_reset_root(int gtid, kmp_root_t *root) {
3867 kmp_team_t *root_team = root->r.r_root_team;
3868 kmp_team_t *hot_team = root->r.r_hot_team;
3869 int n = hot_team->t.t_nproc;
3870 int i;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003871
Jonathan Peyton30419822017-05-12 18:01:32 +00003872 KMP_DEBUG_ASSERT(!root->r.r_active);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003873
Jonathan Peyton30419822017-05-12 18:01:32 +00003874 root->r.r_root_team = NULL;
3875 root->r.r_hot_team = NULL;
3876 // __kmp_free_team() does not free hot teams, so we have to clear r_hot_team
3877 // before call to __kmp_free_team().
3878 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003879#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00003880 if (__kmp_hot_teams_max_level >
3881 0) { // need to free nested hot teams and their threads if any
3882 for (i = 0; i < hot_team->t.t_nproc; ++i) {
3883 kmp_info_t *th = hot_team->t.t_threads[i];
3884 if (__kmp_hot_teams_max_level > 1) {
3885 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level);
3886 }
3887 if (th->th.th_hot_teams) {
3888 __kmp_free(th->th.th_hot_teams);
3889 th->th.th_hot_teams = NULL;
3890 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003891 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003892 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003893#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003894 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003895
Jonathan Peyton30419822017-05-12 18:01:32 +00003896 // Before we can reap the thread, we need to make certain that all other
3897 // threads in the teams that had this root as ancestor have stopped trying to
3898 // steal tasks.
3899 if (__kmp_tasking_mode != tskm_immediate_exec) {
3900 __kmp_wait_to_unref_task_teams();
3901 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003902
Jonathan Peyton30419822017-05-12 18:01:32 +00003903#if KMP_OS_WINDOWS
3904 /* Close Handle of root duplicated in __kmp_create_worker (tr #62919) */
3905 KA_TRACE(
3906 10, ("__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPEC
3907 "\n",
3908 (LPVOID) & (root->r.r_uber_thread->th),
3909 root->r.r_uber_thread->th.th_info.ds.ds_thread));
3910 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread);
3911#endif /* KMP_OS_WINDOWS */
Jim Cownie5e8470a2013-09-27 10:38:44 +00003912
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003913#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00003914 if (ompt_enabled && ompt_callbacks.ompt_callback(ompt_event_thread_end)) {
3915 int gtid = __kmp_get_gtid();
3916 __ompt_thread_end(ompt_thread_initial, gtid);
3917 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003918#endif
3919
Jonathan Peyton30419822017-05-12 18:01:32 +00003920 TCW_4(__kmp_nth,
3921 __kmp_nth - 1); // __kmp_reap_thread will decrement __kmp_all_nth.
Jonathan Peytonf4392462017-07-27 20:58:41 +00003922 root->r.r_cg_nthreads--;
3923
Jonathan Peyton30419822017-05-12 18:01:32 +00003924 __kmp_reap_thread(root->r.r_uber_thread, 1);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003925
Jonathan Peyton30419822017-05-12 18:01:32 +00003926 // We canot put root thread to __kmp_thread_pool, so we have to reap it istead
3927 // of freeing.
3928 root->r.r_uber_thread = NULL;
3929 /* mark root as no longer in use */
3930 root->r.r_begin = FALSE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003931
Jonathan Peyton30419822017-05-12 18:01:32 +00003932 return n;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003933}
3934
Jonathan Peyton30419822017-05-12 18:01:32 +00003935void __kmp_unregister_root_current_thread(int gtid) {
3936 KA_TRACE(1, ("__kmp_unregister_root_current_thread: enter T#%d\n", gtid));
3937 /* this lock should be ok, since unregister_root_current_thread is never
3938 called during an abort, only during a normal close. furthermore, if you
3939 have the forkjoin lock, you should never try to get the initz lock */
3940 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3941 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
3942 KC_TRACE(10, ("__kmp_unregister_root_current_thread: already finished, "
3943 "exiting T#%d\n",
3944 gtid));
3945 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
3946 return;
3947 }
3948 kmp_root_t *root = __kmp_root[gtid];
Jim Cownie77c2a632014-09-03 11:34:33 +00003949
Jonathan Peyton30419822017-05-12 18:01:32 +00003950 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
3951 KMP_ASSERT(KMP_UBER_GTID(gtid));
3952 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
3953 KMP_ASSERT(root->r.r_active == FALSE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003954
Jonathan Peyton30419822017-05-12 18:01:32 +00003955 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00003956
Jonathan Peytondf6818b2016-06-14 17:57:47 +00003957#if OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003958 kmp_info_t *thread = __kmp_threads[gtid];
3959 kmp_team_t *team = thread->th.th_team;
3960 kmp_task_team_t *task_team = thread->th.th_task_team;
Andrey Churbanov535b6fa2015-05-07 17:41:51 +00003961
Jonathan Peyton30419822017-05-12 18:01:32 +00003962 // we need to wait for the proxy tasks before finishing the thread
3963 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks) {
Jonathan Peyton6d247f72015-09-10 21:33:50 +00003964#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00003965 // the runtime is shutting down so we won't report any events
3966 thread->th.ompt_thread_info.state = ompt_state_undefined;
Jonathan Peyton6d247f72015-09-10 21:33:50 +00003967#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003968 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL));
3969 }
Andrey Churbanov535b6fa2015-05-07 17:41:51 +00003970#endif
3971
Jonathan Peyton30419822017-05-12 18:01:32 +00003972 __kmp_reset_root(gtid, root);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003973
Jonathan Peyton30419822017-05-12 18:01:32 +00003974 /* free up this thread slot */
3975 __kmp_gtid_set_specific(KMP_GTID_DNE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003976#ifdef KMP_TDATA_GTID
Jonathan Peyton30419822017-05-12 18:01:32 +00003977 __kmp_gtid = KMP_GTID_DNE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003978#endif
3979
Jonathan Peyton30419822017-05-12 18:01:32 +00003980 KMP_MB();
3981 KC_TRACE(10,
3982 ("__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003983
Jonathan Peyton30419822017-05-12 18:01:32 +00003984 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003985}
3986
Jonathan Peyton2321d572015-06-08 19:25:25 +00003987#if KMP_OS_WINDOWS
Jim Cownie5e8470a2013-09-27 10:38:44 +00003988/* __kmp_forkjoin_lock must be already held
Jonathan Peyton30419822017-05-12 18:01:32 +00003989 Unregisters a root thread that is not the current thread. Returns the number
3990 of __kmp_threads entries freed as a result. */
3991static int __kmp_unregister_root_other_thread(int gtid) {
3992 kmp_root_t *root = __kmp_root[gtid];
3993 int r;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003994
Jonathan Peyton30419822017-05-12 18:01:32 +00003995 KA_TRACE(1, ("__kmp_unregister_root_other_thread: enter T#%d\n", gtid));
3996 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
3997 KMP_ASSERT(KMP_UBER_GTID(gtid));
3998 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
3999 KMP_ASSERT(root->r.r_active == FALSE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004000
Jonathan Peyton30419822017-05-12 18:01:32 +00004001 r = __kmp_reset_root(gtid, root);
4002 KC_TRACE(10,
4003 ("__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid));
4004 return r;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004005}
Jonathan Peyton2321d572015-06-08 19:25:25 +00004006#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00004007
Jim Cownie5e8470a2013-09-27 10:38:44 +00004008#if KMP_DEBUG
4009void __kmp_task_info() {
4010
Jonathan Peyton30419822017-05-12 18:01:32 +00004011 kmp_int32 gtid = __kmp_entry_gtid();
4012 kmp_int32 tid = __kmp_tid_from_gtid(gtid);
4013 kmp_info_t *this_thr = __kmp_threads[gtid];
4014 kmp_team_t *steam = this_thr->th.th_serial_team;
4015 kmp_team_t *team = this_thr->th.th_team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004016
Jonathan Peyton30419822017-05-12 18:01:32 +00004017 __kmp_printf("__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p curtask=%p "
4018 "ptask=%p\n",
4019 gtid, tid, this_thr, team, this_thr->th.th_current_task,
4020 team->t.t_implicit_task_taskdata[tid].td_parent);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004021}
4022#endif // KMP_DEBUG
4023
Jonathan Peyton30419822017-05-12 18:01:32 +00004024/* TODO optimize with one big memclr, take out what isn't needed, split
4025 responsibility to workers as much as possible, and delay initialization of
4026 features as much as possible */
4027static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team,
4028 int tid, int gtid) {
4029 /* this_thr->th.th_info.ds.ds_gtid is setup in
4030 kmp_allocate_thread/create_worker.
4031 this_thr->th.th_serial_team is setup in __kmp_allocate_thread */
4032 kmp_info_t *master = team->t.t_threads[0];
4033 KMP_DEBUG_ASSERT(this_thr != NULL);
4034 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team);
4035 KMP_DEBUG_ASSERT(team);
4036 KMP_DEBUG_ASSERT(team->t.t_threads);
4037 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4038 KMP_DEBUG_ASSERT(master);
4039 KMP_DEBUG_ASSERT(master->th.th_root);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004040
Jonathan Peyton30419822017-05-12 18:01:32 +00004041 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00004042
Jonathan Peyton30419822017-05-12 18:01:32 +00004043 TCW_SYNC_PTR(this_thr->th.th_team, team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004044
Jonathan Peyton30419822017-05-12 18:01:32 +00004045 this_thr->th.th_info.ds.ds_tid = tid;
4046 this_thr->th.th_set_nproc = 0;
4047 if (__kmp_tasking_mode != tskm_immediate_exec)
4048 // When tasking is possible, threads are not safe to reap until they are
4049 // done tasking; this will be set when tasking code is exited in wait
4050 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
4051 else // no tasking --> always safe to reap
4052 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004053#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00004054 this_thr->th.th_set_proc_bind = proc_bind_default;
4055#if KMP_AFFINITY_SUPPORTED
4056 this_thr->th.th_new_place = this_thr->th.th_current_place;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004057#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004058#endif
4059 this_thr->th.th_root = master->th.th_root;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004060
Jonathan Peyton30419822017-05-12 18:01:32 +00004061 /* setup the thread's cache of the team structure */
4062 this_thr->th.th_team_nproc = team->t.t_nproc;
4063 this_thr->th.th_team_master = master;
4064 this_thr->th.th_team_serialized = team->t.t_serialized;
4065 TCW_PTR(this_thr->th.th_sleep_loc, NULL);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004066
Jonathan Peyton30419822017-05-12 18:01:32 +00004067 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004068
Jonathan Peyton30419822017-05-12 18:01:32 +00004069 KF_TRACE(10, ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n",
4070 tid, gtid, this_thr, this_thr->th.th_current_task));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004071
Jonathan Peyton30419822017-05-12 18:01:32 +00004072 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr,
4073 team, tid, TRUE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004074
Jonathan Peyton30419822017-05-12 18:01:32 +00004075 KF_TRACE(10, ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n",
4076 tid, gtid, this_thr, this_thr->th.th_current_task));
4077 // TODO: Initialize ICVs from parent; GEH - isn't that already done in
4078 // __kmp_initialize_team()?
Jim Cownie5e8470a2013-09-27 10:38:44 +00004079
Jonathan Peyton30419822017-05-12 18:01:32 +00004080 /* TODO no worksharing in speculative threads */
4081 this_thr->th.th_dispatch = &team->t.t_dispatch[tid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00004082
Jonathan Peyton30419822017-05-12 18:01:32 +00004083 this_thr->th.th_local.this_construct = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004084
Jonathan Peyton30419822017-05-12 18:01:32 +00004085 if (!this_thr->th.th_pri_common) {
4086 this_thr->th.th_pri_common =
4087 (struct common_table *)__kmp_allocate(sizeof(struct common_table));
4088 if (__kmp_storage_map) {
4089 __kmp_print_storage_map_gtid(
4090 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1,
4091 sizeof(struct common_table), "th_%d.th_pri_common\n", gtid);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00004092 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004093 this_thr->th.th_pri_head = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00004094 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00004095
Jonathan Peyton30419822017-05-12 18:01:32 +00004096 /* Initialize dynamic dispatch */
4097 {
4098 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch;
4099 // Use team max_nproc since this will never change for the team.
4100 size_t disp_size =
4101 sizeof(dispatch_private_info_t) *
4102 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers);
4103 KD_TRACE(10, ("__kmp_initialize_info: T#%d max_nproc: %d\n", gtid,
4104 team->t.t_max_nproc));
4105 KMP_ASSERT(dispatch);
4106 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4107 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004108
Jonathan Peyton30419822017-05-12 18:01:32 +00004109 dispatch->th_disp_index = 0;
Jonathan Peytondf6818b2016-06-14 17:57:47 +00004110#if OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00004111 dispatch->th_doacross_buf_idx = 0;
Jonathan Peyton71909c52016-03-02 22:42:06 +00004112#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004113 if (!dispatch->th_disp_buffer) {
4114 dispatch->th_disp_buffer =
4115 (dispatch_private_info_t *)__kmp_allocate(disp_size);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004116
Jonathan Peyton30419822017-05-12 18:01:32 +00004117 if (__kmp_storage_map) {
4118 __kmp_print_storage_map_gtid(
4119 gtid, &dispatch->th_disp_buffer[0],
4120 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1
4121 ? 1
4122 : __kmp_dispatch_num_buffers],
4123 disp_size, "th_%d.th_dispatch.th_disp_buffer "
4124 "(team_%d.t_dispatch[%d].th_disp_buffer)",
4125 gtid, team->t.t_id, gtid);
4126 }
4127 } else {
4128 memset(&dispatch->th_disp_buffer[0], '\0', disp_size);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004129 }
4130
Jonathan Peyton30419822017-05-12 18:01:32 +00004131 dispatch->th_dispatch_pr_current = 0;
4132 dispatch->th_dispatch_sh_current = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004133
Jonathan Peyton30419822017-05-12 18:01:32 +00004134 dispatch->th_deo_fcn = 0; /* ORDERED */
4135 dispatch->th_dxo_fcn = 0; /* END ORDERED */
4136 }
Andrey Churbanov6d224db2015-02-10 18:37:43 +00004137
Jonathan Peyton30419822017-05-12 18:01:32 +00004138 this_thr->th.th_next_pool = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004139
Jonathan Peyton30419822017-05-12 18:01:32 +00004140 if (!this_thr->th.th_task_state_memo_stack) {
4141 size_t i;
4142 this_thr->th.th_task_state_memo_stack =
4143 (kmp_uint8 *)__kmp_allocate(4 * sizeof(kmp_uint8));
4144 this_thr->th.th_task_state_top = 0;
4145 this_thr->th.th_task_state_stack_sz = 4;
4146 for (i = 0; i < this_thr->th.th_task_state_stack_sz;
4147 ++i) // zero init the stack
4148 this_thr->th.th_task_state_memo_stack[i] = 0;
4149 }
4150
4151 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
4152 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
4153
4154 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00004155}
4156
Jonathan Peyton30419822017-05-12 18:01:32 +00004157/* allocate a new thread for the requesting team. this is only called from
4158 within a forkjoin critical section. we will first try to get an available
4159 thread from the thread pool. if none is available, we will fork a new one
4160 assuming we are able to create a new one. this should be assured, as the
4161 caller should check on this first. */
4162kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team,
4163 int new_tid) {
4164 kmp_team_t *serial_team;
4165 kmp_info_t *new_thr;
4166 int new_gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004167
Jonathan Peyton30419822017-05-12 18:01:32 +00004168 KA_TRACE(20, ("__kmp_allocate_thread: T#%d\n", __kmp_get_gtid()));
4169 KMP_DEBUG_ASSERT(root && team);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004170#if !KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004171 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid()));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004172#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004173 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00004174
Jonathan Peyton30419822017-05-12 18:01:32 +00004175 /* first, try to get one from the thread pool */
4176 if (__kmp_thread_pool) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00004177
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00004178 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool);
Jonathan Peyton30419822017-05-12 18:01:32 +00004179 __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool;
4180 if (new_thr == __kmp_thread_pool_insert_pt) {
4181 __kmp_thread_pool_insert_pt = NULL;
4182 }
4183 TCW_4(new_thr->th.th_in_pool, FALSE);
4184 // Don't touch th_active_in_pool or th_active.
4185 // The worker thread adjusts those flags as it sleeps/awakens.
4186 __kmp_thread_pool_nth--;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004187
Jonathan Peyton30419822017-05-12 18:01:32 +00004188 KA_TRACE(20, ("__kmp_allocate_thread: T#%d using thread T#%d\n",
4189 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid));
4190 KMP_ASSERT(!new_thr->th.th_team);
4191 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity);
4192 KMP_DEBUG_ASSERT(__kmp_thread_pool_nth >= 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004193
Jonathan Peyton30419822017-05-12 18:01:32 +00004194 /* setup the thread structure */
4195 __kmp_initialize_info(new_thr, team, new_tid,
4196 new_thr->th.th_info.ds.ds_gtid);
4197 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004198
Jonathan Peyton30419822017-05-12 18:01:32 +00004199 TCW_4(__kmp_nth, __kmp_nth + 1);
Jonathan Peytonf4392462017-07-27 20:58:41 +00004200 root->r.r_cg_nthreads++;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004201
Jonathan Peyton30419822017-05-12 18:01:32 +00004202 new_thr->th.th_task_state = 0;
4203 new_thr->th.th_task_state_top = 0;
4204 new_thr->th.th_task_state_stack_sz = 4;
Andrey Churbanov6d224db2015-02-10 18:37:43 +00004205
Jim Cownie5e8470a2013-09-27 10:38:44 +00004206#ifdef KMP_ADJUST_BLOCKTIME
Jonathan Peyton30419822017-05-12 18:01:32 +00004207 /* Adjust blocktime back to zero if necessary */
4208 /* Middle initialization might not have occurred yet */
4209 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4210 if (__kmp_nth > __kmp_avail_proc) {
4211 __kmp_zero_bt = TRUE;
4212 }
4213 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00004214#endif /* KMP_ADJUST_BLOCKTIME */
4215
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004216#if KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00004217 // If thread entered pool via __kmp_free_thread, wait_flag should !=
4218 // KMP_BARRIER_PARENT_FLAG.
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004219 int b;
Jonathan Peyton30419822017-05-12 18:01:32 +00004220 kmp_balign_t *balign = new_thr->th.th_bar;
4221 for (b = 0; b < bs_last_barrier; ++b)
4222 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004223#endif
4224
Jonathan Peyton30419822017-05-12 18:01:32 +00004225 KF_TRACE(10, ("__kmp_allocate_thread: T#%d using thread %p T#%d\n",
4226 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004227
Jim Cownie5e8470a2013-09-27 10:38:44 +00004228 KMP_MB();
4229 return new_thr;
Jonathan Peyton30419822017-05-12 18:01:32 +00004230 }
4231
4232 /* no, well fork a new one */
4233 KMP_ASSERT(__kmp_nth == __kmp_all_nth);
4234 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity);
4235
4236#if KMP_USE_MONITOR
4237 // If this is the first worker thread the RTL is creating, then also
4238 // launch the monitor thread. We try to do this as early as possible.
4239 if (!TCR_4(__kmp_init_monitor)) {
4240 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
4241 if (!TCR_4(__kmp_init_monitor)) {
4242 KF_TRACE(10, ("before __kmp_create_monitor\n"));
4243 TCW_4(__kmp_init_monitor, 1);
4244 __kmp_create_monitor(&__kmp_monitor);
4245 KF_TRACE(10, ("after __kmp_create_monitor\n"));
4246#if KMP_OS_WINDOWS
4247 // AC: wait until monitor has started. This is a fix for CQ232808.
4248 // The reason is that if the library is loaded/unloaded in a loop with
4249 // small (parallel) work in between, then there is high probability that
4250 // monitor thread started after the library shutdown. At shutdown it is
4251 // too late to cope with the problem, because when the master is in
4252 // DllMain (process detach) the monitor has no chances to start (it is
4253 // blocked), and master has no means to inform the monitor that the
4254 // library has gone, because all the memory which the monitor can access
4255 // is going to be released/reset.
4256 while (TCR_4(__kmp_init_monitor) < 2) {
4257 KMP_YIELD(TRUE);
4258 }
4259 KF_TRACE(10, ("after monitor thread has started\n"));
4260#endif
4261 }
4262 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
4263 }
4264#endif
4265
4266 KMP_MB();
4267 for (new_gtid = 1; TCR_PTR(__kmp_threads[new_gtid]) != NULL; ++new_gtid) {
4268 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity);
4269 }
4270
4271 /* allocate space for it. */
4272 new_thr = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t));
4273
4274 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr);
4275
4276 if (__kmp_storage_map) {
4277 __kmp_print_thread_storage_map(new_thr, new_gtid);
4278 }
4279
4280 // add the reserve serialized team, initialized from the team's master thread
4281 {
4282 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team);
4283 KF_TRACE(10, ("__kmp_allocate_thread: before th_serial/serial_team\n"));
4284 new_thr->th.th_serial_team = serial_team =
4285 (kmp_team_t *)__kmp_allocate_team(root, 1, 1,
4286#if OMPT_SUPPORT
4287 0, // root parallel id
4288#endif
4289#if OMP_40_ENABLED
4290 proc_bind_default,
4291#endif
4292 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL));
4293 }
4294 KMP_ASSERT(serial_team);
4295 serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for
4296 // execution (it is unused for now).
4297 serial_team->t.t_threads[0] = new_thr;
4298 KF_TRACE(10,
4299 ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n",
4300 new_thr));
4301
4302 /* setup the thread structures */
4303 __kmp_initialize_info(new_thr, team, new_tid, new_gtid);
4304
4305#if USE_FAST_MEMORY
4306 __kmp_initialize_fast_memory(new_thr);
4307#endif /* USE_FAST_MEMORY */
4308
4309#if KMP_USE_BGET
4310 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL);
4311 __kmp_initialize_bget(new_thr);
4312#endif
4313
4314 __kmp_init_random(new_thr); // Initialize random number generator
4315
4316 /* Initialize these only once when thread is grabbed for a team allocation */
4317 KA_TRACE(20,
4318 ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n",
4319 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
4320
4321 int b;
4322 kmp_balign_t *balign = new_thr->th.th_bar;
4323 for (b = 0; b < bs_last_barrier; ++b) {
4324 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE;
4325 balign[b].bb.team = NULL;
4326 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING;
4327 balign[b].bb.use_oncore_barrier = 0;
4328 }
4329
4330 new_thr->th.th_spin_here = FALSE;
4331 new_thr->th.th_next_waiting = 0;
4332
4333#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
4334 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED;
4335 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED;
4336 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED;
4337 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED;
4338#endif
4339
4340 TCW_4(new_thr->th.th_in_pool, FALSE);
4341 new_thr->th.th_active_in_pool = FALSE;
4342 TCW_4(new_thr->th.th_active, TRUE);
4343
4344 /* adjust the global counters */
4345 __kmp_all_nth++;
4346 __kmp_nth++;
4347
Jonathan Peytonf4392462017-07-27 20:58:41 +00004348 root->r.r_cg_nthreads++;
4349
Jonathan Peyton30419822017-05-12 18:01:32 +00004350 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low
4351 // numbers of procs, and method #2 (keyed API call) for higher numbers.
4352 if (__kmp_adjust_gtid_mode) {
4353 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
4354 if (TCR_4(__kmp_gtid_mode) != 2) {
4355 TCW_4(__kmp_gtid_mode, 2);
4356 }
4357 } else {
4358 if (TCR_4(__kmp_gtid_mode) != 1) {
4359 TCW_4(__kmp_gtid_mode, 1);
4360 }
4361 }
4362 }
4363
4364#ifdef KMP_ADJUST_BLOCKTIME
4365 /* Adjust blocktime back to zero if necessary */
4366 /* Middle initialization might not have occurred yet */
4367 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4368 if (__kmp_nth > __kmp_avail_proc) {
4369 __kmp_zero_bt = TRUE;
4370 }
4371 }
4372#endif /* KMP_ADJUST_BLOCKTIME */
4373
4374 /* actually fork it and create the new worker thread */
4375 KF_TRACE(
4376 10, ("__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr));
4377 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize);
4378 KF_TRACE(10,
4379 ("__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr));
4380
4381 KA_TRACE(20, ("__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(),
4382 new_gtid));
4383 KMP_MB();
4384 return new_thr;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004385}
4386
Jonathan Peyton30419822017-05-12 18:01:32 +00004387/* Reinitialize team for reuse.
4388 The hot team code calls this case at every fork barrier, so EPCC barrier
4389 test are extremely sensitive to changes in it, esp. writes to the team
4390 struct, which cause a cache invalidation in all threads.
4391 IF YOU TOUCH THIS ROUTINE, RUN EPCC C SYNCBENCH ON A BIG-IRON MACHINE!!! */
4392static void __kmp_reinitialize_team(kmp_team_t *team,
4393 kmp_internal_control_t *new_icvs,
4394 ident_t *loc) {
4395 KF_TRACE(10, ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n",
4396 team->t.t_threads[0], team));
4397 KMP_DEBUG_ASSERT(team && new_icvs);
4398 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);
4399 KMP_CHECK_UPDATE(team->t.t_ident, loc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004400
Jonathan Peyton30419822017-05-12 18:01:32 +00004401 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID());
Jonathan Peyton30419822017-05-12 18:01:32 +00004402 // Copy ICVs to the master thread's implicit taskdata
4403 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE);
4404 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);
Jim Cownie181b4bb2013-12-23 17:28:57 +00004405
Jonathan Peyton30419822017-05-12 18:01:32 +00004406 KF_TRACE(10, ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n",
4407 team->t.t_threads[0], team));
Jim Cownie181b4bb2013-12-23 17:28:57 +00004408}
4409
Jonathan Peyton30419822017-05-12 18:01:32 +00004410/* Initialize the team data structure.
4411 This assumes the t_threads and t_max_nproc are already set.
4412 Also, we don't touch the arguments */
4413static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
4414 kmp_internal_control_t *new_icvs,
4415 ident_t *loc) {
4416 KF_TRACE(10, ("__kmp_initialize_team: enter: team=%p\n", team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004417
Jonathan Peyton30419822017-05-12 18:01:32 +00004418 /* verify */
4419 KMP_DEBUG_ASSERT(team);
4420 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc);
4421 KMP_DEBUG_ASSERT(team->t.t_threads);
4422 KMP_MB();
Jim Cownie181b4bb2013-12-23 17:28:57 +00004423
Jonathan Peyton30419822017-05-12 18:01:32 +00004424 team->t.t_master_tid = 0; /* not needed */
4425 /* team->t.t_master_bar; not needed */
4426 team->t.t_serialized = new_nproc > 1 ? 0 : 1;
4427 team->t.t_nproc = new_nproc;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004428
Jonathan Peyton30419822017-05-12 18:01:32 +00004429 /* team->t.t_parent = NULL; TODO not needed & would mess up hot team */
4430 team->t.t_next_pool = NULL;
4431 /* memset( team->t.t_threads, 0, sizeof(kmp_info_t*)*new_nproc ); would mess
4432 * up hot team */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004433
Jonathan Peyton30419822017-05-12 18:01:32 +00004434 TCW_SYNC_PTR(team->t.t_pkfn, NULL); /* not needed */
4435 team->t.t_invoke = NULL; /* not needed */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004436
Jonathan Peyton30419822017-05-12 18:01:32 +00004437 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
4438 team->t.t_sched = new_icvs->sched;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004439
4440#if KMP_ARCH_X86 || KMP_ARCH_X86_64
Jonathan Peyton30419822017-05-12 18:01:32 +00004441 team->t.t_fp_control_saved = FALSE; /* not needed */
4442 team->t.t_x87_fpu_control_word = 0; /* not needed */
4443 team->t.t_mxcsr = 0; /* not needed */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004444#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
4445
Jonathan Peyton30419822017-05-12 18:01:32 +00004446 team->t.t_construct = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004447
Jonathan Peyton30419822017-05-12 18:01:32 +00004448 team->t.t_ordered.dt.t_value = 0;
4449 team->t.t_master_active = FALSE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004450
Jonathan Peyton30419822017-05-12 18:01:32 +00004451 memset(&team->t.t_taskq, '\0', sizeof(kmp_taskq_t));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004452
4453#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00004454 team->t.t_copypriv_data = NULL; /* not necessary, but nice for debugging */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004455#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004456 team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004457
Jonathan Peyton30419822017-05-12 18:01:32 +00004458 team->t.t_control_stack_top = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004459
Jonathan Peyton30419822017-05-12 18:01:32 +00004460 __kmp_reinitialize_team(team, new_icvs, loc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004461
Jonathan Peyton30419822017-05-12 18:01:32 +00004462 KMP_MB();
4463 KF_TRACE(10, ("__kmp_initialize_team: exit: team=%p\n", team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004464}
4465
Alp Toker98758b02014-03-02 04:12:06 +00004466#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
Jim Cownie5e8470a2013-09-27 10:38:44 +00004467/* Sets full mask for thread and returns old mask, no changes to structures. */
4468static void
Jonathan Peyton30419822017-05-12 18:01:32 +00004469__kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) {
4470 if (KMP_AFFINITY_CAPABLE()) {
4471 int status;
4472 if (old_mask != NULL) {
4473 status = __kmp_get_system_affinity(old_mask, TRUE);
4474 int error = errno;
4475 if (status != 0) {
Jonathan Peyton6a393f72017-09-05 15:43:58 +00004476 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError), KMP_ERR(error),
4477 __kmp_msg_null);
Jonathan Peyton30419822017-05-12 18:01:32 +00004478 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00004479 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004480 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE);
4481 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00004482}
4483#endif
4484
Alp Toker98758b02014-03-02 04:12:06 +00004485#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
Jim Cownie5e8470a2013-09-27 10:38:44 +00004486
Jim Cownie5e8470a2013-09-27 10:38:44 +00004487// __kmp_partition_places() is the heart of the OpenMP 4.0 affinity mechanism.
4488// It calculats the worker + master thread's partition based upon the parent
Alp Toker8f2d3f02014-02-24 10:40:15 +00004489// thread's partition, and binds each worker to a thread in their partition.
Jim Cownie5e8470a2013-09-27 10:38:44 +00004490// The master thread's partition should already include its current binding.
Jonathan Peyton30419822017-05-12 18:01:32 +00004491static void __kmp_partition_places(kmp_team_t *team, int update_master_only) {
4492 // Copy the master thread's place partion to the team struct
4493 kmp_info_t *master_th = team->t.t_threads[0];
4494 KMP_DEBUG_ASSERT(master_th != NULL);
4495 kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
4496 int first_place = master_th->th.th_first_place;
4497 int last_place = master_th->th.th_last_place;
4498 int masters_place = master_th->th.th_current_place;
4499 team->t.t_first_place = first_place;
4500 team->t.t_last_place = last_place;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004501
Jonathan Peyton30419822017-05-12 18:01:32 +00004502 KA_TRACE(20, ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
4503 "bound to place %d partition = [%d,%d]\n",
4504 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]),
4505 team->t.t_id, masters_place, first_place, last_place));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004506
Jonathan Peyton30419822017-05-12 18:01:32 +00004507 switch (proc_bind) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00004508
Jonathan Peyton30419822017-05-12 18:01:32 +00004509 case proc_bind_default:
4510 // serial teams might have the proc_bind policy set to proc_bind_default. It
4511 // doesn't matter, as we don't rebind master thread for any proc_bind policy
4512 KMP_DEBUG_ASSERT(team->t.t_nproc == 1);
4513 break;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004514
Jonathan Peyton30419822017-05-12 18:01:32 +00004515 case proc_bind_master: {
4516 int f;
4517 int n_th = team->t.t_nproc;
4518 for (f = 1; f < n_th; f++) {
4519 kmp_info_t *th = team->t.t_threads[f];
4520 KMP_DEBUG_ASSERT(th != NULL);
4521 th->th.th_first_place = first_place;
4522 th->th.th_last_place = last_place;
4523 th->th.th_new_place = masters_place;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004524
Jonathan Peyton30419822017-05-12 18:01:32 +00004525 KA_TRACE(100, ("__kmp_partition_places: master: T#%d(%d:%d) place %d "
4526 "partition = [%d,%d]\n",
4527 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
4528 f, masters_place, first_place, last_place));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004529 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004530 } break;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004531
Jonathan Peyton30419822017-05-12 18:01:32 +00004532 case proc_bind_close: {
4533 int f;
4534 int n_th = team->t.t_nproc;
4535 int n_places;
4536 if (first_place <= last_place) {
4537 n_places = last_place - first_place + 1;
4538 } else {
4539 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4540 }
4541 if (n_th <= n_places) {
4542 int place = masters_place;
4543 for (f = 1; f < n_th; f++) {
4544 kmp_info_t *th = team->t.t_threads[f];
4545 KMP_DEBUG_ASSERT(th != NULL);
4546
4547 if (place == last_place) {
4548 place = first_place;
4549 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4550 place = 0;
4551 } else {
4552 place++;
4553 }
4554 th->th.th_first_place = first_place;
4555 th->th.th_last_place = last_place;
4556 th->th.th_new_place = place;
4557
4558 KA_TRACE(100, ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
4559 "partition = [%d,%d]\n",
4560 __kmp_gtid_from_thread(team->t.t_threads[f]),
4561 team->t.t_id, f, place, first_place, last_place));
4562 }
4563 } else {
4564 int S, rem, gap, s_count;
4565 S = n_th / n_places;
4566 s_count = 0;
4567 rem = n_th - (S * n_places);
4568 gap = rem > 0 ? n_places / rem : n_places;
4569 int place = masters_place;
4570 int gap_ct = gap;
4571 for (f = 0; f < n_th; f++) {
4572 kmp_info_t *th = team->t.t_threads[f];
4573 KMP_DEBUG_ASSERT(th != NULL);
4574
4575 th->th.th_first_place = first_place;
4576 th->th.th_last_place = last_place;
4577 th->th.th_new_place = place;
4578 s_count++;
4579
4580 if ((s_count == S) && rem && (gap_ct == gap)) {
4581 // do nothing, add an extra thread to place on next iteration
4582 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4583 // we added an extra thread to this place; move to next place
4584 if (place == last_place) {
4585 place = first_place;
4586 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4587 place = 0;
4588 } else {
4589 place++;
4590 }
4591 s_count = 0;
4592 gap_ct = 1;
4593 rem--;
4594 } else if (s_count == S) { // place full; don't add extra
4595 if (place == last_place) {
4596 place = first_place;
4597 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4598 place = 0;
4599 } else {
4600 place++;
4601 }
4602 gap_ct++;
4603 s_count = 0;
4604 }
4605
4606 KA_TRACE(100,
4607 ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
4608 "partition = [%d,%d]\n",
4609 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f,
4610 th->th.th_new_place, first_place, last_place));
4611 }
4612 KMP_DEBUG_ASSERT(place == masters_place);
4613 }
4614 } break;
4615
4616 case proc_bind_spread: {
4617 int f;
4618 int n_th = team->t.t_nproc;
4619 int n_places;
4620 int thidx;
4621 if (first_place <= last_place) {
4622 n_places = last_place - first_place + 1;
4623 } else {
4624 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4625 }
4626 if (n_th <= n_places) {
Paul Osmialowskia0162792017-08-10 23:04:11 +00004627 int place = -1;
Jonathan Peyton30419822017-05-12 18:01:32 +00004628
Paul Osmialowskia0162792017-08-10 23:04:11 +00004629 if (n_places != static_cast<int>(__kmp_affinity_num_masks)) {
4630 int S = n_places / n_th;
4631 int s_count, rem, gap, gap_ct;
4632
4633 place = masters_place;
4634 rem = n_places - n_th * S;
4635 gap = rem ? n_th / rem : 1;
4636 gap_ct = gap;
4637 thidx = n_th;
4638 if (update_master_only == 1)
4639 thidx = 1;
4640 for (f = 0; f < thidx; f++) {
4641 kmp_info_t *th = team->t.t_threads[f];
4642 KMP_DEBUG_ASSERT(th != NULL);
4643
4644 th->th.th_first_place = place;
4645 th->th.th_new_place = place;
4646 s_count = 1;
4647 while (s_count < S) {
4648 if (place == last_place) {
4649 place = first_place;
4650 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4651 place = 0;
4652 } else {
4653 place++;
4654 }
4655 s_count++;
4656 }
4657 if (rem && (gap_ct == gap)) {
4658 if (place == last_place) {
4659 place = first_place;
4660 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4661 place = 0;
4662 } else {
4663 place++;
4664 }
4665 rem--;
4666 gap_ct = 0;
4667 }
4668 th->th.th_last_place = place;
4669 gap_ct++;
4670
Jonathan Peyton30419822017-05-12 18:01:32 +00004671 if (place == last_place) {
4672 place = first_place;
4673 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4674 place = 0;
4675 } else {
4676 place++;
4677 }
Paul Osmialowskia0162792017-08-10 23:04:11 +00004678
4679 KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
4680 "partition = [%d,%d], __kmp_affinity_num_masks: %u\n",
4681 __kmp_gtid_from_thread(team->t.t_threads[f]),
4682 team->t.t_id, f, th->th.th_new_place,
4683 th->th.th_first_place, th->th.th_last_place,
4684 __kmp_affinity_num_masks));
Jonathan Peyton30419822017-05-12 18:01:32 +00004685 }
Paul Osmialowskia0162792017-08-10 23:04:11 +00004686 } else {
4687 /* Having uniform space of available computation places I can create
4688 T partitions of round(P/T) size and put threads into the first
4689 place of each partition. */
4690 double current = static_cast<double>(masters_place);
4691 double spacing =
4692 (static_cast<double>(n_places + 1) / static_cast<double>(n_th));
4693 int first, last;
4694 kmp_info_t *th;
4695
4696 thidx = n_th + 1;
4697 if (update_master_only == 1)
4698 thidx = 1;
4699 for (f = 0; f < thidx; f++) {
4700 first = static_cast<int>(current);
4701 last = static_cast<int>(current + spacing) - 1;
4702 KMP_DEBUG_ASSERT(last >= first);
4703 if (first >= n_places) {
4704 if (masters_place) {
4705 first -= n_places;
4706 last -= n_places;
4707 if (first == (masters_place + 1)) {
4708 KMP_DEBUG_ASSERT(f == n_th);
4709 first--;
4710 }
4711 if (last == masters_place) {
4712 KMP_DEBUG_ASSERT(f == (n_th - 1));
4713 last--;
4714 }
4715 } else {
4716 KMP_DEBUG_ASSERT(f == n_th);
4717 first = 0;
4718 last = 0;
4719 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004720 }
Paul Osmialowskia0162792017-08-10 23:04:11 +00004721 if (last >= n_places) {
4722 last = (n_places - 1);
4723 }
4724 place = first;
4725 current += spacing;
4726 if (f < n_th) {
4727 KMP_DEBUG_ASSERT(0 <= first);
4728 KMP_DEBUG_ASSERT(n_places > first);
4729 KMP_DEBUG_ASSERT(0 <= last);
4730 KMP_DEBUG_ASSERT(n_places > last);
4731 KMP_DEBUG_ASSERT(last_place >= first_place);
4732 th = team->t.t_threads[f];
4733 KMP_DEBUG_ASSERT(th);
4734 th->th.th_first_place = first;
4735 th->th.th_new_place = place;
4736 th->th.th_last_place = last;
Jonathan Peyton30419822017-05-12 18:01:32 +00004737
Paul Osmialowskia0162792017-08-10 23:04:11 +00004738 KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
4739 "partition = [%d,%d], spacing = %.4f\n",
4740 __kmp_gtid_from_thread(team->t.t_threads[f]),
4741 team->t.t_id, f, th->th.th_new_place,
4742 th->th.th_first_place, th->th.th_last_place,
4743 spacing));
4744 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004745 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004746 }
4747 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
4748 } else {
4749 int S, rem, gap, s_count;
4750 S = n_th / n_places;
4751 s_count = 0;
4752 rem = n_th - (S * n_places);
4753 gap = rem > 0 ? n_places / rem : n_places;
4754 int place = masters_place;
4755 int gap_ct = gap;
4756 thidx = n_th;
4757 if (update_master_only == 1)
4758 thidx = 1;
4759 for (f = 0; f < thidx; f++) {
4760 kmp_info_t *th = team->t.t_threads[f];
4761 KMP_DEBUG_ASSERT(th != NULL);
4762
4763 th->th.th_first_place = place;
4764 th->th.th_last_place = place;
4765 th->th.th_new_place = place;
4766 s_count++;
4767
4768 if ((s_count == S) && rem && (gap_ct == gap)) {
4769 // do nothing, add an extra thread to place on next iteration
4770 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4771 // we added an extra thread to this place; move on to next place
4772 if (place == last_place) {
4773 place = first_place;
4774 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4775 place = 0;
4776 } else {
4777 place++;
4778 }
4779 s_count = 0;
4780 gap_ct = 1;
4781 rem--;
4782 } else if (s_count == S) { // place is full; don't add extra thread
4783 if (place == last_place) {
4784 place = first_place;
4785 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4786 place = 0;
4787 } else {
4788 place++;
4789 }
4790 gap_ct++;
4791 s_count = 0;
4792 }
4793
4794 KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
4795 "partition = [%d,%d]\n",
4796 __kmp_gtid_from_thread(team->t.t_threads[f]),
4797 team->t.t_id, f, th->th.th_new_place,
4798 th->th.th_first_place, th->th.th_last_place));
4799 }
4800 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
4801 }
4802 } break;
4803
4804 default:
4805 break;
4806 }
4807
4808 KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004809}
4810
Alp Toker98758b02014-03-02 04:12:06 +00004811#endif /* OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004812
Jonathan Peyton30419822017-05-12 18:01:32 +00004813/* allocate a new team data structure to use. take one off of the free pool if
4814 available */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004815kmp_team_t *
Jonathan Peyton30419822017-05-12 18:01:32 +00004816__kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00004817#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00004818 ompt_parallel_id_t ompt_parallel_id,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00004819#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00004820#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00004821 kmp_proc_bind_t new_proc_bind,
Jim Cownie5e8470a2013-09-27 10:38:44 +00004822#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004823 kmp_internal_control_t *new_icvs,
4824 int argc USE_NESTED_HOT_ARG(kmp_info_t *master)) {
4825 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team);
4826 int f;
4827 kmp_team_t *team;
4828 int use_hot_team = !root->r.r_active;
4829 int level = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004830
Jonathan Peyton30419822017-05-12 18:01:32 +00004831 KA_TRACE(20, ("__kmp_allocate_team: called\n"));
4832 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0);
4833 KMP_DEBUG_ASSERT(max_nproc >= new_nproc);
4834 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00004835
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004836#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004837 kmp_hot_team_ptr_t *hot_teams;
4838 if (master) {
4839 team = master->th.th_team;
4840 level = team->t.t_active_level;
4841 if (master->th.th_teams_microtask) { // in teams construct?
4842 if (master->th.th_teams_size.nteams > 1 &&
4843 ( // #teams > 1
4844 team->t.t_pkfn ==
4845 (microtask_t)__kmp_teams_master || // inner fork of the teams
4846 master->th.th_teams_level <
4847 team->t.t_level)) { // or nested parallel inside the teams
4848 ++level; // not increment if #teams==1, or for outer fork of the teams;
4849 // increment otherwise
4850 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004851 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004852 hot_teams = master->th.th_hot_teams;
4853 if (level < __kmp_hot_teams_max_level && hot_teams &&
4854 hot_teams[level]
4855 .hot_team) { // hot team has already been allocated for given level
4856 use_hot_team = 1;
4857 } else {
4858 use_hot_team = 0;
4859 }
4860 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004861#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004862 // Optimization to use a "hot" team
4863 if (use_hot_team && new_nproc > 1) {
4864 KMP_DEBUG_ASSERT(new_nproc == max_nproc);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004865#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004866 team = hot_teams[level].hot_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004867#else
Jonathan Peyton30419822017-05-12 18:01:32 +00004868 team = root->r.r_hot_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004869#endif
4870#if KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00004871 if (__kmp_tasking_mode != tskm_immediate_exec) {
4872 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p "
4873 "task_team[1] = %p before reinit\n",
4874 team->t.t_task_team[0], team->t.t_task_team[1]));
4875 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00004876#endif
4877
Jonathan Peyton30419822017-05-12 18:01:32 +00004878 // Has the number of threads changed?
4879 /* Let's assume the most common case is that the number of threads is
4880 unchanged, and put that case first. */
4881 if (team->t.t_nproc == new_nproc) { // Check changes in number of threads
4882 KA_TRACE(20, ("__kmp_allocate_team: reusing hot team\n"));
4883 // This case can mean that omp_set_num_threads() was called and the hot
Jonathan Peyton642688b2017-06-01 16:46:36 +00004884 // team size was already reduced, so we check the special flag
Jonathan Peyton30419822017-05-12 18:01:32 +00004885 if (team->t.t_size_changed == -1) {
4886 team->t.t_size_changed = 1;
4887 } else {
4888 KMP_CHECK_UPDATE(team->t.t_size_changed, 0);
4889 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004890
Jonathan Peyton30419822017-05-12 18:01:32 +00004891 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
4892 kmp_r_sched_t new_sched = new_icvs->sched;
4893 if (team->t.t_sched.r_sched_type != new_sched.r_sched_type ||
4894 team->t.t_sched.chunk != new_sched.chunk)
4895 team->t.t_sched =
4896 new_sched; // set master's schedule as new run-time schedule
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004897
Jonathan Peyton30419822017-05-12 18:01:32 +00004898 __kmp_reinitialize_team(team, new_icvs,
4899 root->r.r_uber_thread->th.th_ident);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004900
Jonathan Peyton30419822017-05-12 18:01:32 +00004901 KF_TRACE(10, ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0,
4902 team->t.t_threads[0], team));
4903 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004904
4905#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00004906#if KMP_AFFINITY_SUPPORTED
4907 if ((team->t.t_size_changed == 0) &&
4908 (team->t.t_proc_bind == new_proc_bind)) {
4909 if (new_proc_bind == proc_bind_spread) {
4910 __kmp_partition_places(
4911 team, 1); // add flag to update only master for spread
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004912 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004913 KA_TRACE(200, ("__kmp_allocate_team: reusing hot team #%d bindings: "
4914 "proc_bind = %d, partition = [%d,%d]\n",
4915 team->t.t_id, new_proc_bind, team->t.t_first_place,
4916 team->t.t_last_place));
4917 } else {
4918 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
4919 __kmp_partition_places(team);
4920 }
4921#else
4922 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
4923#endif /* KMP_AFFINITY_SUPPORTED */
4924#endif /* OMP_40_ENABLED */
4925 } else if (team->t.t_nproc > new_nproc) {
4926 KA_TRACE(20,
4927 ("__kmp_allocate_team: decreasing hot team thread count to %d\n",
4928 new_nproc));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004929
Jonathan Peyton30419822017-05-12 18:01:32 +00004930 team->t.t_size_changed = 1;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004931#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004932 if (__kmp_hot_teams_mode == 0) {
4933 // AC: saved number of threads should correspond to team's value in this
4934 // mode, can be bigger in mode 1, when hot team has threads in reserve
4935 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc);
4936 hot_teams[level].hot_team_nth = new_nproc;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004937#endif // KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004938 /* release the extra threads we don't need any more */
4939 for (f = new_nproc; f < team->t.t_nproc; f++) {
4940 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
4941 if (__kmp_tasking_mode != tskm_immediate_exec) {
4942 // When decreasing team size, threads no longer in the team should
4943 // unref task team.
4944 team->t.t_threads[f]->th.th_task_team = NULL;
4945 }
4946 __kmp_free_thread(team->t.t_threads[f]);
4947 team->t.t_threads[f] = NULL;
4948 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004949#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004950 } // (__kmp_hot_teams_mode == 0)
4951 else {
4952 // When keeping extra threads in team, switch threads to wait on own
4953 // b_go flag
4954 for (f = new_nproc; f < team->t.t_nproc; ++f) {
4955 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
4956 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar;
4957 for (int b = 0; b < bs_last_barrier; ++b) {
4958 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) {
4959 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
Andrey Churbanovd6e1d7e2016-08-11 13:04:00 +00004960 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004961 KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0);
4962 }
4963 }
4964 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004965#endif // KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004966 team->t.t_nproc = new_nproc;
4967 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
4968 if (team->t.t_sched.r_sched_type != new_icvs->sched.r_sched_type ||
4969 team->t.t_sched.chunk != new_icvs->sched.chunk)
4970 team->t.t_sched = new_icvs->sched;
4971 __kmp_reinitialize_team(team, new_icvs,
4972 root->r.r_uber_thread->th.th_ident);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004973
Jonathan Peyton30419822017-05-12 18:01:32 +00004974 /* update the remaining threads */
4975 for (f = 0; f < new_nproc; ++f) {
4976 team->t.t_threads[f]->th.th_team_nproc = new_nproc;
4977 }
4978 // restore the current task state of the master thread: should be the
4979 // implicit task
4980 KF_TRACE(10, ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0,
4981 team->t.t_threads[0], team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004982
Jonathan Peyton30419822017-05-12 18:01:32 +00004983 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004984
4985#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00004986 for (f = 0; f < team->t.t_nproc; f++) {
4987 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
4988 team->t.t_threads[f]->th.th_team_nproc ==
4989 team->t.t_nproc);
4990 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00004991#endif
4992
4993#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00004994 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
4995#if KMP_AFFINITY_SUPPORTED
4996 __kmp_partition_places(team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004997#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004998#endif
4999 } else { // team->t.t_nproc < new_nproc
Alp Toker98758b02014-03-02 04:12:06 +00005000#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00005001 kmp_affin_mask_t *old_mask;
5002 if (KMP_AFFINITY_CAPABLE()) {
5003 KMP_CPU_ALLOC(old_mask);
5004 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005005#endif
5006
Jonathan Peyton30419822017-05-12 18:01:32 +00005007 KA_TRACE(20,
5008 ("__kmp_allocate_team: increasing hot team thread count to %d\n",
5009 new_nproc));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005010
Jonathan Peyton30419822017-05-12 18:01:32 +00005011 team->t.t_size_changed = 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005012
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005013#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00005014 int avail_threads = hot_teams[level].hot_team_nth;
5015 if (new_nproc < avail_threads)
5016 avail_threads = new_nproc;
5017 kmp_info_t **other_threads = team->t.t_threads;
5018 for (f = team->t.t_nproc; f < avail_threads; ++f) {
5019 // Adjust barrier data of reserved threads (if any) of the team
5020 // Other data will be set in __kmp_initialize_info() below.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005021 int b;
Jonathan Peyton30419822017-05-12 18:01:32 +00005022 kmp_balign_t *balign = other_threads[f]->th.th_bar;
5023 for (b = 0; b < bs_last_barrier; ++b) {
5024 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5025 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00005026#if USE_DEBUGGER
Jonathan Peyton30419822017-05-12 18:01:32 +00005027 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00005028#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005029 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005030 }
5031 if (hot_teams[level].hot_team_nth >= new_nproc) {
5032 // we have all needed threads in reserve, no need to allocate any
5033 // this only possible in mode 1, cannot have reserved threads in mode 0
5034 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1);
5035 team->t.t_nproc = new_nproc; // just get reserved threads involved
5036 } else {
5037 // we may have some threads in reserve, but not enough
5038 team->t.t_nproc =
5039 hot_teams[level]
5040 .hot_team_nth; // get reserved threads involved if any
5041 hot_teams[level].hot_team_nth = new_nproc; // adjust hot team max size
5042#endif // KMP_NESTED_HOT_TEAMS
5043 if (team->t.t_max_nproc < new_nproc) {
5044 /* reallocate larger arrays */
5045 __kmp_reallocate_team_arrays(team, new_nproc);
5046 __kmp_reinitialize_team(team, new_icvs, NULL);
5047 }
5048
5049#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
5050 /* Temporarily set full mask for master thread before creation of
5051 workers. The reason is that workers inherit the affinity from master,
5052 so if a lot of workers are created on the single core quickly, they
5053 don't get a chance to set their own affinity for a long time. */
5054 __kmp_set_thread_affinity_mask_full_tmp(old_mask);
5055#endif
5056
5057 /* allocate new threads for the hot team */
5058 for (f = team->t.t_nproc; f < new_nproc; f++) {
5059 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f);
5060 KMP_DEBUG_ASSERT(new_worker);
5061 team->t.t_threads[f] = new_worker;
5062
5063 KA_TRACE(20,
5064 ("__kmp_allocate_team: team %d init T#%d arrived: "
5065 "join=%llu, plain=%llu\n",
5066 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f,
5067 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
5068 team->t.t_bar[bs_plain_barrier].b_arrived));
5069
5070 { // Initialize barrier data for new threads.
5071 int b;
5072 kmp_balign_t *balign = new_worker->th.th_bar;
5073 for (b = 0; b < bs_last_barrier; ++b) {
5074 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5075 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag !=
5076 KMP_BARRIER_PARENT_FLAG);
5077#if USE_DEBUGGER
5078 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5079#endif
5080 }
5081 }
5082 }
5083
5084#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
5085 if (KMP_AFFINITY_CAPABLE()) {
5086 /* Restore initial master thread's affinity mask */
5087 __kmp_set_system_affinity(old_mask, TRUE);
5088 KMP_CPU_FREE(old_mask);
5089 }
5090#endif
5091#if KMP_NESTED_HOT_TEAMS
5092 } // end of check of t_nproc vs. new_nproc vs. hot_team_nth
5093#endif // KMP_NESTED_HOT_TEAMS
5094 /* make sure everyone is syncronized */
5095 int old_nproc = team->t.t_nproc; // save old value and use to update only
5096 // new threads below
5097 __kmp_initialize_team(team, new_nproc, new_icvs,
5098 root->r.r_uber_thread->th.th_ident);
5099
5100 /* reinitialize the threads */
5101 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc);
5102 for (f = 0; f < team->t.t_nproc; ++f)
5103 __kmp_initialize_info(team->t.t_threads[f], team, f,
5104 __kmp_gtid_from_tid(f, team));
5105 if (level) { // set th_task_state for new threads in nested hot team
5106 // __kmp_initialize_info() no longer zeroes th_task_state, so we should
5107 // only need to set the th_task_state for the new threads. th_task_state
5108 // for master thread will not be accurate until after this in
5109 // __kmp_fork_call(), so we look to the master's memo_stack to get the
5110 // correct value.
5111 for (f = old_nproc; f < team->t.t_nproc; ++f)
5112 team->t.t_threads[f]->th.th_task_state =
5113 team->t.t_threads[0]->th.th_task_state_memo_stack[level];
5114 } else { // set th_task_state for new threads in non-nested hot team
5115 int old_state =
5116 team->t.t_threads[0]->th.th_task_state; // copy master's state
5117 for (f = old_nproc; f < team->t.t_nproc; ++f)
5118 team->t.t_threads[f]->th.th_task_state = old_state;
5119 }
5120
5121#ifdef KMP_DEBUG
5122 for (f = 0; f < team->t.t_nproc; ++f) {
5123 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
5124 team->t.t_threads[f]->th.th_team_nproc ==
5125 team->t.t_nproc);
5126 }
5127#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005128
5129#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00005130 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5131#if KMP_AFFINITY_SUPPORTED
5132 __kmp_partition_places(team);
5133#endif
5134#endif
5135 } // Check changes in number of threads
5136
5137#if OMP_40_ENABLED
5138 kmp_info_t *master = team->t.t_threads[0];
5139 if (master->th.th_teams_microtask) {
5140 for (f = 1; f < new_nproc; ++f) {
5141 // propagate teams construct specific info to workers
5142 kmp_info_t *thr = team->t.t_threads[f];
5143 thr->th.th_teams_microtask = master->th.th_teams_microtask;
5144 thr->th.th_teams_level = master->th.th_teams_level;
5145 thr->th.th_teams_size = master->th.th_teams_size;
5146 }
5147 }
5148#endif /* OMP_40_ENABLED */
5149#if KMP_NESTED_HOT_TEAMS
5150 if (level) {
5151 // Sync barrier state for nested hot teams, not needed for outermost hot
5152 // team.
5153 for (f = 1; f < new_nproc; ++f) {
5154 kmp_info_t *thr = team->t.t_threads[f];
5155 int b;
5156 kmp_balign_t *balign = thr->th.th_bar;
5157 for (b = 0; b < bs_last_barrier; ++b) {
5158 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5159 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
5160#if USE_DEBUGGER
5161 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5162#endif
5163 }
5164 }
5165 }
5166#endif // KMP_NESTED_HOT_TEAMS
5167
5168 /* reallocate space for arguments if necessary */
5169 __kmp_alloc_argv_entries(argc, team, TRUE);
5170 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5171 // The hot team re-uses the previous task team,
5172 // if untouched during the previous release->gather phase.
5173
5174 KF_TRACE(10, (" hot_team = %p\n", team));
5175
5176#if KMP_DEBUG
5177 if (__kmp_tasking_mode != tskm_immediate_exec) {
5178 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p "
5179 "task_team[1] = %p after reinit\n",
5180 team->t.t_task_team[0], team->t.t_task_team[1]));
5181 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005182#endif
5183
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005184#if OMPT_SUPPORT
5185 __ompt_team_assign_id(team, ompt_parallel_id);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005186#endif
5187
Jim Cownie5e8470a2013-09-27 10:38:44 +00005188 KMP_MB();
5189
Jim Cownie5e8470a2013-09-27 10:38:44 +00005190 return team;
Jonathan Peyton30419822017-05-12 18:01:32 +00005191 }
5192
5193 /* next, let's try to take one from the team pool */
5194 KMP_MB();
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00005195 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) {
Jonathan Peyton30419822017-05-12 18:01:32 +00005196 /* TODO: consider resizing undersized teams instead of reaping them, now
5197 that we have a resizing mechanism */
5198 if (team->t.t_max_nproc >= max_nproc) {
5199 /* take this team from the team pool */
5200 __kmp_team_pool = team->t.t_next_pool;
5201
5202 /* setup the team for fresh use */
5203 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5204
5205 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and "
5206 "task_team[1] %p to NULL\n",
5207 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5208 team->t.t_task_team[0] = NULL;
5209 team->t.t_task_team[1] = NULL;
5210
5211 /* reallocate space for arguments if necessary */
5212 __kmp_alloc_argv_entries(argc, team, TRUE);
5213 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5214
5215 KA_TRACE(
5216 20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5217 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5218 { // Initialize barrier data.
5219 int b;
5220 for (b = 0; b < bs_last_barrier; ++b) {
5221 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5222#if USE_DEBUGGER
5223 team->t.t_bar[b].b_master_arrived = 0;
5224 team->t.t_bar[b].b_team_arrived = 0;
5225#endif
5226 }
5227 }
5228
5229#if OMP_40_ENABLED
5230 team->t.t_proc_bind = new_proc_bind;
5231#endif
5232
5233 KA_TRACE(20, ("__kmp_allocate_team: using team from pool %d.\n",
5234 team->t.t_id));
5235
5236#if OMPT_SUPPORT
5237 __ompt_team_assign_id(team, ompt_parallel_id);
5238#endif
5239
5240 KMP_MB();
5241
5242 return team;
5243 }
5244
5245/* reap team if it is too small, then loop back and check the next one */
5246// not sure if this is wise, but, will be redone during the hot-teams rewrite.
5247/* TODO: Use technique to find the right size hot-team, don't reap them */
5248 team = __kmp_reap_team(team);
5249 __kmp_team_pool = team;
5250 }
5251
5252 /* nothing available in the pool, no matter, make a new team! */
5253 KMP_MB();
5254 team = (kmp_team_t *)__kmp_allocate(sizeof(kmp_team_t));
5255
5256 /* and set it up */
5257 team->t.t_max_nproc = max_nproc;
5258 /* NOTE well, for some reason allocating one big buffer and dividing it up
5259 seems to really hurt performance a lot on the P4, so, let's not use this */
5260 __kmp_allocate_team_arrays(team, max_nproc);
5261
5262 KA_TRACE(20, ("__kmp_allocate_team: making a new team\n"));
5263 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5264
5265 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] "
5266 "%p to NULL\n",
5267 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5268 team->t.t_task_team[0] = NULL; // to be removed, as __kmp_allocate zeroes
5269 // memory, no need to duplicate
5270 team->t.t_task_team[1] = NULL; // to be removed, as __kmp_allocate zeroes
5271 // memory, no need to duplicate
5272
5273 if (__kmp_storage_map) {
5274 __kmp_print_team_storage_map("team", team, team->t.t_id, new_nproc);
5275 }
5276
5277 /* allocate space for arguments */
5278 __kmp_alloc_argv_entries(argc, team, FALSE);
5279 team->t.t_argc = argc;
5280
5281 KA_TRACE(20,
5282 ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5283 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5284 { // Initialize barrier data.
5285 int b;
5286 for (b = 0; b < bs_last_barrier; ++b) {
5287 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5288#if USE_DEBUGGER
5289 team->t.t_bar[b].b_master_arrived = 0;
5290 team->t.t_bar[b].b_team_arrived = 0;
5291#endif
5292 }
5293 }
5294
5295#if OMP_40_ENABLED
5296 team->t.t_proc_bind = new_proc_bind;
5297#endif
5298
5299#if OMPT_SUPPORT
5300 __ompt_team_assign_id(team, ompt_parallel_id);
5301 team->t.ompt_serialized_team_info = NULL;
5302#endif
5303
5304 KMP_MB();
5305
5306 KA_TRACE(20, ("__kmp_allocate_team: done creating a new team %d.\n",
5307 team->t.t_id));
5308
5309 return team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005310}
5311
5312/* TODO implement hot-teams at all levels */
5313/* TODO implement lazy thread release on demand (disband request) */
5314
5315/* free the team. return it to the team pool. release all the threads
5316 * associated with it */
Jonathan Peyton30419822017-05-12 18:01:32 +00005317void __kmp_free_team(kmp_root_t *root,
5318 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) {
5319 int f;
5320 KA_TRACE(20, ("__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(),
5321 team->t.t_id));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005322
Jonathan Peyton30419822017-05-12 18:01:32 +00005323 /* verify state */
5324 KMP_DEBUG_ASSERT(root);
5325 KMP_DEBUG_ASSERT(team);
5326 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc);
5327 KMP_DEBUG_ASSERT(team->t.t_threads);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005328
Jonathan Peyton30419822017-05-12 18:01:32 +00005329 int use_hot_team = team == root->r.r_hot_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005330#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00005331 int level;
5332 kmp_hot_team_ptr_t *hot_teams;
5333 if (master) {
5334 level = team->t.t_active_level - 1;
5335 if (master->th.th_teams_microtask) { // in teams construct?
5336 if (master->th.th_teams_size.nteams > 1) {
5337 ++level; // level was not increased in teams construct for
5338 // team_of_masters
5339 }
5340 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
5341 master->th.th_teams_level == team->t.t_level) {
5342 ++level; // level was not increased in teams construct for
5343 // team_of_workers before the parallel
5344 } // team->t.t_level will be increased inside parallel
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005345 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005346 hot_teams = master->th.th_hot_teams;
5347 if (level < __kmp_hot_teams_max_level) {
5348 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team);
5349 use_hot_team = 1;
5350 }
5351 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005352#endif // KMP_NESTED_HOT_TEAMS
5353
Jonathan Peyton30419822017-05-12 18:01:32 +00005354 /* team is done working */
5355 TCW_SYNC_PTR(team->t.t_pkfn,
5356 NULL); // Important for Debugging Support Library.
5357 team->t.t_copyin_counter = 0; // init counter for possible reuse
5358 // Do not reset pointer to parent team to NULL for hot teams.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005359
Jonathan Peyton30419822017-05-12 18:01:32 +00005360 /* if we are non-hot team, release our threads */
5361 if (!use_hot_team) {
5362 if (__kmp_tasking_mode != tskm_immediate_exec) {
5363 // Wait for threads to reach reapable state
5364 for (f = 1; f < team->t.t_nproc; ++f) {
5365 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5366 kmp_info_t *th = team->t.t_threads[f];
5367 volatile kmp_uint32 *state = &th->th.th_reap_state;
5368 while (*state != KMP_SAFE_TO_REAP) {
Andrey Churbanov581490e2017-02-06 18:53:32 +00005369#if KMP_OS_WINDOWS
Jonathan Peyton30419822017-05-12 18:01:32 +00005370 // On Windows a thread can be killed at any time, check this
5371 DWORD ecode;
5372 if (!__kmp_is_thread_alive(th, &ecode)) {
5373 *state = KMP_SAFE_TO_REAP; // reset the flag for dead thread
5374 break;
5375 }
Andrey Churbanov581490e2017-02-06 18:53:32 +00005376#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00005377 // first check if thread is sleeping
5378 kmp_flag_64 fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th);
5379 if (fl.is_sleeping())
5380 fl.resume(__kmp_gtid_from_thread(th));
5381 KMP_CPU_PAUSE();
5382 }
5383 }
Andrey Churbanov581490e2017-02-06 18:53:32 +00005384
Jonathan Peyton30419822017-05-12 18:01:32 +00005385 // Delete task teams
5386 int tt_idx;
5387 for (tt_idx = 0; tt_idx < 2; ++tt_idx) {
5388 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
5389 if (task_team != NULL) {
5390 for (f = 0; f < team->t.t_nproc;
5391 ++f) { // Have all threads unref task teams
5392 team->t.t_threads[f]->th.th_task_team = NULL;
5393 }
5394 KA_TRACE(
5395 20,
5396 ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n",
5397 __kmp_get_gtid(), task_team, team->t.t_id));
Jonathan Peytonbaaccfa2015-11-16 22:48:41 +00005398#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00005399 __kmp_free_task_team(master, task_team);
Jonathan Peytonbaaccfa2015-11-16 22:48:41 +00005400#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00005401 team->t.t_task_team[tt_idx] = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005402 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005403 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005404 }
5405
Jonathan Peyton30419822017-05-12 18:01:32 +00005406 // Reset pointer to parent team only for non-hot teams.
5407 team->t.t_parent = NULL;
5408 team->t.t_level = 0;
5409 team->t.t_active_level = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005410
Jonathan Peyton30419822017-05-12 18:01:32 +00005411 /* free the worker threads */
5412 for (f = 1; f < team->t.t_nproc; ++f) {
5413 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5414 __kmp_free_thread(team->t.t_threads[f]);
5415 team->t.t_threads[f] = NULL;
5416 }
5417
5418 /* put the team back in the team pool */
5419 /* TODO limit size of team pool, call reap_team if pool too large */
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00005420 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool);
Jonathan Peyton30419822017-05-12 18:01:32 +00005421 __kmp_team_pool = (volatile kmp_team_t *)team;
5422 }
5423
5424 KMP_MB();
5425}
Jim Cownie5e8470a2013-09-27 10:38:44 +00005426
5427/* reap the team. destroy it, reclaim all its resources and free its memory */
Jonathan Peyton30419822017-05-12 18:01:32 +00005428kmp_team_t *__kmp_reap_team(kmp_team_t *team) {
5429 kmp_team_t *next_pool = team->t.t_next_pool;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005430
Jonathan Peyton30419822017-05-12 18:01:32 +00005431 KMP_DEBUG_ASSERT(team);
5432 KMP_DEBUG_ASSERT(team->t.t_dispatch);
5433 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
5434 KMP_DEBUG_ASSERT(team->t.t_threads);
5435 KMP_DEBUG_ASSERT(team->t.t_argv);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005436
Jonathan Peyton30419822017-05-12 18:01:32 +00005437 /* TODO clean the threads that are a part of this? */
Jim Cownie5e8470a2013-09-27 10:38:44 +00005438
Jonathan Peyton30419822017-05-12 18:01:32 +00005439 /* free stuff */
5440 __kmp_free_team_arrays(team);
5441 if (team->t.t_argv != &team->t.t_inline_argv[0])
5442 __kmp_free((void *)team->t.t_argv);
5443 __kmp_free(team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005444
Jonathan Peyton30419822017-05-12 18:01:32 +00005445 KMP_MB();
5446 return next_pool;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005447}
5448
Jim Cownie5e8470a2013-09-27 10:38:44 +00005449// Free the thread. Don't reap it, just place it on the pool of available
5450// threads.
5451//
5452// Changes for Quad issue 527845: We need a predictable OMP tid <-> gtid
5453// binding for the affinity mechanism to be useful.
5454//
5455// Now, we always keep the free list (__kmp_thread_pool) sorted by gtid.
5456// However, we want to avoid a potential performance problem by always
5457// scanning through the list to find the correct point at which to insert
5458// the thread (potential N**2 behavior). To do this we keep track of the
5459// last place a thread struct was inserted (__kmp_thread_pool_insert_pt).
5460// With single-level parallelism, threads will always be added to the tail
5461// of the list, kept track of by __kmp_thread_pool_insert_pt. With nested
5462// parallelism, all bets are off and we may need to scan through the entire
5463// free list.
5464//
5465// This change also has a potentially large performance benefit, for some
5466// applications. Previously, as threads were freed from the hot team, they
5467// would be placed back on the free list in inverse order. If the hot team
5468// grew back to it's original size, then the freed thread would be placed
5469// back on the hot team in reverse order. This could cause bad cache
5470// locality problems on programs where the size of the hot team regularly
5471// grew and shrunk.
5472//
5473// Now, for single-level parallelism, the OMP tid is alway == gtid.
Jonathan Peyton30419822017-05-12 18:01:32 +00005474void __kmp_free_thread(kmp_info_t *this_th) {
5475 int gtid;
5476 kmp_info_t **scan;
Jonathan Peytonf4392462017-07-27 20:58:41 +00005477 kmp_root_t *root = this_th->th.th_root;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005478
Jonathan Peyton30419822017-05-12 18:01:32 +00005479 KA_TRACE(20, ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n",
5480 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005481
Jonathan Peyton30419822017-05-12 18:01:32 +00005482 KMP_DEBUG_ASSERT(this_th);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005483
Jonathan Peyton30419822017-05-12 18:01:32 +00005484 // When moving thread to pool, switch thread to wait on own b_go flag, and
5485 // uninitialized (NULL team).
5486 int b;
5487 kmp_balign_t *balign = this_th->th.th_bar;
5488 for (b = 0; b < bs_last_barrier; ++b) {
5489 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG)
5490 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
5491 balign[b].bb.team = NULL;
5492 balign[b].bb.leaf_kids = 0;
5493 }
5494 this_th->th.th_task_state = 0;
5495
5496 /* put thread back on the free pool */
5497 TCW_PTR(this_th->th.th_team, NULL);
5498 TCW_PTR(this_th->th.th_root, NULL);
5499 TCW_PTR(this_th->th.th_dispatch, NULL); /* NOT NEEDED */
5500
5501 // If the __kmp_thread_pool_insert_pt is already past the new insert
5502 // point, then we need to re-scan the entire list.
5503 gtid = this_th->th.th_info.ds.ds_gtid;
5504 if (__kmp_thread_pool_insert_pt != NULL) {
5505 KMP_DEBUG_ASSERT(__kmp_thread_pool != NULL);
5506 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) {
5507 __kmp_thread_pool_insert_pt = NULL;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005508 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005509 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005510
Jonathan Peyton30419822017-05-12 18:01:32 +00005511 // Scan down the list to find the place to insert the thread.
5512 // scan is the address of a link in the list, possibly the address of
5513 // __kmp_thread_pool itself.
5514 //
5515 // In the absence of nested parallism, the for loop will have 0 iterations.
5516 if (__kmp_thread_pool_insert_pt != NULL) {
5517 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool);
5518 } else {
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00005519 scan = CCAST(kmp_info_t **, &__kmp_thread_pool);
Jonathan Peyton30419822017-05-12 18:01:32 +00005520 }
5521 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid);
5522 scan = &((*scan)->th.th_next_pool))
5523 ;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005524
Jonathan Peyton30419822017-05-12 18:01:32 +00005525 // Insert the new element on the list, and set __kmp_thread_pool_insert_pt
5526 // to its address.
5527 TCW_PTR(this_th->th.th_next_pool, *scan);
5528 __kmp_thread_pool_insert_pt = *scan = this_th;
5529 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) ||
5530 (this_th->th.th_info.ds.ds_gtid <
5531 this_th->th.th_next_pool->th.th_info.ds.ds_gtid));
5532 TCW_4(this_th->th.th_in_pool, TRUE);
5533 __kmp_thread_pool_nth++;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005534
Jonathan Peyton30419822017-05-12 18:01:32 +00005535 TCW_4(__kmp_nth, __kmp_nth - 1);
Jonathan Peytonf4392462017-07-27 20:58:41 +00005536 root->r.r_cg_nthreads--;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005537
5538#ifdef KMP_ADJUST_BLOCKTIME
Jonathan Peyton30419822017-05-12 18:01:32 +00005539 /* Adjust blocktime back to user setting or default if necessary */
5540 /* Middle initialization might never have occurred */
5541 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5542 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
5543 if (__kmp_nth <= __kmp_avail_proc) {
5544 __kmp_zero_bt = FALSE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005545 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005546 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005547#endif /* KMP_ADJUST_BLOCKTIME */
5548
Jonathan Peyton30419822017-05-12 18:01:32 +00005549 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00005550}
5551
Jim Cownie5e8470a2013-09-27 10:38:44 +00005552/* ------------------------------------------------------------------------ */
5553
Jonathan Peyton30419822017-05-12 18:01:32 +00005554void *__kmp_launch_thread(kmp_info_t *this_thr) {
5555 int gtid = this_thr->th.th_info.ds.ds_gtid;
5556 /* void *stack_data;*/
5557 kmp_team_t *(*volatile pteam);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005558
Jonathan Peyton30419822017-05-12 18:01:32 +00005559 KMP_MB();
5560 KA_TRACE(10, ("__kmp_launch_thread: T#%d start\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005561
Jonathan Peyton30419822017-05-12 18:01:32 +00005562 if (__kmp_env_consistency_check) {
5563 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid); // ATT: Memory leak?
5564 }
5565
5566#if OMPT_SUPPORT
5567 if (ompt_enabled) {
5568 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5569 this_thr->th.ompt_thread_info.wait_id = 0;
5570 this_thr->th.ompt_thread_info.idle_frame = __builtin_frame_address(0);
5571 if (ompt_callbacks.ompt_callback(ompt_event_thread_begin)) {
5572 __ompt_thread_begin(ompt_thread_worker, gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005573 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005574 }
5575#endif
5576
5577 /* This is the place where threads wait for work */
5578 while (!TCR_4(__kmp_global.g.g_done)) {
5579 KMP_DEBUG_ASSERT(this_thr == __kmp_threads[gtid]);
5580 KMP_MB();
5581
5582 /* wait for work to do */
5583 KA_TRACE(20, ("__kmp_launch_thread: T#%d waiting for work\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005584
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005585#if OMPT_SUPPORT
Jonathan Peytonb68a85d2015-09-21 18:11:22 +00005586 if (ompt_enabled) {
Jonathan Peyton30419822017-05-12 18:01:32 +00005587 this_thr->th.ompt_thread_info.state = ompt_state_idle;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005588 }
5589#endif
5590
Jonathan Peyton30419822017-05-12 18:01:32 +00005591 /* No tid yet since not part of a team */
5592 __kmp_fork_barrier(gtid, KMP_GTID_DNE);
5593
5594#if OMPT_SUPPORT
5595 if (ompt_enabled) {
5596 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5597 }
5598#endif
5599
5600 pteam = (kmp_team_t * (*))(&this_thr->th.th_team);
5601
5602 /* have we been allocated? */
5603 if (TCR_SYNC_PTR(*pteam) && !TCR_4(__kmp_global.g.g_done)) {
5604#if OMPT_SUPPORT
5605 ompt_task_info_t *task_info;
5606 ompt_parallel_id_t my_parallel_id;
5607 if (ompt_enabled) {
5608 task_info = __ompt_get_taskinfo(0);
5609 my_parallel_id = (*pteam)->t.ompt_team_info.parallel_id;
5610 }
5611#endif
5612 /* we were just woken up, so run our new task */
5613 if (TCR_SYNC_PTR((*pteam)->t.t_pkfn) != NULL) {
5614 int rc;
5615 KA_TRACE(20,
5616 ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n",
5617 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5618 (*pteam)->t.t_pkfn));
5619
5620 updateHWFPControl(*pteam);
5621
5622#if OMPT_SUPPORT
5623 if (ompt_enabled) {
5624 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
5625 // Initialize OMPT task id for implicit task.
5626 int tid = __kmp_tid_from_gtid(gtid);
5627 task_info->task_id = __ompt_task_id_new(tid);
5628 }
5629#endif
5630
5631 {
5632 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
5633 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
5634 rc = (*pteam)->t.t_invoke(gtid);
5635 }
5636 KMP_ASSERT(rc);
5637
5638#if OMPT_SUPPORT
5639 if (ompt_enabled) {
5640 /* no frame set while outside task */
5641 task_info->frame.exit_runtime_frame = NULL;
5642
5643 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5644 }
5645#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005646 KMP_MB();
Jonathan Peyton30419822017-05-12 18:01:32 +00005647 KA_TRACE(20, ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n",
5648 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5649 (*pteam)->t.t_pkfn));
5650 }
5651 /* join barrier after parallel region */
5652 __kmp_join_barrier(gtid);
Jonathan Peytonb4c73d82016-01-26 21:45:21 +00005653#if OMPT_SUPPORT && OMPT_TRACE
Jonathan Peyton30419822017-05-12 18:01:32 +00005654 if (ompt_enabled) {
5655 if (ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)) {
5656 // don't access *pteam here: it may have already been freed
5657 // by the master thread behind the barrier (possible race)
5658 ompt_callbacks.ompt_callback(ompt_event_implicit_task_end)(
5659 my_parallel_id, task_info->task_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005660 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005661 task_info->frame.exit_runtime_frame = NULL;
5662 task_info->task_id = 0;
5663 }
5664#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005665 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005666 }
5667 TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005668
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005669#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00005670 if (ompt_enabled && ompt_callbacks.ompt_callback(ompt_event_thread_end)) {
5671 __ompt_thread_end(ompt_thread_worker, gtid);
5672 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005673#endif
5674
Jonathan Peyton30419822017-05-12 18:01:32 +00005675 this_thr->th.th_task_team = NULL;
5676 /* run the destructors for the threadprivate data for this thread */
5677 __kmp_common_destroy_gtid(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005678
Jonathan Peyton30419822017-05-12 18:01:32 +00005679 KA_TRACE(10, ("__kmp_launch_thread: T#%d done\n", gtid));
5680 KMP_MB();
5681 return this_thr;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005682}
5683
5684/* ------------------------------------------------------------------------ */
Jim Cownie5e8470a2013-09-27 10:38:44 +00005685
Jonathan Peyton30419822017-05-12 18:01:32 +00005686void __kmp_internal_end_dest(void *specific_gtid) {
5687#if KMP_COMPILER_ICC
5688#pragma warning(push)
5689#pragma warning(disable : 810) // conversion from "void *" to "int" may lose
5690// significant bits
5691#endif
5692 // Make sure no significant bits are lost
5693 int gtid = (kmp_intptr_t)specific_gtid - 1;
5694#if KMP_COMPILER_ICC
5695#pragma warning(pop)
5696#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005697
Jonathan Peyton30419822017-05-12 18:01:32 +00005698 KA_TRACE(30, ("__kmp_internal_end_dest: T#%d\n", gtid));
5699 /* NOTE: the gtid is stored as gitd+1 in the thread-local-storage
5700 * this is because 0 is reserved for the nothing-stored case */
Jim Cownie5e8470a2013-09-27 10:38:44 +00005701
Jonathan Peyton30419822017-05-12 18:01:32 +00005702 /* josh: One reason for setting the gtid specific data even when it is being
5703 destroyed by pthread is to allow gtid lookup through thread specific data
5704 (__kmp_gtid_get_specific). Some of the code, especially stat code,
5705 that gets executed in the call to __kmp_internal_end_thread, actually
5706 gets the gtid through the thread specific data. Setting it here seems
5707 rather inelegant and perhaps wrong, but allows __kmp_internal_end_thread
5708 to run smoothly.
5709 todo: get rid of this after we remove the dependence on
5710 __kmp_gtid_get_specific */
5711 if (gtid >= 0 && KMP_UBER_GTID(gtid))
5712 __kmp_gtid_set_specific(gtid);
5713#ifdef KMP_TDATA_GTID
5714 __kmp_gtid = gtid;
5715#endif
5716 __kmp_internal_end_thread(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005717}
5718
Jonathan Peyton99016992015-05-26 17:32:53 +00005719#if KMP_OS_UNIX && KMP_DYNAMIC_LIB
Jim Cownie5e8470a2013-09-27 10:38:44 +00005720
Jonathan Peyton30419822017-05-12 18:01:32 +00005721// 2009-09-08 (lev): It looks the destructor does not work. In simple test cases
5722// destructors work perfectly, but in real libomp.so I have no evidence it is
5723// ever called. However, -fini linker option in makefile.mk works fine.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005724
Jonathan Peyton30419822017-05-12 18:01:32 +00005725__attribute__((destructor)) void __kmp_internal_end_dtor(void) {
5726 __kmp_internal_end_atexit();
Jim Cownie5e8470a2013-09-27 10:38:44 +00005727}
5728
Jonathan Peyton30419822017-05-12 18:01:32 +00005729void __kmp_internal_end_fini(void) { __kmp_internal_end_atexit(); }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005730
5731#endif
5732
Jonathan Peyton30419822017-05-12 18:01:32 +00005733/* [Windows] josh: when the atexit handler is called, there may still be more
5734 than one thread alive */
5735void __kmp_internal_end_atexit(void) {
5736 KA_TRACE(30, ("__kmp_internal_end_atexit\n"));
5737 /* [Windows]
5738 josh: ideally, we want to completely shutdown the library in this atexit
5739 handler, but stat code that depends on thread specific data for gtid fails
5740 because that data becomes unavailable at some point during the shutdown, so
5741 we call __kmp_internal_end_thread instead. We should eventually remove the
5742 dependency on __kmp_get_specific_gtid in the stat code and use
5743 __kmp_internal_end_library to cleanly shutdown the library.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005744
Jonathan Peyton30419822017-05-12 18:01:32 +00005745 // TODO: Can some of this comment about GVS be removed?
5746 I suspect that the offending stat code is executed when the calling thread
5747 tries to clean up a dead root thread's data structures, resulting in GVS
5748 code trying to close the GVS structures for that thread, but since the stat
5749 code uses __kmp_get_specific_gtid to get the gtid with the assumption that
5750 the calling thread is cleaning up itself instead of another thread, it get
5751 confused. This happens because allowing a thread to unregister and cleanup
5752 another thread is a recent modification for addressing an issue.
5753 Based on the current design (20050722), a thread may end up
5754 trying to unregister another thread only if thread death does not trigger
5755 the calling of __kmp_internal_end_thread. For Linux* OS, there is the
5756 thread specific data destructor function to detect thread death. For
5757 Windows dynamic, there is DllMain(THREAD_DETACH). For Windows static, there
5758 is nothing. Thus, the workaround is applicable only for Windows static
5759 stat library. */
5760 __kmp_internal_end_library(-1);
5761#if KMP_OS_WINDOWS
5762 __kmp_close_console();
5763#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005764}
5765
Jonathan Peyton30419822017-05-12 18:01:32 +00005766static void __kmp_reap_thread(kmp_info_t *thread, int is_root) {
5767 // It is assumed __kmp_forkjoin_lock is acquired.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005768
Jonathan Peyton30419822017-05-12 18:01:32 +00005769 int gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005770
Jonathan Peyton30419822017-05-12 18:01:32 +00005771 KMP_DEBUG_ASSERT(thread != NULL);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005772
Jonathan Peyton30419822017-05-12 18:01:32 +00005773 gtid = thread->th.th_info.ds.ds_gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005774
Jonathan Peyton30419822017-05-12 18:01:32 +00005775 if (!is_root) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00005776
Jonathan Peyton30419822017-05-12 18:01:32 +00005777 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
5778 /* Assume the threads are at the fork barrier here */
5779 KA_TRACE(
5780 20, ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n",
5781 gtid));
5782 /* Need release fence here to prevent seg faults for tree forkjoin barrier
5783 * (GEH) */
5784 ANNOTATE_HAPPENS_BEFORE(thread);
5785 kmp_flag_64 flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread);
5786 __kmp_release_64(&flag);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005787 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005788
Jonathan Peyton30419822017-05-12 18:01:32 +00005789 // Terminate OS thread.
5790 __kmp_reap_worker(thread);
Jonathan Peyton7ca7ef02016-11-21 16:18:57 +00005791
Jonathan Peyton30419822017-05-12 18:01:32 +00005792 // The thread was killed asynchronously. If it was actively
5793 // spinning in the thread pool, decrement the global count.
5794 //
5795 // There is a small timing hole here - if the worker thread was just waking
5796 // up after sleeping in the pool, had reset it's th_active_in_pool flag but
5797 // not decremented the global counter __kmp_thread_pool_active_nth yet, then
5798 // the global counter might not get updated.
5799 //
5800 // Currently, this can only happen as the library is unloaded,
5801 // so there are no harmful side effects.
5802 if (thread->th.th_active_in_pool) {
5803 thread->th.th_active_in_pool = FALSE;
Andrey Churbanov5ba90c72017-07-17 09:03:14 +00005804 KMP_TEST_THEN_DEC32(&__kmp_thread_pool_active_nth);
Jonathan Peyton30419822017-05-12 18:01:32 +00005805 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
5806 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005807
Jonathan Peyton30419822017-05-12 18:01:32 +00005808 // Decrement # of [worker] threads in the pool.
5809 KMP_DEBUG_ASSERT(__kmp_thread_pool_nth > 0);
5810 --__kmp_thread_pool_nth;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005811 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005812
Jonathan Peyton30419822017-05-12 18:01:32 +00005813 __kmp_free_implicit_task(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005814
Jonathan Peyton30419822017-05-12 18:01:32 +00005815// Free the fast memory for tasking
5816#if USE_FAST_MEMORY
5817 __kmp_free_fast_memory(thread);
5818#endif /* USE_FAST_MEMORY */
5819
5820 __kmp_suspend_uninitialize_thread(thread);
5821
5822 KMP_DEBUG_ASSERT(__kmp_threads[gtid] == thread);
5823 TCW_SYNC_PTR(__kmp_threads[gtid], NULL);
5824
5825 --__kmp_all_nth;
5826// __kmp_nth was decremented when thread is added to the pool.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005827
5828#ifdef KMP_ADJUST_BLOCKTIME
Jonathan Peyton30419822017-05-12 18:01:32 +00005829 /* Adjust blocktime back to user setting or default if necessary */
5830 /* Middle initialization might never have occurred */
5831 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5832 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
5833 if (__kmp_nth <= __kmp_avail_proc) {
5834 __kmp_zero_bt = FALSE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005835 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005836 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005837#endif /* KMP_ADJUST_BLOCKTIME */
5838
Jonathan Peyton30419822017-05-12 18:01:32 +00005839 /* free the memory being used */
5840 if (__kmp_env_consistency_check) {
5841 if (thread->th.th_cons) {
5842 __kmp_free_cons_stack(thread->th.th_cons);
5843 thread->th.th_cons = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005844 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005845 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005846
Jonathan Peyton30419822017-05-12 18:01:32 +00005847 if (thread->th.th_pri_common != NULL) {
5848 __kmp_free(thread->th.th_pri_common);
5849 thread->th.th_pri_common = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005850 }
Andrey Churbanov6d224db2015-02-10 18:37:43 +00005851
Jonathan Peyton30419822017-05-12 18:01:32 +00005852 if (thread->th.th_task_state_memo_stack != NULL) {
5853 __kmp_free(thread->th.th_task_state_memo_stack);
5854 thread->th.th_task_state_memo_stack = NULL;
5855 }
5856
5857#if KMP_USE_BGET
5858 if (thread->th.th_local.bget_data != NULL) {
5859 __kmp_finalize_bget(thread);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005860 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005861#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005862
Alp Toker98758b02014-03-02 04:12:06 +00005863#if KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00005864 if (thread->th.th_affin_mask != NULL) {
5865 KMP_CPU_FREE(thread->th.th_affin_mask);
5866 thread->th.th_affin_mask = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005867 }
Alp Toker98758b02014-03-02 04:12:06 +00005868#endif /* KMP_AFFINITY_SUPPORTED */
Jim Cownie5e8470a2013-09-27 10:38:44 +00005869
Jonathan Peyton30419822017-05-12 18:01:32 +00005870 __kmp_reap_team(thread->th.th_serial_team);
5871 thread->th.th_serial_team = NULL;
5872 __kmp_free(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005873
Jonathan Peyton30419822017-05-12 18:01:32 +00005874 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00005875
5876} // __kmp_reap_thread
5877
Jonathan Peyton30419822017-05-12 18:01:32 +00005878static void __kmp_internal_end(void) {
5879 int i;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005880
Jonathan Peyton30419822017-05-12 18:01:32 +00005881 /* First, unregister the library */
5882 __kmp_unregister_library();
Jim Cownie5e8470a2013-09-27 10:38:44 +00005883
Jonathan Peyton30419822017-05-12 18:01:32 +00005884#if KMP_OS_WINDOWS
5885 /* In Win static library, we can't tell when a root actually dies, so we
5886 reclaim the data structures for any root threads that have died but not
5887 unregistered themselves, in order to shut down cleanly.
5888 In Win dynamic library we also can't tell when a thread dies. */
5889 __kmp_reclaim_dead_roots(); // AC: moved here to always clean resources of
5890// dead roots
Jonathan Peytonb66d1aa2016-09-27 17:11:17 +00005891#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005892
Jonathan Peyton30419822017-05-12 18:01:32 +00005893 for (i = 0; i < __kmp_threads_capacity; i++)
5894 if (__kmp_root[i])
5895 if (__kmp_root[i]->r.r_active)
5896 break;
5897 KMP_MB(); /* Flush all pending memory write invalidates. */
5898 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
5899
5900 if (i < __kmp_threads_capacity) {
5901#if KMP_USE_MONITOR
5902 // 2009-09-08 (lev): Other alive roots found. Why do we kill the monitor??
5903 KMP_MB(); /* Flush all pending memory write invalidates. */
5904
5905// Need to check that monitor was initialized before reaping it. If we are
5906// called form __kmp_atfork_child (which sets __kmp_init_parallel = 0), then
5907// __kmp_monitor will appear to contain valid data, but it is only valid in the
5908// parent process, not the child.
5909 // New behavior (201008): instead of keying off of the flag
5910 // __kmp_init_parallel, the monitor thread creation is keyed off
5911 // of the new flag __kmp_init_monitor.
5912 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
5913 if (TCR_4(__kmp_init_monitor)) {
5914 __kmp_reap_monitor(&__kmp_monitor);
5915 TCW_4(__kmp_init_monitor, 0);
5916 }
5917 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
5918 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n"));
5919#endif // KMP_USE_MONITOR
5920 } else {
5921/* TODO move this to cleanup code */
5922#ifdef KMP_DEBUG
5923 /* make sure that everything has properly ended */
5924 for (i = 0; i < __kmp_threads_capacity; i++) {
5925 if (__kmp_root[i]) {
5926 // KMP_ASSERT( ! KMP_UBER_GTID( i ) ); // AC:
5927 // there can be uber threads alive here
5928 KMP_ASSERT(!__kmp_root[i]->r.r_active); // TODO: can they be active?
5929 }
5930 }
5931#endif
5932
5933 KMP_MB();
5934
5935 // Reap the worker threads.
5936 // This is valid for now, but be careful if threads are reaped sooner.
5937 while (__kmp_thread_pool != NULL) { // Loop thru all the thread in the pool.
5938 // Get the next thread from the pool.
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00005939 kmp_info_t *thread = CCAST(kmp_info_t *, __kmp_thread_pool);
Jonathan Peyton30419822017-05-12 18:01:32 +00005940 __kmp_thread_pool = thread->th.th_next_pool;
5941 // Reap it.
5942 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP);
5943 thread->th.th_next_pool = NULL;
5944 thread->th.th_in_pool = FALSE;
5945 __kmp_reap_thread(thread, 0);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005946 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005947 __kmp_thread_pool_insert_pt = NULL;
5948
5949 // Reap teams.
5950 while (__kmp_team_pool != NULL) { // Loop thru all the teams in the pool.
5951 // Get the next team from the pool.
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00005952 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool);
Jonathan Peyton30419822017-05-12 18:01:32 +00005953 __kmp_team_pool = team->t.t_next_pool;
5954 // Reap it.
5955 team->t.t_next_pool = NULL;
5956 __kmp_reap_team(team);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005957 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005958
5959 __kmp_reap_task_teams();
5960
5961 for (i = 0; i < __kmp_threads_capacity; ++i) {
5962 // TBD: Add some checking...
5963 // Something like KMP_DEBUG_ASSERT( __kmp_thread[ i ] == NULL );
5964 }
5965
5966 /* Make sure all threadprivate destructors get run by joining with all
5967 worker threads before resetting this flag */
5968 TCW_SYNC_4(__kmp_init_common, FALSE);
5969
5970 KA_TRACE(10, ("__kmp_internal_end: all workers reaped\n"));
5971 KMP_MB();
5972
5973#if KMP_USE_MONITOR
5974 // See note above: One of the possible fixes for CQ138434 / CQ140126
5975 //
5976 // FIXME: push both code fragments down and CSE them?
5977 // push them into __kmp_cleanup() ?
5978 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
5979 if (TCR_4(__kmp_init_monitor)) {
5980 __kmp_reap_monitor(&__kmp_monitor);
5981 TCW_4(__kmp_init_monitor, 0);
5982 }
5983 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
5984 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n"));
5985#endif
5986 } /* else !__kmp_global.t_active */
5987 TCW_4(__kmp_init_gtid, FALSE);
5988 KMP_MB(); /* Flush all pending memory write invalidates. */
5989
5990 __kmp_cleanup();
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005991#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00005992 ompt_fini();
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005993#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005994}
5995
Jonathan Peyton30419822017-05-12 18:01:32 +00005996void __kmp_internal_end_library(int gtid_req) {
5997 /* if we have already cleaned up, don't try again, it wouldn't be pretty */
5998 /* this shouldn't be a race condition because __kmp_internal_end() is the
5999 only place to clear __kmp_serial_init */
6000 /* we'll check this later too, after we get the lock */
6001 // 2009-09-06: We do not set g_abort without setting g_done. This check looks
6002 // redundaant, because the next check will work in any case.
6003 if (__kmp_global.g.g_abort) {
6004 KA_TRACE(11, ("__kmp_internal_end_library: abort, exiting\n"));
6005 /* TODO abort? */
6006 return;
6007 }
6008 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6009 KA_TRACE(10, ("__kmp_internal_end_library: already finished\n"));
6010 return;
6011 }
6012
6013 KMP_MB(); /* Flush all pending memory write invalidates. */
6014
6015 /* find out who we are and what we should do */
6016 {
6017 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6018 KA_TRACE(
6019 10, ("__kmp_internal_end_library: enter T#%d (%d)\n", gtid, gtid_req));
6020 if (gtid == KMP_GTID_SHUTDOWN) {
6021 KA_TRACE(10, ("__kmp_internal_end_library: !__kmp_init_runtime, system "
6022 "already shutdown\n"));
6023 return;
6024 } else if (gtid == KMP_GTID_MONITOR) {
6025 KA_TRACE(10, ("__kmp_internal_end_library: monitor thread, gtid not "
6026 "registered, or system shutdown\n"));
6027 return;
6028 } else if (gtid == KMP_GTID_DNE) {
6029 KA_TRACE(10, ("__kmp_internal_end_library: gtid not registered or system "
6030 "shutdown\n"));
6031 /* we don't know who we are, but we may still shutdown the library */
6032 } else if (KMP_UBER_GTID(gtid)) {
6033 /* unregister ourselves as an uber thread. gtid is no longer valid */
6034 if (__kmp_root[gtid]->r.r_active) {
6035 __kmp_global.g.g_abort = -1;
6036 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6037 KA_TRACE(10,
6038 ("__kmp_internal_end_library: root still active, abort T#%d\n",
6039 gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006040 return;
Jonathan Peyton30419822017-05-12 18:01:32 +00006041 } else {
6042 KA_TRACE(
6043 10,
6044 ("__kmp_internal_end_library: unregistering sibling T#%d\n", gtid));
6045 __kmp_unregister_root_current_thread(gtid);
6046 }
6047 } else {
6048/* worker threads may call this function through the atexit handler, if they
6049 * call exit() */
6050/* For now, skip the usual subsequent processing and just dump the debug buffer.
6051 TODO: do a thorough shutdown instead */
6052#ifdef DUMP_DEBUG_ON_EXIT
6053 if (__kmp_debug_buf)
6054 __kmp_dump_debug_buffer();
6055#endif
6056 return;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006057 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006058 }
6059 /* synchronize the termination process */
6060 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006061
Jonathan Peyton30419822017-05-12 18:01:32 +00006062 /* have we already finished */
6063 if (__kmp_global.g.g_abort) {
6064 KA_TRACE(10, ("__kmp_internal_end_library: abort, exiting\n"));
6065 /* TODO abort? */
6066 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6067 return;
6068 }
6069 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6070 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6071 return;
6072 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006073
Jonathan Peyton30419822017-05-12 18:01:32 +00006074 /* We need this lock to enforce mutex between this reading of
6075 __kmp_threads_capacity and the writing by __kmp_register_root.
6076 Alternatively, we can use a counter of roots that is atomically updated by
6077 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and
6078 __kmp_internal_end_*. */
6079 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006080
Jonathan Peyton30419822017-05-12 18:01:32 +00006081 /* now we can safely conduct the actual termination */
6082 __kmp_internal_end();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006083
Jonathan Peyton30419822017-05-12 18:01:32 +00006084 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6085 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006086
Jonathan Peyton30419822017-05-12 18:01:32 +00006087 KA_TRACE(10, ("__kmp_internal_end_library: exit\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006088
Jonathan Peyton30419822017-05-12 18:01:32 +00006089#ifdef DUMP_DEBUG_ON_EXIT
6090 if (__kmp_debug_buf)
6091 __kmp_dump_debug_buffer();
6092#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006093
Jonathan Peyton30419822017-05-12 18:01:32 +00006094#if KMP_OS_WINDOWS
6095 __kmp_close_console();
6096#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006097
Jonathan Peyton30419822017-05-12 18:01:32 +00006098 __kmp_fini_allocator();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006099
6100} // __kmp_internal_end_library
6101
Jonathan Peyton30419822017-05-12 18:01:32 +00006102void __kmp_internal_end_thread(int gtid_req) {
6103 int i;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006104
Jonathan Peyton30419822017-05-12 18:01:32 +00006105 /* if we have already cleaned up, don't try again, it wouldn't be pretty */
6106 /* this shouldn't be a race condition because __kmp_internal_end() is the
6107 * only place to clear __kmp_serial_init */
6108 /* we'll check this later too, after we get the lock */
6109 // 2009-09-06: We do not set g_abort without setting g_done. This check looks
6110 // redundant, because the next check will work in any case.
6111 if (__kmp_global.g.g_abort) {
6112 KA_TRACE(11, ("__kmp_internal_end_thread: abort, exiting\n"));
6113 /* TODO abort? */
6114 return;
6115 }
6116 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6117 KA_TRACE(10, ("__kmp_internal_end_thread: already finished\n"));
6118 return;
6119 }
6120
6121 KMP_MB(); /* Flush all pending memory write invalidates. */
6122
6123 /* find out who we are and what we should do */
6124 {
6125 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6126 KA_TRACE(10,
6127 ("__kmp_internal_end_thread: enter T#%d (%d)\n", gtid, gtid_req));
6128 if (gtid == KMP_GTID_SHUTDOWN) {
6129 KA_TRACE(10, ("__kmp_internal_end_thread: !__kmp_init_runtime, system "
6130 "already shutdown\n"));
6131 return;
6132 } else if (gtid == KMP_GTID_MONITOR) {
6133 KA_TRACE(10, ("__kmp_internal_end_thread: monitor thread, gtid not "
6134 "registered, or system shutdown\n"));
6135 return;
6136 } else if (gtid == KMP_GTID_DNE) {
6137 KA_TRACE(10, ("__kmp_internal_end_thread: gtid not registered or system "
6138 "shutdown\n"));
6139 return;
6140 /* we don't know who we are */
6141 } else if (KMP_UBER_GTID(gtid)) {
6142 /* unregister ourselves as an uber thread. gtid is no longer valid */
6143 if (__kmp_root[gtid]->r.r_active) {
6144 __kmp_global.g.g_abort = -1;
6145 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6146 KA_TRACE(10,
6147 ("__kmp_internal_end_thread: root still active, abort T#%d\n",
6148 gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006149 return;
Jonathan Peyton30419822017-05-12 18:01:32 +00006150 } else {
6151 KA_TRACE(10, ("__kmp_internal_end_thread: unregistering sibling T#%d\n",
6152 gtid));
6153 __kmp_unregister_root_current_thread(gtid);
6154 }
6155 } else {
6156 /* just a worker thread, let's leave */
6157 KA_TRACE(10, ("__kmp_internal_end_thread: worker thread T#%d\n", gtid));
6158
6159 if (gtid >= 0) {
6160 __kmp_threads[gtid]->th.th_task_team = NULL;
6161 }
6162
6163 KA_TRACE(10,
6164 ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n",
6165 gtid));
6166 return;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006167 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006168 }
6169#if defined KMP_DYNAMIC_LIB
6170 // AC: lets not shutdown the Linux* OS dynamic library at the exit of uber
6171 // thread, because we will better shutdown later in the library destructor.
6172 // The reason of this change is performance problem when non-openmp thread in
6173 // a loop forks and joins many openmp threads. We can save a lot of time
6174 // keeping worker threads alive until the program shutdown.
6175 // OM: Removed Linux* OS restriction to fix the crash on OS X* (DPD200239966)
6176 // and Windows(DPD200287443) that occurs when using critical sections from
6177 // foreign threads.
6178 KA_TRACE(10, ("__kmp_internal_end_thread: exiting T#%d\n", gtid_req));
6179 return;
6180#endif
6181 /* synchronize the termination process */
6182 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006183
Jonathan Peyton30419822017-05-12 18:01:32 +00006184 /* have we already finished */
6185 if (__kmp_global.g.g_abort) {
6186 KA_TRACE(10, ("__kmp_internal_end_thread: abort, exiting\n"));
6187 /* TODO abort? */
6188 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6189 return;
6190 }
6191 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6192 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6193 return;
6194 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006195
Jonathan Peyton30419822017-05-12 18:01:32 +00006196 /* We need this lock to enforce mutex between this reading of
6197 __kmp_threads_capacity and the writing by __kmp_register_root.
6198 Alternatively, we can use a counter of roots that is atomically updated by
6199 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and
6200 __kmp_internal_end_*. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006201
Jonathan Peyton30419822017-05-12 18:01:32 +00006202 /* should we finish the run-time? are all siblings done? */
6203 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006204
Jonathan Peyton30419822017-05-12 18:01:32 +00006205 for (i = 0; i < __kmp_threads_capacity; ++i) {
6206 if (KMP_UBER_GTID(i)) {
6207 KA_TRACE(
6208 10,
6209 ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i));
6210 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6211 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6212 return;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006213 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006214 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006215
Jonathan Peyton30419822017-05-12 18:01:32 +00006216 /* now we can safely conduct the actual termination */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006217
Jonathan Peyton30419822017-05-12 18:01:32 +00006218 __kmp_internal_end();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006219
Jonathan Peyton30419822017-05-12 18:01:32 +00006220 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6221 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006222
Jonathan Peyton30419822017-05-12 18:01:32 +00006223 KA_TRACE(10, ("__kmp_internal_end_thread: exit T#%d\n", gtid_req));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006224
Jonathan Peyton30419822017-05-12 18:01:32 +00006225#ifdef DUMP_DEBUG_ON_EXIT
6226 if (__kmp_debug_buf)
6227 __kmp_dump_debug_buffer();
6228#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006229} // __kmp_internal_end_thread
6230
Jonathan Peyton30419822017-05-12 18:01:32 +00006231// -----------------------------------------------------------------------------
Jim Cownie5e8470a2013-09-27 10:38:44 +00006232// Library registration stuff.
6233
Jonathan Peyton30419822017-05-12 18:01:32 +00006234static long __kmp_registration_flag = 0;
6235// Random value used to indicate library initialization.
6236static char *__kmp_registration_str = NULL;
6237// Value to be saved in env var __KMP_REGISTERED_LIB_<pid>.
Jim Cownie5e8470a2013-09-27 10:38:44 +00006238
Jonathan Peyton30419822017-05-12 18:01:32 +00006239static inline char *__kmp_reg_status_name() {
6240 /* On RHEL 3u5 if linked statically, getpid() returns different values in
6241 each thread. If registration and unregistration go in different threads
6242 (omp_misc_other_root_exit.cpp test case), the name of registered_lib_env
6243 env var can not be found, because the name will contain different pid. */
6244 return __kmp_str_format("__KMP_REGISTERED_LIB_%d", (int)getpid());
Jim Cownie5e8470a2013-09-27 10:38:44 +00006245} // __kmp_reg_status_get
6246
Jonathan Peyton30419822017-05-12 18:01:32 +00006247void __kmp_register_library_startup(void) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006248
Jonathan Peyton30419822017-05-12 18:01:32 +00006249 char *name = __kmp_reg_status_name(); // Name of the environment variable.
6250 int done = 0;
6251 union {
6252 double dtime;
6253 long ltime;
6254 } time;
6255#if KMP_ARCH_X86 || KMP_ARCH_X86_64
6256 __kmp_initialize_system_tick();
6257#endif
6258 __kmp_read_system_time(&time.dtime);
6259 __kmp_registration_flag = 0xCAFE0000L | (time.ltime & 0x0000FFFFL);
6260 __kmp_registration_str =
6261 __kmp_str_format("%p-%lx-%s", &__kmp_registration_flag,
6262 __kmp_registration_flag, KMP_LIBRARY_FILE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006263
Jonathan Peyton30419822017-05-12 18:01:32 +00006264 KA_TRACE(50, ("__kmp_register_library_startup: %s=\"%s\"\n", name,
6265 __kmp_registration_str));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006266
Jonathan Peyton30419822017-05-12 18:01:32 +00006267 while (!done) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006268
Jonathan Peyton30419822017-05-12 18:01:32 +00006269 char *value = NULL; // Actual value of the environment variable.
Jim Cownie5e8470a2013-09-27 10:38:44 +00006270
Jonathan Peyton30419822017-05-12 18:01:32 +00006271 // Set environment variable, but do not overwrite if it is exist.
6272 __kmp_env_set(name, __kmp_registration_str, 0);
6273 // Check the variable is written.
6274 value = __kmp_env_get(name);
6275 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006276
Jonathan Peyton30419822017-05-12 18:01:32 +00006277 done = 1; // Ok, environment variable set successfully, exit the loop.
Jim Cownie5e8470a2013-09-27 10:38:44 +00006278
Jonathan Peyton30419822017-05-12 18:01:32 +00006279 } else {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006280
Jonathan Peyton30419822017-05-12 18:01:32 +00006281 // Oops. Write failed. Another copy of OpenMP RTL is in memory.
6282 // Check whether it alive or dead.
6283 int neighbor = 0; // 0 -- unknown status, 1 -- alive, 2 -- dead.
6284 char *tail = value;
6285 char *flag_addr_str = NULL;
6286 char *flag_val_str = NULL;
6287 char const *file_name = NULL;
6288 __kmp_str_split(tail, '-', &flag_addr_str, &tail);
6289 __kmp_str_split(tail, '-', &flag_val_str, &tail);
6290 file_name = tail;
6291 if (tail != NULL) {
6292 long *flag_addr = 0;
6293 long flag_val = 0;
6294 KMP_SSCANF(flag_addr_str, "%p", &flag_addr);
6295 KMP_SSCANF(flag_val_str, "%lx", &flag_val);
6296 if (flag_addr != 0 && flag_val != 0 && strcmp(file_name, "") != 0) {
6297 // First, check whether environment-encoded address is mapped into
6298 // addr space.
6299 // If so, dereference it to see if it still has the right value.
6300 if (__kmp_is_address_mapped(flag_addr) && *flag_addr == flag_val) {
6301 neighbor = 1;
6302 } else {
6303 // If not, then we know the other copy of the library is no longer
6304 // running.
6305 neighbor = 2;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006306 }
6307 }
6308 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006309 switch (neighbor) {
6310 case 0: // Cannot parse environment variable -- neighbor status unknown.
6311 // Assume it is the incompatible format of future version of the
6312 // library. Assume the other library is alive.
6313 // WARN( ... ); // TODO: Issue a warning.
6314 file_name = "unknown library";
6315 // Attention! Falling to the next case. That's intentional.
6316 case 1: { // Neighbor is alive.
6317 // Check it is allowed.
6318 char *duplicate_ok = __kmp_env_get("KMP_DUPLICATE_LIB_OK");
6319 if (!__kmp_str_match_true(duplicate_ok)) {
6320 // That's not allowed. Issue fatal error.
Jonathan Peyton6a393f72017-09-05 15:43:58 +00006321 __kmp_fatal(KMP_MSG(DuplicateLibrary, KMP_LIBRARY_FILE, file_name),
6322 KMP_HNT(DuplicateLibrary), __kmp_msg_null);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006323 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006324 KMP_INTERNAL_FREE(duplicate_ok);
6325 __kmp_duplicate_library_ok = 1;
6326 done = 1; // Exit the loop.
6327 } break;
6328 case 2: { // Neighbor is dead.
6329 // Clear the variable and try to register library again.
6330 __kmp_env_unset(name);
6331 } break;
6332 default: { KMP_DEBUG_ASSERT(0); } break;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006333 }
6334 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006335 KMP_INTERNAL_FREE((void *)value);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006336 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006337 KMP_INTERNAL_FREE((void *)name);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006338
6339} // func __kmp_register_library_startup
6340
Jonathan Peyton30419822017-05-12 18:01:32 +00006341void __kmp_unregister_library(void) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006342
Jonathan Peyton30419822017-05-12 18:01:32 +00006343 char *name = __kmp_reg_status_name();
6344 char *value = __kmp_env_get(name);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006345
Jonathan Peyton30419822017-05-12 18:01:32 +00006346 KMP_DEBUG_ASSERT(__kmp_registration_flag != 0);
6347 KMP_DEBUG_ASSERT(__kmp_registration_str != NULL);
6348 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
6349 // Ok, this is our variable. Delete it.
6350 __kmp_env_unset(name);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006351 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006352
Jonathan Peyton30419822017-05-12 18:01:32 +00006353 KMP_INTERNAL_FREE(__kmp_registration_str);
6354 KMP_INTERNAL_FREE(value);
6355 KMP_INTERNAL_FREE(name);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006356
Jonathan Peyton30419822017-05-12 18:01:32 +00006357 __kmp_registration_flag = 0;
6358 __kmp_registration_str = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006359
6360} // __kmp_unregister_library
6361
Jim Cownie5e8470a2013-09-27 10:38:44 +00006362// End of Library registration stuff.
Jonathan Peyton30419822017-05-12 18:01:32 +00006363// -----------------------------------------------------------------------------
Jim Cownie5e8470a2013-09-27 10:38:44 +00006364
Jonathan Peyton492e0a32017-06-13 17:17:26 +00006365#if KMP_MIC_SUPPORTED
Andrey Churbanov613edeb2015-02-20 18:14:43 +00006366
Jonathan Peyton30419822017-05-12 18:01:32 +00006367static void __kmp_check_mic_type() {
6368 kmp_cpuid_t cpuid_state = {0};
6369 kmp_cpuid_t *cs_p = &cpuid_state;
6370 __kmp_x86_cpuid(1, 0, cs_p);
6371 // We don't support mic1 at the moment
6372 if ((cs_p->eax & 0xff0) == 0xB10) {
6373 __kmp_mic_type = mic2;
6374 } else if ((cs_p->eax & 0xf0ff0) == 0x50670) {
6375 __kmp_mic_type = mic3;
6376 } else {
6377 __kmp_mic_type = non_mic;
6378 }
Andrey Churbanov613edeb2015-02-20 18:14:43 +00006379}
6380
Jonathan Peyton492e0a32017-06-13 17:17:26 +00006381#endif /* KMP_MIC_SUPPORTED */
Andrey Churbanov613edeb2015-02-20 18:14:43 +00006382
Jonathan Peyton30419822017-05-12 18:01:32 +00006383static void __kmp_do_serial_initialize(void) {
6384 int i, gtid;
6385 int size;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006386
Jonathan Peyton30419822017-05-12 18:01:32 +00006387 KA_TRACE(10, ("__kmp_do_serial_initialize: enter\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006388
Jonathan Peyton30419822017-05-12 18:01:32 +00006389 KMP_DEBUG_ASSERT(sizeof(kmp_int32) == 4);
6390 KMP_DEBUG_ASSERT(sizeof(kmp_uint32) == 4);
6391 KMP_DEBUG_ASSERT(sizeof(kmp_int64) == 8);
6392 KMP_DEBUG_ASSERT(sizeof(kmp_uint64) == 8);
6393 KMP_DEBUG_ASSERT(sizeof(kmp_intptr_t) == sizeof(void *));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006394
Jonathan Peyton82a13bf2015-09-21 18:01:02 +00006395#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00006396 ompt_pre_init();
Jonathan Peyton82a13bf2015-09-21 18:01:02 +00006397#endif
6398
Jonathan Peyton30419822017-05-12 18:01:32 +00006399 __kmp_validate_locks();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006400
Jonathan Peyton30419822017-05-12 18:01:32 +00006401 /* Initialize internal memory allocator */
6402 __kmp_init_allocator();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006403
Jonathan Peyton30419822017-05-12 18:01:32 +00006404 /* Register the library startup via an environment variable and check to see
6405 whether another copy of the library is already registered. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006406
Jonathan Peyton30419822017-05-12 18:01:32 +00006407 __kmp_register_library_startup();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006408
Jonathan Peyton30419822017-05-12 18:01:32 +00006409 /* TODO reinitialization of library */
6410 if (TCR_4(__kmp_global.g.g_done)) {
6411 KA_TRACE(10, ("__kmp_do_serial_initialize: reinitialization of library\n"));
6412 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006413
Jonathan Peyton30419822017-05-12 18:01:32 +00006414 __kmp_global.g.g_abort = 0;
6415 TCW_SYNC_4(__kmp_global.g.g_done, FALSE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006416
Jonathan Peyton30419822017-05-12 18:01:32 +00006417/* initialize the locks */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006418#if KMP_USE_ADAPTIVE_LOCKS
6419#if KMP_DEBUG_ADAPTIVE_LOCKS
Jonathan Peyton30419822017-05-12 18:01:32 +00006420 __kmp_init_speculative_stats();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006421#endif
6422#endif
Jonathan Peytonad579922015-12-17 16:19:05 +00006423#if KMP_STATS_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00006424 __kmp_stats_init();
Jonathan Peytonad579922015-12-17 16:19:05 +00006425#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006426 __kmp_init_lock(&__kmp_global_lock);
6427 __kmp_init_queuing_lock(&__kmp_dispatch_lock);
6428 __kmp_init_lock(&__kmp_debug_lock);
6429 __kmp_init_atomic_lock(&__kmp_atomic_lock);
6430 __kmp_init_atomic_lock(&__kmp_atomic_lock_1i);
6431 __kmp_init_atomic_lock(&__kmp_atomic_lock_2i);
6432 __kmp_init_atomic_lock(&__kmp_atomic_lock_4i);
6433 __kmp_init_atomic_lock(&__kmp_atomic_lock_4r);
6434 __kmp_init_atomic_lock(&__kmp_atomic_lock_8i);
6435 __kmp_init_atomic_lock(&__kmp_atomic_lock_8r);
6436 __kmp_init_atomic_lock(&__kmp_atomic_lock_8c);
6437 __kmp_init_atomic_lock(&__kmp_atomic_lock_10r);
6438 __kmp_init_atomic_lock(&__kmp_atomic_lock_16r);
6439 __kmp_init_atomic_lock(&__kmp_atomic_lock_16c);
6440 __kmp_init_atomic_lock(&__kmp_atomic_lock_20c);
6441 __kmp_init_atomic_lock(&__kmp_atomic_lock_32c);
6442 __kmp_init_bootstrap_lock(&__kmp_forkjoin_lock);
6443 __kmp_init_bootstrap_lock(&__kmp_exit_lock);
Jonathan Peytonb66d1aa2016-09-27 17:11:17 +00006444#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00006445 __kmp_init_bootstrap_lock(&__kmp_monitor_lock);
Jonathan Peytonb66d1aa2016-09-27 17:11:17 +00006446#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006447 __kmp_init_bootstrap_lock(&__kmp_tp_cached_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006448
Jonathan Peyton30419822017-05-12 18:01:32 +00006449 /* conduct initialization and initial setup of configuration */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006450
Jonathan Peyton30419822017-05-12 18:01:32 +00006451 __kmp_runtime_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006452
Jonathan Peyton492e0a32017-06-13 17:17:26 +00006453#if KMP_MIC_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00006454 __kmp_check_mic_type();
Andrey Churbanov613edeb2015-02-20 18:14:43 +00006455#endif
6456
Jonathan Peyton30419822017-05-12 18:01:32 +00006457// Some global variable initialization moved here from kmp_env_initialize()
Jim Cownie5e8470a2013-09-27 10:38:44 +00006458#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00006459 kmp_diag = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006460#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006461 __kmp_abort_delay = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006462
Jonathan Peyton30419822017-05-12 18:01:32 +00006463 // From __kmp_init_dflt_team_nth()
6464 /* assume the entire machine will be used */
6465 __kmp_dflt_team_nth_ub = __kmp_xproc;
6466 if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH) {
6467 __kmp_dflt_team_nth_ub = KMP_MIN_NTH;
6468 }
6469 if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) {
6470 __kmp_dflt_team_nth_ub = __kmp_sys_max_nth;
6471 }
6472 __kmp_max_nth = __kmp_sys_max_nth;
Jonathan Peytonf4392462017-07-27 20:58:41 +00006473 __kmp_cg_max_nth = __kmp_sys_max_nth;
Jonathan Peyton4f90c822017-08-02 20:04:45 +00006474 __kmp_teams_max_nth = __kmp_xproc; // set a "reasonable" default
6475 if (__kmp_teams_max_nth > __kmp_sys_max_nth) {
6476 __kmp_teams_max_nth = __kmp_sys_max_nth;
6477 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006478
Jonathan Peyton30419822017-05-12 18:01:32 +00006479 // Three vars below moved here from __kmp_env_initialize() "KMP_BLOCKTIME"
6480 // part
6481 __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME;
Jonathan Peytone1c7c132016-10-07 18:12:19 +00006482#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00006483 __kmp_monitor_wakeups =
6484 KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
6485 __kmp_bt_intervals =
6486 KMP_INTERVALS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
Jonathan Peytone1c7c132016-10-07 18:12:19 +00006487#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006488 // From "KMP_LIBRARY" part of __kmp_env_initialize()
6489 __kmp_library = library_throughput;
6490 // From KMP_SCHEDULE initialization
6491 __kmp_static = kmp_sch_static_balanced;
6492// AC: do not use analytical here, because it is non-monotonous
6493//__kmp_guided = kmp_sch_guided_iterative_chunked;
6494//__kmp_auto = kmp_sch_guided_analytical_chunked; // AC: it is the default, no
6495// need to repeat assignment
6496// Barrier initialization. Moved here from __kmp_env_initialize() Barrier branch
6497// bit control and barrier method control parts
Jim Cownie4cc4bb42014-10-07 16:25:50 +00006498#if KMP_FAST_REDUCTION_BARRIER
Jonathan Peyton30419822017-05-12 18:01:32 +00006499#define kmp_reduction_barrier_gather_bb ((int)1)
6500#define kmp_reduction_barrier_release_bb ((int)1)
6501#define kmp_reduction_barrier_gather_pat bp_hyper_bar
6502#define kmp_reduction_barrier_release_pat bp_hyper_bar
6503#endif // KMP_FAST_REDUCTION_BARRIER
6504 for (i = bs_plain_barrier; i < bs_last_barrier; i++) {
6505 __kmp_barrier_gather_branch_bits[i] = __kmp_barrier_gather_bb_dflt;
6506 __kmp_barrier_release_branch_bits[i] = __kmp_barrier_release_bb_dflt;
6507 __kmp_barrier_gather_pattern[i] = __kmp_barrier_gather_pat_dflt;
6508 __kmp_barrier_release_pattern[i] = __kmp_barrier_release_pat_dflt;
6509#if KMP_FAST_REDUCTION_BARRIER
6510 if (i == bs_reduction_barrier) { // tested and confirmed on ALTIX only (
6511 // lin_64 ): hyper,1
6512 __kmp_barrier_gather_branch_bits[i] = kmp_reduction_barrier_gather_bb;
6513 __kmp_barrier_release_branch_bits[i] = kmp_reduction_barrier_release_bb;
6514 __kmp_barrier_gather_pattern[i] = kmp_reduction_barrier_gather_pat;
6515 __kmp_barrier_release_pattern[i] = kmp_reduction_barrier_release_pat;
Andrey Churbanov613edeb2015-02-20 18:14:43 +00006516 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006517#endif // KMP_FAST_REDUCTION_BARRIER
6518 }
6519#if KMP_FAST_REDUCTION_BARRIER
6520#undef kmp_reduction_barrier_release_pat
6521#undef kmp_reduction_barrier_gather_pat
6522#undef kmp_reduction_barrier_release_bb
6523#undef kmp_reduction_barrier_gather_bb
6524#endif // KMP_FAST_REDUCTION_BARRIER
Jonathan Peyton492e0a32017-06-13 17:17:26 +00006525#if KMP_MIC_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00006526 if (__kmp_mic_type == mic2) { // KNC
6527 // AC: plane=3,2, forkjoin=2,1 are optimal for 240 threads on KNC
6528 __kmp_barrier_gather_branch_bits[bs_plain_barrier] = 3; // plain gather
6529 __kmp_barrier_release_branch_bits[bs_forkjoin_barrier] =
6530 1; // forkjoin release
6531 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6532 __kmp_barrier_release_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6533 }
6534#if KMP_FAST_REDUCTION_BARRIER
6535 if (__kmp_mic_type == mic2) { // KNC
6536 __kmp_barrier_gather_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6537 __kmp_barrier_release_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6538 }
Jonathan Peyton492e0a32017-06-13 17:17:26 +00006539#endif // KMP_FAST_REDUCTION_BARRIER
6540#endif // KMP_MIC_SUPPORTED
Jim Cownie5e8470a2013-09-27 10:38:44 +00006541
Jonathan Peyton30419822017-05-12 18:01:32 +00006542// From KMP_CHECKS initialization
Jim Cownie5e8470a2013-09-27 10:38:44 +00006543#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00006544 __kmp_env_checks = TRUE; /* development versions have the extra checks */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006545#else
Jonathan Peyton30419822017-05-12 18:01:32 +00006546 __kmp_env_checks = FALSE; /* port versions do not have the extra checks */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006547#endif
6548
Jonathan Peyton30419822017-05-12 18:01:32 +00006549 // From "KMP_FOREIGN_THREADS_THREADPRIVATE" initialization
6550 __kmp_foreign_tp = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006551
Jonathan Peyton30419822017-05-12 18:01:32 +00006552 __kmp_global.g.g_dynamic = FALSE;
6553 __kmp_global.g.g_dynamic_mode = dynamic_default;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006554
Jonathan Peyton30419822017-05-12 18:01:32 +00006555 __kmp_env_initialize(NULL);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00006556
Jonathan Peyton30419822017-05-12 18:01:32 +00006557// Print all messages in message catalog for testing purposes.
6558#ifdef KMP_DEBUG
6559 char const *val = __kmp_env_get("KMP_DUMP_CATALOG");
6560 if (__kmp_str_match_true(val)) {
6561 kmp_str_buf_t buffer;
6562 __kmp_str_buf_init(&buffer);
6563 __kmp_i18n_dump_catalog(&buffer);
6564 __kmp_printf("%s", buffer.str);
6565 __kmp_str_buf_free(&buffer);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006566 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006567 __kmp_env_free(&val);
6568#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006569
Jonathan Peyton30419822017-05-12 18:01:32 +00006570 __kmp_threads_capacity =
6571 __kmp_initial_threads_capacity(__kmp_dflt_team_nth_ub);
6572 // Moved here from __kmp_env_initialize() "KMP_ALL_THREADPRIVATE" part
6573 __kmp_tp_capacity = __kmp_default_tp_capacity(
6574 __kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006575
Jonathan Peyton30419822017-05-12 18:01:32 +00006576 // If the library is shut down properly, both pools must be NULL. Just in
6577 // case, set them to NULL -- some memory may leak, but subsequent code will
6578 // work even if pools are not freed.
6579 KMP_DEBUG_ASSERT(__kmp_thread_pool == NULL);
6580 KMP_DEBUG_ASSERT(__kmp_thread_pool_insert_pt == NULL);
6581 KMP_DEBUG_ASSERT(__kmp_team_pool == NULL);
6582 __kmp_thread_pool = NULL;
6583 __kmp_thread_pool_insert_pt = NULL;
6584 __kmp_team_pool = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006585
Jonathan Peyton30419822017-05-12 18:01:32 +00006586 /* Allocate all of the variable sized records */
6587 /* NOTE: __kmp_threads_capacity entries are allocated, but the arrays are
6588 * expandable */
6589 /* Since allocation is cache-aligned, just add extra padding at the end */
6590 size =
6591 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * __kmp_threads_capacity +
6592 CACHE_LINE;
6593 __kmp_threads = (kmp_info_t **)__kmp_allocate(size);
6594 __kmp_root = (kmp_root_t **)((char *)__kmp_threads +
6595 sizeof(kmp_info_t *) * __kmp_threads_capacity);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006596
Jonathan Peyton30419822017-05-12 18:01:32 +00006597 /* init thread counts */
6598 KMP_DEBUG_ASSERT(__kmp_all_nth ==
6599 0); // Asserts fail if the library is reinitializing and
6600 KMP_DEBUG_ASSERT(__kmp_nth == 0); // something was wrong in termination.
6601 __kmp_all_nth = 0;
6602 __kmp_nth = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006603
Jonathan Peyton30419822017-05-12 18:01:32 +00006604 /* setup the uber master thread and hierarchy */
6605 gtid = __kmp_register_root(TRUE);
6606 KA_TRACE(10, ("__kmp_do_serial_initialize T#%d\n", gtid));
6607 KMP_ASSERT(KMP_UBER_GTID(gtid));
6608 KMP_ASSERT(KMP_INITIAL_GTID(gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006609
Jonathan Peyton30419822017-05-12 18:01:32 +00006610 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006611
Jonathan Peyton30419822017-05-12 18:01:32 +00006612 __kmp_common_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006613
Jonathan Peyton30419822017-05-12 18:01:32 +00006614#if KMP_OS_UNIX
6615 /* invoke the child fork handler */
6616 __kmp_register_atfork();
6617#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006618
Jonathan Peyton30419822017-05-12 18:01:32 +00006619#if !defined KMP_DYNAMIC_LIB
6620 {
6621 /* Invoke the exit handler when the program finishes, only for static
6622 library. For dynamic library, we already have _fini and DllMain. */
6623 int rc = atexit(__kmp_internal_end_atexit);
6624 if (rc != 0) {
Jonathan Peyton6a393f72017-09-05 15:43:58 +00006625 __kmp_fatal(KMP_MSG(FunctionError, "atexit()"), KMP_ERR(rc),
6626 __kmp_msg_null);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006627 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006628 }
6629#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006630
Jonathan Peyton30419822017-05-12 18:01:32 +00006631#if KMP_HANDLE_SIGNALS
6632#if KMP_OS_UNIX
6633 /* NOTE: make sure that this is called before the user installs their own
6634 signal handlers so that the user handlers are called first. this way they
6635 can return false, not call our handler, avoid terminating the library, and
6636 continue execution where they left off. */
6637 __kmp_install_signals(FALSE);
6638#endif /* KMP_OS_UNIX */
6639#if KMP_OS_WINDOWS
6640 __kmp_install_signals(TRUE);
6641#endif /* KMP_OS_WINDOWS */
6642#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006643
Jonathan Peyton30419822017-05-12 18:01:32 +00006644 /* we have finished the serial initialization */
6645 __kmp_init_counter++;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006646
Jonathan Peyton30419822017-05-12 18:01:32 +00006647 __kmp_init_serial = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006648
Jonathan Peyton30419822017-05-12 18:01:32 +00006649 if (__kmp_settings) {
6650 __kmp_env_print();
6651 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006652
6653#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00006654 if (__kmp_display_env || __kmp_display_env_verbose) {
6655 __kmp_env_print_2();
6656 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006657#endif // OMP_40_ENABLED
6658
Jonathan Peyton82a13bf2015-09-21 18:01:02 +00006659#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00006660 ompt_post_init();
Jonathan Peyton82a13bf2015-09-21 18:01:02 +00006661#endif
6662
Jonathan Peyton30419822017-05-12 18:01:32 +00006663 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006664
Jonathan Peyton30419822017-05-12 18:01:32 +00006665 KA_TRACE(10, ("__kmp_do_serial_initialize: exit\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006666}
6667
Jonathan Peyton30419822017-05-12 18:01:32 +00006668void __kmp_serial_initialize(void) {
6669 if (__kmp_init_serial) {
6670 return;
6671 }
6672 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6673 if (__kmp_init_serial) {
6674 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6675 return;
6676 }
6677 __kmp_do_serial_initialize();
6678 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6679}
6680
6681static void __kmp_do_middle_initialize(void) {
6682 int i, j;
6683 int prev_dflt_team_nth;
6684
6685 if (!__kmp_init_serial) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006686 __kmp_do_serial_initialize();
Jonathan Peyton30419822017-05-12 18:01:32 +00006687 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006688
Jonathan Peyton30419822017-05-12 18:01:32 +00006689 KA_TRACE(10, ("__kmp_middle_initialize: enter\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006690
Jonathan Peyton30419822017-05-12 18:01:32 +00006691 // Save the previous value for the __kmp_dflt_team_nth so that
6692 // we can avoid some reinitialization if it hasn't changed.
6693 prev_dflt_team_nth = __kmp_dflt_team_nth;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006694
Alp Toker98758b02014-03-02 04:12:06 +00006695#if KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00006696 // __kmp_affinity_initialize() will try to set __kmp_ncores to the
6697 // number of cores on the machine.
6698 __kmp_affinity_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006699
Jonathan Peyton30419822017-05-12 18:01:32 +00006700 // Run through the __kmp_threads array and set the affinity mask
6701 // for each root thread that is currently registered with the RTL.
6702 for (i = 0; i < __kmp_threads_capacity; i++) {
6703 if (TCR_PTR(__kmp_threads[i]) != NULL) {
6704 __kmp_affinity_set_init_mask(i, TRUE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006705 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006706 }
Alp Toker98758b02014-03-02 04:12:06 +00006707#endif /* KMP_AFFINITY_SUPPORTED */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006708
Jonathan Peyton30419822017-05-12 18:01:32 +00006709 KMP_ASSERT(__kmp_xproc > 0);
6710 if (__kmp_avail_proc == 0) {
6711 __kmp_avail_proc = __kmp_xproc;
6712 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006713
Jonathan Peyton30419822017-05-12 18:01:32 +00006714 // If there were empty places in num_threads list (OMP_NUM_THREADS=,,2,3),
6715 // correct them now
6716 j = 0;
6717 while ((j < __kmp_nested_nth.used) && !__kmp_nested_nth.nth[j]) {
6718 __kmp_nested_nth.nth[j] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub =
6719 __kmp_avail_proc;
6720 j++;
6721 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006722
Jonathan Peyton30419822017-05-12 18:01:32 +00006723 if (__kmp_dflt_team_nth == 0) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006724#ifdef KMP_DFLT_NTH_CORES
Jonathan Peyton30419822017-05-12 18:01:32 +00006725 // Default #threads = #cores
6726 __kmp_dflt_team_nth = __kmp_ncores;
6727 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
6728 "__kmp_ncores (%d)\n",
6729 __kmp_dflt_team_nth));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006730#else
Jonathan Peyton30419822017-05-12 18:01:32 +00006731 // Default #threads = #available OS procs
6732 __kmp_dflt_team_nth = __kmp_avail_proc;
6733 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
6734 "__kmp_avail_proc(%d)\n",
6735 __kmp_dflt_team_nth));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006736#endif /* KMP_DFLT_NTH_CORES */
Jonathan Peyton30419822017-05-12 18:01:32 +00006737 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006738
Jonathan Peyton30419822017-05-12 18:01:32 +00006739 if (__kmp_dflt_team_nth < KMP_MIN_NTH) {
6740 __kmp_dflt_team_nth = KMP_MIN_NTH;
6741 }
6742 if (__kmp_dflt_team_nth > __kmp_sys_max_nth) {
6743 __kmp_dflt_team_nth = __kmp_sys_max_nth;
6744 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006745
Jonathan Peyton30419822017-05-12 18:01:32 +00006746 // There's no harm in continuing if the following check fails,
6747 // but it indicates an error in the previous logic.
6748 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006749
Jonathan Peyton30419822017-05-12 18:01:32 +00006750 if (__kmp_dflt_team_nth != prev_dflt_team_nth) {
6751 // Run through the __kmp_threads array and set the num threads icv for each
6752 // root thread that is currently registered with the RTL (which has not
6753 // already explicitly set its nthreads-var with a call to
6754 // omp_set_num_threads()).
6755 for (i = 0; i < __kmp_threads_capacity; i++) {
6756 kmp_info_t *thread = __kmp_threads[i];
6757 if (thread == NULL)
6758 continue;
6759 if (thread->th.th_current_task->td_icvs.nproc != 0)
6760 continue;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006761
Jonathan Peyton30419822017-05-12 18:01:32 +00006762 set__nproc(__kmp_threads[i], __kmp_dflt_team_nth);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006763 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006764 }
6765 KA_TRACE(
6766 20,
6767 ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n",
6768 __kmp_dflt_team_nth));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006769
6770#ifdef KMP_ADJUST_BLOCKTIME
Jonathan Peyton30419822017-05-12 18:01:32 +00006771 /* Adjust blocktime to zero if necessary now that __kmp_avail_proc is set */
6772 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
6773 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
6774 if (__kmp_nth > __kmp_avail_proc) {
6775 __kmp_zero_bt = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006776 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006777 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006778#endif /* KMP_ADJUST_BLOCKTIME */
6779
Jonathan Peyton30419822017-05-12 18:01:32 +00006780 /* we have finished middle initialization */
6781 TCW_SYNC_4(__kmp_init_middle, TRUE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006782
Jonathan Peyton30419822017-05-12 18:01:32 +00006783 KA_TRACE(10, ("__kmp_do_middle_initialize: exit\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006784}
6785
Jonathan Peyton30419822017-05-12 18:01:32 +00006786void __kmp_middle_initialize(void) {
6787 if (__kmp_init_middle) {
6788 return;
6789 }
6790 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6791 if (__kmp_init_middle) {
6792 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6793 return;
6794 }
6795 __kmp_do_middle_initialize();
6796 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6797}
6798
6799void __kmp_parallel_initialize(void) {
6800 int gtid = __kmp_entry_gtid(); // this might be a new root
6801
6802 /* synchronize parallel initialization (for sibling) */
6803 if (TCR_4(__kmp_init_parallel))
6804 return;
6805 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6806 if (TCR_4(__kmp_init_parallel)) {
6807 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6808 return;
6809 }
6810
6811 /* TODO reinitialization after we have already shut down */
6812 if (TCR_4(__kmp_global.g.g_done)) {
6813 KA_TRACE(
6814 10,
6815 ("__kmp_parallel_initialize: attempt to init while shutting down\n"));
6816 __kmp_infinite_loop();
6817 }
6818
6819 /* jc: The lock __kmp_initz_lock is already held, so calling
6820 __kmp_serial_initialize would cause a deadlock. So we call
6821 __kmp_do_serial_initialize directly. */
6822 if (!__kmp_init_middle) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006823 __kmp_do_middle_initialize();
Jonathan Peyton30419822017-05-12 18:01:32 +00006824 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006825
Jonathan Peyton30419822017-05-12 18:01:32 +00006826 /* begin initialization */
6827 KA_TRACE(10, ("__kmp_parallel_initialize: enter\n"));
6828 KMP_ASSERT(KMP_UBER_GTID(gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006829
6830#if KMP_ARCH_X86 || KMP_ARCH_X86_64
Jonathan Peyton30419822017-05-12 18:01:32 +00006831 // Save the FP control regs.
6832 // Worker threads will set theirs to these values at thread startup.
6833 __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
6834 __kmp_store_mxcsr(&__kmp_init_mxcsr);
6835 __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006836#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
6837
6838#if KMP_OS_UNIX
Jonathan Peyton30419822017-05-12 18:01:32 +00006839#if KMP_HANDLE_SIGNALS
6840 /* must be after __kmp_serial_initialize */
6841 __kmp_install_signals(TRUE);
6842#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006843#endif
6844
Jonathan Peyton30419822017-05-12 18:01:32 +00006845 __kmp_suspend_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006846
Jonathan Peyton749b4d52016-01-27 21:02:04 +00006847#if defined(USE_LOAD_BALANCE)
Jonathan Peyton30419822017-05-12 18:01:32 +00006848 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
6849 __kmp_global.g.g_dynamic_mode = dynamic_load_balance;
6850 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006851#else
Jonathan Peyton30419822017-05-12 18:01:32 +00006852 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
6853 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
6854 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006855#endif
6856
Jonathan Peyton30419822017-05-12 18:01:32 +00006857 if (__kmp_version) {
6858 __kmp_print_version_2();
6859 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006860
Jonathan Peyton30419822017-05-12 18:01:32 +00006861 /* we have finished parallel initialization */
6862 TCW_SYNC_4(__kmp_init_parallel, TRUE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006863
Jonathan Peyton30419822017-05-12 18:01:32 +00006864 KMP_MB();
6865 KA_TRACE(10, ("__kmp_parallel_initialize: exit\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006866
Jonathan Peyton30419822017-05-12 18:01:32 +00006867 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006868}
6869
Jim Cownie5e8470a2013-09-27 10:38:44 +00006870/* ------------------------------------------------------------------------ */
6871
Jonathan Peyton30419822017-05-12 18:01:32 +00006872void __kmp_run_before_invoked_task(int gtid, int tid, kmp_info_t *this_thr,
6873 kmp_team_t *team) {
6874 kmp_disp_t *dispatch;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006875
Jonathan Peyton30419822017-05-12 18:01:32 +00006876 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006877
Jonathan Peyton30419822017-05-12 18:01:32 +00006878 /* none of the threads have encountered any constructs, yet. */
6879 this_thr->th.th_local.this_construct = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006880#if KMP_CACHE_MANAGE
Jonathan Peyton30419822017-05-12 18:01:32 +00006881 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006882#endif /* KMP_CACHE_MANAGE */
Jonathan Peyton30419822017-05-12 18:01:32 +00006883 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch);
6884 KMP_DEBUG_ASSERT(dispatch);
6885 KMP_DEBUG_ASSERT(team->t.t_dispatch);
6886 // KMP_DEBUG_ASSERT( this_thr->th.th_dispatch == &team->t.t_dispatch[
6887 // this_thr->th.th_info.ds.ds_tid ] );
Jim Cownie5e8470a2013-09-27 10:38:44 +00006888
Jonathan Peyton30419822017-05-12 18:01:32 +00006889 dispatch->th_disp_index = 0; /* reset the dispatch buffer counter */
Jonathan Peytondf6818b2016-06-14 17:57:47 +00006890#if OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00006891 dispatch->th_doacross_buf_idx =
6892 0; /* reset the doacross dispatch buffer counter */
Jonathan Peyton71909c52016-03-02 22:42:06 +00006893#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006894 if (__kmp_env_consistency_check)
6895 __kmp_push_parallel(gtid, team->t.t_ident);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006896
Jonathan Peyton30419822017-05-12 18:01:32 +00006897 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006898}
6899
Jonathan Peyton30419822017-05-12 18:01:32 +00006900void __kmp_run_after_invoked_task(int gtid, int tid, kmp_info_t *this_thr,
6901 kmp_team_t *team) {
6902 if (__kmp_env_consistency_check)
6903 __kmp_pop_parallel(gtid, team->t.t_ident);
Andrey Churbanovdf0d75e2016-10-27 11:43:07 +00006904
Jonathan Peyton30419822017-05-12 18:01:32 +00006905 __kmp_finish_implicit_task(this_thr);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006906}
6907
Jonathan Peyton30419822017-05-12 18:01:32 +00006908int __kmp_invoke_task_func(int gtid) {
6909 int rc;
6910 int tid = __kmp_tid_from_gtid(gtid);
6911 kmp_info_t *this_thr = __kmp_threads[gtid];
6912 kmp_team_t *team = this_thr->th.th_team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006913
Jonathan Peyton30419822017-05-12 18:01:32 +00006914 __kmp_run_before_invoked_task(gtid, tid, this_thr, team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006915#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +00006916 if (__itt_stack_caller_create_ptr) {
6917 __kmp_itt_stack_callee_enter(
6918 (__itt_caller)
6919 team->t.t_stack_id); // inform ittnotify about entering user's code
6920 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006921#endif /* USE_ITT_BUILD */
Jim Cownie4cc4bb42014-10-07 16:25:50 +00006922#if INCLUDE_SSC_MARKS
Jonathan Peyton30419822017-05-12 18:01:32 +00006923 SSC_MARK_INVOKING();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00006924#endif
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006925
6926#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00006927 void *dummy;
6928 void **exit_runtime_p;
6929 ompt_task_id_t my_task_id;
6930 ompt_parallel_id_t my_parallel_id;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006931
Jonathan Peyton30419822017-05-12 18:01:32 +00006932 if (ompt_enabled) {
6933 exit_runtime_p = &(team->t.t_implicit_task_taskdata[tid]
6934 .ompt_task_info.frame.exit_runtime_frame);
6935 } else {
6936 exit_runtime_p = &dummy;
6937 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006938
6939#if OMPT_TRACE
Jonathan Peyton30419822017-05-12 18:01:32 +00006940 my_task_id = team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_id;
6941 my_parallel_id = team->t.ompt_team_info.parallel_id;
6942 if (ompt_enabled &&
6943 ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)) {
6944 ompt_callbacks.ompt_callback(ompt_event_implicit_task_begin)(my_parallel_id,
6945 my_task_id);
6946 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006947#endif
6948#endif
6949
Jonathan Peyton30419822017-05-12 18:01:32 +00006950 {
6951 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
6952 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
6953 rc =
6954 __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid,
6955 tid, (int)team->t.t_argc, (void **)team->t.t_argv
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006956#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00006957 ,
6958 exit_runtime_p
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006959#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006960 );
Jonas Hahnfeld8a270642016-09-14 13:59:19 +00006961#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00006962 *exit_runtime_p = NULL;
Jonas Hahnfeld8a270642016-09-14 13:59:19 +00006963#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006964 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006965
Jim Cownie5e8470a2013-09-27 10:38:44 +00006966#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +00006967 if (__itt_stack_caller_create_ptr) {
6968 __kmp_itt_stack_callee_leave(
6969 (__itt_caller)
6970 team->t.t_stack_id); // inform ittnotify about leaving user's code
6971 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006972#endif /* USE_ITT_BUILD */
Jonathan Peyton30419822017-05-12 18:01:32 +00006973 __kmp_run_after_invoked_task(gtid, tid, this_thr, team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006974
Jonathan Peyton30419822017-05-12 18:01:32 +00006975 return rc;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006976}
6977
6978#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00006979void __kmp_teams_master(int gtid) {
6980 // This routine is called by all master threads in teams construct
6981 kmp_info_t *thr = __kmp_threads[gtid];
6982 kmp_team_t *team = thr->th.th_team;
6983 ident_t *loc = team->t.t_ident;
6984 thr->th.th_set_nproc = thr->th.th_teams_size.nth;
6985 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask);
6986 KMP_DEBUG_ASSERT(thr->th.th_set_nproc);
6987 KA_TRACE(20, ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid,
6988 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask));
6989// Launch league of teams now, but not let workers execute
6990// (they hang on fork barrier until next parallel)
Jim Cownie4cc4bb42014-10-07 16:25:50 +00006991#if INCLUDE_SSC_MARKS
Jonathan Peyton30419822017-05-12 18:01:32 +00006992 SSC_MARK_FORKING();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00006993#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006994 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006995#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00006996 (void *)thr->th.th_teams_microtask, // "unwrapped" task
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006997#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006998 (microtask_t)thr->th.th_teams_microtask, // "wrapped" task
6999 VOLATILE_CAST(launch_t) __kmp_invoke_task_func, NULL);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007000#if INCLUDE_SSC_MARKS
Jonathan Peyton30419822017-05-12 18:01:32 +00007001 SSC_MARK_JOINING();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007002#endif
Jonathan Peyton61118492016-05-20 19:03:38 +00007003
Jonathan Peyton30419822017-05-12 18:01:32 +00007004 // AC: last parameter "1" eliminates join barrier which won't work because
7005 // worker threads are in a fork barrier waiting for more parallel regions
7006 __kmp_join_call(loc, gtid
Jonathan Peytonf89fbbb2015-08-31 18:15:00 +00007007#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00007008 ,
7009 fork_context_intel
Jonathan Peytonf89fbbb2015-08-31 18:15:00 +00007010#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00007011 ,
7012 1);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007013}
7014
Jonathan Peyton30419822017-05-12 18:01:32 +00007015int __kmp_invoke_teams_master(int gtid) {
7016 kmp_info_t *this_thr = __kmp_threads[gtid];
7017 kmp_team_t *team = this_thr->th.th_team;
7018#if KMP_DEBUG
7019 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized)
7020 KMP_DEBUG_ASSERT((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn ==
7021 (void *)__kmp_teams_master);
7022#endif
7023 __kmp_run_before_invoked_task(gtid, 0, this_thr, team);
7024 __kmp_teams_master(gtid);
7025 __kmp_run_after_invoked_task(gtid, 0, this_thr, team);
7026 return 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007027}
7028#endif /* OMP_40_ENABLED */
7029
7030/* this sets the requested number of threads for the next parallel region
Jonathan Peyton30419822017-05-12 18:01:32 +00007031 encountered by this team. since this should be enclosed in the forkjoin
7032 critical section it should avoid race conditions with assymmetrical nested
7033 parallelism */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007034
Jonathan Peyton30419822017-05-12 18:01:32 +00007035void __kmp_push_num_threads(ident_t *id, int gtid, int num_threads) {
7036 kmp_info_t *thr = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00007037
Jonathan Peyton30419822017-05-12 18:01:32 +00007038 if (num_threads > 0)
7039 thr->th.th_set_nproc = num_threads;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007040}
7041
7042#if OMP_40_ENABLED
7043
7044/* this sets the requested number of teams for the teams region and/or
Jonathan Peyton30419822017-05-12 18:01:32 +00007045 the number of threads for the next parallel region encountered */
7046void __kmp_push_num_teams(ident_t *id, int gtid, int num_teams,
7047 int num_threads) {
7048 kmp_info_t *thr = __kmp_threads[gtid];
7049 KMP_DEBUG_ASSERT(num_teams >= 0);
7050 KMP_DEBUG_ASSERT(num_threads >= 0);
Jonathan Peyton1be692e2015-11-30 20:14:05 +00007051
Jonathan Peyton30419822017-05-12 18:01:32 +00007052 if (num_teams == 0)
7053 num_teams = 1; // default number of teams is 1.
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007054 if (num_teams > __kmp_teams_max_nth) { // if too many teams requested?
Jonathan Peyton30419822017-05-12 18:01:32 +00007055 if (!__kmp_reserve_warn) {
7056 __kmp_reserve_warn = 1;
7057 __kmp_msg(kmp_ms_warning,
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007058 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth),
Jonathan Peyton30419822017-05-12 18:01:32 +00007059 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007060 }
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007061 num_teams = __kmp_teams_max_nth;
Jonathan Peyton30419822017-05-12 18:01:32 +00007062 }
7063 // Set number of teams (number of threads in the outer "parallel" of the
7064 // teams)
7065 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007066
Jonathan Peyton30419822017-05-12 18:01:32 +00007067 // Remember the number of threads for inner parallel regions
7068 if (num_threads == 0) {
7069 if (!TCR_4(__kmp_init_middle))
7070 __kmp_middle_initialize(); // get __kmp_avail_proc calculated
7071 num_threads = __kmp_avail_proc / num_teams;
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007072 if (num_teams * num_threads > __kmp_teams_max_nth) {
Jonathan Peyton30419822017-05-12 18:01:32 +00007073 // adjust num_threads w/o warning as it is not user setting
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007074 num_threads = __kmp_teams_max_nth / num_teams;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007075 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007076 } else {
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007077 if (num_teams * num_threads > __kmp_teams_max_nth) {
7078 int new_threads = __kmp_teams_max_nth / num_teams;
Jonathan Peyton30419822017-05-12 18:01:32 +00007079 if (!__kmp_reserve_warn) { // user asked for too many threads
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007080 __kmp_reserve_warn = 1; // that conflicts with KMP_TEAMS_THREAD_LIMIT
Jonathan Peyton30419822017-05-12 18:01:32 +00007081 __kmp_msg(kmp_ms_warning,
7082 KMP_MSG(CantFormThrTeam, num_threads, new_threads),
7083 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
7084 }
7085 num_threads = new_threads;
7086 }
7087 }
7088 thr->th.th_teams_size.nth = num_threads;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007089}
7090
Jim Cownie5e8470a2013-09-27 10:38:44 +00007091// Set the proc_bind var to use in the following parallel region.
Jonathan Peyton30419822017-05-12 18:01:32 +00007092void __kmp_push_proc_bind(ident_t *id, int gtid, kmp_proc_bind_t proc_bind) {
7093 kmp_info_t *thr = __kmp_threads[gtid];
7094 thr->th.th_set_proc_bind = proc_bind;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007095}
7096
7097#endif /* OMP_40_ENABLED */
7098
7099/* Launch the worker threads into the microtask. */
7100
Jonathan Peyton30419822017-05-12 18:01:32 +00007101void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team) {
7102 kmp_info_t *this_thr = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00007103
7104#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00007105 int f;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007106#endif /* KMP_DEBUG */
7107
Jonathan Peyton30419822017-05-12 18:01:32 +00007108 KMP_DEBUG_ASSERT(team);
7109 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7110 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7111 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007112
Jonathan Peyton30419822017-05-12 18:01:32 +00007113 team->t.t_construct = 0; /* no single directives seen yet */
7114 team->t.t_ordered.dt.t_value =
7115 0; /* thread 0 enters the ordered section first */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007116
Jonathan Peyton30419822017-05-12 18:01:32 +00007117 /* Reset the identifiers on the dispatch buffer */
7118 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
7119 if (team->t.t_max_nproc > 1) {
7120 int i;
7121 for (i = 0; i < __kmp_dispatch_num_buffers; ++i) {
7122 team->t.t_disp_buffer[i].buffer_index = i;
Jonathan Peytondf6818b2016-06-14 17:57:47 +00007123#if OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00007124 team->t.t_disp_buffer[i].doacross_buf_idx = i;
Jonathan Peyton71909c52016-03-02 22:42:06 +00007125#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00007126 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007127 } else {
7128 team->t.t_disp_buffer[0].buffer_index = 0;
7129#if OMP_45_ENABLED
7130 team->t.t_disp_buffer[0].doacross_buf_idx = 0;
7131#endif
7132 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007133
Jonathan Peyton30419822017-05-12 18:01:32 +00007134 KMP_MB(); /* Flush all pending memory write invalidates. */
7135 KMP_ASSERT(this_thr->th.th_team == team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007136
7137#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00007138 for (f = 0; f < team->t.t_nproc; f++) {
7139 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
7140 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc);
7141 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007142#endif /* KMP_DEBUG */
7143
Jonathan Peyton30419822017-05-12 18:01:32 +00007144 /* release the worker threads so they may begin working */
7145 __kmp_fork_barrier(gtid, 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007146}
7147
Jonathan Peyton30419822017-05-12 18:01:32 +00007148void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team) {
7149 kmp_info_t *this_thr = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00007150
Jonathan Peyton30419822017-05-12 18:01:32 +00007151 KMP_DEBUG_ASSERT(team);
7152 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7153 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7154 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007155
Jonathan Peyton30419822017-05-12 18:01:32 +00007156/* Join barrier after fork */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007157
7158#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00007159 if (__kmp_threads[gtid] &&
7160 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) {
7161 __kmp_printf("GTID: %d, __kmp_threads[%d]=%p\n", gtid, gtid,
7162 __kmp_threads[gtid]);
7163 __kmp_printf("__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, "
7164 "team->t.t_nproc=%d\n",
7165 gtid, __kmp_threads[gtid]->th.th_team_nproc, team,
7166 team->t.t_nproc);
7167 __kmp_print_structure();
7168 }
7169 KMP_DEBUG_ASSERT(__kmp_threads[gtid] &&
7170 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007171#endif /* KMP_DEBUG */
7172
Jonathan Peyton30419822017-05-12 18:01:32 +00007173 __kmp_join_barrier(gtid); /* wait for everyone */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007174
Jonathan Peyton30419822017-05-12 18:01:32 +00007175 KMP_MB(); /* Flush all pending memory write invalidates. */
7176 KMP_ASSERT(this_thr->th.th_team == team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007177}
7178
Jim Cownie5e8470a2013-09-27 10:38:44 +00007179/* ------------------------------------------------------------------------ */
7180
7181#ifdef USE_LOAD_BALANCE
7182
Jim Cownie5e8470a2013-09-27 10:38:44 +00007183// Return the worker threads actively spinning in the hot team, if we
7184// are at the outermost level of parallelism. Otherwise, return 0.
Jonathan Peyton30419822017-05-12 18:01:32 +00007185static int __kmp_active_hot_team_nproc(kmp_root_t *root) {
7186 int i;
7187 int retval;
7188 kmp_team_t *hot_team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007189
Jonathan Peyton30419822017-05-12 18:01:32 +00007190 if (root->r.r_active) {
7191 return 0;
7192 }
7193 hot_team = root->r.r_hot_team;
7194 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
7195 return hot_team->t.t_nproc - 1; // Don't count master thread
7196 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007197
Jonathan Peyton30419822017-05-12 18:01:32 +00007198 // Skip the master thread - it is accounted for elsewhere.
7199 retval = 0;
7200 for (i = 1; i < hot_team->t.t_nproc; i++) {
7201 if (hot_team->t.t_threads[i]->th.th_active) {
7202 retval++;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007203 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007204 }
7205 return retval;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007206}
7207
Jim Cownie5e8470a2013-09-27 10:38:44 +00007208// Perform an automatic adjustment to the number of
7209// threads used by the next parallel region.
Jonathan Peyton30419822017-05-12 18:01:32 +00007210static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc) {
7211 int retval;
7212 int pool_active;
7213 int hot_team_active;
7214 int team_curr_active;
7215 int system_active;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007216
Jonathan Peyton30419822017-05-12 18:01:32 +00007217 KB_TRACE(20, ("__kmp_load_balance_nproc: called root:%p set_nproc:%d\n", root,
7218 set_nproc));
7219 KMP_DEBUG_ASSERT(root);
7220 KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0]
7221 ->th.th_current_task->td_icvs.dynamic == TRUE);
7222 KMP_DEBUG_ASSERT(set_nproc > 1);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007223
Jonathan Peyton30419822017-05-12 18:01:32 +00007224 if (set_nproc == 1) {
7225 KB_TRACE(20, ("__kmp_load_balance_nproc: serial execution.\n"));
7226 return 1;
7227 }
7228
7229 // Threads that are active in the thread pool, active in the hot team for this
7230 // particular root (if we are at the outer par level), and the currently
7231 // executing thread (to become the master) are available to add to the new
7232 // team, but are currently contributing to the system load, and must be
7233 // accounted for.
7234 pool_active = TCR_4(__kmp_thread_pool_active_nth);
7235 hot_team_active = __kmp_active_hot_team_nproc(root);
7236 team_curr_active = pool_active + hot_team_active + 1;
7237
7238 // Check the system load.
7239 system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active);
7240 KB_TRACE(30, ("__kmp_load_balance_nproc: system active = %d pool active = %d "
7241 "hot team active = %d\n",
7242 system_active, pool_active, hot_team_active));
7243
7244 if (system_active < 0) {
7245 // There was an error reading the necessary info from /proc, so use the
7246 // thread limit algorithm instead. Once we set __kmp_global.g.g_dynamic_mode
7247 // = dynamic_thread_limit, we shouldn't wind up getting back here.
7248 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
7249 KMP_WARNING(CantLoadBalUsing, "KMP_DYNAMIC_MODE=thread limit");
7250
7251 // Make this call behave like the thread limit algorithm.
7252 retval = __kmp_avail_proc - __kmp_nth +
7253 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
7254 if (retval > set_nproc) {
7255 retval = set_nproc;
7256 }
7257 if (retval < KMP_MIN_NTH) {
7258 retval = KMP_MIN_NTH;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007259 }
7260
Jonathan Peyton30419822017-05-12 18:01:32 +00007261 KB_TRACE(20, ("__kmp_load_balance_nproc: thread limit exit. retval:%d\n",
7262 retval));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007263 return retval;
Jonathan Peyton30419822017-05-12 18:01:32 +00007264 }
7265
7266 // There is a slight delay in the load balance algorithm in detecting new
7267 // running procs. The real system load at this instant should be at least as
7268 // large as the #active omp thread that are available to add to the team.
7269 if (system_active < team_curr_active) {
7270 system_active = team_curr_active;
7271 }
7272 retval = __kmp_avail_proc - system_active + team_curr_active;
7273 if (retval > set_nproc) {
7274 retval = set_nproc;
7275 }
7276 if (retval < KMP_MIN_NTH) {
7277 retval = KMP_MIN_NTH;
7278 }
7279
7280 KB_TRACE(20, ("__kmp_load_balance_nproc: exit. retval:%d\n", retval));
7281 return retval;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007282} // __kmp_load_balance_nproc()
7283
7284#endif /* USE_LOAD_BALANCE */
7285
Jim Cownie5e8470a2013-09-27 10:38:44 +00007286/* ------------------------------------------------------------------------ */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007287
7288/* NOTE: this is called with the __kmp_init_lock held */
Jonathan Peyton30419822017-05-12 18:01:32 +00007289void __kmp_cleanup(void) {
7290 int f;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007291
Jonathan Peyton30419822017-05-12 18:01:32 +00007292 KA_TRACE(10, ("__kmp_cleanup: enter\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007293
Jonathan Peyton30419822017-05-12 18:01:32 +00007294 if (TCR_4(__kmp_init_parallel)) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00007295#if KMP_HANDLE_SIGNALS
Jonathan Peyton30419822017-05-12 18:01:32 +00007296 __kmp_remove_signals();
Jim Cownie5e8470a2013-09-27 10:38:44 +00007297#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00007298 TCW_4(__kmp_init_parallel, FALSE);
7299 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007300
Jonathan Peyton30419822017-05-12 18:01:32 +00007301 if (TCR_4(__kmp_init_middle)) {
Alp Toker763b9392014-02-28 09:42:41 +00007302#if KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00007303 __kmp_affinity_uninitialize();
Alp Toker763b9392014-02-28 09:42:41 +00007304#endif /* KMP_AFFINITY_SUPPORTED */
Jonathan Peyton30419822017-05-12 18:01:32 +00007305 __kmp_cleanup_hierarchy();
7306 TCW_4(__kmp_init_middle, FALSE);
7307 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007308
Jonathan Peyton30419822017-05-12 18:01:32 +00007309 KA_TRACE(10, ("__kmp_cleanup: go serial cleanup\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007310
Jonathan Peyton30419822017-05-12 18:01:32 +00007311 if (__kmp_init_serial) {
7312 __kmp_runtime_destroy();
7313 __kmp_init_serial = FALSE;
7314 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007315
Jonathan Peyton30419822017-05-12 18:01:32 +00007316 for (f = 0; f < __kmp_threads_capacity; f++) {
7317 if (__kmp_root[f] != NULL) {
7318 __kmp_free(__kmp_root[f]);
7319 __kmp_root[f] = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007320 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007321 }
7322 __kmp_free(__kmp_threads);
7323 // __kmp_threads and __kmp_root were allocated at once, as single block, so
7324 // there is no need in freeing __kmp_root.
7325 __kmp_threads = NULL;
7326 __kmp_root = NULL;
7327 __kmp_threads_capacity = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007328
Andrey Churbanov5c56fb52015-02-20 18:05:17 +00007329#if KMP_USE_DYNAMIC_LOCK
Jonathan Peyton30419822017-05-12 18:01:32 +00007330 __kmp_cleanup_indirect_user_locks();
Andrey Churbanov5c56fb52015-02-20 18:05:17 +00007331#else
Jonathan Peyton30419822017-05-12 18:01:32 +00007332 __kmp_cleanup_user_locks();
Andrey Churbanov5c56fb52015-02-20 18:05:17 +00007333#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00007334
Jonathan Peyton30419822017-05-12 18:01:32 +00007335#if KMP_AFFINITY_SUPPORTED
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00007336 KMP_INTERNAL_FREE(CCAST(char *, __kmp_cpuinfo_file));
Jonathan Peyton30419822017-05-12 18:01:32 +00007337 __kmp_cpuinfo_file = NULL;
7338#endif /* KMP_AFFINITY_SUPPORTED */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007339
Jonathan Peyton30419822017-05-12 18:01:32 +00007340#if KMP_USE_ADAPTIVE_LOCKS
7341#if KMP_DEBUG_ADAPTIVE_LOCKS
7342 __kmp_print_speculative_stats();
7343#endif
7344#endif
7345 KMP_INTERNAL_FREE(__kmp_nested_nth.nth);
7346 __kmp_nested_nth.nth = NULL;
7347 __kmp_nested_nth.size = 0;
7348 __kmp_nested_nth.used = 0;
7349 KMP_INTERNAL_FREE(__kmp_nested_proc_bind.bind_types);
7350 __kmp_nested_proc_bind.bind_types = NULL;
7351 __kmp_nested_proc_bind.size = 0;
7352 __kmp_nested_proc_bind.used = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007353
Jonathan Peyton30419822017-05-12 18:01:32 +00007354 __kmp_i18n_catclose();
Jim Cownie5e8470a2013-09-27 10:38:44 +00007355
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007356#if KMP_STATS_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00007357 __kmp_stats_fini();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007358#endif
7359
Jonathan Peyton30419822017-05-12 18:01:32 +00007360 KA_TRACE(10, ("__kmp_cleanup: exit\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007361}
7362
7363/* ------------------------------------------------------------------------ */
Jonathan Peyton30419822017-05-12 18:01:32 +00007364
7365int __kmp_ignore_mppbeg(void) {
7366 char *env;
7367
7368 if ((env = getenv("KMP_IGNORE_MPPBEG")) != NULL) {
7369 if (__kmp_str_match_false(env))
7370 return FALSE;
7371 }
7372 // By default __kmpc_begin() is no-op.
7373 return TRUE;
7374}
7375
7376int __kmp_ignore_mppend(void) {
7377 char *env;
7378
7379 if ((env = getenv("KMP_IGNORE_MPPEND")) != NULL) {
7380 if (__kmp_str_match_false(env))
7381 return FALSE;
7382 }
7383 // By default __kmpc_end() is no-op.
7384 return TRUE;
7385}
7386
7387void __kmp_internal_begin(void) {
7388 int gtid;
7389 kmp_root_t *root;
7390
7391 /* this is a very important step as it will register new sibling threads
7392 and assign these new uber threads a new gtid */
7393 gtid = __kmp_entry_gtid();
7394 root = __kmp_threads[gtid]->th.th_root;
7395 KMP_ASSERT(KMP_UBER_GTID(gtid));
7396
7397 if (root->r.r_begin)
7398 return;
7399 __kmp_acquire_lock(&root->r.r_begin_lock, gtid);
7400 if (root->r.r_begin) {
7401 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7402 return;
7403 }
7404
7405 root->r.r_begin = TRUE;
7406
7407 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7408}
7409
Jim Cownie5e8470a2013-09-27 10:38:44 +00007410/* ------------------------------------------------------------------------ */
7411
Jonathan Peyton30419822017-05-12 18:01:32 +00007412void __kmp_user_set_library(enum library_type arg) {
7413 int gtid;
7414 kmp_root_t *root;
7415 kmp_info_t *thread;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007416
Jonathan Peyton30419822017-05-12 18:01:32 +00007417 /* first, make sure we are initialized so we can get our gtid */
7418
7419 gtid = __kmp_entry_gtid();
7420 thread = __kmp_threads[gtid];
7421
7422 root = thread->th.th_root;
7423
7424 KA_TRACE(20, ("__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg,
7425 library_serial));
7426 if (root->r.r_in_parallel) { /* Must be called in serial section of top-level
7427 thread */
7428 KMP_WARNING(SetLibraryIncorrectCall);
7429 return;
7430 }
7431
7432 switch (arg) {
7433 case library_serial:
7434 thread->th.th_set_nproc = 0;
7435 set__nproc(thread, 1);
7436 break;
7437 case library_turnaround:
7438 thread->th.th_set_nproc = 0;
7439 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
7440 : __kmp_dflt_team_nth_ub);
7441 break;
7442 case library_throughput:
7443 thread->th.th_set_nproc = 0;
7444 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
7445 : __kmp_dflt_team_nth_ub);
7446 break;
7447 default:
7448 KMP_FATAL(UnknownLibraryType, arg);
7449 }
7450
7451 __kmp_aux_set_library(arg);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007452}
7453
Jonathan Peyton30419822017-05-12 18:01:32 +00007454void __kmp_aux_set_stacksize(size_t arg) {
7455 if (!__kmp_init_serial)
7456 __kmp_serial_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00007457
7458#if KMP_OS_DARWIN
Jonathan Peyton30419822017-05-12 18:01:32 +00007459 if (arg & (0x1000 - 1)) {
7460 arg &= ~(0x1000 - 1);
7461 if (arg + 0x1000) /* check for overflow if we round up */
7462 arg += 0x1000;
7463 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007464#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00007465 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007466
Jonathan Peyton30419822017-05-12 18:01:32 +00007467 /* only change the default stacksize before the first parallel region */
7468 if (!TCR_4(__kmp_init_parallel)) {
7469 size_t value = arg; /* argument is in bytes */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007470
Jonathan Peyton30419822017-05-12 18:01:32 +00007471 if (value < __kmp_sys_min_stksize)
7472 value = __kmp_sys_min_stksize;
7473 else if (value > KMP_MAX_STKSIZE)
7474 value = KMP_MAX_STKSIZE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007475
Jonathan Peyton30419822017-05-12 18:01:32 +00007476 __kmp_stksize = value;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007477
Jonathan Peyton30419822017-05-12 18:01:32 +00007478 __kmp_env_stksize = TRUE; /* was KMP_STACKSIZE specified? */
7479 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007480
Jonathan Peyton30419822017-05-12 18:01:32 +00007481 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007482}
7483
7484/* set the behaviour of the runtime library */
7485/* TODO this can cause some odd behaviour with sibling parallelism... */
Jonathan Peyton30419822017-05-12 18:01:32 +00007486void __kmp_aux_set_library(enum library_type arg) {
7487 __kmp_library = arg;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007488
Jonathan Peyton30419822017-05-12 18:01:32 +00007489 switch (__kmp_library) {
7490 case library_serial: {
7491 KMP_INFORM(LibraryIsSerial);
7492 (void)__kmp_change_library(TRUE);
7493 } break;
7494 case library_turnaround:
7495 (void)__kmp_change_library(TRUE);
7496 break;
7497 case library_throughput:
7498 (void)__kmp_change_library(FALSE);
7499 break;
7500 default:
7501 KMP_FATAL(UnknownLibraryType, arg);
7502 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007503}
7504
7505/* ------------------------------------------------------------------------ */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007506
Jonathan Peyton30419822017-05-12 18:01:32 +00007507void __kmp_aux_set_blocktime(int arg, kmp_info_t *thread, int tid) {
7508 int blocktime = arg; /* argument is in milliseconds */
Jonathan Peytone1c7c132016-10-07 18:12:19 +00007509#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00007510 int bt_intervals;
Jonathan Peytone1c7c132016-10-07 18:12:19 +00007511#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00007512 int bt_set;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007513
Jonathan Peyton30419822017-05-12 18:01:32 +00007514 __kmp_save_internal_controls(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007515
Jonathan Peyton30419822017-05-12 18:01:32 +00007516 /* Normalize and set blocktime for the teams */
7517 if (blocktime < KMP_MIN_BLOCKTIME)
7518 blocktime = KMP_MIN_BLOCKTIME;
7519 else if (blocktime > KMP_MAX_BLOCKTIME)
7520 blocktime = KMP_MAX_BLOCKTIME;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007521
Jonathan Peyton30419822017-05-12 18:01:32 +00007522 set__blocktime_team(thread->th.th_team, tid, blocktime);
7523 set__blocktime_team(thread->th.th_serial_team, 0, blocktime);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007524
Jonathan Peytone1c7c132016-10-07 18:12:19 +00007525#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00007526 /* Calculate and set blocktime intervals for the teams */
7527 bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007528
Jonathan Peyton30419822017-05-12 18:01:32 +00007529 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals);
7530 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals);
Jonathan Peytone1c7c132016-10-07 18:12:19 +00007531#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00007532
Jonathan Peyton30419822017-05-12 18:01:32 +00007533 /* Set whether blocktime has been set to "TRUE" */
7534 bt_set = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007535
Jonathan Peyton30419822017-05-12 18:01:32 +00007536 set__bt_set_team(thread->th.th_team, tid, bt_set);
7537 set__bt_set_team(thread->th.th_serial_team, 0, bt_set);
Jonathan Peytone1c7c132016-10-07 18:12:19 +00007538#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00007539 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
7540 "bt_intervals=%d, monitor_updates=%d\n",
7541 __kmp_gtid_from_tid(tid, thread->th.th_team),
7542 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals,
7543 __kmp_monitor_wakeups));
Samuel Antao33515192016-10-20 13:20:17 +00007544#else
Jonathan Peyton30419822017-05-12 18:01:32 +00007545 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n",
7546 __kmp_gtid_from_tid(tid, thread->th.th_team),
7547 thread->th.th_team->t.t_id, tid, blocktime));
Jonathan Peytone1c7c132016-10-07 18:12:19 +00007548#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00007549}
7550
Jonathan Peyton30419822017-05-12 18:01:32 +00007551void __kmp_aux_set_defaults(char const *str, int len) {
7552 if (!__kmp_init_serial) {
7553 __kmp_serial_initialize();
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00007554 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007555 __kmp_env_initialize(str);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007556
Jonathan Peyton30419822017-05-12 18:01:32 +00007557 if (__kmp_settings
Jim Cownie5e8470a2013-09-27 10:38:44 +00007558#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00007559 || __kmp_display_env || __kmp_display_env_verbose
Jim Cownie5e8470a2013-09-27 10:38:44 +00007560#endif // OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00007561 ) {
7562 __kmp_env_print();
7563 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007564} // __kmp_aux_set_defaults
7565
7566/* ------------------------------------------------------------------------ */
Jonathan Peyton30419822017-05-12 18:01:32 +00007567/* internal fast reduction routines */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007568
Jim Cownie5e8470a2013-09-27 10:38:44 +00007569PACKED_REDUCTION_METHOD_T
Jonathan Peyton30419822017-05-12 18:01:32 +00007570__kmp_determine_reduction_method(
7571 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size,
7572 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),
7573 kmp_critical_name *lck) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00007574
Jonathan Peyton30419822017-05-12 18:01:32 +00007575 // Default reduction method: critical construct ( lck != NULL, like in current
7576 // PAROPT )
7577 // If ( reduce_data!=NULL && reduce_func!=NULL ): the tree-reduction method
7578 // can be selected by RTL
7579 // If loc->flags contains KMP_IDENT_ATOMIC_REDUCE, the atomic reduce method
7580 // can be selected by RTL
7581 // Finally, it's up to OpenMP RTL to make a decision on which method to select
7582 // among generated by PAROPT.
Jim Cownie5e8470a2013-09-27 10:38:44 +00007583
Jonathan Peyton30419822017-05-12 18:01:32 +00007584 PACKED_REDUCTION_METHOD_T retval;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007585
Jonathan Peyton30419822017-05-12 18:01:32 +00007586 int team_size;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007587
Jonathan Peyton30419822017-05-12 18:01:32 +00007588 KMP_DEBUG_ASSERT(loc); // it would be nice to test ( loc != 0 )
7589 KMP_DEBUG_ASSERT(lck); // it would be nice to test ( lck != 0 )
Jim Cownie5e8470a2013-09-27 10:38:44 +00007590
Jonathan Peyton30419822017-05-12 18:01:32 +00007591#define FAST_REDUCTION_ATOMIC_METHOD_GENERATED \
7592 ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE)) == (KMP_IDENT_ATOMIC_REDUCE))
7593#define FAST_REDUCTION_TREE_METHOD_GENERATED ((reduce_data) && (reduce_func))
Jim Cownie5e8470a2013-09-27 10:38:44 +00007594
Jonathan Peyton30419822017-05-12 18:01:32 +00007595 retval = critical_reduce_block;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007596
Jonathan Peyton30419822017-05-12 18:01:32 +00007597 // another choice of getting a team size (with 1 dynamic deference) is slower
7598 team_size = __kmp_get_team_num_threads(global_tid);
7599 if (team_size == 1) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00007600
Jonathan Peyton30419822017-05-12 18:01:32 +00007601 retval = empty_reduce_block;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007602
Jonathan Peyton30419822017-05-12 18:01:32 +00007603 } else {
Jim Cownie5e8470a2013-09-27 10:38:44 +00007604
Jonathan Peyton30419822017-05-12 18:01:32 +00007605 int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
7606 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007607
Jonathan Peyton30419822017-05-12 18:01:32 +00007608#if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64
Jim Cownie5e8470a2013-09-27 10:38:44 +00007609
Jonathan Peyton30419822017-05-12 18:01:32 +00007610#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_WINDOWS || \
7611 KMP_OS_DARWIN
Jim Cownie5e8470a2013-09-27 10:38:44 +00007612
Jonathan Peyton30419822017-05-12 18:01:32 +00007613 int teamsize_cutoff = 4;
Jonathan Peyton91b78702015-06-08 19:39:07 +00007614
Jonathan Peyton492e0a32017-06-13 17:17:26 +00007615#if KMP_MIC_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00007616 if (__kmp_mic_type != non_mic) {
7617 teamsize_cutoff = 8;
7618 }
Andrey Churbanov613edeb2015-02-20 18:14:43 +00007619#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00007620 if (tree_available) {
7621 if (team_size <= teamsize_cutoff) {
7622 if (atomic_available) {
7623 retval = atomic_reduce_block;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007624 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007625 } else {
7626 retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
7627 }
7628 } else if (atomic_available) {
7629 retval = atomic_reduce_block;
7630 }
7631#else
7632#error "Unknown or unsupported OS"
7633#endif // KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_WINDOWS ||
7634// KMP_OS_DARWIN
Jim Cownie5e8470a2013-09-27 10:38:44 +00007635
Jonathan Peyton30419822017-05-12 18:01:32 +00007636#elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS
7637
7638#if KMP_OS_LINUX || KMP_OS_WINDOWS
7639
7640 // basic tuning
7641
7642 if (atomic_available) {
7643 if (num_vars <= 2) { // && ( team_size <= 8 ) due to false-sharing ???
7644 retval = atomic_reduce_block;
7645 }
7646 } // otherwise: use critical section
7647
7648#elif KMP_OS_DARWIN
7649
7650 if (atomic_available && (num_vars <= 3)) {
7651 retval = atomic_reduce_block;
7652 } else if (tree_available) {
7653 if ((reduce_size > (9 * sizeof(kmp_real64))) &&
7654 (reduce_size < (2000 * sizeof(kmp_real64)))) {
7655 retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER;
7656 }
7657 } // otherwise: use critical section
7658
7659#else
7660#error "Unknown or unsupported OS"
7661#endif
7662
7663#else
7664#error "Unknown or unsupported architecture"
7665#endif
7666 }
7667
7668 // KMP_FORCE_REDUCTION
7669
7670 // If the team is serialized (team_size == 1), ignore the forced reduction
7671 // method and stay with the unsynchronized method (empty_reduce_block)
7672 if (__kmp_force_reduction_method != reduction_method_not_defined &&
7673 team_size != 1) {
7674
7675 PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block;
7676
7677 int atomic_available, tree_available;
7678
7679 switch ((forced_retval = __kmp_force_reduction_method)) {
7680 case critical_reduce_block:
7681 KMP_ASSERT(lck); // lck should be != 0
7682 break;
7683
7684 case atomic_reduce_block:
7685 atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
7686 if (!atomic_available) {
7687 KMP_WARNING(RedMethodNotSupported, "atomic");
7688 forced_retval = critical_reduce_block;
7689 }
7690 break;
7691
7692 case tree_reduce_block:
7693 tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
7694 if (!tree_available) {
7695 KMP_WARNING(RedMethodNotSupported, "tree");
7696 forced_retval = critical_reduce_block;
7697 } else {
7698#if KMP_FAST_REDUCTION_BARRIER
7699 forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
7700#endif
7701 }
7702 break;
7703
7704 default:
7705 KMP_ASSERT(0); // "unsupported method specified"
Jim Cownie5e8470a2013-09-27 10:38:44 +00007706 }
7707
Jonathan Peyton30419822017-05-12 18:01:32 +00007708 retval = forced_retval;
7709 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007710
Jonathan Peyton30419822017-05-12 18:01:32 +00007711 KA_TRACE(10, ("reduction method selected=%08x\n", retval));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007712
Jonathan Peyton30419822017-05-12 18:01:32 +00007713#undef FAST_REDUCTION_TREE_METHOD_GENERATED
7714#undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED
7715
7716 return (retval);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007717}
7718
7719// this function is for testing set/get/determine reduce method
Jonathan Peyton30419822017-05-12 18:01:32 +00007720kmp_int32 __kmp_get_reduce_method(void) {
7721 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007722}