blob: 66726312d105952326f1513548efed5093e5b0d7 [file] [log] [blame]
Jim Cownie5e8470a2013-09-27 10:38:44 +00001/*
Jonathan Peytonde4749b2016-12-14 23:01:24 +00002 * kmp_runtime.cpp -- KPTS runtime support library
Jim Cownie5e8470a2013-09-27 10:38:44 +00003 */
4
Jim Cownie5e8470a2013-09-27 10:38:44 +00005//===----------------------------------------------------------------------===//
6//
Chandler Carruth57b08b02019-01-19 10:56:40 +00007// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8// See https://llvm.org/LICENSE.txt for license information.
9// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
Jim Cownie5e8470a2013-09-27 10:38:44 +000010//
11//===----------------------------------------------------------------------===//
12
Jim Cownie5e8470a2013-09-27 10:38:44 +000013#include "kmp.h"
Jonathan Peyton30419822017-05-12 18:01:32 +000014#include "kmp_affinity.h"
Jim Cownie5e8470a2013-09-27 10:38:44 +000015#include "kmp_atomic.h"
Jim Cownie5e8470a2013-09-27 10:38:44 +000016#include "kmp_environment.h"
Jonathan Peyton30419822017-05-12 18:01:32 +000017#include "kmp_error.h"
Jim Cownie5e8470a2013-09-27 10:38:44 +000018#include "kmp_i18n.h"
19#include "kmp_io.h"
Jonathan Peyton30419822017-05-12 18:01:32 +000020#include "kmp_itt.h"
21#include "kmp_settings.h"
Jim Cownie4cc4bb42014-10-07 16:25:50 +000022#include "kmp_stats.h"
Jonathan Peyton30419822017-05-12 18:01:32 +000023#include "kmp_str.h"
Jim Cownie4cc4bb42014-10-07 16:25:50 +000024#include "kmp_wait_release.h"
Jonathan Peyton30419822017-05-12 18:01:32 +000025#include "kmp_wrapper_getpid.h"
Jonathan Peytonf6399362018-07-09 17:51:13 +000026#include "kmp_dispatch.h"
27#if KMP_USE_HIER_SCHED
28#include "kmp_dispatch_hier.h"
29#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +000030
Andrey Churbanovd7d088f2015-04-29 16:42:24 +000031#if OMPT_SUPPORT
32#include "ompt-specific.h"
33#endif
34
Jim Cownie5e8470a2013-09-27 10:38:44 +000035/* these are temporary issues to be dealt with */
36#define KMP_USE_PRCTL 0
Jim Cownie5e8470a2013-09-27 10:38:44 +000037
Jim Cownie5e8470a2013-09-27 10:38:44 +000038#if KMP_OS_WINDOWS
39#include <process.h>
40#endif
41
Jonas Hahnfeld50fed042016-11-07 15:58:36 +000042#include "tsan_annotations.h"
Jim Cownie5e8470a2013-09-27 10:38:44 +000043
44#if defined(KMP_GOMP_COMPAT)
Jonathan Peyton30419822017-05-12 18:01:32 +000045char const __kmp_version_alt_comp[] =
46 KMP_VERSION_PREFIX "alternative compiler support: yes";
Jim Cownie5e8470a2013-09-27 10:38:44 +000047#endif /* defined(KMP_GOMP_COMPAT) */
48
49char const __kmp_version_omp_api[] = KMP_VERSION_PREFIX "API version: "
Jonathan Peytone844a542017-03-06 22:07:40 +000050#if OMP_50_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +000051 "5.0 (201611)";
Jonathan Peytone844a542017-03-06 22:07:40 +000052#elif OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +000053 "4.5 (201511)";
Jonathan Peyton74f3ffc2016-09-30 15:50:14 +000054#elif OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +000055 "4.0 (201307)";
Jim Cownie5e8470a2013-09-27 10:38:44 +000056#else
Jonathan Peyton30419822017-05-12 18:01:32 +000057 "3.1 (201107)";
Jim Cownie5e8470a2013-09-27 10:38:44 +000058#endif
59
60#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +000061char const __kmp_version_lock[] =
62 KMP_VERSION_PREFIX "lock type: run time selectable";
Jim Cownie5e8470a2013-09-27 10:38:44 +000063#endif /* KMP_DEBUG */
64
Jonathan Peyton30419822017-05-12 18:01:32 +000065#define KMP_MIN(x, y) ((x) < (y) ? (x) : (y))
Jim Cownie181b4bb2013-12-23 17:28:57 +000066
Jim Cownie5e8470a2013-09-27 10:38:44 +000067/* ------------------------------------------------------------------------ */
Jim Cownie5e8470a2013-09-27 10:38:44 +000068
Jonathan Peyton37e2ef52018-07-09 17:36:22 +000069#if KMP_USE_MONITOR
Jim Cownie5e8470a2013-09-27 10:38:44 +000070kmp_info_t __kmp_monitor;
Jonathan Peyton37e2ef52018-07-09 17:36:22 +000071#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +000072
Jim Cownie5e8470a2013-09-27 10:38:44 +000073/* Forward declarations */
74
Jonathan Peyton30419822017-05-12 18:01:32 +000075void __kmp_cleanup(void);
Jim Cownie5e8470a2013-09-27 10:38:44 +000076
Jonathan Peyton30419822017-05-12 18:01:32 +000077static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *, int tid,
78 int gtid);
79static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
80 kmp_internal_control_t *new_icvs,
81 ident_t *loc);
Jonathan Peyton2321d572015-06-08 19:25:25 +000082#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +000083static void __kmp_partition_places(kmp_team_t *team,
84 int update_master_only = 0);
Jonathan Peyton2321d572015-06-08 19:25:25 +000085#endif
Jonathan Peyton30419822017-05-12 18:01:32 +000086static void __kmp_do_serial_initialize(void);
87void __kmp_fork_barrier(int gtid, int tid);
88void __kmp_join_barrier(int gtid);
89void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc,
90 kmp_internal_control_t *new_icvs, ident_t *loc);
Jim Cownie5e8470a2013-09-27 10:38:44 +000091
Jim Cownie5e8470a2013-09-27 10:38:44 +000092#ifdef USE_LOAD_BALANCE
Jonathan Peyton30419822017-05-12 18:01:32 +000093static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc);
Jim Cownie5e8470a2013-09-27 10:38:44 +000094#endif
95
Jonathan Peyton1800ece2018-01-10 18:27:01 +000096static int __kmp_expand_threads(int nNeed);
Jonathan Peyton2321d572015-06-08 19:25:25 +000097#if KMP_OS_WINDOWS
Jonathan Peyton30419822017-05-12 18:01:32 +000098static int __kmp_unregister_root_other_thread(int gtid);
Jonathan Peyton2321d572015-06-08 19:25:25 +000099#endif
Jonathan Peyton30419822017-05-12 18:01:32 +0000100static void __kmp_unregister_library(void); // called by __kmp_internal_end()
101static void __kmp_reap_thread(kmp_info_t *thread, int is_root);
Jonathan Peytoneaa9e402018-01-10 18:21:48 +0000102kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000103
Jim Cownie5e8470a2013-09-27 10:38:44 +0000104/* Calculate the identifier of the current thread */
Jonathan Peyton30419822017-05-12 18:01:32 +0000105/* fast (and somewhat portable) way to get unique identifier of executing
106 thread. Returns KMP_GTID_DNE if we haven't been assigned a gtid. */
Jonathan Peyton30419822017-05-12 18:01:32 +0000107int __kmp_get_global_thread_id() {
108 int i;
109 kmp_info_t **other_threads;
110 size_t stack_data;
111 char *stack_addr;
112 size_t stack_size;
113 char *stack_base;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000114
Jonathan Peyton30419822017-05-12 18:01:32 +0000115 KA_TRACE(
116 1000,
117 ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n",
118 __kmp_nth, __kmp_all_nth));
Jim Cownie5e8470a2013-09-27 10:38:44 +0000119
Jonathan Peyton30419822017-05-12 18:01:32 +0000120 /* JPH - to handle the case where __kmpc_end(0) is called immediately prior to
121 a parallel region, made it return KMP_GTID_DNE to force serial_initialize
122 by caller. Had to handle KMP_GTID_DNE at all call-sites, or else guarantee
123 __kmp_init_gtid for this to work. */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000124
Jonathan Peyton30419822017-05-12 18:01:32 +0000125 if (!TCR_4(__kmp_init_gtid))
126 return KMP_GTID_DNE;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000127
128#ifdef KMP_TDATA_GTID
Jonathan Peyton30419822017-05-12 18:01:32 +0000129 if (TCR_4(__kmp_gtid_mode) >= 3) {
130 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using TDATA\n"));
131 return __kmp_gtid;
132 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000133#endif
Jonathan Peyton30419822017-05-12 18:01:32 +0000134 if (TCR_4(__kmp_gtid_mode) >= 2) {
135 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using keyed TLS\n"));
136 return __kmp_gtid_get_specific();
137 }
138 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using internal alg.\n"));
139
140 stack_addr = (char *)&stack_data;
141 other_threads = __kmp_threads;
142
143 /* ATT: The code below is a source of potential bugs due to unsynchronized
144 access to __kmp_threads array. For example:
145 1. Current thread loads other_threads[i] to thr and checks it, it is
146 non-NULL.
147 2. Current thread is suspended by OS.
148 3. Another thread unregisters and finishes (debug versions of free()
149 may fill memory with something like 0xEF).
150 4. Current thread is resumed.
151 5. Current thread reads junk from *thr.
152 TODO: Fix it. --ln */
153
154 for (i = 0; i < __kmp_threads_capacity; i++) {
155
156 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i]);
157 if (!thr)
158 continue;
159
160 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize);
161 stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase);
162
163 /* stack grows down -- search through all of the active threads */
164
165 if (stack_addr <= stack_base) {
166 size_t stack_diff = stack_base - stack_addr;
167
168 if (stack_diff <= stack_size) {
169 /* The only way we can be closer than the allocated */
170 /* stack size is if we are running on this thread. */
171 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i);
172 return i;
173 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000174 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000175 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000176
Jonathan Peyton30419822017-05-12 18:01:32 +0000177 /* get specific to try and determine our gtid */
178 KA_TRACE(1000,
179 ("*** __kmp_get_global_thread_id: internal alg. failed to find "
180 "thread, using TLS\n"));
181 i = __kmp_gtid_get_specific();
Jim Cownie5e8470a2013-09-27 10:38:44 +0000182
Jonathan Peyton30419822017-05-12 18:01:32 +0000183 /*fprintf( stderr, "=== %d\n", i ); */ /* GROO */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000184
Jonathan Peyton30419822017-05-12 18:01:32 +0000185 /* if we havn't been assigned a gtid, then return code */
186 if (i < 0)
Jim Cownie5e8470a2013-09-27 10:38:44 +0000187 return i;
Jonathan Peyton30419822017-05-12 18:01:32 +0000188
189 /* dynamically updated stack window for uber threads to avoid get_specific
190 call */
191 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) {
192 KMP_FATAL(StackOverflow, i);
193 }
194
195 stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
196 if (stack_addr > stack_base) {
197 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr);
198 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
199 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr -
200 stack_base);
201 } else {
202 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
203 stack_base - stack_addr);
204 }
205
206 /* Reprint stack bounds for ubermaster since they have been refined */
207 if (__kmp_storage_map) {
208 char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
209 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize;
210 __kmp_print_storage_map_gtid(i, stack_beg, stack_end,
211 other_threads[i]->th.th_info.ds.ds_stacksize,
212 "th_%d stack (refinement)", i);
213 }
214 return i;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000215}
216
Jonathan Peyton30419822017-05-12 18:01:32 +0000217int __kmp_get_global_thread_id_reg() {
218 int gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000219
Jonathan Peyton30419822017-05-12 18:01:32 +0000220 if (!__kmp_init_serial) {
221 gtid = KMP_GTID_DNE;
222 } else
Jim Cownie5e8470a2013-09-27 10:38:44 +0000223#ifdef KMP_TDATA_GTID
Jonathan Peyton30419822017-05-12 18:01:32 +0000224 if (TCR_4(__kmp_gtid_mode) >= 3) {
225 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using TDATA\n"));
226 gtid = __kmp_gtid;
227 } else
Jim Cownie5e8470a2013-09-27 10:38:44 +0000228#endif
Jonathan Peyton30419822017-05-12 18:01:32 +0000229 if (TCR_4(__kmp_gtid_mode) >= 2) {
230 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n"));
231 gtid = __kmp_gtid_get_specific();
232 } else {
233 KA_TRACE(1000,
234 ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"));
235 gtid = __kmp_get_global_thread_id();
236 }
237
238 /* we must be a new uber master sibling thread */
239 if (gtid == KMP_GTID_DNE) {
240 KA_TRACE(10,
241 ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
242 "Registering a new gtid.\n"));
243 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
244 if (!__kmp_init_serial) {
245 __kmp_do_serial_initialize();
246 gtid = __kmp_gtid_get_specific();
Jim Cownie5e8470a2013-09-27 10:38:44 +0000247 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +0000248 gtid = __kmp_register_root(FALSE);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000249 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000250 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
251 /*__kmp_printf( "+++ %d\n", gtid ); */ /* GROO */
252 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000253
Jonathan Peyton30419822017-05-12 18:01:32 +0000254 KMP_DEBUG_ASSERT(gtid >= 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000255
Jonathan Peyton30419822017-05-12 18:01:32 +0000256 return gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000257}
258
259/* caller must hold forkjoin_lock */
Jonathan Peyton30419822017-05-12 18:01:32 +0000260void __kmp_check_stack_overlap(kmp_info_t *th) {
261 int f;
262 char *stack_beg = NULL;
263 char *stack_end = NULL;
264 int gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000265
Jonathan Peyton30419822017-05-12 18:01:32 +0000266 KA_TRACE(10, ("__kmp_check_stack_overlap: called\n"));
267 if (__kmp_storage_map) {
268 stack_end = (char *)th->th.th_info.ds.ds_stackbase;
269 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000270
Jonathan Peyton30419822017-05-12 18:01:32 +0000271 gtid = __kmp_gtid_from_thread(th);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000272
Jonathan Peyton30419822017-05-12 18:01:32 +0000273 if (gtid == KMP_GTID_MONITOR) {
274 __kmp_print_storage_map_gtid(
275 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
276 "th_%s stack (%s)", "mon",
277 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
Jim Cownie5e8470a2013-09-27 10:38:44 +0000278 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +0000279 __kmp_print_storage_map_gtid(
280 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
281 "th_%d stack (%s)", gtid,
282 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
283 }
284 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000285
Jonathan Peyton30419822017-05-12 18:01:32 +0000286 /* No point in checking ubermaster threads since they use refinement and
287 * cannot overlap */
288 gtid = __kmp_gtid_from_thread(th);
289 if (__kmp_env_checks == TRUE && !KMP_UBER_GTID(gtid)) {
290 KA_TRACE(10,
291 ("__kmp_check_stack_overlap: performing extensive checking\n"));
292 if (stack_beg == NULL) {
293 stack_end = (char *)th->th.th_info.ds.ds_stackbase;
294 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
295 }
296
297 for (f = 0; f < __kmp_threads_capacity; f++) {
298 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f]);
299
300 if (f_th && f_th != th) {
301 char *other_stack_end =
302 (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase);
303 char *other_stack_beg =
304 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize);
305 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) ||
306 (stack_end > other_stack_beg && stack_end < other_stack_end)) {
307
308 /* Print the other stack values before the abort */
309 if (__kmp_storage_map)
310 __kmp_print_storage_map_gtid(
311 -1, other_stack_beg, other_stack_end,
312 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize),
313 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th));
314
Jonathan Peyton6a393f72017-09-05 15:43:58 +0000315 __kmp_fatal(KMP_MSG(StackOverlap), KMP_HNT(ChangeStackLimit),
316 __kmp_msg_null);
Jonathan Peyton30419822017-05-12 18:01:32 +0000317 }
318 }
319 }
320 }
321 KA_TRACE(10, ("__kmp_check_stack_overlap: returning\n"));
322}
323
324/* ------------------------------------------------------------------------ */
325
326void __kmp_infinite_loop(void) {
327 static int done = FALSE;
328
329 while (!done) {
Jonathan Peytone47d32f2019-02-28 19:11:29 +0000330 KMP_YIELD(TRUE);
Jonathan Peyton30419822017-05-12 18:01:32 +0000331 }
332}
333
334#define MAX_MESSAGE 512
335
336void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2, size_t size,
337 char const *format, ...) {
338 char buffer[MAX_MESSAGE];
339 va_list ap;
340
341 va_start(ap, format);
342 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP storage map: %p %p%8lu %s\n", p1,
343 p2, (unsigned long)size, format);
344 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
345 __kmp_vprintf(kmp_err, buffer, ap);
346#if KMP_PRINT_DATA_PLACEMENT
347 int node;
348 if (gtid >= 0) {
349 if (p1 <= p2 && (char *)p2 - (char *)p1 == size) {
350 if (__kmp_storage_map_verbose) {
351 node = __kmp_get_host_node(p1);
352 if (node < 0) /* doesn't work, so don't try this next time */
353 __kmp_storage_map_verbose = FALSE;
354 else {
355 char *last;
356 int lastNode;
357 int localProc = __kmp_get_cpu_from_gtid(gtid);
358
359 const int page_size = KMP_GET_PAGE_SIZE();
360
361 p1 = (void *)((size_t)p1 & ~((size_t)page_size - 1));
362 p2 = (void *)(((size_t)p2 - 1) & ~((size_t)page_size - 1));
363 if (localProc >= 0)
364 __kmp_printf_no_lock(" GTID %d localNode %d\n", gtid,
365 localProc >> 1);
366 else
367 __kmp_printf_no_lock(" GTID %d\n", gtid);
368#if KMP_USE_PRCTL
369 /* The more elaborate format is disabled for now because of the prctl
370 * hanging bug. */
371 do {
372 last = p1;
373 lastNode = node;
374 /* This loop collates adjacent pages with the same host node. */
375 do {
376 (char *)p1 += page_size;
377 } while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode);
378 __kmp_printf_no_lock(" %p-%p memNode %d\n", last, (char *)p1 - 1,
379 lastNode);
380 } while (p1 <= p2);
381#else
382 __kmp_printf_no_lock(" %p-%p memNode %d\n", p1,
383 (char *)p1 + (page_size - 1),
384 __kmp_get_host_node(p1));
385 if (p1 < p2) {
386 __kmp_printf_no_lock(" %p-%p memNode %d\n", p2,
387 (char *)p2 + (page_size - 1),
388 __kmp_get_host_node(p2));
389 }
390#endif
391 }
392 }
393 } else
394 __kmp_printf_no_lock(" %s\n", KMP_I18N_STR(StorageMapWarning));
395 }
396#endif /* KMP_PRINT_DATA_PLACEMENT */
397 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
398}
399
400void __kmp_warn(char const *format, ...) {
401 char buffer[MAX_MESSAGE];
402 va_list ap;
403
404 if (__kmp_generate_warnings == kmp_warnings_off) {
405 return;
406 }
407
408 va_start(ap, format);
409
410 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP warning: %s\n", format);
411 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
412 __kmp_vprintf(kmp_err, buffer, ap);
413 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
414
415 va_end(ap);
416}
417
418void __kmp_abort_process() {
419 // Later threads may stall here, but that's ok because abort() will kill them.
420 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock);
421
422 if (__kmp_debug_buf) {
423 __kmp_dump_debug_buffer();
Jonathan Peytonbd3a7632017-09-27 20:36:27 +0000424 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000425
426 if (KMP_OS_WINDOWS) {
427 // Let other threads know of abnormal termination and prevent deadlock
428 // if abort happened during library initialization or shutdown
429 __kmp_global.g.g_abort = SIGABRT;
430
431 /* On Windows* OS by default abort() causes pop-up error box, which stalls
432 nightly testing. Unfortunately, we cannot reliably suppress pop-up error
433 boxes. _set_abort_behavior() works well, but this function is not
434 available in VS7 (this is not problem for DLL, but it is a problem for
435 static OpenMP RTL). SetErrorMode (and so, timelimit utility) does not
436 help, at least in some versions of MS C RTL.
437
438 It seems following sequence is the only way to simulate abort() and
439 avoid pop-up error box. */
440 raise(SIGABRT);
441 _exit(3); // Just in case, if signal ignored, exit anyway.
442 } else {
443 abort();
Jonathan Peytonbd3a7632017-09-27 20:36:27 +0000444 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000445
446 __kmp_infinite_loop();
447 __kmp_release_bootstrap_lock(&__kmp_exit_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000448
449} // __kmp_abort_process
450
Jonathan Peyton30419822017-05-12 18:01:32 +0000451void __kmp_abort_thread(void) {
452 // TODO: Eliminate g_abort global variable and this function.
453 // In case of abort just call abort(), it will kill all the threads.
454 __kmp_infinite_loop();
Jim Cownie5e8470a2013-09-27 10:38:44 +0000455} // __kmp_abort_thread
456
Jonathan Peyton30419822017-05-12 18:01:32 +0000457/* Print out the storage map for the major kmp_info_t thread data structures
458 that are allocated together. */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000459
Jonathan Peyton30419822017-05-12 18:01:32 +0000460static void __kmp_print_thread_storage_map(kmp_info_t *thr, int gtid) {
461 __kmp_print_storage_map_gtid(gtid, thr, thr + 1, sizeof(kmp_info_t), "th_%d",
462 gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000463
Jonathan Peyton30419822017-05-12 18:01:32 +0000464 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team,
465 sizeof(kmp_desc_t), "th_%d.th_info", gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000466
Jonathan Peyton30419822017-05-12 18:01:32 +0000467 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head,
468 sizeof(kmp_local_t), "th_%d.th_local", gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000469
Jonathan Peyton30419822017-05-12 18:01:32 +0000470 __kmp_print_storage_map_gtid(
471 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier],
472 sizeof(kmp_balign_t) * bs_last_barrier, "th_%d.th_bar", gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000473
Jonathan Peyton30419822017-05-12 18:01:32 +0000474 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier],
475 &thr->th.th_bar[bs_plain_barrier + 1],
476 sizeof(kmp_balign_t), "th_%d.th_bar[plain]",
477 gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000478
Jonathan Peyton30419822017-05-12 18:01:32 +0000479 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier],
480 &thr->th.th_bar[bs_forkjoin_barrier + 1],
481 sizeof(kmp_balign_t), "th_%d.th_bar[forkjoin]",
482 gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000483
Jonathan Peyton30419822017-05-12 18:01:32 +0000484#if KMP_FAST_REDUCTION_BARRIER
485 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier],
486 &thr->th.th_bar[bs_reduction_barrier + 1],
487 sizeof(kmp_balign_t), "th_%d.th_bar[reduction]",
488 gtid);
489#endif // KMP_FAST_REDUCTION_BARRIER
Jim Cownie5e8470a2013-09-27 10:38:44 +0000490}
491
Jonathan Peyton30419822017-05-12 18:01:32 +0000492/* Print out the storage map for the major kmp_team_t team data structures
493 that are allocated together. */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000494
Jonathan Peyton30419822017-05-12 18:01:32 +0000495static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team,
496 int team_id, int num_thr) {
497 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;
498 __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d",
499 header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000500
Jonathan Peyton30419822017-05-12 18:01:32 +0000501 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0],
502 &team->t.t_bar[bs_last_barrier],
503 sizeof(kmp_balign_team_t) * bs_last_barrier,
504 "%s_%d.t_bar", header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000505
Jonathan Peyton30419822017-05-12 18:01:32 +0000506 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier],
507 &team->t.t_bar[bs_plain_barrier + 1],
508 sizeof(kmp_balign_team_t), "%s_%d.t_bar[plain]",
509 header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000510
Jonathan Peyton30419822017-05-12 18:01:32 +0000511 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier],
512 &team->t.t_bar[bs_forkjoin_barrier + 1],
513 sizeof(kmp_balign_team_t),
514 "%s_%d.t_bar[forkjoin]", header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000515
Jonathan Peyton30419822017-05-12 18:01:32 +0000516#if KMP_FAST_REDUCTION_BARRIER
517 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier],
518 &team->t.t_bar[bs_reduction_barrier + 1],
519 sizeof(kmp_balign_team_t),
520 "%s_%d.t_bar[reduction]", header, team_id);
521#endif // KMP_FAST_REDUCTION_BARRIER
Jim Cownie5e8470a2013-09-27 10:38:44 +0000522
Jonathan Peyton30419822017-05-12 18:01:32 +0000523 __kmp_print_storage_map_gtid(
524 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr],
525 sizeof(kmp_disp_t) * num_thr, "%s_%d.t_dispatch", header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000526
Jonathan Peyton30419822017-05-12 18:01:32 +0000527 __kmp_print_storage_map_gtid(
528 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr],
529 sizeof(kmp_info_t *) * num_thr, "%s_%d.t_threads", header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000530
Jonathan Peyton30419822017-05-12 18:01:32 +0000531 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0],
532 &team->t.t_disp_buffer[num_disp_buff],
533 sizeof(dispatch_shared_info_t) * num_disp_buff,
534 "%s_%d.t_disp_buffer", header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000535}
536
Jonathan Peyton92ca6182018-09-07 18:25:49 +0000537static void __kmp_init_allocator() {
538#if OMP_50_ENABLED
539 __kmp_init_memkind();
540#endif
541}
542static void __kmp_fini_allocator() {
543#if OMP_50_ENABLED
544 __kmp_fini_memkind();
545#endif
546}
Jim Cownie5e8470a2013-09-27 10:38:44 +0000547
548/* ------------------------------------------------------------------------ */
549
Jonathan Peyton8b3842f2018-10-05 17:59:39 +0000550#if KMP_DYNAMIC_LIB
Jonathan Peyton30419822017-05-12 18:01:32 +0000551#if KMP_OS_WINDOWS
Jim Cownie5e8470a2013-09-27 10:38:44 +0000552
Jonathan Peyton30419822017-05-12 18:01:32 +0000553static void __kmp_reset_lock(kmp_bootstrap_lock_t *lck) {
554 // TODO: Change to __kmp_break_bootstrap_lock().
555 __kmp_init_bootstrap_lock(lck); // make the lock released
Jim Cownie5e8470a2013-09-27 10:38:44 +0000556}
557
Jonathan Peyton30419822017-05-12 18:01:32 +0000558static void __kmp_reset_locks_on_process_detach(int gtid_req) {
559 int i;
560 int thread_count;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000561
Jonathan Peyton30419822017-05-12 18:01:32 +0000562 // PROCESS_DETACH is expected to be called by a thread that executes
563 // ProcessExit() or FreeLibrary(). OS terminates other threads (except the one
564 // calling ProcessExit or FreeLibrary). So, it might be safe to access the
565 // __kmp_threads[] without taking the forkjoin_lock. However, in fact, some
566 // threads can be still alive here, although being about to be terminated. The
567 // threads in the array with ds_thread==0 are most suspicious. Actually, it
568 // can be not safe to access the __kmp_threads[].
Jim Cownie5e8470a2013-09-27 10:38:44 +0000569
Jonathan Peyton30419822017-05-12 18:01:32 +0000570 // TODO: does it make sense to check __kmp_roots[] ?
Jim Cownie5e8470a2013-09-27 10:38:44 +0000571
Jonathan Peyton30419822017-05-12 18:01:32 +0000572 // Let's check that there are no other alive threads registered with the OMP
573 // lib.
574 while (1) {
575 thread_count = 0;
576 for (i = 0; i < __kmp_threads_capacity; ++i) {
577 if (!__kmp_threads)
578 continue;
579 kmp_info_t *th = __kmp_threads[i];
580 if (th == NULL)
581 continue;
582 int gtid = th->th.th_info.ds.ds_gtid;
583 if (gtid == gtid_req)
584 continue;
585 if (gtid < 0)
586 continue;
587 DWORD exit_val;
588 int alive = __kmp_is_thread_alive(th, &exit_val);
589 if (alive) {
590 ++thread_count;
591 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000592 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000593 if (thread_count == 0)
594 break; // success
595 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000596
Jonathan Peyton30419822017-05-12 18:01:32 +0000597 // Assume that I'm alone. Now it might be safe to check and reset locks.
598 // __kmp_forkjoin_lock and __kmp_stdio_lock are expected to be reset.
599 __kmp_reset_lock(&__kmp_forkjoin_lock);
600#ifdef KMP_DEBUG
601 __kmp_reset_lock(&__kmp_stdio_lock);
602#endif // KMP_DEBUG
Jim Cownie5e8470a2013-09-27 10:38:44 +0000603}
604
Jonathan Peyton30419822017-05-12 18:01:32 +0000605BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) {
606 //__kmp_acquire_bootstrap_lock( &__kmp_initz_lock );
Jim Cownie5e8470a2013-09-27 10:38:44 +0000607
Jonathan Peyton30419822017-05-12 18:01:32 +0000608 switch (fdwReason) {
Jim Cownie5e8470a2013-09-27 10:38:44 +0000609
Jonathan Peyton30419822017-05-12 18:01:32 +0000610 case DLL_PROCESS_ATTACH:
611 KA_TRACE(10, ("DllMain: PROCESS_ATTACH\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +0000612
613 return TRUE;
Jonathan Peyton30419822017-05-12 18:01:32 +0000614
615 case DLL_PROCESS_DETACH:
616 KA_TRACE(10, ("DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific()));
617
618 if (lpReserved != NULL) {
619 // lpReserved is used for telling the difference:
620 // lpReserved == NULL when FreeLibrary() was called,
621 // lpReserved != NULL when the process terminates.
622 // When FreeLibrary() is called, worker threads remain alive. So they will
623 // release the forkjoin lock by themselves. When the process terminates,
624 // worker threads disappear triggering the problem of unreleased forkjoin
625 // lock as described below.
626
627 // A worker thread can take the forkjoin lock. The problem comes up if
628 // that worker thread becomes dead before it releases the forkjoin lock.
629 // The forkjoin lock remains taken, while the thread executing
630 // DllMain()->PROCESS_DETACH->__kmp_internal_end_library() below will try
631 // to take the forkjoin lock and will always fail, so that the application
632 // will never finish [normally]. This scenario is possible if
633 // __kmpc_end() has not been executed. It looks like it's not a corner
634 // case, but common cases:
635 // - the main function was compiled by an alternative compiler;
636 // - the main function was compiled by icl but without /Qopenmp
637 // (application with plugins);
638 // - application terminates by calling C exit(), Fortran CALL EXIT() or
639 // Fortran STOP.
640 // - alive foreign thread prevented __kmpc_end from doing cleanup.
641 //
642 // This is a hack to work around the problem.
643 // TODO: !!! figure out something better.
644 __kmp_reset_locks_on_process_detach(__kmp_gtid_get_specific());
645 }
646
647 __kmp_internal_end_library(__kmp_gtid_get_specific());
648
649 return TRUE;
650
651 case DLL_THREAD_ATTACH:
652 KA_TRACE(10, ("DllMain: THREAD_ATTACH\n"));
653
654 /* if we want to register new siblings all the time here call
655 * __kmp_get_gtid(); */
656 return TRUE;
657
658 case DLL_THREAD_DETACH:
659 KA_TRACE(10, ("DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific()));
660
661 __kmp_internal_end_thread(__kmp_gtid_get_specific());
662 return TRUE;
663 }
664
665 return TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000666}
667
Jonathan Peyton30419822017-05-12 18:01:32 +0000668#endif /* KMP_OS_WINDOWS */
Jonathan Peyton99016992015-05-26 17:32:53 +0000669#endif /* KMP_DYNAMIC_LIB */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000670
Jonathan Peyton30419822017-05-12 18:01:32 +0000671/* __kmp_parallel_deo -- Wait until it's our turn. */
672void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
673 int gtid = *gtid_ref;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000674#ifdef BUILD_PARALLEL_ORDERED
Jonathan Peyton30419822017-05-12 18:01:32 +0000675 kmp_team_t *team = __kmp_team_from_gtid(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000676#endif /* BUILD_PARALLEL_ORDERED */
677
Jonathan Peyton30419822017-05-12 18:01:32 +0000678 if (__kmp_env_consistency_check) {
679 if (__kmp_threads[gtid]->th.th_root->r.r_active)
Andrey Churbanov5c56fb52015-02-20 18:05:17 +0000680#if KMP_USE_DYNAMIC_LOCK
Jonathan Peyton30419822017-05-12 18:01:32 +0000681 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL, 0);
Andrey Churbanov5c56fb52015-02-20 18:05:17 +0000682#else
Jonathan Peyton30419822017-05-12 18:01:32 +0000683 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL);
Andrey Churbanov5c56fb52015-02-20 18:05:17 +0000684#endif
Jonathan Peyton30419822017-05-12 18:01:32 +0000685 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000686#ifdef BUILD_PARALLEL_ORDERED
Jonathan Peyton30419822017-05-12 18:01:32 +0000687 if (!team->t.t_serialized) {
688 KMP_MB();
Jonathan Peytone47d32f2019-02-28 19:11:29 +0000689 KMP_WAIT(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), KMP_EQ,
690 NULL);
Jonathan Peyton30419822017-05-12 18:01:32 +0000691 KMP_MB();
692 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000693#endif /* BUILD_PARALLEL_ORDERED */
694}
695
Jonathan Peyton30419822017-05-12 18:01:32 +0000696/* __kmp_parallel_dxo -- Signal the next task. */
697void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
698 int gtid = *gtid_ref;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000699#ifdef BUILD_PARALLEL_ORDERED
Jonathan Peyton30419822017-05-12 18:01:32 +0000700 int tid = __kmp_tid_from_gtid(gtid);
701 kmp_team_t *team = __kmp_team_from_gtid(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000702#endif /* BUILD_PARALLEL_ORDERED */
703
Jonathan Peyton30419822017-05-12 18:01:32 +0000704 if (__kmp_env_consistency_check) {
705 if (__kmp_threads[gtid]->th.th_root->r.r_active)
706 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref);
707 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000708#ifdef BUILD_PARALLEL_ORDERED
Jonathan Peyton30419822017-05-12 18:01:32 +0000709 if (!team->t.t_serialized) {
710 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000711
Jonathan Peyton30419822017-05-12 18:01:32 +0000712 /* use the tid of the next thread in this team */
713 /* TODO replace with general release procedure */
714 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000715
Jonathan Peyton30419822017-05-12 18:01:32 +0000716 KMP_MB(); /* Flush all pending memory write invalidates. */
717 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000718#endif /* BUILD_PARALLEL_ORDERED */
719}
720
721/* ------------------------------------------------------------------------ */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000722/* The BARRIER for a SINGLE process section is always explicit */
723
Jonathan Peyton30419822017-05-12 18:01:32 +0000724int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws) {
725 int status;
726 kmp_info_t *th;
727 kmp_team_t *team;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000728
Jonathan Peyton30419822017-05-12 18:01:32 +0000729 if (!TCR_4(__kmp_init_parallel))
730 __kmp_parallel_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +0000731
Jonathan Peyton9b8bb322019-01-16 20:07:39 +0000732#if OMP_50_ENABLED
733 __kmp_resume_if_soft_paused();
734#endif
735
Jonathan Peyton30419822017-05-12 18:01:32 +0000736 th = __kmp_threads[gtid];
737 team = th->th.th_team;
738 status = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000739
Jonathan Peyton30419822017-05-12 18:01:32 +0000740 th->th.th_ident = id_ref;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000741
Jonathan Peyton30419822017-05-12 18:01:32 +0000742 if (team->t.t_serialized) {
743 status = 1;
744 } else {
745 kmp_int32 old_this = th->th.th_local.this_construct;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000746
Jonathan Peyton30419822017-05-12 18:01:32 +0000747 ++th->th.th_local.this_construct;
748 /* try to set team count to thread count--success means thread got the
749 single block */
750 /* TODO: Should this be acquire or release? */
751 if (team->t.t_construct == old_this) {
Jonathan Peyton37e2ef52018-07-09 17:36:22 +0000752 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this,
753 th->th.th_local.this_construct);
Jonathan Peyton30419822017-05-12 18:01:32 +0000754 }
Andrey Churbanov51aecb82015-05-06 19:22:36 +0000755#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +0000756 if (__itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 &&
757 KMP_MASTER_GTID(gtid) &&
Andrey Churbanov51aecb82015-05-06 19:22:36 +0000758#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +0000759 th->th.th_teams_microtask == NULL &&
Andrey Churbanov51aecb82015-05-06 19:22:36 +0000760#endif
Jonathan Peyton30419822017-05-12 18:01:32 +0000761 team->t.t_active_level ==
762 1) { // Only report metadata by master of active team at level 1
763 __kmp_itt_metadata_single(id_ref);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000764 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000765#endif /* USE_ITT_BUILD */
766 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000767
Jonathan Peyton30419822017-05-12 18:01:32 +0000768 if (__kmp_env_consistency_check) {
769 if (status && push_ws) {
770 __kmp_push_workshare(gtid, ct_psingle, id_ref);
771 } else {
772 __kmp_check_workshare(gtid, ct_psingle, id_ref);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000773 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000774 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000775#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +0000776 if (status) {
777 __kmp_itt_single_start(gtid);
778 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000779#endif /* USE_ITT_BUILD */
Jonathan Peyton30419822017-05-12 18:01:32 +0000780 return status;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000781}
782
Jonathan Peyton30419822017-05-12 18:01:32 +0000783void __kmp_exit_single(int gtid) {
Jim Cownie5e8470a2013-09-27 10:38:44 +0000784#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +0000785 __kmp_itt_single_end(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000786#endif /* USE_ITT_BUILD */
Jonathan Peyton30419822017-05-12 18:01:32 +0000787 if (__kmp_env_consistency_check)
788 __kmp_pop_workshare(gtid, ct_psingle, NULL);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000789}
790
Jonathan Peyton30419822017-05-12 18:01:32 +0000791/* determine if we can go parallel or must use a serialized parallel region and
Jim Cownie5e8470a2013-09-27 10:38:44 +0000792 * how many threads we can use
793 * set_nproc is the number of threads requested for the team
794 * returns 0 if we should serialize or only use one thread,
795 * otherwise the number of threads to use
Jonathan Peyton30419822017-05-12 18:01:32 +0000796 * The forkjoin lock is held by the caller. */
797static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team,
798 int master_tid, int set_nthreads
Jim Cownie5e8470a2013-09-27 10:38:44 +0000799#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +0000800 ,
801 int enter_teams
Jim Cownie5e8470a2013-09-27 10:38:44 +0000802#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +0000803 ) {
804 int capacity;
805 int new_nthreads;
806 KMP_DEBUG_ASSERT(__kmp_init_serial);
807 KMP_DEBUG_ASSERT(root && parent_team);
Jonathan Peyton65ebfee2019-02-11 21:04:23 +0000808 kmp_info_t *this_thr = parent_team->t.t_threads[master_tid];
Jim Cownie5e8470a2013-09-27 10:38:44 +0000809
Jonathan Peyton30419822017-05-12 18:01:32 +0000810 // If dyn-var is set, dynamically adjust the number of desired threads,
811 // according to the method specified by dynamic_mode.
812 new_nthreads = set_nthreads;
813 if (!get__dynamic_2(parent_team, master_tid)) {
814 ;
815 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000816#ifdef USE_LOAD_BALANCE
Jonathan Peyton30419822017-05-12 18:01:32 +0000817 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) {
818 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads);
819 if (new_nthreads == 1) {
820 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "
821 "reservation to 1 thread\n",
822 master_tid));
823 return 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000824 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000825 if (new_nthreads < set_nthreads) {
826 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "
827 "reservation to %d threads\n",
828 master_tid, new_nthreads));
829 }
830 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000831#endif /* USE_LOAD_BALANCE */
Jonathan Peyton30419822017-05-12 18:01:32 +0000832 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) {
833 new_nthreads = __kmp_avail_proc - __kmp_nth +
834 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
835 if (new_nthreads <= 1) {
836 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "
837 "reservation to 1 thread\n",
838 master_tid));
839 return 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000840 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000841 if (new_nthreads < set_nthreads) {
842 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "
843 "reservation to %d threads\n",
844 master_tid, new_nthreads));
845 } else {
846 new_nthreads = set_nthreads;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000847 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000848 } else if (__kmp_global.g.g_dynamic_mode == dynamic_random) {
849 if (set_nthreads > 2) {
850 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]);
851 new_nthreads = (new_nthreads % set_nthreads) + 1;
852 if (new_nthreads == 1) {
853 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "
854 "reservation to 1 thread\n",
855 master_tid));
Jim Cownie5e8470a2013-09-27 10:38:44 +0000856 return 1;
Jonathan Peyton30419822017-05-12 18:01:32 +0000857 }
858 if (new_nthreads < set_nthreads) {
859 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "
860 "reservation to %d threads\n",
861 master_tid, new_nthreads));
862 }
863 }
864 } else {
865 KMP_ASSERT(0);
866 }
867
Jonathan Peytonf4392462017-07-27 20:58:41 +0000868 // Respect KMP_ALL_THREADS/KMP_DEVICE_THREAD_LIMIT.
Jonathan Peyton30419822017-05-12 18:01:32 +0000869 if (__kmp_nth + new_nthreads -
870 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
871 __kmp_max_nth) {
872 int tl_nthreads = __kmp_max_nth - __kmp_nth +
873 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
874 if (tl_nthreads <= 0) {
875 tl_nthreads = 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000876 }
877
Jonathan Peyton30419822017-05-12 18:01:32 +0000878 // If dyn-var is false, emit a 1-time warning.
879 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
880 __kmp_reserve_warn = 1;
881 __kmp_msg(kmp_ms_warning,
882 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
883 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
884 }
885 if (tl_nthreads == 1) {
Jonathan Peytonf4392462017-07-27 20:58:41 +0000886 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
887 "reduced reservation to 1 thread\n",
Jonathan Peyton30419822017-05-12 18:01:32 +0000888 master_tid));
889 return 1;
890 }
Jonathan Peytonf4392462017-07-27 20:58:41 +0000891 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
892 "reservation to %d threads\n",
893 master_tid, tl_nthreads));
894 new_nthreads = tl_nthreads;
895 }
896
897 // Respect OMP_THREAD_LIMIT
Jonathan Peyton65ebfee2019-02-11 21:04:23 +0000898 int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads;
899 int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit;
900 if (cg_nthreads + new_nthreads -
Jonathan Peytonf4392462017-07-27 20:58:41 +0000901 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
Jonathan Peyton65ebfee2019-02-11 21:04:23 +0000902 max_cg_threads) {
903 int tl_nthreads = max_cg_threads - cg_nthreads +
Jonathan Peytonf4392462017-07-27 20:58:41 +0000904 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
905 if (tl_nthreads <= 0) {
906 tl_nthreads = 1;
907 }
908
909 // If dyn-var is false, emit a 1-time warning.
910 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
911 __kmp_reserve_warn = 1;
912 __kmp_msg(kmp_ms_warning,
913 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
914 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
915 }
916 if (tl_nthreads == 1) {
917 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
918 "reduced reservation to 1 thread\n",
919 master_tid));
920 return 1;
921 }
922 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
Jonathan Peyton30419822017-05-12 18:01:32 +0000923 "reservation to %d threads\n",
924 master_tid, tl_nthreads));
925 new_nthreads = tl_nthreads;
926 }
927
928 // Check if the threads array is large enough, or needs expanding.
Jonathan Peyton30419822017-05-12 18:01:32 +0000929 // See comment in __kmp_register_root() about the adjustment if
930 // __kmp_threads[0] == NULL.
931 capacity = __kmp_threads_capacity;
932 if (TCR_PTR(__kmp_threads[0]) == NULL) {
933 --capacity;
934 }
935 if (__kmp_nth + new_nthreads -
936 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
937 capacity) {
938 // Expand the threads array.
939 int slotsRequired = __kmp_nth + new_nthreads -
940 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) -
941 capacity;
Jonathan Peyton1800ece2018-01-10 18:27:01 +0000942 int slotsAdded = __kmp_expand_threads(slotsRequired);
Jonathan Peyton30419822017-05-12 18:01:32 +0000943 if (slotsAdded < slotsRequired) {
944 // The threads array was not expanded enough.
945 new_nthreads -= (slotsRequired - slotsAdded);
946 KMP_ASSERT(new_nthreads >= 1);
947
948 // If dyn-var is false, emit a 1-time warning.
949 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
950 __kmp_reserve_warn = 1;
951 if (__kmp_tp_cached) {
952 __kmp_msg(kmp_ms_warning,
953 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
954 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
955 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
956 } else {
957 __kmp_msg(kmp_ms_warning,
958 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
959 KMP_HNT(SystemLimitOnThreads), __kmp_msg_null);
960 }
961 }
962 }
963 }
964
Jonathan Peyton642688b2017-06-01 16:46:36 +0000965#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +0000966 if (new_nthreads == 1) {
967 KC_TRACE(10,
968 ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
969 "dead roots and rechecking; requested %d threads\n",
970 __kmp_get_gtid(), set_nthreads));
Jonathan Peyton642688b2017-06-01 16:46:36 +0000971 } else {
972 KC_TRACE(10, ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
973 " %d threads\n",
974 __kmp_get_gtid(), new_nthreads, set_nthreads));
Jonathan Peyton30419822017-05-12 18:01:32 +0000975 }
Jonathan Peyton642688b2017-06-01 16:46:36 +0000976#endif // KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +0000977 return new_nthreads;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000978}
979
Jonathan Peyton30419822017-05-12 18:01:32 +0000980/* Allocate threads from the thread pool and assign them to the new team. We are
981 assured that there are enough threads available, because we checked on that
982 earlier within critical section forkjoin */
983static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
984 kmp_info_t *master_th, int master_gtid) {
985 int i;
986 int use_hot_team;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000987
Jonathan Peyton30419822017-05-12 18:01:32 +0000988 KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc));
989 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid());
990 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +0000991
Jonathan Peyton30419822017-05-12 18:01:32 +0000992 /* first, let's setup the master thread */
993 master_th->th.th_info.ds.ds_tid = 0;
994 master_th->th.th_team = team;
995 master_th->th.th_team_nproc = team->t.t_nproc;
996 master_th->th.th_team_master = master_th;
997 master_th->th.th_team_serialized = FALSE;
998 master_th->th.th_dispatch = &team->t.t_dispatch[0];
Jim Cownie5e8470a2013-09-27 10:38:44 +0000999
Jonathan Peyton30419822017-05-12 18:01:32 +00001000/* make sure we are not the optimized hot team */
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001001#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00001002 use_hot_team = 0;
1003 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams;
1004 if (hot_teams) { // hot teams array is not allocated if
1005 // KMP_HOT_TEAMS_MAX_LEVEL=0
1006 int level = team->t.t_active_level - 1; // index in array of hot teams
1007 if (master_th->th.th_teams_microtask) { // are we inside the teams?
1008 if (master_th->th.th_teams_size.nteams > 1) {
1009 ++level; // level was not increased in teams construct for
1010 // team_of_masters
1011 }
1012 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
1013 master_th->th.th_teams_level == team->t.t_level) {
1014 ++level; // level was not increased in teams construct for
1015 // team_of_workers before the parallel
1016 } // team->t.t_level will be increased inside parallel
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001017 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001018 if (level < __kmp_hot_teams_max_level) {
1019 if (hot_teams[level].hot_team) {
1020 // hot team has already been allocated for given level
1021 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team);
1022 use_hot_team = 1; // the team is ready to use
1023 } else {
1024 use_hot_team = 0; // AC: threads are not allocated yet
1025 hot_teams[level].hot_team = team; // remember new hot team
1026 hot_teams[level].hot_team_nth = team->t.t_nproc;
1027 }
1028 } else {
1029 use_hot_team = 0;
1030 }
1031 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001032#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001033 use_hot_team = team == root->r.r_hot_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001034#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001035 if (!use_hot_team) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00001036
Jonathan Peyton30419822017-05-12 18:01:32 +00001037 /* install the master thread */
1038 team->t.t_threads[0] = master_th;
1039 __kmp_initialize_info(master_th, team, 0, master_gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001040
Jonathan Peyton30419822017-05-12 18:01:32 +00001041 /* now, install the worker threads */
1042 for (i = 1; i < team->t.t_nproc; i++) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00001043
Jonathan Peyton30419822017-05-12 18:01:32 +00001044 /* fork or reallocate a new thread and install it in team */
1045 kmp_info_t *thr = __kmp_allocate_thread(root, team, i);
1046 team->t.t_threads[i] = thr;
1047 KMP_DEBUG_ASSERT(thr);
1048 KMP_DEBUG_ASSERT(thr->th.th_team == team);
1049 /* align team and thread arrived states */
1050 KA_TRACE(20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
1051 "T#%d(%d:%d) join =%llu, plain=%llu\n",
1052 __kmp_gtid_from_tid(0, team), team->t.t_id, 0,
1053 __kmp_gtid_from_tid(i, team), team->t.t_id, i,
1054 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
1055 team->t.t_bar[bs_plain_barrier].b_arrived));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001056#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001057 thr->th.th_teams_microtask = master_th->th.th_teams_microtask;
1058 thr->th.th_teams_level = master_th->th.th_teams_level;
1059 thr->th.th_teams_size = master_th->th.th_teams_size;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001060#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001061 { // Initialize threads' barrier data.
1062 int b;
1063 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar;
1064 for (b = 0; b < bs_last_barrier; ++b) {
1065 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
1066 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00001067#if USE_DEBUGGER
Jonathan Peyton30419822017-05-12 18:01:32 +00001068 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00001069#endif
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00001070 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001071 }
1072 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001073
Alp Toker98758b02014-03-02 04:12:06 +00001074#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00001075 __kmp_partition_places(team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001076#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001077 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001078
Jonathan Peyton6d88e042018-12-13 23:14:24 +00001079#if OMP_50_ENABLED
1080 if (__kmp_display_affinity && team->t.t_display_affinity != 1) {
1081 for (i = 0; i < team->t.t_nproc; i++) {
1082 kmp_info_t *thr = team->t.t_threads[i];
1083 if (thr->th.th_prev_num_threads != team->t.t_nproc ||
1084 thr->th.th_prev_level != team->t.t_level) {
1085 team->t.t_display_affinity = 1;
1086 break;
1087 }
1088 }
1089 }
1090#endif
1091
Jonathan Peyton30419822017-05-12 18:01:32 +00001092 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00001093}
1094
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001095#if KMP_ARCH_X86 || KMP_ARCH_X86_64
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001096// Propagate any changes to the floating point control registers out to the team
Jonathan Peyton30419822017-05-12 18:01:32 +00001097// We try to avoid unnecessary writes to the relevant cache line in the team
1098// structure, so we don't make changes unless they are needed.
1099inline static void propagateFPControl(kmp_team_t *team) {
1100 if (__kmp_inherit_fp_control) {
1101 kmp_int16 x87_fpu_control_word;
1102 kmp_uint32 mxcsr;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001103
Jonathan Peyton30419822017-05-12 18:01:32 +00001104 // Get master values of FPU control flags (both X87 and vector)
1105 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1106 __kmp_store_mxcsr(&mxcsr);
1107 mxcsr &= KMP_X86_MXCSR_MASK;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001108
Jonathan Peyton94a114f2017-10-20 19:30:57 +00001109 // There is no point looking at t_fp_control_saved here.
1110 // If it is TRUE, we still have to update the values if they are different
Jonas Hahnfeldf0a1c652017-11-03 18:28:19 +00001111 // from those we now have. If it is FALSE we didn't save anything yet, but
1112 // our objective is the same. We have to ensure that the values in the team
1113 // are the same as those we have.
Jonathan Peyton94a114f2017-10-20 19:30:57 +00001114 // So, this code achieves what we need whether or not t_fp_control_saved is
1115 // true. By checking whether the value needs updating we avoid unnecessary
1116 // writes that would put the cache-line into a written state, causing all
1117 // threads in the team to have to read it again.
Jonathan Peyton30419822017-05-12 18:01:32 +00001118 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word);
1119 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr);
1120 // Although we don't use this value, other code in the runtime wants to know
1121 // whether it should restore them. So we must ensure it is correct.
1122 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE);
1123 } else {
1124 // Similarly here. Don't write to this cache-line in the team structure
1125 // unless we have to.
1126 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE);
1127 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001128}
1129
Jonathan Peyton30419822017-05-12 18:01:32 +00001130// Do the opposite, setting the hardware registers to the updated values from
1131// the team.
1132inline static void updateHWFPControl(kmp_team_t *team) {
1133 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) {
1134 // Only reset the fp control regs if they have been changed in the team.
1135 // the parallel region that we are exiting.
1136 kmp_int16 x87_fpu_control_word;
1137 kmp_uint32 mxcsr;
1138 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1139 __kmp_store_mxcsr(&mxcsr);
1140 mxcsr &= KMP_X86_MXCSR_MASK;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001141
Jonathan Peyton30419822017-05-12 18:01:32 +00001142 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) {
1143 __kmp_clear_x87_fpu_status_word();
1144 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001145 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001146
1147 if (team->t.t_mxcsr != mxcsr) {
1148 __kmp_load_mxcsr(&team->t.t_mxcsr);
1149 }
1150 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001151}
1152#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001153#define propagateFPControl(x) ((void)0)
1154#define updateHWFPControl(x) ((void)0)
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001155#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1156
Jonathan Peyton30419822017-05-12 18:01:32 +00001157static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team,
1158 int realloc); // forward declaration
Jim Cownie5e8470a2013-09-27 10:38:44 +00001159
Jonathan Peyton30419822017-05-12 18:01:32 +00001160/* Run a parallel region that has been serialized, so runs only in a team of the
1161 single master thread. */
1162void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
1163 kmp_info_t *this_thr;
1164 kmp_team_t *serial_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001165
Jonathan Peyton30419822017-05-12 18:01:32 +00001166 KC_TRACE(10, ("__kmpc_serialized_parallel: called by T#%d\n", global_tid));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001167
Jonathan Peyton30419822017-05-12 18:01:32 +00001168 /* Skip all this code for autopar serialized loops since it results in
1169 unacceptable overhead */
1170 if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR))
1171 return;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001172
Jonathan Peyton30419822017-05-12 18:01:32 +00001173 if (!TCR_4(__kmp_init_parallel))
1174 __kmp_parallel_initialize();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001175
Jonathan Peyton9b8bb322019-01-16 20:07:39 +00001176#if OMP_50_ENABLED
1177 __kmp_resume_if_soft_paused();
1178#endif
1179
Jonathan Peyton30419822017-05-12 18:01:32 +00001180 this_thr = __kmp_threads[global_tid];
1181 serial_team = this_thr->th.th_serial_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001182
Jonathan Peyton30419822017-05-12 18:01:32 +00001183 /* utilize the serialized team held by this thread */
1184 KMP_DEBUG_ASSERT(serial_team);
1185 KMP_MB();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001186
Jonathan Peyton30419822017-05-12 18:01:32 +00001187 if (__kmp_tasking_mode != tskm_immediate_exec) {
1188 KMP_DEBUG_ASSERT(
1189 this_thr->th.th_task_team ==
1190 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]);
1191 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] ==
1192 NULL);
1193 KA_TRACE(20, ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
1194 "team %p, new task_team = NULL\n",
1195 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
1196 this_thr->th.th_task_team = NULL;
1197 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001198
1199#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001200 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
1201 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1202 proc_bind = proc_bind_false;
1203 } else if (proc_bind == proc_bind_default) {
1204 // No proc_bind clause was specified, so use the current value
1205 // of proc-bind-var for this parallel region.
1206 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
1207 }
1208 // Reset for next parallel region
1209 this_thr->th.th_set_proc_bind = proc_bind_default;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001210#endif /* OMP_40_ENABLED */
1211
Joachim Protze82e94a52017-11-01 10:08:30 +00001212#if OMPT_SUPPORT
Jonathan Peyton3574f282018-10-04 14:57:04 +00001213 ompt_data_t ompt_parallel_data = ompt_data_none;
Joachim Protze82e94a52017-11-01 10:08:30 +00001214 ompt_data_t *implicit_task_data;
1215 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
1216 if (ompt_enabled.enabled &&
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001217 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
Joachim Protze82e94a52017-11-01 10:08:30 +00001218
1219 ompt_task_info_t *parent_task_info;
1220 parent_task_info = OMPT_CUR_TASK_INFO(this_thr);
1221
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001222 parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
Joachim Protze82e94a52017-11-01 10:08:30 +00001223 if (ompt_enabled.ompt_callback_parallel_begin) {
1224 int team_size = 1;
1225
1226 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
1227 &(parent_task_info->task_data), &(parent_task_info->frame),
Joachim Protze489cdb72018-09-10 14:34:54 +00001228 &ompt_parallel_data, team_size, ompt_parallel_invoker_program,
1229 codeptr);
Joachim Protze82e94a52017-11-01 10:08:30 +00001230 }
1231 }
1232#endif // OMPT_SUPPORT
1233
Jonathan Peyton30419822017-05-12 18:01:32 +00001234 if (this_thr->th.th_team != serial_team) {
1235 // Nested level will be an index in the nested nthreads array
1236 int level = this_thr->th.th_team->t.t_level;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001237
Jonathan Peyton30419822017-05-12 18:01:32 +00001238 if (serial_team->t.t_serialized) {
1239 /* this serial team was already used
1240 TODO increase performance by making this locks more specific */
1241 kmp_team_t *new_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001242
Jonathan Peyton30419822017-05-12 18:01:32 +00001243 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001244
Jonathan Peyton30419822017-05-12 18:01:32 +00001245 new_team = __kmp_allocate_team(this_thr->th.th_root, 1, 1,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001246#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00001247 ompt_parallel_data,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001248#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001249#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001250 proc_bind,
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001251#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001252 &this_thr->th.th_current_task->td_icvs,
1253 0 USE_NESTED_HOT_ARG(NULL));
1254 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1255 KMP_ASSERT(new_team);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001256
Jonathan Peyton30419822017-05-12 18:01:32 +00001257 /* setup new serialized team and install it */
1258 new_team->t.t_threads[0] = this_thr;
1259 new_team->t.t_parent = this_thr->th.th_team;
1260 serial_team = new_team;
1261 this_thr->th.th_serial_team = serial_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001262
Jonathan Peyton30419822017-05-12 18:01:32 +00001263 KF_TRACE(
1264 10,
1265 ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",
1266 global_tid, serial_team));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001267
Jonathan Peyton30419822017-05-12 18:01:32 +00001268 /* TODO the above breaks the requirement that if we run out of resources,
1269 then we can still guarantee that serialized teams are ok, since we may
1270 need to allocate a new one */
1271 } else {
1272 KF_TRACE(
1273 10,
1274 ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",
1275 global_tid, serial_team));
1276 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001277
Jonathan Peyton30419822017-05-12 18:01:32 +00001278 /* we have to initialize this serial team */
1279 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1280 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1281 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team);
1282 serial_team->t.t_ident = loc;
1283 serial_team->t.t_serialized = 1;
1284 serial_team->t.t_nproc = 1;
1285 serial_team->t.t_parent = this_thr->th.th_team;
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00001286 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched;
Jonathan Peyton30419822017-05-12 18:01:32 +00001287 this_thr->th.th_team = serial_team;
1288 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001289
Jonathan Peyton30419822017-05-12 18:01:32 +00001290 KF_TRACE(10, ("__kmpc_serialized_parallel: T#d curtask=%p\n", global_tid,
1291 this_thr->th.th_current_task));
1292 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1);
1293 this_thr->th.th_current_task->td_flags.executing = 0;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001294
Jonathan Peyton30419822017-05-12 18:01:32 +00001295 __kmp_push_current_task_to_thread(this_thr, serial_team, 0);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001296
Jonathan Peyton30419822017-05-12 18:01:32 +00001297 /* TODO: GEH: do ICVs work for nested serialized teams? Don't we need an
1298 implicit task for each serialized task represented by
1299 team->t.t_serialized? */
1300 copy_icvs(&this_thr->th.th_current_task->td_icvs,
1301 &this_thr->th.th_current_task->td_parent->td_icvs);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001302
Jonathan Peyton30419822017-05-12 18:01:32 +00001303 // Thread value exists in the nested nthreads array for the next nested
1304 // level
1305 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1306 this_thr->th.th_current_task->td_icvs.nproc =
1307 __kmp_nested_nth.nth[level + 1];
1308 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001309
1310#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001311 if (__kmp_nested_proc_bind.used &&
1312 (level + 1 < __kmp_nested_proc_bind.used)) {
1313 this_thr->th.th_current_task->td_icvs.proc_bind =
1314 __kmp_nested_proc_bind.bind_types[level + 1];
1315 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001316#endif /* OMP_40_ENABLED */
1317
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00001318#if USE_DEBUGGER
Jonathan Peyton30419822017-05-12 18:01:32 +00001319 serial_team->t.t_pkfn = (microtask_t)(~0); // For the debugger.
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00001320#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001321 this_thr->th.th_info.ds.ds_tid = 0;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001322
Jonathan Peyton30419822017-05-12 18:01:32 +00001323 /* set thread cache values */
1324 this_thr->th.th_team_nproc = 1;
1325 this_thr->th.th_team_master = this_thr;
1326 this_thr->th.th_team_serialized = 1;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001327
Jonathan Peyton30419822017-05-12 18:01:32 +00001328 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1;
1329 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level;
Jonathan Peyton92ca6182018-09-07 18:25:49 +00001330#if OMP_50_ENABLED
1331 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator; // save
1332#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001333
Jonathan Peyton30419822017-05-12 18:01:32 +00001334 propagateFPControl(serial_team);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001335
Jonathan Peyton30419822017-05-12 18:01:32 +00001336 /* check if we need to allocate dispatch buffers stack */
1337 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1338 if (!serial_team->t.t_dispatch->th_disp_buffer) {
1339 serial_team->t.t_dispatch->th_disp_buffer =
1340 (dispatch_private_info_t *)__kmp_allocate(
1341 sizeof(dispatch_private_info_t));
1342 }
1343 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001344
Jonathan Peyton30419822017-05-12 18:01:32 +00001345 KMP_MB();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001346
Jonathan Peyton30419822017-05-12 18:01:32 +00001347 } else {
1348 /* this serialized team is already being used,
1349 * that's fine, just add another nested level */
1350 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
1351 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1352 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1353 ++serial_team->t.t_serialized;
1354 this_thr->th.th_team_serialized = serial_team->t.t_serialized;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001355
Jonathan Peyton30419822017-05-12 18:01:32 +00001356 // Nested level will be an index in the nested nthreads array
1357 int level = this_thr->th.th_team->t.t_level;
1358 // Thread value exists in the nested nthreads array for the next nested
1359 // level
1360 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1361 this_thr->th.th_current_task->td_icvs.nproc =
1362 __kmp_nested_nth.nth[level + 1];
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001363 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001364 serial_team->t.t_level++;
1365 KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d increasing nesting level "
1366 "of serial team %p to %d\n",
1367 global_tid, serial_team, serial_team->t.t_level));
1368
1369 /* allocate/push dispatch buffers stack */
1370 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1371 {
1372 dispatch_private_info_t *disp_buffer =
1373 (dispatch_private_info_t *)__kmp_allocate(
1374 sizeof(dispatch_private_info_t));
1375 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;
1376 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;
1377 }
1378 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1379
1380 KMP_MB();
1381 }
Olga Malyshevadbdcfa12017-04-04 13:56:50 +00001382#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001383 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq);
Olga Malyshevadbdcfa12017-04-04 13:56:50 +00001384#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001385
Jonathan Peyton6d88e042018-12-13 23:14:24 +00001386#if OMP_50_ENABLED
1387 // Perform the display affinity functionality for
1388 // serialized parallel regions
1389 if (__kmp_display_affinity) {
1390 if (this_thr->th.th_prev_level != serial_team->t.t_level ||
1391 this_thr->th.th_prev_num_threads != 1) {
1392 // NULL means use the affinity-format-var ICV
1393 __kmp_aux_display_affinity(global_tid, NULL);
1394 this_thr->th.th_prev_level = serial_team->t.t_level;
1395 this_thr->th.th_prev_num_threads = 1;
1396 }
1397 }
1398#endif
1399
Jonathan Peyton30419822017-05-12 18:01:32 +00001400 if (__kmp_env_consistency_check)
1401 __kmp_push_parallel(global_tid, NULL);
Joachim Protze82e94a52017-11-01 10:08:30 +00001402#if OMPT_SUPPORT
1403 serial_team->t.ompt_team_info.master_return_address = codeptr;
1404 if (ompt_enabled.enabled &&
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001405 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
1406 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
Joachim Protze82e94a52017-11-01 10:08:30 +00001407
1408 ompt_lw_taskteam_t lw_taskteam;
1409 __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid,
1410 &ompt_parallel_data, codeptr);
1411
1412 __ompt_lw_taskteam_link(&lw_taskteam, this_thr, 1);
1413 // don't use lw_taskteam after linking. content was swaped
1414
1415 /* OMPT implicit task begin */
1416 implicit_task_data = OMPT_CUR_TASK_DATA(this_thr);
1417 if (ompt_enabled.ompt_callback_implicit_task) {
1418 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1419 ompt_scope_begin, OMPT_CUR_TEAM_DATA(this_thr),
Joachim Protze2b46d302019-01-15 15:36:53 +00001420 OMPT_CUR_TASK_DATA(this_thr), 1, __kmp_tid_from_gtid(global_tid), ompt_task_implicit); // TODO: Can this be ompt_task_initial?
Joachim Protze9be9cf22018-05-07 12:42:21 +00001421 OMPT_CUR_TASK_INFO(this_thr)
1422 ->thread_num = __kmp_tid_from_gtid(global_tid);
Joachim Protze82e94a52017-11-01 10:08:30 +00001423 }
1424
1425 /* OMPT state */
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001426 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
1427 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
Joachim Protze82e94a52017-11-01 10:08:30 +00001428 }
1429#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001430}
Jim Cownie181b4bb2013-12-23 17:28:57 +00001431
Jim Cownie5e8470a2013-09-27 10:38:44 +00001432/* most of the work for a fork */
1433/* return true if we really went parallel, false if serialized */
Jonathan Peyton30419822017-05-12 18:01:32 +00001434int __kmp_fork_call(ident_t *loc, int gtid,
1435 enum fork_context_e call_context, // Intel, GNU, ...
Joachim Protze82e94a52017-11-01 10:08:30 +00001436 kmp_int32 argc, microtask_t microtask, launch_t invoker,
Jim Cownie5e8470a2013-09-27 10:38:44 +00001437/* TODO: revert workaround for Intel(R) 64 tracker #96 */
Andrey Churbanovcbda8682015-01-13 14:43:35 +00001438#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
Jonathan Peyton30419822017-05-12 18:01:32 +00001439 va_list *ap
Jim Cownie5e8470a2013-09-27 10:38:44 +00001440#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001441 va_list ap
Jim Cownie5e8470a2013-09-27 10:38:44 +00001442#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001443 ) {
1444 void **argv;
1445 int i;
1446 int master_tid;
1447 int master_this_cons;
1448 kmp_team_t *team;
1449 kmp_team_t *parent_team;
1450 kmp_info_t *master_th;
1451 kmp_root_t *root;
1452 int nthreads;
1453 int master_active;
1454 int master_set_numthreads;
1455 int level;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001456#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001457 int active_level;
1458 int teams_level;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001459#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001460#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00001461 kmp_hot_team_ptr_t **p_hot_teams;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001462#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001463 { // KMP_TIME_BLOCK
Jonathan Peyton5375fe82016-11-14 21:13:44 +00001464 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call);
Jonathan Peyton45be4502015-08-11 21:36:41 +00001465 KMP_COUNT_VALUE(OMP_PARALLEL_args, argc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001466
Jonathan Peyton30419822017-05-12 18:01:32 +00001467 KA_TRACE(20, ("__kmp_fork_call: enter T#%d\n", gtid));
1468 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL) {
1469 /* Some systems prefer the stack for the root thread(s) to start with */
1470 /* some gap from the parent stack to prevent false sharing. */
1471 void *dummy = KMP_ALLOCA(__kmp_stkpadding);
1472 /* These 2 lines below are so this does not get optimized out */
1473 if (__kmp_stkpadding > KMP_MAX_STKPADDING)
1474 __kmp_stkpadding += (short)((kmp_int64)dummy);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001475 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001476
1477 /* initialize if needed */
Jonathan Peyton30419822017-05-12 18:01:32 +00001478 KMP_DEBUG_ASSERT(
1479 __kmp_init_serial); // AC: potentially unsafe, not in sync with shutdown
1480 if (!TCR_4(__kmp_init_parallel))
1481 __kmp_parallel_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00001482
Jonathan Peyton9b8bb322019-01-16 20:07:39 +00001483#if OMP_50_ENABLED
1484 __kmp_resume_if_soft_paused();
1485#endif
1486
Jim Cownie5e8470a2013-09-27 10:38:44 +00001487 /* setup current data */
Jonathan Peyton30419822017-05-12 18:01:32 +00001488 master_th = __kmp_threads[gtid]; // AC: potentially unsafe, not in sync with
1489 // shutdown
1490 parent_team = master_th->th.th_team;
1491 master_tid = master_th->th.th_info.ds.ds_tid;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001492 master_this_cons = master_th->th.th_local.this_construct;
Jonathan Peyton30419822017-05-12 18:01:32 +00001493 root = master_th->th.th_root;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001494 master_active = root->r.r_active;
1495 master_set_numthreads = master_th->th.th_set_nproc;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001496
1497#if OMPT_SUPPORT
Jonathan Peyton3574f282018-10-04 14:57:04 +00001498 ompt_data_t ompt_parallel_data = ompt_data_none;
Joachim Protze82e94a52017-11-01 10:08:30 +00001499 ompt_data_t *parent_task_data;
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001500 ompt_frame_t *ompt_frame;
Joachim Protze82e94a52017-11-01 10:08:30 +00001501 ompt_data_t *implicit_task_data;
1502 void *return_address = NULL;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001503
Joachim Protze82e94a52017-11-01 10:08:30 +00001504 if (ompt_enabled.enabled) {
1505 __ompt_get_task_info_internal(0, NULL, &parent_task_data, &ompt_frame,
1506 NULL, NULL);
1507 return_address = OMPT_LOAD_RETURN_ADDRESS(gtid);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001508 }
1509#endif
1510
Jim Cownie5e8470a2013-09-27 10:38:44 +00001511 // Nested level will be an index in the nested nthreads array
Jonathan Peyton30419822017-05-12 18:01:32 +00001512 level = parent_team->t.t_level;
1513 // used to launch non-serial teams even if nested is not allowed
1514 active_level = parent_team->t.t_active_level;
Jonathan Peytonc76f9f02016-06-21 19:12:07 +00001515#if OMP_40_ENABLED
Jonathan Peyton642688b2017-06-01 16:46:36 +00001516 // needed to check nesting inside the teams
1517 teams_level = master_th->th.th_teams_level;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001518#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001519#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00001520 p_hot_teams = &master_th->th.th_hot_teams;
1521 if (*p_hot_teams == NULL && __kmp_hot_teams_max_level > 0) {
1522 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate(
1523 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level);
1524 (*p_hot_teams)[0].hot_team = root->r.r_hot_team;
Jonathan Peyton642688b2017-06-01 16:46:36 +00001525 // it is either actual or not needed (when active_level > 0)
1526 (*p_hot_teams)[0].hot_team_nth = 1;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001527 }
1528#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001529
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001530#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00001531 if (ompt_enabled.enabled) {
1532 if (ompt_enabled.ompt_callback_parallel_begin) {
1533 int team_size = master_set_numthreads
1534 ? master_set_numthreads
1535 : get__nproc_2(parent_team, master_tid);
1536 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
1537 parent_task_data, ompt_frame, &ompt_parallel_data, team_size,
1538 OMPT_INVOKER(call_context), return_address);
1539 }
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001540 master_th->th.ompt_thread_info.state = ompt_state_overhead;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001541 }
1542#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001543
Jim Cownie5e8470a2013-09-27 10:38:44 +00001544 master_th->th.th_ident = loc;
1545
1546#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001547 if (master_th->th.th_teams_microtask && ap &&
1548 microtask != (microtask_t)__kmp_teams_master && level == teams_level) {
1549 // AC: This is start of parallel that is nested inside teams construct.
1550 // The team is actual (hot), all workers are ready at the fork barrier.
1551 // No lock needed to initialize the team a bit, then free workers.
1552 parent_team->t.t_ident = loc;
1553 __kmp_alloc_argv_entries(argc, parent_team, TRUE);
1554 parent_team->t.t_argc = argc;
1555 argv = (void **)parent_team->t.t_argv;
1556 for (i = argc - 1; i >= 0; --i)
Jim Cownie5e8470a2013-09-27 10:38:44 +00001557/* TODO: revert workaround for Intel(R) 64 tracker #96 */
Andrey Churbanovcbda8682015-01-13 14:43:35 +00001558#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
Jonathan Peyton30419822017-05-12 18:01:32 +00001559 *argv++ = va_arg(*ap, void *);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001560#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001561 *argv++ = va_arg(ap, void *);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001562#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001563 // Increment our nested depth levels, but not increase the serialization
1564 if (parent_team == master_th->th.th_serial_team) {
1565 // AC: we are in serialized parallel
1566 __kmpc_serialized_parallel(loc, gtid);
1567 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1);
1568 // AC: need this in order enquiry functions work
1569 // correctly, will restore at join time
1570 parent_team->t.t_serialized--;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001571#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001572 void *dummy;
1573 void **exit_runtime_p;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001574
Jonathan Peyton30419822017-05-12 18:01:32 +00001575 ompt_lw_taskteam_t lw_taskteam;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001576
Joachim Protze82e94a52017-11-01 10:08:30 +00001577 if (ompt_enabled.enabled) {
1578 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1579 &ompt_parallel_data, return_address);
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001580 exit_runtime_p = &(lw_taskteam.ompt_task_info.frame.exit_frame.ptr);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001581
Joachim Protze82e94a52017-11-01 10:08:30 +00001582 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1583 // don't use lw_taskteam after linking. content was swaped
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001584
Jonathan Peyton30419822017-05-12 18:01:32 +00001585 /* OMPT implicit task begin */
Joachim Protze82e94a52017-11-01 10:08:30 +00001586 implicit_task_data = OMPT_CUR_TASK_DATA(master_th);
1587 if (ompt_enabled.ompt_callback_implicit_task) {
1588 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1589 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
Joachim Protze2b46d302019-01-15 15:36:53 +00001590 implicit_task_data, 1, __kmp_tid_from_gtid(gtid), ompt_task_implicit); // TODO: Can this be ompt_task_initial?
Joachim Protze9be9cf22018-05-07 12:42:21 +00001591 OMPT_CUR_TASK_INFO(master_th)
1592 ->thread_num = __kmp_tid_from_gtid(gtid);
Jonathan Peyton30419822017-05-12 18:01:32 +00001593 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001594
Jonathan Peyton30419822017-05-12 18:01:32 +00001595 /* OMPT state */
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001596 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
Jonathan Peyton30419822017-05-12 18:01:32 +00001597 } else {
1598 exit_runtime_p = &dummy;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001599 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001600#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001601
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001602 {
Jonathan Peyton30419822017-05-12 18:01:32 +00001603 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1604 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1605 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv
1606#if OMPT_SUPPORT
1607 ,
1608 exit_runtime_p
1609#endif
1610 );
Jim Cownie5e8470a2013-09-27 10:38:44 +00001611 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001612
Jonathan Peyton30419822017-05-12 18:01:32 +00001613#if OMPT_SUPPORT
1614 *exit_runtime_p = NULL;
Joachim Protze82e94a52017-11-01 10:08:30 +00001615 if (ompt_enabled.enabled) {
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001616 OMPT_CUR_TASK_INFO(master_th)->frame.exit_frame = ompt_data_none;
Joachim Protze82e94a52017-11-01 10:08:30 +00001617 if (ompt_enabled.ompt_callback_implicit_task) {
1618 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1619 ompt_scope_end, NULL, implicit_task_data, 1,
Joachim Protze2b46d302019-01-15 15:36:53 +00001620 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit); // TODO: Can this be ompt_task_initial?
Jonathan Peyton30419822017-05-12 18:01:32 +00001621 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001622 __ompt_lw_taskteam_unlink(master_th);
Jonathan Peyton30419822017-05-12 18:01:32 +00001623
Joachim Protze82e94a52017-11-01 10:08:30 +00001624 if (ompt_enabled.ompt_callback_parallel_end) {
1625 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1626 OMPT_CUR_TEAM_DATA(master_th), OMPT_CUR_TASK_DATA(master_th),
1627 OMPT_INVOKER(call_context), return_address);
Jonathan Peyton30419822017-05-12 18:01:32 +00001628 }
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001629 master_th->th.ompt_thread_info.state = ompt_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00001630 }
1631#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001632 return TRUE;
Jonathan Peyton30419822017-05-12 18:01:32 +00001633 }
1634
1635 parent_team->t.t_pkfn = microtask;
Jonathan Peyton30419822017-05-12 18:01:32 +00001636 parent_team->t.t_invoke = invoker;
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00001637 KMP_ATOMIC_INC(&root->r.r_in_parallel);
Jonathan Peyton30419822017-05-12 18:01:32 +00001638 parent_team->t.t_active_level++;
1639 parent_team->t.t_level++;
Jonathan Peyton92ca6182018-09-07 18:25:49 +00001640#if OMP_50_ENABLED
1641 parent_team->t.t_def_allocator = master_th->th.th_def_allocator; // save
1642#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001643
1644 /* Change number of threads in the team if requested */
1645 if (master_set_numthreads) { // The parallel has num_threads clause
1646 if (master_set_numthreads < master_th->th.th_teams_size.nth) {
1647 // AC: only can reduce number of threads dynamically, can't increase
1648 kmp_info_t **other_threads = parent_team->t.t_threads;
1649 parent_team->t.t_nproc = master_set_numthreads;
1650 for (i = 0; i < master_set_numthreads; ++i) {
1651 other_threads[i]->th.th_team_nproc = master_set_numthreads;
1652 }
1653 // Keep extra threads hot in the team for possible next parallels
1654 }
1655 master_th->th.th_set_nproc = 0;
1656 }
1657
1658#if USE_DEBUGGER
1659 if (__kmp_debugging) { // Let debugger override number of threads.
1660 int nth = __kmp_omp_num_threads(loc);
Jonathan Peyton642688b2017-06-01 16:46:36 +00001661 if (nth > 0) { // 0 means debugger doesn't want to change num threads
Jonathan Peyton30419822017-05-12 18:01:32 +00001662 master_set_numthreads = nth;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00001663 }
1664 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001665#endif
1666
1667 KF_TRACE(10, ("__kmp_fork_call: before internal fork: root=%p, team=%p, "
1668 "master_th=%p, gtid=%d\n",
1669 root, parent_team, master_th, gtid));
1670 __kmp_internal_fork(loc, gtid, parent_team);
1671 KF_TRACE(10, ("__kmp_fork_call: after internal fork: root=%p, team=%p, "
1672 "master_th=%p, gtid=%d\n",
1673 root, parent_team, master_th, gtid));
1674
1675 /* Invoke microtask for MASTER thread */
1676 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
1677 parent_team->t.t_id, parent_team->t.t_pkfn));
1678
Jonathan Peytonf0682ac2018-07-30 17:41:08 +00001679 if (!parent_team->t.t_invoke(gtid)) {
1680 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread");
Jonathan Peyton30419822017-05-12 18:01:32 +00001681 }
1682 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
1683 parent_team->t.t_id, parent_team->t.t_pkfn));
1684 KMP_MB(); /* Flush all pending memory write invalidates. */
1685
1686 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid));
1687
1688 return TRUE;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001689 } // Parallel closely nested in teams construct
Jim Cownie5e8470a2013-09-27 10:38:44 +00001690#endif /* OMP_40_ENABLED */
1691
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001692#if KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00001693 if (__kmp_tasking_mode != tskm_immediate_exec) {
1694 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
1695 parent_team->t.t_task_team[master_th->th.th_task_state]);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001696 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001697#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001698
Jonathan Peyton30419822017-05-12 18:01:32 +00001699 if (parent_team->t.t_active_level >=
1700 master_th->th.th_current_task->td_icvs.max_active_levels) {
1701 nthreads = 1;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001702 } else {
Andrey Churbanov92effc42015-08-18 10:08:27 +00001703#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001704 int enter_teams = ((ap == NULL && active_level == 0) ||
1705 (ap && teams_level > 0 && teams_level == level));
Andrey Churbanov92effc42015-08-18 10:08:27 +00001706#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001707 nthreads =
1708 master_set_numthreads
1709 ? master_set_numthreads
1710 : get__nproc_2(
1711 parent_team,
1712 master_tid); // TODO: get nproc directly from current task
Andrey Churbanov92effc42015-08-18 10:08:27 +00001713
Jonathan Peyton30419822017-05-12 18:01:32 +00001714 // Check if we need to take forkjoin lock? (no need for serialized
1715 // parallel out of teams construct). This code moved here from
1716 // __kmp_reserve_threads() to speedup nested serialized parallels.
1717 if (nthreads > 1) {
Jonathan Peyton76b45e82019-02-28 20:47:21 +00001718 if ((get__max_active_levels(master_th) == 1 && (root->r.r_in_parallel
Andrey Churbanov92effc42015-08-18 10:08:27 +00001719#if OMP_40_ENABLED
Jonathan Peyton76b45e82019-02-28 20:47:21 +00001720 && !enter_teams
Andrey Churbanov92effc42015-08-18 10:08:27 +00001721#endif /* OMP_40_ENABLED */
Jonathan Peyton76b45e82019-02-28 20:47:21 +00001722 )) ||
Jonathan Peyton30419822017-05-12 18:01:32 +00001723 (__kmp_library == library_serial)) {
Jonathan Peyton642688b2017-06-01 16:46:36 +00001724 KC_TRACE(10, ("__kmp_fork_call: T#%d serializing team; requested %d"
1725 " threads\n",
1726 gtid, nthreads));
Jonathan Peyton30419822017-05-12 18:01:32 +00001727 nthreads = 1;
Andrey Churbanov92effc42015-08-18 10:08:27 +00001728 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001729 }
1730 if (nthreads > 1) {
1731 /* determine how many new threads we can use */
1732 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
Jonathan Peyton30419822017-05-12 18:01:32 +00001733 nthreads = __kmp_reserve_threads(
1734 root, parent_team, master_tid, nthreads
Jim Cownie5e8470a2013-09-27 10:38:44 +00001735#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001736 /* AC: If we execute teams from parallel region (on host), then
1737 teams should be created but each can only have 1 thread if
1738 nesting is disabled. If teams called from serial region, then
1739 teams and their threads should be created regardless of the
1740 nesting setting. */
1741 ,
1742 enter_teams
Jim Cownie5e8470a2013-09-27 10:38:44 +00001743#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00001744 );
1745 if (nthreads == 1) {
1746 // Free lock for single thread execution here; for multi-thread
1747 // execution it will be freed later after team of threads created
1748 // and initialized
1749 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
Andrey Churbanov92effc42015-08-18 10:08:27 +00001750 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001751 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001752 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001753 KMP_DEBUG_ASSERT(nthreads > 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001754
Jonathan Peyton30419822017-05-12 18:01:32 +00001755 // If we temporarily changed the set number of threads then restore it now
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001756 master_th->th.th_set_nproc = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001757
Jim Cownie5e8470a2013-09-27 10:38:44 +00001758 /* create a serialized parallel region? */
Jonathan Peyton30419822017-05-12 18:01:32 +00001759 if (nthreads == 1) {
1760/* josh todo: hypothetical question: what do we do for OS X*? */
1761#if KMP_OS_LINUX && \
1762 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1763 void *args[argc];
Jim Cownie5e8470a2013-09-27 10:38:44 +00001764#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001765 void **args = (void **)KMP_ALLOCA(argc * sizeof(void *));
1766#endif /* KMP_OS_LINUX && ( KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || \
1767 KMP_ARCH_AARCH64) */
Jim Cownie5e8470a2013-09-27 10:38:44 +00001768
Jonathan Peyton30419822017-05-12 18:01:32 +00001769 KA_TRACE(20,
1770 ("__kmp_fork_call: T#%d serializing parallel region\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00001771
Jonathan Peyton30419822017-05-12 18:01:32 +00001772 __kmpc_serialized_parallel(loc, gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001773
Jonathan Peyton30419822017-05-12 18:01:32 +00001774 if (call_context == fork_context_intel) {
1775 /* TODO this sucks, use the compiler itself to pass args! :) */
1776 master_th->th.th_serial_team->t.t_ident = loc;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001777#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001778 if (!ap) {
1779 // revert change made in __kmpc_serialized_parallel()
1780 master_th->th.th_serial_team->t.t_level--;
1781// Get args from parent team for teams construct
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001782
1783#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001784 void *dummy;
1785 void **exit_runtime_p;
Joachim Protze82e94a52017-11-01 10:08:30 +00001786 ompt_task_info_t *task_info;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001787
Jonathan Peyton30419822017-05-12 18:01:32 +00001788 ompt_lw_taskteam_t lw_taskteam;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001789
Joachim Protze82e94a52017-11-01 10:08:30 +00001790 if (ompt_enabled.enabled) {
Jonathan Peyton30419822017-05-12 18:01:32 +00001791 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
Joachim Protze82e94a52017-11-01 10:08:30 +00001792 &ompt_parallel_data, return_address);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001793
Joachim Protze82e94a52017-11-01 10:08:30 +00001794 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1795 // don't use lw_taskteam after linking. content was swaped
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001796
Joachim Protze82e94a52017-11-01 10:08:30 +00001797 task_info = OMPT_CUR_TASK_INFO(master_th);
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001798 exit_runtime_p = &(task_info->frame.exit_frame.ptr);
Joachim Protze82e94a52017-11-01 10:08:30 +00001799 if (ompt_enabled.ompt_callback_implicit_task) {
1800 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1801 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
Joachim Protze2b46d302019-01-15 15:36:53 +00001802 &(task_info->task_data), 1, __kmp_tid_from_gtid(gtid), ompt_task_implicit); // TODO: Can this be ompt_task_initial?
Joachim Protze9be9cf22018-05-07 12:42:21 +00001803 OMPT_CUR_TASK_INFO(master_th)
1804 ->thread_num = __kmp_tid_from_gtid(gtid);
Jonathan Peyton30419822017-05-12 18:01:32 +00001805 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001806
Jonathan Peyton30419822017-05-12 18:01:32 +00001807 /* OMPT state */
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001808 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
Jonathan Peyton30419822017-05-12 18:01:32 +00001809 } else {
1810 exit_runtime_p = &dummy;
1811 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001812#endif
1813
Jonathan Peyton30419822017-05-12 18:01:32 +00001814 {
1815 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1816 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1817 __kmp_invoke_microtask(microtask, gtid, 0, argc,
1818 parent_team->t.t_argv
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001819#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001820 ,
1821 exit_runtime_p
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001822#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001823 );
1824 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001825
1826#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00001827 if (ompt_enabled.enabled) {
1828 exit_runtime_p = NULL;
1829 if (ompt_enabled.ompt_callback_implicit_task) {
1830 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1831 ompt_scope_end, NULL, &(task_info->task_data), 1,
Joachim Protze2b46d302019-01-15 15:36:53 +00001832 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit); // TODO: Can this be ompt_task_initial?
Jonathan Peyton30419822017-05-12 18:01:32 +00001833 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001834
Jonathan Peyton30419822017-05-12 18:01:32 +00001835 __ompt_lw_taskteam_unlink(master_th);
Joachim Protze82e94a52017-11-01 10:08:30 +00001836 if (ompt_enabled.ompt_callback_parallel_end) {
1837 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1838 OMPT_CUR_TEAM_DATA(master_th), parent_task_data,
1839 OMPT_INVOKER(call_context), return_address);
Jonathan Peyton30419822017-05-12 18:01:32 +00001840 }
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001841 master_th->th.ompt_thread_info.state = ompt_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00001842 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001843#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001844 } else if (microtask == (microtask_t)__kmp_teams_master) {
1845 KMP_DEBUG_ASSERT(master_th->th.th_team ==
1846 master_th->th.th_serial_team);
1847 team = master_th->th.th_team;
1848 // team->t.t_pkfn = microtask;
1849 team->t.t_invoke = invoker;
1850 __kmp_alloc_argv_entries(argc, team, TRUE);
1851 team->t.t_argc = argc;
1852 argv = (void **)team->t.t_argv;
1853 if (ap) {
1854 for (i = argc - 1; i >= 0; --i)
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001855// TODO: revert workaround for Intel(R) 64 tracker #96
Andrey Churbanovcbda8682015-01-13 14:43:35 +00001856#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
Jonathan Peyton30419822017-05-12 18:01:32 +00001857 *argv++ = va_arg(*ap, void *);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001858#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001859 *argv++ = va_arg(ap, void *);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001860#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001861 } else {
1862 for (i = 0; i < argc; ++i)
1863 // Get args from parent team for teams construct
1864 argv[i] = parent_team->t.t_argv[i];
1865 }
1866 // AC: revert change made in __kmpc_serialized_parallel()
1867 // because initial code in teams should have level=0
1868 team->t.t_level--;
1869 // AC: call special invoker for outer "parallel" of teams construct
Jonathan Peytonf0682ac2018-07-30 17:41:08 +00001870 invoker(gtid);
Jonathan Peyton30419822017-05-12 18:01:32 +00001871 } else {
1872#endif /* OMP_40_ENABLED */
1873 argv = args;
1874 for (i = argc - 1; i >= 0; --i)
1875// TODO: revert workaround for Intel(R) 64 tracker #96
1876#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
1877 *argv++ = va_arg(*ap, void *);
1878#else
1879 *argv++ = va_arg(ap, void *);
1880#endif
1881 KMP_MB();
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001882
1883#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001884 void *dummy;
1885 void **exit_runtime_p;
Joachim Protze82e94a52017-11-01 10:08:30 +00001886 ompt_task_info_t *task_info;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001887
Jonathan Peyton30419822017-05-12 18:01:32 +00001888 ompt_lw_taskteam_t lw_taskteam;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001889
Joachim Protze82e94a52017-11-01 10:08:30 +00001890 if (ompt_enabled.enabled) {
Jonathan Peyton30419822017-05-12 18:01:32 +00001891 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
Joachim Protze82e94a52017-11-01 10:08:30 +00001892 &ompt_parallel_data, return_address);
1893 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1894 // don't use lw_taskteam after linking. content was swaped
1895 task_info = OMPT_CUR_TASK_INFO(master_th);
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001896 exit_runtime_p = &(task_info->frame.exit_frame.ptr);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001897
Jonathan Peyton30419822017-05-12 18:01:32 +00001898 /* OMPT implicit task begin */
Joachim Protze82e94a52017-11-01 10:08:30 +00001899 implicit_task_data = OMPT_CUR_TASK_DATA(master_th);
1900 if (ompt_enabled.ompt_callback_implicit_task) {
1901 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1902 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
Joachim Protze2b46d302019-01-15 15:36:53 +00001903 implicit_task_data, 1, __kmp_tid_from_gtid(gtid), ompt_task_implicit); // TODO: Can this be ompt_task_initial?
Joachim Protze9be9cf22018-05-07 12:42:21 +00001904 OMPT_CUR_TASK_INFO(master_th)
1905 ->thread_num = __kmp_tid_from_gtid(gtid);
Jonathan Peyton30419822017-05-12 18:01:32 +00001906 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001907
Jonathan Peyton30419822017-05-12 18:01:32 +00001908 /* OMPT state */
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001909 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
Jonathan Peyton30419822017-05-12 18:01:32 +00001910 } else {
1911 exit_runtime_p = &dummy;
1912 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001913#endif
1914
Jonathan Peyton30419822017-05-12 18:01:32 +00001915 {
1916 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1917 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1918 __kmp_invoke_microtask(microtask, gtid, 0, argc, args
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001919#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001920 ,
1921 exit_runtime_p
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001922#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001923 );
1924 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001925
1926#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00001927 if (ompt_enabled.enabled) {
1928 *exit_runtime_p = NULL;
1929 if (ompt_enabled.ompt_callback_implicit_task) {
1930 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1931 ompt_scope_end, NULL, &(task_info->task_data), 1,
Joachim Protze2b46d302019-01-15 15:36:53 +00001932 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit); // TODO: Can this be ompt_task_initial?
Jonathan Peyton30419822017-05-12 18:01:32 +00001933 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001934
Joachim Protze82e94a52017-11-01 10:08:30 +00001935 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
Jonathan Peyton30419822017-05-12 18:01:32 +00001936 __ompt_lw_taskteam_unlink(master_th);
Joachim Protze82e94a52017-11-01 10:08:30 +00001937 if (ompt_enabled.ompt_callback_parallel_end) {
1938 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1939 &ompt_parallel_data, parent_task_data,
1940 OMPT_INVOKER(call_context), return_address);
Jonathan Peyton30419822017-05-12 18:01:32 +00001941 }
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001942 master_th->th.ompt_thread_info.state = ompt_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00001943 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001944#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001945#if OMP_40_ENABLED
Jim Cownie5e8470a2013-09-27 10:38:44 +00001946 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001947#endif /* OMP_40_ENABLED */
1948 } else if (call_context == fork_context_gnu) {
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001949#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00001950 ompt_lw_taskteam_t lwt;
1951 __ompt_lw_taskteam_init(&lwt, master_th, gtid, &ompt_parallel_data,
1952 return_address);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001953
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00001954 lwt.ompt_task_info.frame.exit_frame = ompt_data_none;
Joachim Protze82e94a52017-11-01 10:08:30 +00001955 __ompt_lw_taskteam_link(&lwt, master_th, 1);
1956// don't use lw_taskteam after linking. content was swaped
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001957#endif
1958
Jonathan Peyton30419822017-05-12 18:01:32 +00001959 // we were called from GNU native code
1960 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00001961 return FALSE;
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00001962 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00001963 KMP_ASSERT2(call_context < fork_context_last,
1964 "__kmp_fork_call: unknown fork_context parameter");
1965 }
1966
1967 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid));
1968 KMP_MB();
1969 return FALSE;
Jonathan Peyton6d88e042018-12-13 23:14:24 +00001970 } // if (nthreads == 1)
Jim Cownie5e8470a2013-09-27 10:38:44 +00001971
Jim Cownie5e8470a2013-09-27 10:38:44 +00001972 // GEH: only modify the executing flag in the case when not serialized
1973 // serialized case is handled in kmpc_serialized_parallel
Jonathan Peyton30419822017-05-12 18:01:32 +00001974 KF_TRACE(10, ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
1975 "curtask=%p, curtask_max_aclevel=%d\n",
1976 parent_team->t.t_active_level, master_th,
1977 master_th->th.th_current_task,
1978 master_th->th.th_current_task->td_icvs.max_active_levels));
1979 // TODO: GEH - cannot do this assertion because root thread not set up as
1980 // executing
Jim Cownie5e8470a2013-09-27 10:38:44 +00001981 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 1 );
1982 master_th->th.th_current_task->td_flags.executing = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001983
1984#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001985 if (!master_th->th.th_teams_microtask || level > teams_level)
Jim Cownie5e8470a2013-09-27 10:38:44 +00001986#endif /* OMP_40_ENABLED */
1987 {
Jonathan Peyton30419822017-05-12 18:01:32 +00001988 /* Increment our nested depth level */
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00001989 KMP_ATOMIC_INC(&root->r.r_in_parallel);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001990 }
1991
Jim Cownie5e8470a2013-09-27 10:38:44 +00001992 // See if we need to make a copy of the ICVs.
Jim Cownie5e8470a2013-09-27 10:38:44 +00001993 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc;
Jonathan Peyton30419822017-05-12 18:01:32 +00001994 if ((level + 1 < __kmp_nested_nth.used) &&
1995 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) {
1996 nthreads_icv = __kmp_nested_nth.nth[level + 1];
1997 } else {
1998 nthreads_icv = 0; // don't update
Jim Cownie5e8470a2013-09-27 10:38:44 +00001999 }
2000
2001#if OMP_40_ENABLED
Jim Cownie5e8470a2013-09-27 10:38:44 +00002002 // Figure out the proc_bind_policy for the new team.
Jim Cownie5e8470a2013-09-27 10:38:44 +00002003 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;
Jonathan Peyton30419822017-05-12 18:01:32 +00002004 kmp_proc_bind_t proc_bind_icv =
2005 proc_bind_default; // proc_bind_default means don't update
2006 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
2007 proc_bind = proc_bind_false;
2008 } else {
2009 if (proc_bind == proc_bind_default) {
2010 // No proc_bind clause specified; use current proc-bind-var for this
2011 // parallel region
2012 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;
2013 }
2014 /* else: The proc_bind policy was specified explicitly on parallel clause.
2015 This overrides proc-bind-var for this parallel region, but does not
2016 change proc-bind-var. */
2017 // Figure the value of proc-bind-var for the child threads.
2018 if ((level + 1 < __kmp_nested_proc_bind.used) &&
2019 (__kmp_nested_proc_bind.bind_types[level + 1] !=
2020 master_th->th.th_current_task->td_icvs.proc_bind)) {
2021 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1];
2022 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002023 }
2024
Jim Cownie5e8470a2013-09-27 10:38:44 +00002025 // Reset for next parallel region
Jim Cownie5e8470a2013-09-27 10:38:44 +00002026 master_th->th.th_set_proc_bind = proc_bind_default;
2027#endif /* OMP_40_ENABLED */
2028
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002029 if ((nthreads_icv > 0)
Jim Cownie5e8470a2013-09-27 10:38:44 +00002030#if OMP_40_ENABLED
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002031 || (proc_bind_icv != proc_bind_default)
Jim Cownie5e8470a2013-09-27 10:38:44 +00002032#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00002033 ) {
2034 kmp_internal_control_t new_icvs;
2035 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs);
2036 new_icvs.next = NULL;
2037 if (nthreads_icv > 0) {
2038 new_icvs.nproc = nthreads_icv;
2039 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002040
2041#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002042 if (proc_bind_icv != proc_bind_default) {
2043 new_icvs.proc_bind = proc_bind_icv;
2044 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002045#endif /* OMP_40_ENABLED */
2046
Jonathan Peyton30419822017-05-12 18:01:32 +00002047 /* allocate a new parallel team */
2048 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n"));
2049 team = __kmp_allocate_team(root, nthreads, nthreads,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002050#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002051 ompt_parallel_data,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002052#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002053#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002054 proc_bind,
Jim Cownie5e8470a2013-09-27 10:38:44 +00002055#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002056 &new_icvs, argc USE_NESTED_HOT_ARG(master_th));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002057 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00002058 /* allocate a new parallel team */
2059 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n"));
2060 team = __kmp_allocate_team(root, nthreads, nthreads,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002061#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002062 ompt_parallel_data,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002063#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002064#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002065 proc_bind,
Jim Cownie5e8470a2013-09-27 10:38:44 +00002066#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002067 &master_th->th.th_current_task->td_icvs,
2068 argc USE_NESTED_HOT_ARG(master_th));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002069 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002070 KF_TRACE(
2071 10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002072
2073 /* setup the new team */
Jonathan Peytonb044e4f2016-05-23 18:01:19 +00002074 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid);
2075 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons);
2076 KMP_CHECK_UPDATE(team->t.t_ident, loc);
2077 KMP_CHECK_UPDATE(team->t.t_parent, parent_team);
2078 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002079#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002080 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address,
2081 return_address);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002082#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002083 KMP_CHECK_UPDATE(team->t.t_invoke, invoker); // TODO move to root, maybe
2084// TODO: parent_team->t.t_level == INT_MAX ???
Jim Cownie5e8470a2013-09-27 10:38:44 +00002085#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002086 if (!master_th->th.th_teams_microtask || level > teams_level) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002087#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00002088 int new_level = parent_team->t.t_level + 1;
2089 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2090 new_level = parent_team->t.t_active_level + 1;
2091 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002092#if OMP_40_ENABLED
2093 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00002094 // AC: Do not increase parallel level at start of the teams construct
2095 int new_level = parent_team->t.t_level;
2096 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2097 new_level = parent_team->t.t_active_level;
2098 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002099 }
2100#endif /* OMP_40_ENABLED */
Jonathan Peytonb044e4f2016-05-23 18:01:19 +00002101 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid);
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00002102 // set master's schedule as new run-time schedule
2103 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002104
Jonathan Peyton45ca5da2015-10-19 19:33:38 +00002105#if OMP_40_ENABLED
Jonathan Peytonb044e4f2016-05-23 18:01:19 +00002106 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq);
Jonathan Peyton45ca5da2015-10-19 19:33:38 +00002107#endif
Jonathan Peyton92ca6182018-09-07 18:25:49 +00002108#if OMP_50_ENABLED
2109 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator);
2110#endif
Jonathan Peyton45ca5da2015-10-19 19:33:38 +00002111
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002112 // Update the floating point rounding in the team if required.
2113 propagateFPControl(team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002114
Jonathan Peyton30419822017-05-12 18:01:32 +00002115 if (__kmp_tasking_mode != tskm_immediate_exec) {
2116 // Set master's task team to team's task team. Unless this is hot team, it
2117 // should be NULL.
Jonathan Peyton30419822017-05-12 18:01:32 +00002118 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2119 parent_team->t.t_task_team[master_th->th.th_task_state]);
Jonathan Peyton30419822017-05-12 18:01:32 +00002120 KA_TRACE(20, ("__kmp_fork_call: Master T#%d pushing task_team %p / team "
2121 "%p, new task_team %p / team %p\n",
2122 __kmp_gtid_from_thread(master_th),
2123 master_th->th.th_task_team, parent_team,
2124 team->t.t_task_team[master_th->th.th_task_state], team));
Jonathan Peytond3f2b942016-02-09 22:32:41 +00002125
Jonathan Peyton30419822017-05-12 18:01:32 +00002126 if (active_level || master_th->th.th_task_team) {
2127 // Take a memo of master's task_state
2128 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2129 if (master_th->th.th_task_state_top >=
2130 master_th->th.th_task_state_stack_sz) { // increase size
2131 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz;
2132 kmp_uint8 *old_stack, *new_stack;
2133 kmp_uint32 i;
2134 new_stack = (kmp_uint8 *)__kmp_allocate(new_size);
2135 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) {
2136 new_stack[i] = master_th->th.th_task_state_memo_stack[i];
2137 }
2138 for (i = master_th->th.th_task_state_stack_sz; i < new_size;
2139 ++i) { // zero-init rest of stack
2140 new_stack[i] = 0;
2141 }
2142 old_stack = master_th->th.th_task_state_memo_stack;
2143 master_th->th.th_task_state_memo_stack = new_stack;
2144 master_th->th.th_task_state_stack_sz = new_size;
2145 __kmp_free(old_stack);
Andrey Churbanov6d224db2015-02-10 18:37:43 +00002146 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002147 // Store master's task_state on stack
2148 master_th->th
2149 .th_task_state_memo_stack[master_th->th.th_task_state_top] =
2150 master_th->th.th_task_state;
2151 master_th->th.th_task_state_top++;
2152#if KMP_NESTED_HOT_TEAMS
Jonathan Peytonca10a762018-08-24 18:05:00 +00002153 if (master_th->th.th_hot_teams &&
Jonathan Peytonf4c07202018-11-28 20:15:11 +00002154 active_level < __kmp_hot_teams_max_level &&
Jonathan Peytonca10a762018-08-24 18:05:00 +00002155 team == master_th->th.th_hot_teams[active_level].hot_team) {
Jonathan Peyton642688b2017-06-01 16:46:36 +00002156 // Restore master's nested state if nested hot team
Jonathan Peyton30419822017-05-12 18:01:32 +00002157 master_th->th.th_task_state =
2158 master_th->th
2159 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2160 } else {
2161#endif
2162 master_th->th.th_task_state = 0;
2163#if KMP_NESTED_HOT_TEAMS
2164 }
2165#endif
2166 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002167#if !KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00002168 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) ||
2169 (team == root->r.r_hot_team));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002170#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002171 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002172
Jonathan Peyton30419822017-05-12 18:01:32 +00002173 KA_TRACE(
2174 20,
2175 ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",
2176 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id,
2177 team->t.t_nproc));
2178 KMP_DEBUG_ASSERT(team != root->r.r_hot_team ||
2179 (team->t.t_master_tid == 0 &&
2180 (team->t.t_parent == root->r.r_root_team ||
2181 team->t.t_parent->t.t_serialized)));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002182 KMP_MB();
2183
2184 /* now, setup the arguments */
Jonathan Peyton30419822017-05-12 18:01:32 +00002185 argv = (void **)team->t.t_argv;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002186#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002187 if (ap) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002188#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00002189 for (i = argc - 1; i >= 0; --i) {
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002190// TODO: revert workaround for Intel(R) 64 tracker #96
Andrey Churbanovcbda8682015-01-13 14:43:35 +00002191#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
Jonathan Peyton30419822017-05-12 18:01:32 +00002192 void *new_argv = va_arg(*ap, void *);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002193#else
Jonathan Peyton30419822017-05-12 18:01:32 +00002194 void *new_argv = va_arg(ap, void *);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002195#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002196 KMP_CHECK_UPDATE(*argv, new_argv);
2197 argv++;
2198 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002199#if OMP_40_ENABLED
2200 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00002201 for (i = 0; i < argc; ++i) {
2202 // Get args from parent team for teams construct
2203 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]);
2204 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002205 }
2206#endif /* OMP_40_ENABLED */
2207
2208 /* now actually fork the threads */
Jonathan Peytonb044e4f2016-05-23 18:01:19 +00002209 KMP_CHECK_UPDATE(team->t.t_master_active, master_active);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002210 if (!root->r.r_active) // Only do assignment if it prevents cache ping-pong
Jonathan Peyton30419822017-05-12 18:01:32 +00002211 root->r.r_active = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002212
Jonathan Peyton30419822017-05-12 18:01:32 +00002213 __kmp_fork_team_threads(root, team, master_th, gtid);
2214 __kmp_setup_icv_copy(team, nthreads,
2215 &master_th->th.th_current_task->td_icvs, loc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002216
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002217#if OMPT_SUPPORT
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00002218 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002219#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002220
Jonathan Peyton30419822017-05-12 18:01:32 +00002221 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002222
Jim Cownie5e8470a2013-09-27 10:38:44 +00002223#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +00002224 if (team->t.t_active_level == 1 // only report frames at level 1
2225#if OMP_40_ENABLED
Andrey Churbanov51aecb82015-05-06 19:22:36 +00002226 && !master_th->th.th_teams_microtask // not in teams construct
Jonathan Peyton30419822017-05-12 18:01:32 +00002227#endif /* OMP_40_ENABLED */
2228 ) {
Andrey Churbanov51aecb82015-05-06 19:22:36 +00002229#if USE_ITT_NOTIFY
Jonathan Peyton30419822017-05-12 18:01:32 +00002230 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2231 (__kmp_forkjoin_frames_mode == 3 ||
2232 __kmp_forkjoin_frames_mode == 1)) {
2233 kmp_uint64 tmp_time = 0;
2234 if (__itt_get_timestamp_ptr)
2235 tmp_time = __itt_get_timestamp();
2236 // Internal fork - report frame begin
2237 master_th->th.th_frame_time = tmp_time;
2238 if (__kmp_forkjoin_frames_mode == 3)
2239 team->t.t_region_time = tmp_time;
Jonathan Peyton642688b2017-06-01 16:46:36 +00002240 } else
2241// only one notification scheme (either "submit" or "forking/joined", not both)
Andrey Churbanov51aecb82015-05-06 19:22:36 +00002242#endif /* USE_ITT_NOTIFY */
Jonathan Peyton30419822017-05-12 18:01:32 +00002243 if ((__itt_frame_begin_v3_ptr || KMP_ITT_DEBUG) &&
2244 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) {
Jonathan Peyton8c432f22018-01-04 22:56:47 +00002245 // Mark start of "parallel" region for Intel(R) VTune(TM) analyzer.
Jonathan Peyton30419822017-05-12 18:01:32 +00002246 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
2247 }
Andrey Churbanovf6451d92015-01-16 15:58:03 +00002248 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002249#endif /* USE_ITT_BUILD */
2250
2251 /* now go on and do the work */
Jonathan Peyton30419822017-05-12 18:01:32 +00002252 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002253 KMP_MB();
Jonathan Peyton30419822017-05-12 18:01:32 +00002254 KF_TRACE(10,
2255 ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n",
2256 root, team, master_th, gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002257
2258#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +00002259 if (__itt_stack_caller_create_ptr) {
2260 team->t.t_stack_id =
2261 __kmp_itt_stack_caller_create(); // create new stack stitching id
2262 // before entering fork barrier
Jim Cownie5e8470a2013-09-27 10:38:44 +00002263 }
2264#endif /* USE_ITT_BUILD */
2265
2266#if OMP_40_ENABLED
Jonathan Peyton642688b2017-06-01 16:46:36 +00002267 // AC: skip __kmp_internal_fork at teams construct, let only master
2268 // threads execute
2269 if (ap)
Jim Cownie5e8470a2013-09-27 10:38:44 +00002270#endif /* OMP_40_ENABLED */
2271 {
Jonathan Peyton30419822017-05-12 18:01:32 +00002272 __kmp_internal_fork(loc, gtid, team);
2273 KF_TRACE(10, ("__kmp_internal_fork : after : root=%p, team=%p, "
2274 "master_th=%p, gtid=%d\n",
2275 root, team, master_th, gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002276 }
2277
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002278 if (call_context == fork_context_gnu) {
Jonathan Peyton30419822017-05-12 18:01:32 +00002279 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid));
2280 return TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002281 }
2282
2283 /* Invoke microtask for MASTER thread */
Jonathan Peyton30419822017-05-12 18:01:32 +00002284 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
2285 team->t.t_id, team->t.t_pkfn));
2286 } // END of timer KMP_fork_call block
Jim Cownie5e8470a2013-09-27 10:38:44 +00002287
Jonathan Peytond2b53ca2019-04-03 18:53:26 +00002288#if KMP_STATS_ENABLED && OMP_40_ENABLED
2289 // If beginning a teams construct, then change thread state
2290 stats_state_e previous_state = KMP_GET_THREAD_STATE();
2291 if (!ap) {
2292 KMP_SET_THREAD_STATE(stats_state_e::TEAMS_REGION);
2293 }
2294#endif
2295
Jonathan Peytonf0682ac2018-07-30 17:41:08 +00002296 if (!team->t.t_invoke(gtid)) {
2297 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread");
Jonathan Peyton30419822017-05-12 18:01:32 +00002298 }
Jonathan Peytond2b53ca2019-04-03 18:53:26 +00002299
2300#if KMP_STATS_ENABLED && OMP_40_ENABLED
2301 // If was beginning of a teams construct, then reset thread state
2302 if (!ap) {
2303 KMP_SET_THREAD_STATE(previous_state);
2304 }
2305#endif
2306
Jonathan Peyton30419822017-05-12 18:01:32 +00002307 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
2308 team->t.t_id, team->t.t_pkfn));
2309 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00002310
Jonathan Peyton30419822017-05-12 18:01:32 +00002311 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002312
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002313#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002314 if (ompt_enabled.enabled) {
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00002315 master_th->th.ompt_thread_info.state = ompt_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00002316 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002317#endif
2318
Jonathan Peyton30419822017-05-12 18:01:32 +00002319 return TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002320}
2321
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002322#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00002323static inline void __kmp_join_restore_state(kmp_info_t *thread,
2324 kmp_team_t *team) {
2325 // restore state outside the region
2326 thread->th.ompt_thread_info.state =
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00002327 ((team->t.t_serialized) ? ompt_state_work_serial
2328 : ompt_state_work_parallel);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002329}
2330
Joachim Protze82e94a52017-11-01 10:08:30 +00002331static inline void __kmp_join_ompt(int gtid, kmp_info_t *thread,
2332 kmp_team_t *team, ompt_data_t *parallel_data,
2333 fork_context_e fork_context, void *codeptr) {
2334 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2335 if (ompt_enabled.ompt_callback_parallel_end) {
2336 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
2337 parallel_data, &(task_info->task_data), OMPT_INVOKER(fork_context),
2338 codeptr);
Jonathan Peyton30419822017-05-12 18:01:32 +00002339 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002340
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00002341 task_info->frame.enter_frame = ompt_data_none;
Jonathan Peyton30419822017-05-12 18:01:32 +00002342 __kmp_join_restore_state(thread, team);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002343}
2344#endif
2345
Jonathan Peyton30419822017-05-12 18:01:32 +00002346void __kmp_join_call(ident_t *loc, int gtid
Jonathan Peytonf89fbbb2015-08-31 18:15:00 +00002347#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00002348 ,
2349 enum fork_context_e fork_context
Jonathan Peytonf89fbbb2015-08-31 18:15:00 +00002350#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002351#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002352 ,
2353 int exit_teams
Jim Cownie5e8470a2013-09-27 10:38:44 +00002354#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00002355 ) {
2356 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call);
2357 kmp_team_t *team;
2358 kmp_team_t *parent_team;
2359 kmp_info_t *master_th;
2360 kmp_root_t *root;
2361 int master_active;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002362
Jonathan Peyton30419822017-05-12 18:01:32 +00002363 KA_TRACE(20, ("__kmp_join_call: enter T#%d\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002364
Jonathan Peyton30419822017-05-12 18:01:32 +00002365 /* setup current data */
2366 master_th = __kmp_threads[gtid];
2367 root = master_th->th.th_root;
2368 team = master_th->th.th_team;
2369 parent_team = team->t.t_parent;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002370
Jonathan Peyton30419822017-05-12 18:01:32 +00002371 master_th->th.th_ident = loc;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002372
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002373#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002374 if (ompt_enabled.enabled) {
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00002375 master_th->th.ompt_thread_info.state = ompt_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00002376 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002377#endif
2378
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002379#if KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00002380 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) {
2381 KA_TRACE(20, ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
2382 "th_task_team = %p\n",
2383 __kmp_gtid_from_thread(master_th), team,
2384 team->t.t_task_team[master_th->th.th_task_state],
2385 master_th->th.th_task_team));
2386 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2387 team->t.t_task_team[master_th->th.th_task_state]);
2388 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002389#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002390
Jonathan Peyton30419822017-05-12 18:01:32 +00002391 if (team->t.t_serialized) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002392#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002393 if (master_th->th.th_teams_microtask) {
2394 // We are in teams construct
2395 int level = team->t.t_level;
2396 int tlevel = master_th->th.th_teams_level;
2397 if (level == tlevel) {
2398 // AC: we haven't incremented it earlier at start of teams construct,
2399 // so do it here - at the end of teams construct
2400 team->t.t_level++;
2401 } else if (level == tlevel + 1) {
2402 // AC: we are exiting parallel inside teams, need to increment
2403 // serialization in order to restore it in the next call to
2404 // __kmpc_end_serialized_parallel
2405 team->t.t_serialized++;
2406 }
Andrey Churbanov6d224db2015-02-10 18:37:43 +00002407 }
Jonathan Peyton441f3372015-09-21 17:24:46 +00002408#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00002409 __kmpc_end_serialized_parallel(loc, gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002410
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002411#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002412 if (ompt_enabled.enabled) {
Jonathan Peyton30419822017-05-12 18:01:32 +00002413 __kmp_join_restore_state(master_th, parent_team);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002414 }
2415#endif
2416
Jonathan Peyton30419822017-05-12 18:01:32 +00002417 return;
2418 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002419
Jonathan Peyton30419822017-05-12 18:01:32 +00002420 master_active = team->t.t_master_active;
2421
2422#if OMP_40_ENABLED
2423 if (!exit_teams)
2424#endif /* OMP_40_ENABLED */
2425 {
2426 // AC: No barrier for internal teams at exit from teams construct.
2427 // But there is barrier for external team (league).
2428 __kmp_internal_join(loc, gtid, team);
2429 }
2430#if OMP_40_ENABLED
2431 else {
2432 master_th->th.th_task_state =
2433 0; // AC: no tasking in teams (out of any parallel)
2434 }
2435#endif /* OMP_40_ENABLED */
2436
2437 KMP_MB();
2438
2439#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002440 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data);
2441 void *codeptr = team->t.ompt_team_info.master_return_address;
Jonathan Peyton30419822017-05-12 18:01:32 +00002442#endif
2443
2444#if USE_ITT_BUILD
2445 if (__itt_stack_caller_create_ptr) {
2446 __kmp_itt_stack_caller_destroy(
2447 (__itt_caller)team->t
2448 .t_stack_id); // destroy the stack stitching id after join barrier
2449 }
2450
Jonathan Peyton8c432f22018-01-04 22:56:47 +00002451 // Mark end of "parallel" region for Intel(R) VTune(TM) analyzer.
Jonathan Peyton30419822017-05-12 18:01:32 +00002452 if (team->t.t_active_level == 1
2453#if OMP_40_ENABLED
2454 && !master_th->th.th_teams_microtask /* not in teams construct */
2455#endif /* OMP_40_ENABLED */
2456 ) {
2457 master_th->th.th_ident = loc;
2458 // only one notification scheme (either "submit" or "forking/joined", not
2459 // both)
2460 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2461 __kmp_forkjoin_frames_mode == 3)
2462 __kmp_itt_frame_submit(gtid, team->t.t_region_time,
2463 master_th->th.th_frame_time, 0, loc,
2464 master_th->th.th_team_nproc, 1);
2465 else if ((__itt_frame_end_v3_ptr || KMP_ITT_DEBUG) &&
2466 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames)
2467 __kmp_itt_region_joined(gtid);
2468 } // active_level == 1
2469#endif /* USE_ITT_BUILD */
2470
2471#if OMP_40_ENABLED
2472 if (master_th->th.th_teams_microtask && !exit_teams &&
2473 team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
2474 team->t.t_level == master_th->th.th_teams_level + 1) {
2475 // AC: We need to leave the team structure intact at the end of parallel
2476 // inside the teams construct, so that at the next parallel same (hot) team
2477 // works, only adjust nesting levels
2478
2479 /* Decrement our nested depth level */
2480 team->t.t_level--;
2481 team->t.t_active_level--;
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00002482 KMP_ATOMIC_DEC(&root->r.r_in_parallel);
Jonathan Peyton30419822017-05-12 18:01:32 +00002483
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00002484 // Restore number of threads in the team if needed. This code relies on
2485 // the proper adjustment of th_teams_size.nth after the fork in
2486 // __kmp_teams_master on each teams master in the case that
2487 // __kmp_reserve_threads reduced it.
Jonathan Peyton30419822017-05-12 18:01:32 +00002488 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) {
2489 int old_num = master_th->th.th_team_nproc;
2490 int new_num = master_th->th.th_teams_size.nth;
2491 kmp_info_t **other_threads = team->t.t_threads;
2492 team->t.t_nproc = new_num;
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00002493 for (int i = 0; i < old_num; ++i) {
Jonathan Peyton30419822017-05-12 18:01:32 +00002494 other_threads[i]->th.th_team_nproc = new_num;
2495 }
2496 // Adjust states of non-used threads of the team
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00002497 for (int i = old_num; i < new_num; ++i) {
Jonathan Peyton30419822017-05-12 18:01:32 +00002498 // Re-initialize thread's barrier data.
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00002499 KMP_DEBUG_ASSERT(other_threads[i]);
Jonathan Peyton30419822017-05-12 18:01:32 +00002500 kmp_balign_t *balign = other_threads[i]->th.th_bar;
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00002501 for (int b = 0; b < bs_last_barrier; ++b) {
Jonathan Peyton30419822017-05-12 18:01:32 +00002502 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
2503 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
2504#if USE_DEBUGGER
2505 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
2506#endif
2507 }
2508 if (__kmp_tasking_mode != tskm_immediate_exec) {
2509 // Synchronize thread's task state
2510 other_threads[i]->th.th_task_state = master_th->th.th_task_state;
2511 }
2512 }
2513 }
2514
2515#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002516 if (ompt_enabled.enabled) {
2517 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context,
2518 codeptr);
Jonathan Peyton30419822017-05-12 18:01:32 +00002519 }
2520#endif
2521
2522 return;
2523 }
2524#endif /* OMP_40_ENABLED */
2525
2526 /* do cleanup and restore the parent team */
2527 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid;
2528 master_th->th.th_local.this_construct = team->t.t_master_this_cons;
2529
2530 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid];
2531
2532 /* jc: The following lock has instructions with REL and ACQ semantics,
2533 separating the parallel user code called in this parallel region
2534 from the serial user code called after this function returns. */
2535 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2536
2537#if OMP_40_ENABLED
2538 if (!master_th->th.th_teams_microtask ||
2539 team->t.t_level > master_th->th.th_teams_level)
2540#endif /* OMP_40_ENABLED */
2541 {
2542 /* Decrement our nested depth level */
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00002543 KMP_ATOMIC_DEC(&root->r.r_in_parallel);
Jonathan Peyton30419822017-05-12 18:01:32 +00002544 }
2545 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0);
2546
Joachim Protze82e94a52017-11-01 10:08:30 +00002547#if OMPT_SUPPORT
2548 if (ompt_enabled.enabled) {
2549 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2550 if (ompt_enabled.ompt_callback_implicit_task) {
2551 int ompt_team_size = team->t.t_nproc;
2552 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
2553 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size,
Joachim Protze2b46d302019-01-15 15:36:53 +00002554 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit); // TODO: Can this be ompt_task_initial?
Jonathan Peyton30419822017-05-12 18:01:32 +00002555 }
Joachim Protze82e94a52017-11-01 10:08:30 +00002556
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00002557 task_info->frame.exit_frame = ompt_data_none;
Joachim Protze82e94a52017-11-01 10:08:30 +00002558 task_info->task_data = ompt_data_none;
Jonathan Peyton30419822017-05-12 18:01:32 +00002559 }
2560#endif
2561
2562 KF_TRACE(10, ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0,
2563 master_th, team));
2564 __kmp_pop_current_task_from_thread(master_th);
2565
2566#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
2567 // Restore master thread's partition.
2568 master_th->th.th_first_place = team->t.t_first_place;
2569 master_th->th.th_last_place = team->t.t_last_place;
2570#endif /* OMP_40_ENABLED */
Jonathan Peyton92ca6182018-09-07 18:25:49 +00002571#if OMP_50_ENABLED
2572 master_th->th.th_def_allocator = team->t.t_def_allocator;
2573#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002574
2575 updateHWFPControl(team);
2576
2577 if (root->r.r_active != master_active)
2578 root->r.r_active = master_active;
2579
2580 __kmp_free_team(root, team USE_NESTED_HOT_ARG(
2581 master_th)); // this will free worker threads
2582
2583 /* this race was fun to find. make sure the following is in the critical
2584 region otherwise assertions may fail occasionally since the old team may be
2585 reallocated and the hierarchy appears inconsistent. it is actually safe to
2586 run and won't cause any bugs, but will cause those assertion failures. it's
2587 only one deref&assign so might as well put this in the critical region */
2588 master_th->th.th_team = parent_team;
2589 master_th->th.th_team_nproc = parent_team->t.t_nproc;
2590 master_th->th.th_team_master = parent_team->t.t_threads[0];
2591 master_th->th.th_team_serialized = parent_team->t.t_serialized;
2592
2593 /* restore serialized team, if need be */
2594 if (parent_team->t.t_serialized &&
2595 parent_team != master_th->th.th_serial_team &&
2596 parent_team != root->r.r_root_team) {
2597 __kmp_free_team(root,
2598 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL));
2599 master_th->th.th_serial_team = parent_team;
2600 }
2601
2602 if (__kmp_tasking_mode != tskm_immediate_exec) {
2603 if (master_th->th.th_task_state_top >
2604 0) { // Restore task state from memo stack
2605 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2606 // Remember master's state if we re-use this nested hot team
2607 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] =
2608 master_th->th.th_task_state;
2609 --master_th->th.th_task_state_top; // pop
2610 // Now restore state at this level
2611 master_th->th.th_task_state =
2612 master_th->th
2613 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2614 }
2615 // Copy the task team from the parent team to the master thread
2616 master_th->th.th_task_team =
2617 parent_team->t.t_task_team[master_th->th.th_task_state];
2618 KA_TRACE(20,
2619 ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n",
2620 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,
2621 parent_team));
2622 }
2623
2624 // TODO: GEH - cannot do this assertion because root thread not set up as
2625 // executing
2626 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 );
2627 master_th->th.th_current_task->td_flags.executing = 1;
2628
2629 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2630
2631#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002632 if (ompt_enabled.enabled) {
2633 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context,
2634 codeptr);
Jonathan Peyton30419822017-05-12 18:01:32 +00002635 }
2636#endif
2637
2638 KMP_MB();
2639 KA_TRACE(20, ("__kmp_join_call: exit T#%d\n", gtid));
2640}
Jim Cownie5e8470a2013-09-27 10:38:44 +00002641
2642/* Check whether we should push an internal control record onto the
2643 serial team stack. If so, do it. */
Jonathan Peyton30419822017-05-12 18:01:32 +00002644void __kmp_save_internal_controls(kmp_info_t *thread) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002645
Jonathan Peyton30419822017-05-12 18:01:32 +00002646 if (thread->th.th_team != thread->th.th_serial_team) {
2647 return;
2648 }
2649 if (thread->th.th_team->t.t_serialized > 1) {
2650 int push = 0;
2651
2652 if (thread->th.th_team->t.t_control_stack_top == NULL) {
2653 push = 1;
2654 } else {
2655 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level !=
2656 thread->th.th_team->t.t_serialized) {
2657 push = 1;
2658 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002659 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002660 if (push) { /* push a record on the serial team's stack */
2661 kmp_internal_control_t *control =
2662 (kmp_internal_control_t *)__kmp_allocate(
2663 sizeof(kmp_internal_control_t));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002664
Jonathan Peyton30419822017-05-12 18:01:32 +00002665 copy_icvs(control, &thread->th.th_current_task->td_icvs);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002666
Jonathan Peyton30419822017-05-12 18:01:32 +00002667 control->serial_nesting_level = thread->th.th_team->t.t_serialized;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002668
Jonathan Peyton30419822017-05-12 18:01:32 +00002669 control->next = thread->th.th_team->t.t_control_stack_top;
2670 thread->th.th_team->t.t_control_stack_top = control;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002671 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002672 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002673}
2674
2675/* Changes set_nproc */
Jonathan Peyton30419822017-05-12 18:01:32 +00002676void __kmp_set_num_threads(int new_nth, int gtid) {
2677 kmp_info_t *thread;
2678 kmp_root_t *root;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002679
Jonathan Peyton30419822017-05-12 18:01:32 +00002680 KF_TRACE(10, ("__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth));
2681 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002682
Jonathan Peyton30419822017-05-12 18:01:32 +00002683 if (new_nth < 1)
2684 new_nth = 1;
2685 else if (new_nth > __kmp_max_nth)
2686 new_nth = __kmp_max_nth;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002687
Jonathan Peyton30419822017-05-12 18:01:32 +00002688 KMP_COUNT_VALUE(OMP_set_numthreads, new_nth);
2689 thread = __kmp_threads[gtid];
Andrey Churbanov82318c62018-11-14 13:49:41 +00002690 if (thread->th.th_current_task->td_icvs.nproc == new_nth)
2691 return; // nothing to do
Jim Cownie5e8470a2013-09-27 10:38:44 +00002692
Jonathan Peyton30419822017-05-12 18:01:32 +00002693 __kmp_save_internal_controls(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002694
Jonathan Peyton30419822017-05-12 18:01:32 +00002695 set__nproc(thread, new_nth);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002696
Jonathan Peyton30419822017-05-12 18:01:32 +00002697 // If this omp_set_num_threads() call will cause the hot team size to be
2698 // reduced (in the absence of a num_threads clause), then reduce it now,
2699 // rather than waiting for the next parallel region.
2700 root = thread->th.th_root;
2701 if (__kmp_init_parallel && (!root->r.r_active) &&
2702 (root->r.r_hot_team->t.t_nproc > new_nth)
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002703#if KMP_NESTED_HOT_TEAMS
2704 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode
2705#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002706 ) {
2707 kmp_team_t *hot_team = root->r.r_hot_team;
2708 int f;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002709
Jonathan Peyton30419822017-05-12 18:01:32 +00002710 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002711
Jonathan Peyton30419822017-05-12 18:01:32 +00002712 // Release the extra threads we don't need any more.
2713 for (f = new_nth; f < hot_team->t.t_nproc; f++) {
2714 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2715 if (__kmp_tasking_mode != tskm_immediate_exec) {
2716 // When decreasing team size, threads no longer in the team should unref
2717 // task team.
2718 hot_team->t.t_threads[f]->th.th_task_team = NULL;
2719 }
2720 __kmp_free_thread(hot_team->t.t_threads[f]);
2721 hot_team->t.t_threads[f] = NULL;
2722 }
2723 hot_team->t.t_nproc = new_nth;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002724#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00002725 if (thread->th.th_hot_teams) {
2726 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team);
2727 thread->th.th_hot_teams[0].hot_team_nth = new_nth;
2728 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002729#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002730
Jonathan Peyton30419822017-05-12 18:01:32 +00002731 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002732
Jonathan Peyton30419822017-05-12 18:01:32 +00002733 // Update the t_nproc field in the threads that are still active.
2734 for (f = 0; f < new_nth; f++) {
2735 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2736 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002737 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002738 // Special flag in case omp_set_num_threads() call
2739 hot_team->t.t_size_changed = -1;
2740 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002741}
2742
Jim Cownie5e8470a2013-09-27 10:38:44 +00002743/* Changes max_active_levels */
Jonathan Peyton30419822017-05-12 18:01:32 +00002744void __kmp_set_max_active_levels(int gtid, int max_active_levels) {
2745 kmp_info_t *thread;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002746
Jonathan Peyton30419822017-05-12 18:01:32 +00002747 KF_TRACE(10, ("__kmp_set_max_active_levels: new max_active_levels for thread "
2748 "%d = (%d)\n",
2749 gtid, max_active_levels));
2750 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002751
Jonathan Peyton30419822017-05-12 18:01:32 +00002752 // validate max_active_levels
2753 if (max_active_levels < 0) {
2754 KMP_WARNING(ActiveLevelsNegative, max_active_levels);
2755 // We ignore this call if the user has specified a negative value.
2756 // The current setting won't be changed. The last valid setting will be
2757 // used. A warning will be issued (if warnings are allowed as controlled by
2758 // the KMP_WARNINGS env var).
2759 KF_TRACE(10, ("__kmp_set_max_active_levels: the call is ignored: new "
2760 "max_active_levels for thread %d = (%d)\n",
2761 gtid, max_active_levels));
2762 return;
2763 }
2764 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT) {
2765 // it's OK, the max_active_levels is within the valid range: [ 0;
2766 // KMP_MAX_ACTIVE_LEVELS_LIMIT ]
2767 // We allow a zero value. (implementation defined behavior)
2768 } else {
2769 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels,
2770 KMP_MAX_ACTIVE_LEVELS_LIMIT);
2771 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT;
2772 // Current upper limit is MAX_INT. (implementation defined behavior)
2773 // If the input exceeds the upper limit, we correct the input to be the
2774 // upper limit. (implementation defined behavior)
2775 // Actually, the flow should never get here until we use MAX_INT limit.
2776 }
2777 KF_TRACE(10, ("__kmp_set_max_active_levels: after validation: new "
2778 "max_active_levels for thread %d = (%d)\n",
2779 gtid, max_active_levels));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002780
Jonathan Peyton30419822017-05-12 18:01:32 +00002781 thread = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00002782
Jonathan Peyton30419822017-05-12 18:01:32 +00002783 __kmp_save_internal_controls(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002784
Jonathan Peyton30419822017-05-12 18:01:32 +00002785 set__max_active_levels(thread, max_active_levels);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002786}
2787
2788/* Gets max_active_levels */
Jonathan Peyton30419822017-05-12 18:01:32 +00002789int __kmp_get_max_active_levels(int gtid) {
2790 kmp_info_t *thread;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002791
Jonathan Peyton30419822017-05-12 18:01:32 +00002792 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d\n", gtid));
2793 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002794
Jonathan Peyton30419822017-05-12 18:01:32 +00002795 thread = __kmp_threads[gtid];
2796 KMP_DEBUG_ASSERT(thread->th.th_current_task);
2797 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
2798 "curtask_maxaclevel=%d\n",
2799 gtid, thread->th.th_current_task,
2800 thread->th.th_current_task->td_icvs.max_active_levels));
2801 return thread->th.th_current_task->td_icvs.max_active_levels;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002802}
2803
2804/* Changes def_sched_var ICV values (run-time schedule kind and chunk) */
Jonathan Peyton30419822017-05-12 18:01:32 +00002805void __kmp_set_schedule(int gtid, kmp_sched_t kind, int chunk) {
2806 kmp_info_t *thread;
2807 // kmp_team_t *team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002808
Jonathan Peyton30419822017-05-12 18:01:32 +00002809 KF_TRACE(10, ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n",
2810 gtid, (int)kind, chunk));
2811 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002812
Jonathan Peyton30419822017-05-12 18:01:32 +00002813 // Check if the kind parameter is valid, correct if needed.
2814 // Valid parameters should fit in one of two intervals - standard or extended:
2815 // <lower>, <valid>, <upper_std>, <lower_ext>, <valid>, <upper>
2816 // 2008-01-25: 0, 1 - 4, 5, 100, 101 - 102, 103
2817 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper ||
2818 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) {
2819 // TODO: Hint needs attention in case we change the default schedule.
2820 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind),
2821 KMP_HNT(DefaultScheduleKindUsed, "static, no chunk"),
2822 __kmp_msg_null);
2823 kind = kmp_sched_default;
2824 chunk = 0; // ignore chunk value in case of bad kind
2825 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002826
Jonathan Peyton30419822017-05-12 18:01:32 +00002827 thread = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00002828
Jonathan Peyton30419822017-05-12 18:01:32 +00002829 __kmp_save_internal_controls(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002830
Jonathan Peyton30419822017-05-12 18:01:32 +00002831 if (kind < kmp_sched_upper_std) {
2832 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK) {
2833 // differ static chunked vs. unchunked: chunk should be invalid to
2834 // indicate unchunked schedule (which is the default)
2835 thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002836 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00002837 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2838 __kmp_sch_map[kind - kmp_sched_lower - 1];
Jim Cownie5e8470a2013-09-27 10:38:44 +00002839 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002840 } else {
2841 // __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2842 // kmp_sched_lower - 2 ];
2843 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2844 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2845 kmp_sched_lower - 2];
2846 }
Andrey Churbanovd454c732017-06-05 17:17:33 +00002847 if (kind == kmp_sched_auto || chunk < 1) {
Jonathan Peyton30419822017-05-12 18:01:32 +00002848 // ignore parameter chunk for schedule auto
2849 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK;
2850 } else {
2851 thread->th.th_current_task->td_icvs.sched.chunk = chunk;
2852 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002853}
2854
2855/* Gets def_sched_var ICV values */
Jonathan Peyton30419822017-05-12 18:01:32 +00002856void __kmp_get_schedule(int gtid, kmp_sched_t *kind, int *chunk) {
2857 kmp_info_t *thread;
2858 enum sched_type th_type;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002859
Jonathan Peyton30419822017-05-12 18:01:32 +00002860 KF_TRACE(10, ("__kmp_get_schedule: thread %d\n", gtid));
2861 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002862
Jonathan Peyton30419822017-05-12 18:01:32 +00002863 thread = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00002864
Jonathan Peyton30419822017-05-12 18:01:32 +00002865 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002866
Jonathan Peyton30419822017-05-12 18:01:32 +00002867 switch (th_type) {
2868 case kmp_sch_static:
2869 case kmp_sch_static_greedy:
2870 case kmp_sch_static_balanced:
2871 *kind = kmp_sched_static;
2872 *chunk = 0; // chunk was not set, try to show this fact via zero value
2873 return;
2874 case kmp_sch_static_chunked:
2875 *kind = kmp_sched_static;
2876 break;
2877 case kmp_sch_dynamic_chunked:
2878 *kind = kmp_sched_dynamic;
2879 break;
2880 case kmp_sch_guided_chunked:
2881 case kmp_sch_guided_iterative_chunked:
2882 case kmp_sch_guided_analytical_chunked:
2883 *kind = kmp_sched_guided;
2884 break;
2885 case kmp_sch_auto:
2886 *kind = kmp_sched_auto;
2887 break;
2888 case kmp_sch_trapezoidal:
2889 *kind = kmp_sched_trapezoidal;
2890 break;
Jonathan Peytona1234cf2016-10-07 18:01:35 +00002891#if KMP_STATIC_STEAL_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002892 case kmp_sch_static_steal:
2893 *kind = kmp_sched_static_steal;
2894 break;
Jonathan Peytona1234cf2016-10-07 18:01:35 +00002895#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002896 default:
2897 KMP_FATAL(UnknownSchedulingType, th_type);
2898 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002899
Jonathan Peyton30419822017-05-12 18:01:32 +00002900 *chunk = thread->th.th_current_task->td_icvs.sched.chunk;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002901}
2902
Jonathan Peyton30419822017-05-12 18:01:32 +00002903int __kmp_get_ancestor_thread_num(int gtid, int level) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002904
Jonathan Peyton30419822017-05-12 18:01:32 +00002905 int ii, dd;
2906 kmp_team_t *team;
2907 kmp_info_t *thr;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002908
Jonathan Peyton30419822017-05-12 18:01:32 +00002909 KF_TRACE(10, ("__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level));
2910 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002911
Jonathan Peyton30419822017-05-12 18:01:32 +00002912 // validate level
2913 if (level == 0)
2914 return 0;
2915 if (level < 0)
2916 return -1;
2917 thr = __kmp_threads[gtid];
2918 team = thr->th.th_team;
2919 ii = team->t.t_level;
2920 if (level > ii)
2921 return -1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002922
2923#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002924 if (thr->th.th_teams_microtask) {
2925 // AC: we are in teams region where multiple nested teams have same level
2926 int tlevel = thr->th.th_teams_level; // the level of the teams construct
2927 if (level <=
2928 tlevel) { // otherwise usual algorithm works (will not touch the teams)
2929 KMP_DEBUG_ASSERT(ii >= tlevel);
2930 // AC: As we need to pass by the teams league, we need to artificially
2931 // increase ii
2932 if (ii == tlevel) {
2933 ii += 2; // three teams have same level
2934 } else {
2935 ii++; // two teams have same level
2936 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002937 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002938 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002939#endif
2940
Jonathan Peyton30419822017-05-12 18:01:32 +00002941 if (ii == level)
2942 return __kmp_tid_from_gtid(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002943
Jonathan Peyton30419822017-05-12 18:01:32 +00002944 dd = team->t.t_serialized;
2945 level++;
2946 while (ii > level) {
2947 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002948 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002949 if ((team->t.t_serialized) && (!dd)) {
2950 team = team->t.t_parent;
2951 continue;
2952 }
2953 if (ii > level) {
2954 team = team->t.t_parent;
2955 dd = team->t.t_serialized;
2956 ii--;
2957 }
2958 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002959
Jonathan Peyton30419822017-05-12 18:01:32 +00002960 return (dd > 1) ? (0) : (team->t.t_master_tid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002961}
2962
Jonathan Peyton30419822017-05-12 18:01:32 +00002963int __kmp_get_team_size(int gtid, int level) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002964
Jonathan Peyton30419822017-05-12 18:01:32 +00002965 int ii, dd;
2966 kmp_team_t *team;
2967 kmp_info_t *thr;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002968
Jonathan Peyton30419822017-05-12 18:01:32 +00002969 KF_TRACE(10, ("__kmp_get_team_size: thread %d %d\n", gtid, level));
2970 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002971
Jonathan Peyton30419822017-05-12 18:01:32 +00002972 // validate level
2973 if (level == 0)
2974 return 1;
2975 if (level < 0)
2976 return -1;
2977 thr = __kmp_threads[gtid];
2978 team = thr->th.th_team;
2979 ii = team->t.t_level;
2980 if (level > ii)
2981 return -1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002982
2983#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002984 if (thr->th.th_teams_microtask) {
2985 // AC: we are in teams region where multiple nested teams have same level
2986 int tlevel = thr->th.th_teams_level; // the level of the teams construct
2987 if (level <=
2988 tlevel) { // otherwise usual algorithm works (will not touch the teams)
2989 KMP_DEBUG_ASSERT(ii >= tlevel);
2990 // AC: As we need to pass by the teams league, we need to artificially
2991 // increase ii
2992 if (ii == tlevel) {
2993 ii += 2; // three teams have same level
2994 } else {
2995 ii++; // two teams have same level
2996 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002997 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002998 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002999#endif
3000
Jonathan Peyton30419822017-05-12 18:01:32 +00003001 while (ii > level) {
3002 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003003 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003004 if (team->t.t_serialized && (!dd)) {
3005 team = team->t.t_parent;
3006 continue;
3007 }
3008 if (ii > level) {
3009 team = team->t.t_parent;
3010 ii--;
3011 }
3012 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003013
Jonathan Peyton30419822017-05-12 18:01:32 +00003014 return team->t.t_nproc;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003015}
3016
Jonathan Peyton30419822017-05-12 18:01:32 +00003017kmp_r_sched_t __kmp_get_schedule_global() {
3018 // This routine created because pairs (__kmp_sched, __kmp_chunk) and
3019 // (__kmp_static, __kmp_guided) may be changed by kmp_set_defaults
3020 // independently. So one can get the updated schedule here.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003021
Jonathan Peyton30419822017-05-12 18:01:32 +00003022 kmp_r_sched_t r_sched;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003023
Jonathan Peyton30419822017-05-12 18:01:32 +00003024 // create schedule from 4 globals: __kmp_sched, __kmp_chunk, __kmp_static,
3025 // __kmp_guided. __kmp_sched should keep original value, so that user can set
3026 // KMP_SCHEDULE multiple times, and thus have different run-time schedules in
3027 // different roots (even in OMP 2.5)
3028 if (__kmp_sched == kmp_sch_static) {
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00003029 // replace STATIC with more detailed schedule (balanced or greedy)
3030 r_sched.r_sched_type = __kmp_static;
Jonathan Peyton30419822017-05-12 18:01:32 +00003031 } else if (__kmp_sched == kmp_sch_guided_chunked) {
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00003032 // replace GUIDED with more detailed schedule (iterative or analytical)
3033 r_sched.r_sched_type = __kmp_guided;
3034 } else { // (STATIC_CHUNKED), or (DYNAMIC_CHUNKED), or other
3035 r_sched.r_sched_type = __kmp_sched;
Jonathan Peyton30419822017-05-12 18:01:32 +00003036 }
3037
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00003038 if (__kmp_chunk < KMP_DEFAULT_CHUNK) {
3039 // __kmp_chunk may be wrong here (if it was not ever set)
Jonathan Peyton30419822017-05-12 18:01:32 +00003040 r_sched.chunk = KMP_DEFAULT_CHUNK;
3041 } else {
3042 r_sched.chunk = __kmp_chunk;
3043 }
3044
3045 return r_sched;
3046}
3047
3048/* Allocate (realloc == FALSE) * or reallocate (realloc == TRUE)
3049 at least argc number of *t_argv entries for the requested team. */
3050static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, int realloc) {
3051
3052 KMP_DEBUG_ASSERT(team);
3053 if (!realloc || argc > team->t.t_max_argc) {
3054
3055 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "
3056 "current entries=%d\n",
3057 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0));
3058 /* if previously allocated heap space for args, free them */
3059 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0])
3060 __kmp_free((void *)team->t.t_argv);
3061
3062 if (argc <= KMP_INLINE_ARGV_ENTRIES) {
3063 /* use unused space in the cache line for arguments */
3064 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES;
3065 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: inline allocate %d "
3066 "argv entries\n",
3067 team->t.t_id, team->t.t_max_argc));
3068 team->t.t_argv = &team->t.t_inline_argv[0];
3069 if (__kmp_storage_map) {
3070 __kmp_print_storage_map_gtid(
3071 -1, &team->t.t_inline_argv[0],
3072 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES],
3073 (sizeof(void *) * KMP_INLINE_ARGV_ENTRIES), "team_%d.t_inline_argv",
3074 team->t.t_id);
3075 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003076 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00003077 /* allocate space for arguments in the heap */
3078 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1))
3079 ? KMP_MIN_MALLOC_ARGV_ENTRIES
3080 : 2 * argc;
3081 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
3082 "argv entries\n",
3083 team->t.t_id, team->t.t_max_argc));
3084 team->t.t_argv =
3085 (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc);
3086 if (__kmp_storage_map) {
3087 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0],
3088 &team->t.t_argv[team->t.t_max_argc],
3089 sizeof(void *) * team->t.t_max_argc,
3090 "team_%d.t_argv", team->t.t_id);
3091 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003092 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003093 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003094}
3095
Jonathan Peyton30419822017-05-12 18:01:32 +00003096static void __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth) {
3097 int i;
3098 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2;
3099 team->t.t_threads =
3100 (kmp_info_t **)__kmp_allocate(sizeof(kmp_info_t *) * max_nth);
3101 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate(
3102 sizeof(dispatch_shared_info_t) * num_disp_buff);
3103 team->t.t_dispatch =
3104 (kmp_disp_t *)__kmp_allocate(sizeof(kmp_disp_t) * max_nth);
3105 team->t.t_implicit_task_taskdata =
3106 (kmp_taskdata_t *)__kmp_allocate(sizeof(kmp_taskdata_t) * max_nth);
3107 team->t.t_max_nproc = max_nth;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003108
Jonathan Peyton30419822017-05-12 18:01:32 +00003109 /* setup dispatch buffers */
3110 for (i = 0; i < num_disp_buff; ++i) {
3111 team->t.t_disp_buffer[i].buffer_index = i;
Jonathan Peytondf6818b2016-06-14 17:57:47 +00003112#if OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003113 team->t.t_disp_buffer[i].doacross_buf_idx = i;
Jonathan Peyton71909c52016-03-02 22:42:06 +00003114#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003115 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003116}
3117
Jonathan Peyton30419822017-05-12 18:01:32 +00003118static void __kmp_free_team_arrays(kmp_team_t *team) {
3119 /* Note: this does not free the threads in t_threads (__kmp_free_threads) */
3120 int i;
3121 for (i = 0; i < team->t.t_max_nproc; ++i) {
3122 if (team->t.t_dispatch[i].th_disp_buffer != NULL) {
3123 __kmp_free(team->t.t_dispatch[i].th_disp_buffer);
3124 team->t.t_dispatch[i].th_disp_buffer = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003125 }
3126 }
Jonathan Peytonf6399362018-07-09 17:51:13 +00003127#if KMP_USE_HIER_SCHED
3128 __kmp_dispatch_free_hierarchies(team);
3129#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003130 __kmp_free(team->t.t_threads);
3131 __kmp_free(team->t.t_disp_buffer);
3132 __kmp_free(team->t.t_dispatch);
3133 __kmp_free(team->t.t_implicit_task_taskdata);
3134 team->t.t_threads = NULL;
3135 team->t.t_disp_buffer = NULL;
3136 team->t.t_dispatch = NULL;
3137 team->t.t_implicit_task_taskdata = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003138}
3139
Jonathan Peyton30419822017-05-12 18:01:32 +00003140static void __kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) {
3141 kmp_info_t **oldThreads = team->t.t_threads;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003142
Jonathan Peyton30419822017-05-12 18:01:32 +00003143 __kmp_free(team->t.t_disp_buffer);
3144 __kmp_free(team->t.t_dispatch);
3145 __kmp_free(team->t.t_implicit_task_taskdata);
3146 __kmp_allocate_team_arrays(team, max_nth);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003147
Jonathan Peyton30419822017-05-12 18:01:32 +00003148 KMP_MEMCPY(team->t.t_threads, oldThreads,
3149 team->t.t_nproc * sizeof(kmp_info_t *));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003150
Jonathan Peyton30419822017-05-12 18:01:32 +00003151 __kmp_free(oldThreads);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003152}
3153
Jonathan Peyton30419822017-05-12 18:01:32 +00003154static kmp_internal_control_t __kmp_get_global_icvs(void) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003155
Jonathan Peyton30419822017-05-12 18:01:32 +00003156 kmp_r_sched_t r_sched =
3157 __kmp_get_schedule_global(); // get current state of scheduling globals
Jim Cownie5e8470a2013-09-27 10:38:44 +00003158
3159#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003160 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003161#endif /* OMP_40_ENABLED */
3162
Jonathan Peyton30419822017-05-12 18:01:32 +00003163 kmp_internal_control_t g_icvs = {
3164 0, // int serial_nesting_level; //corresponds to value of th_team_serialized
Jonathan Peyton30419822017-05-12 18:01:32 +00003165 (kmp_int8)__kmp_global.g.g_dynamic, // internal control for dynamic
3166 // adjustment of threads (per thread)
3167 (kmp_int8)__kmp_env_blocktime, // int bt_set; //internal control for
3168 // whether blocktime is explicitly set
3169 __kmp_dflt_blocktime, // int blocktime; //internal control for blocktime
Jonathan Peytone1c7c132016-10-07 18:12:19 +00003170#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00003171 __kmp_bt_intervals, // int bt_intervals; //internal control for blocktime
3172// intervals
Jonathan Peytone1c7c132016-10-07 18:12:19 +00003173#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003174 __kmp_dflt_team_nth, // int nproc; //internal control for # of threads for
3175 // next parallel region (per thread)
3176 // (use a max ub on value if __kmp_parallel_initialize not called yet)
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00003177 __kmp_cg_max_nth, // int thread_limit;
Jonathan Peyton30419822017-05-12 18:01:32 +00003178 __kmp_dflt_max_active_levels, // int max_active_levels; //internal control
3179 // for max_active_levels
3180 r_sched, // kmp_r_sched_t sched; //internal control for runtime schedule
3181// {sched,chunk} pair
Jim Cownie5e8470a2013-09-27 10:38:44 +00003182#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003183 __kmp_nested_proc_bind.bind_types[0],
3184 __kmp_default_device,
Jim Cownie5e8470a2013-09-27 10:38:44 +00003185#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00003186 NULL // struct kmp_internal_control *next;
3187 };
Jim Cownie5e8470a2013-09-27 10:38:44 +00003188
Jonathan Peyton30419822017-05-12 18:01:32 +00003189 return g_icvs;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003190}
3191
Jonathan Peyton30419822017-05-12 18:01:32 +00003192static kmp_internal_control_t __kmp_get_x_global_icvs(const kmp_team_t *team) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003193
Jonathan Peyton30419822017-05-12 18:01:32 +00003194 kmp_internal_control_t gx_icvs;
3195 gx_icvs.serial_nesting_level =
3196 0; // probably =team->t.t_serial like in save_inter_controls
3197 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs);
3198 gx_icvs.next = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003199
Jonathan Peyton30419822017-05-12 18:01:32 +00003200 return gx_icvs;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003201}
3202
Jonathan Peyton30419822017-05-12 18:01:32 +00003203static void __kmp_initialize_root(kmp_root_t *root) {
3204 int f;
3205 kmp_team_t *root_team;
3206 kmp_team_t *hot_team;
3207 int hot_team_max_nth;
3208 kmp_r_sched_t r_sched =
3209 __kmp_get_schedule_global(); // get current state of scheduling globals
3210 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3211 KMP_DEBUG_ASSERT(root);
3212 KMP_ASSERT(!root->r.r_begin);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003213
Jonathan Peyton30419822017-05-12 18:01:32 +00003214 /* setup the root state structure */
3215 __kmp_init_lock(&root->r.r_begin_lock);
3216 root->r.r_begin = FALSE;
3217 root->r.r_active = FALSE;
3218 root->r.r_in_parallel = 0;
3219 root->r.r_blocktime = __kmp_dflt_blocktime;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003220
Jonathan Peyton30419822017-05-12 18:01:32 +00003221 /* setup the root team for this task */
3222 /* allocate the root team structure */
3223 KF_TRACE(10, ("__kmp_initialize_root: before root_team\n"));
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003224
Jonathan Peyton30419822017-05-12 18:01:32 +00003225 root_team =
3226 __kmp_allocate_team(root,
3227 1, // new_nproc
3228 1, // max_nproc
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003229#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00003230 ompt_data_none, // root parallel id
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003231#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00003232#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003233 __kmp_nested_proc_bind.bind_types[0],
Jim Cownie5e8470a2013-09-27 10:38:44 +00003234#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003235 &r_icvs,
3236 0 // argc
3237 USE_NESTED_HOT_ARG(NULL) // master thread is unknown
3238 );
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00003239#if USE_DEBUGGER
Jonathan Peyton30419822017-05-12 18:01:32 +00003240 // Non-NULL value should be assigned to make the debugger display the root
3241 // team.
3242 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0));
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00003243#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00003244
Jonathan Peyton30419822017-05-12 18:01:32 +00003245 KF_TRACE(10, ("__kmp_initialize_root: after root_team = %p\n", root_team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003246
Jonathan Peyton30419822017-05-12 18:01:32 +00003247 root->r.r_root_team = root_team;
3248 root_team->t.t_control_stack_top = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003249
Jonathan Peyton30419822017-05-12 18:01:32 +00003250 /* initialize root team */
3251 root_team->t.t_threads[0] = NULL;
3252 root_team->t.t_nproc = 1;
3253 root_team->t.t_serialized = 1;
3254 // TODO???: root_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00003255 root_team->t.t_sched.sched = r_sched.sched;
Jonathan Peyton30419822017-05-12 18:01:32 +00003256 KA_TRACE(
3257 20,
3258 ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n",
3259 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003260
Jonathan Peyton30419822017-05-12 18:01:32 +00003261 /* setup the hot team for this task */
3262 /* allocate the hot team structure */
3263 KF_TRACE(10, ("__kmp_initialize_root: before hot_team\n"));
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003264
Jonathan Peyton30419822017-05-12 18:01:32 +00003265 hot_team =
3266 __kmp_allocate_team(root,
3267 1, // new_nproc
3268 __kmp_dflt_team_nth_ub * 2, // max_nproc
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003269#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00003270 ompt_data_none, // root parallel id
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003271#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00003272#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003273 __kmp_nested_proc_bind.bind_types[0],
Jim Cownie5e8470a2013-09-27 10:38:44 +00003274#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003275 &r_icvs,
3276 0 // argc
3277 USE_NESTED_HOT_ARG(NULL) // master thread is unknown
3278 );
3279 KF_TRACE(10, ("__kmp_initialize_root: after hot_team = %p\n", hot_team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003280
Jonathan Peyton30419822017-05-12 18:01:32 +00003281 root->r.r_hot_team = hot_team;
3282 root_team->t.t_control_stack_top = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003283
Jonathan Peyton30419822017-05-12 18:01:32 +00003284 /* first-time initialization */
3285 hot_team->t.t_parent = root_team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003286
Jonathan Peyton30419822017-05-12 18:01:32 +00003287 /* initialize hot team */
3288 hot_team_max_nth = hot_team->t.t_max_nproc;
3289 for (f = 0; f < hot_team_max_nth; ++f) {
3290 hot_team->t.t_threads[f] = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003291 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003292 hot_team->t.t_nproc = 1;
3293 // TODO???: hot_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00003294 hot_team->t.t_sched.sched = r_sched.sched;
Jonathan Peyton30419822017-05-12 18:01:32 +00003295 hot_team->t.t_size_changed = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003296}
3297
3298#ifdef KMP_DEBUG
3299
Jim Cownie5e8470a2013-09-27 10:38:44 +00003300typedef struct kmp_team_list_item {
Jonathan Peyton30419822017-05-12 18:01:32 +00003301 kmp_team_p const *entry;
3302 struct kmp_team_list_item *next;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003303} kmp_team_list_item_t;
Jonathan Peyton30419822017-05-12 18:01:32 +00003304typedef kmp_team_list_item_t *kmp_team_list_t;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003305
Jonathan Peyton30419822017-05-12 18:01:32 +00003306static void __kmp_print_structure_team_accum( // Add team to list of teams.
3307 kmp_team_list_t list, // List of teams.
3308 kmp_team_p const *team // Team to add.
3309 ) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003310
Jonathan Peyton30419822017-05-12 18:01:32 +00003311 // List must terminate with item where both entry and next are NULL.
3312 // Team is added to the list only once.
3313 // List is sorted in ascending order by team id.
3314 // Team id is *not* a key.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003315
Jonathan Peyton30419822017-05-12 18:01:32 +00003316 kmp_team_list_t l;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003317
Jonathan Peyton30419822017-05-12 18:01:32 +00003318 KMP_DEBUG_ASSERT(list != NULL);
3319 if (team == NULL) {
3320 return;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003321 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003322
Jonathan Peyton30419822017-05-12 18:01:32 +00003323 __kmp_print_structure_team_accum(list, team->t.t_parent);
3324 __kmp_print_structure_team_accum(list, team->t.t_next_pool);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003325
Jonathan Peyton30419822017-05-12 18:01:32 +00003326 // Search list for the team.
3327 l = list;
3328 while (l->next != NULL && l->entry != team) {
3329 l = l->next;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003330 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003331 if (l->next != NULL) {
3332 return; // Team has been added before, exit.
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003333 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003334
Jonathan Peyton30419822017-05-12 18:01:32 +00003335 // Team is not found. Search list again for insertion point.
3336 l = list;
3337 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) {
3338 l = l->next;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003339 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003340
Jonathan Peyton30419822017-05-12 18:01:32 +00003341 // Insert team.
3342 {
3343 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(
3344 sizeof(kmp_team_list_item_t));
3345 *item = *l;
3346 l->entry = team;
3347 l->next = item;
3348 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003349}
3350
Jonathan Peyton30419822017-05-12 18:01:32 +00003351static void __kmp_print_structure_team(char const *title, kmp_team_p const *team
Jim Cownie5e8470a2013-09-27 10:38:44 +00003352
Jonathan Peyton30419822017-05-12 18:01:32 +00003353 ) {
3354 __kmp_printf("%s", title);
3355 if (team != NULL) {
3356 __kmp_printf("%2x %p\n", team->t.t_id, team);
3357 } else {
3358 __kmp_printf(" - (nil)\n");
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003359 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003360}
3361
Jonathan Peyton30419822017-05-12 18:01:32 +00003362static void __kmp_print_structure_thread(char const *title,
3363 kmp_info_p const *thread) {
3364 __kmp_printf("%s", title);
3365 if (thread != NULL) {
3366 __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread);
3367 } else {
3368 __kmp_printf(" - (nil)\n");
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003369 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003370}
3371
Jonathan Peyton30419822017-05-12 18:01:32 +00003372void __kmp_print_structure(void) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003373
Jonathan Peyton30419822017-05-12 18:01:32 +00003374 kmp_team_list_t list;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003375
Jonathan Peyton30419822017-05-12 18:01:32 +00003376 // Initialize list of teams.
3377 list =
3378 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(sizeof(kmp_team_list_item_t));
3379 list->entry = NULL;
3380 list->next = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003381
Jonathan Peyton30419822017-05-12 18:01:32 +00003382 __kmp_printf("\n------------------------------\nGlobal Thread "
3383 "Table\n------------------------------\n");
3384 {
3385 int gtid;
3386 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3387 __kmp_printf("%2d", gtid);
3388 if (__kmp_threads != NULL) {
3389 __kmp_printf(" %p", __kmp_threads[gtid]);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003390 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003391 if (__kmp_root != NULL) {
3392 __kmp_printf(" %p", __kmp_root[gtid]);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003393 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003394 __kmp_printf("\n");
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003395 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003396 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003397
Jonathan Peyton30419822017-05-12 18:01:32 +00003398 // Print out __kmp_threads array.
3399 __kmp_printf("\n------------------------------\nThreads\n--------------------"
3400 "----------\n");
3401 if (__kmp_threads != NULL) {
3402 int gtid;
3403 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3404 kmp_info_t const *thread = __kmp_threads[gtid];
3405 if (thread != NULL) {
3406 __kmp_printf("GTID %2d %p:\n", gtid, thread);
3407 __kmp_printf(" Our Root: %p\n", thread->th.th_root);
3408 __kmp_print_structure_team(" Our Team: ", thread->th.th_team);
3409 __kmp_print_structure_team(" Serial Team: ",
3410 thread->th.th_serial_team);
3411 __kmp_printf(" Threads: %2d\n", thread->th.th_team_nproc);
3412 __kmp_print_structure_thread(" Master: ",
3413 thread->th.th_team_master);
3414 __kmp_printf(" Serialized?: %2d\n", thread->th.th_team_serialized);
3415 __kmp_printf(" Set NProc: %2d\n", thread->th.th_set_nproc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003416#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003417 __kmp_printf(" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003418#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003419 __kmp_print_structure_thread(" Next in pool: ",
3420 thread->th.th_next_pool);
3421 __kmp_printf("\n");
3422 __kmp_print_structure_team_accum(list, thread->th.th_team);
3423 __kmp_print_structure_team_accum(list, thread->th.th_serial_team);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003424 }
3425 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003426 } else {
3427 __kmp_printf("Threads array is not allocated.\n");
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003428 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003429
Jonathan Peyton30419822017-05-12 18:01:32 +00003430 // Print out __kmp_root array.
3431 __kmp_printf("\n------------------------------\nUbers\n----------------------"
3432 "--------\n");
3433 if (__kmp_root != NULL) {
3434 int gtid;
3435 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3436 kmp_root_t const *root = __kmp_root[gtid];
3437 if (root != NULL) {
3438 __kmp_printf("GTID %2d %p:\n", gtid, root);
3439 __kmp_print_structure_team(" Root Team: ", root->r.r_root_team);
3440 __kmp_print_structure_team(" Hot Team: ", root->r.r_hot_team);
3441 __kmp_print_structure_thread(" Uber Thread: ",
3442 root->r.r_uber_thread);
3443 __kmp_printf(" Active?: %2d\n", root->r.r_active);
Jonathan Peyton61d44f12018-07-09 18:09:25 +00003444 __kmp_printf(" In Parallel: %2d\n",
3445 KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel));
Jonathan Peyton30419822017-05-12 18:01:32 +00003446 __kmp_printf("\n");
3447 __kmp_print_structure_team_accum(list, root->r.r_root_team);
3448 __kmp_print_structure_team_accum(list, root->r.r_hot_team);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003449 }
3450 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003451 } else {
3452 __kmp_printf("Ubers array is not allocated.\n");
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003453 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003454
Jonathan Peyton30419822017-05-12 18:01:32 +00003455 __kmp_printf("\n------------------------------\nTeams\n----------------------"
3456 "--------\n");
3457 while (list->next != NULL) {
3458 kmp_team_p const *team = list->entry;
3459 int i;
3460 __kmp_printf("Team %2x %p:\n", team->t.t_id, team);
3461 __kmp_print_structure_team(" Parent Team: ", team->t.t_parent);
3462 __kmp_printf(" Master TID: %2d\n", team->t.t_master_tid);
3463 __kmp_printf(" Max threads: %2d\n", team->t.t_max_nproc);
3464 __kmp_printf(" Levels of serial: %2d\n", team->t.t_serialized);
3465 __kmp_printf(" Number threads: %2d\n", team->t.t_nproc);
3466 for (i = 0; i < team->t.t_nproc; ++i) {
3467 __kmp_printf(" Thread %2d: ", i);
3468 __kmp_print_structure_thread("", team->t.t_threads[i]);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003469 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003470 __kmp_print_structure_team(" Next in pool: ", team->t.t_next_pool);
3471 __kmp_printf("\n");
3472 list = list->next;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003473 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003474
Jonathan Peyton30419822017-05-12 18:01:32 +00003475 // Print out __kmp_thread_pool and __kmp_team_pool.
3476 __kmp_printf("\n------------------------------\nPools\n----------------------"
3477 "--------\n");
3478 __kmp_print_structure_thread("Thread pool: ",
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00003479 CCAST(kmp_info_t *, __kmp_thread_pool));
Jonathan Peyton30419822017-05-12 18:01:32 +00003480 __kmp_print_structure_team("Team pool: ",
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00003481 CCAST(kmp_team_t *, __kmp_team_pool));
Jonathan Peyton30419822017-05-12 18:01:32 +00003482 __kmp_printf("\n");
Jim Cownie5e8470a2013-09-27 10:38:44 +00003483
Jonathan Peyton30419822017-05-12 18:01:32 +00003484 // Free team list.
3485 while (list != NULL) {
3486 kmp_team_list_item_t *item = list;
3487 list = list->next;
3488 KMP_INTERNAL_FREE(item);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003489 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003490}
3491
3492#endif
3493
Jim Cownie5e8470a2013-09-27 10:38:44 +00003494//---------------------------------------------------------------------------
3495// Stuff for per-thread fast random number generator
3496// Table of primes
Jim Cownie5e8470a2013-09-27 10:38:44 +00003497static const unsigned __kmp_primes[] = {
Jonathan Peyton30419822017-05-12 18:01:32 +00003498 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877,
3499 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231,
3500 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201,
3501 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3,
3502 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7,
3503 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9,
3504 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45,
3505 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7,
3506 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363,
3507 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3,
3508 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f};
Jim Cownie5e8470a2013-09-27 10:38:44 +00003509
3510//---------------------------------------------------------------------------
3511// __kmp_get_random: Get a random number using a linear congruential method.
Jonathan Peyton30419822017-05-12 18:01:32 +00003512unsigned short __kmp_get_random(kmp_info_t *thread) {
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003513 unsigned x = thread->th.th_x;
Jonathan Peyton30419822017-05-12 18:01:32 +00003514 unsigned short r = x >> 16;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003515
Jonathan Peyton30419822017-05-12 18:01:32 +00003516 thread->th.th_x = x * thread->th.th_a + 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003517
3518 KA_TRACE(30, ("__kmp_get_random: THREAD: %d, RETURN: %u\n",
Jonathan Peyton30419822017-05-12 18:01:32 +00003519 thread->th.th_info.ds.ds_tid, r));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003520
3521 return r;
3522}
3523//--------------------------------------------------------
3524// __kmp_init_random: Initialize a random number generator
Jonathan Peyton30419822017-05-12 18:01:32 +00003525void __kmp_init_random(kmp_info_t *thread) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003526 unsigned seed = thread->th.th_info.ds.ds_tid;
3527
Jonathan Peyton30419822017-05-12 18:01:32 +00003528 thread->th.th_a =
3529 __kmp_primes[seed % (sizeof(__kmp_primes) / sizeof(__kmp_primes[0]))];
3530 thread->th.th_x = (seed + 1) * thread->th.th_a + 1;
3531 KA_TRACE(30,
3532 ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003533}
3534
Jim Cownie5e8470a2013-09-27 10:38:44 +00003535#if KMP_OS_WINDOWS
Jonathan Peyton30419822017-05-12 18:01:32 +00003536/* reclaim array entries for root threads that are already dead, returns number
3537 * reclaimed */
3538static int __kmp_reclaim_dead_roots(void) {
3539 int i, r = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003540
Jonathan Peyton30419822017-05-12 18:01:32 +00003541 for (i = 0; i < __kmp_threads_capacity; ++i) {
3542 if (KMP_UBER_GTID(i) &&
3543 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])) &&
3544 !__kmp_root[i]
3545 ->r.r_active) { // AC: reclaim only roots died in non-active state
3546 r += __kmp_unregister_root_other_thread(i);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003547 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003548 }
3549 return r;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003550}
3551#endif
3552
Jonathan Peyton30419822017-05-12 18:01:32 +00003553/* This function attempts to create free entries in __kmp_threads and
3554 __kmp_root, and returns the number of free entries generated.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003555
Jonathan Peyton30419822017-05-12 18:01:32 +00003556 For Windows* OS static library, the first mechanism used is to reclaim array
3557 entries for root threads that are already dead.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003558
Jonathan Peyton30419822017-05-12 18:01:32 +00003559 On all platforms, expansion is attempted on the arrays __kmp_threads_ and
3560 __kmp_root, with appropriate update to __kmp_threads_capacity. Array
3561 capacity is increased by doubling with clipping to __kmp_tp_capacity, if
3562 threadprivate cache array has been created. Synchronization with
3563 __kmpc_threadprivate_cached is done using __kmp_tp_cached_lock.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003564
Jonathan Peyton30419822017-05-12 18:01:32 +00003565 After any dead root reclamation, if the clipping value allows array expansion
Jonathan Peyton1800ece2018-01-10 18:27:01 +00003566 to result in the generation of a total of nNeed free slots, the function does
3567 that expansion. If not, nothing is done beyond the possible initial root
3568 thread reclamation.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003569
Jonathan Peyton30419822017-05-12 18:01:32 +00003570 If any argument is negative, the behavior is undefined. */
Jonathan Peyton1800ece2018-01-10 18:27:01 +00003571static int __kmp_expand_threads(int nNeed) {
Jonathan Peyton30419822017-05-12 18:01:32 +00003572 int added = 0;
Andrey Churbanov9e9333a2018-03-05 18:42:01 +00003573 int minimumRequiredCapacity;
3574 int newCapacity;
3575 kmp_info_t **newThreads;
3576 kmp_root_t **newRoot;
3577
3578// All calls to __kmp_expand_threads should be under __kmp_forkjoin_lock, so
3579// resizing __kmp_threads does not need additional protection if foreign
3580// threads are present
Jim Cownie5e8470a2013-09-27 10:38:44 +00003581
Jonathan Peyton8b3842f2018-10-05 17:59:39 +00003582#if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB
Jonathan Peyton30419822017-05-12 18:01:32 +00003583 /* only for Windows static library */
3584 /* reclaim array entries for root threads that are already dead */
3585 added = __kmp_reclaim_dead_roots();
Jim Cownie5e8470a2013-09-27 10:38:44 +00003586
Jonathan Peyton30419822017-05-12 18:01:32 +00003587 if (nNeed) {
3588 nNeed -= added;
3589 if (nNeed < 0)
3590 nNeed = 0;
3591 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003592#endif
Jonathan Peyton1800ece2018-01-10 18:27:01 +00003593 if (nNeed <= 0)
Jim Cownie5e8470a2013-09-27 10:38:44 +00003594 return added;
Jonathan Peyton30419822017-05-12 18:01:32 +00003595
Andrey Churbanov9e9333a2018-03-05 18:42:01 +00003596 // Note that __kmp_threads_capacity is not bounded by __kmp_max_nth. If
3597 // __kmp_max_nth is set to some value less than __kmp_sys_max_nth by the
3598 // user via KMP_DEVICE_THREAD_LIMIT, then __kmp_threads_capacity may become
3599 // > __kmp_max_nth in one of two ways:
3600 //
3601 // 1) The initialization thread (gtid = 0) exits. __kmp_threads[0]
3602 // may not be resused by another thread, so we may need to increase
3603 // __kmp_threads_capacity to __kmp_max_nth + 1.
3604 //
3605 // 2) New foreign root(s) are encountered. We always register new foreign
3606 // roots. This may cause a smaller # of threads to be allocated at
3607 // subsequent parallel regions, but the worker threads hang around (and
3608 // eventually go to sleep) and need slots in the __kmp_threads[] array.
3609 //
3610 // Anyway, that is the reason for moving the check to see if
3611 // __kmp_max_nth was exceeded into __kmp_reserve_threads()
3612 // instead of having it performed here. -BB
Jonathan Peyton30419822017-05-12 18:01:32 +00003613
Andrey Churbanov9e9333a2018-03-05 18:42:01 +00003614 KMP_DEBUG_ASSERT(__kmp_sys_max_nth >= __kmp_threads_capacity);
Jonathan Peyton30419822017-05-12 18:01:32 +00003615
Andrey Churbanov9e9333a2018-03-05 18:42:01 +00003616 /* compute expansion headroom to check if we can expand */
3617 if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) {
3618 /* possible expansion too small -- give up */
3619 return added;
Jonathan Peyton30419822017-05-12 18:01:32 +00003620 }
Andrey Churbanov9e9333a2018-03-05 18:42:01 +00003621 minimumRequiredCapacity = __kmp_threads_capacity + nNeed;
3622
3623 newCapacity = __kmp_threads_capacity;
3624 do {
3625 newCapacity = newCapacity <= (__kmp_sys_max_nth >> 1) ? (newCapacity << 1)
3626 : __kmp_sys_max_nth;
3627 } while (newCapacity < minimumRequiredCapacity);
3628 newThreads = (kmp_info_t **)__kmp_allocate(
3629 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * newCapacity + CACHE_LINE);
3630 newRoot =
3631 (kmp_root_t **)((char *)newThreads + sizeof(kmp_info_t *) * newCapacity);
3632 KMP_MEMCPY(newThreads, __kmp_threads,
3633 __kmp_threads_capacity * sizeof(kmp_info_t *));
3634 KMP_MEMCPY(newRoot, __kmp_root,
3635 __kmp_threads_capacity * sizeof(kmp_root_t *));
3636
3637 kmp_info_t **temp_threads = __kmp_threads;
3638 *(kmp_info_t * *volatile *)&__kmp_threads = newThreads;
3639 *(kmp_root_t * *volatile *)&__kmp_root = newRoot;
3640 __kmp_free(temp_threads);
3641 added += newCapacity - __kmp_threads_capacity;
3642 *(volatile int *)&__kmp_threads_capacity = newCapacity;
3643
3644 if (newCapacity > __kmp_tp_capacity) {
3645 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
3646 if (__kmp_tp_cached && newCapacity > __kmp_tp_capacity) {
3647 __kmp_threadprivate_resize_cache(newCapacity);
3648 } else { // increase __kmp_tp_capacity to correspond with kmp_threads size
3649 *(volatile int *)&__kmp_tp_capacity = newCapacity;
3650 }
3651 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
3652 }
3653
Jonathan Peyton30419822017-05-12 18:01:32 +00003654 return added;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003655}
3656
Jonathan Peyton30419822017-05-12 18:01:32 +00003657/* Register the current thread as a root thread and obtain our gtid. We must
3658 have the __kmp_initz_lock held at this point. Argument TRUE only if are the
3659 thread that calls from __kmp_do_serial_initialize() */
3660int __kmp_register_root(int initial_thread) {
3661 kmp_info_t *root_thread;
3662 kmp_root_t *root;
3663 int gtid;
3664 int capacity;
3665 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3666 KA_TRACE(20, ("__kmp_register_root: entered\n"));
3667 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00003668
Jonathan Peyton30419822017-05-12 18:01:32 +00003669 /* 2007-03-02:
3670 If initial thread did not invoke OpenMP RTL yet, and this thread is not an
3671 initial one, "__kmp_all_nth >= __kmp_threads_capacity" condition does not
3672 work as expected -- it may return false (that means there is at least one
3673 empty slot in __kmp_threads array), but it is possible the only free slot
3674 is #0, which is reserved for initial thread and so cannot be used for this
3675 one. Following code workarounds this bug.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003676
Jonathan Peyton30419822017-05-12 18:01:32 +00003677 However, right solution seems to be not reserving slot #0 for initial
3678 thread because:
3679 (1) there is no magic in slot #0,
3680 (2) we cannot detect initial thread reliably (the first thread which does
3681 serial initialization may be not a real initial thread).
3682 */
3683 capacity = __kmp_threads_capacity;
3684 if (!initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) {
3685 --capacity;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003686 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003687
Jonathan Peyton30419822017-05-12 18:01:32 +00003688 /* see if there are too many threads */
Jonathan Peyton1800ece2018-01-10 18:27:01 +00003689 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) {
Jonathan Peyton30419822017-05-12 18:01:32 +00003690 if (__kmp_tp_cached) {
Jonathan Peyton6a393f72017-09-05 15:43:58 +00003691 __kmp_fatal(KMP_MSG(CantRegisterNewThread),
3692 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
3693 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
Jonathan Peyton30419822017-05-12 18:01:32 +00003694 } else {
Jonathan Peyton6a393f72017-09-05 15:43:58 +00003695 __kmp_fatal(KMP_MSG(CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads),
3696 __kmp_msg_null);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003697 }
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003698 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003699
3700 /* find an available thread slot */
3701 /* Don't reassign the zero slot since we need that to only be used by initial
3702 thread */
3703 for (gtid = (initial_thread ? 0 : 1); TCR_PTR(__kmp_threads[gtid]) != NULL;
3704 gtid++)
3705 ;
3706 KA_TRACE(1,
3707 ("__kmp_register_root: found slot in threads array: T#%d\n", gtid));
3708 KMP_ASSERT(gtid < __kmp_threads_capacity);
3709
3710 /* update global accounting */
3711 __kmp_all_nth++;
3712 TCW_4(__kmp_nth, __kmp_nth + 1);
3713
3714 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low
3715 // numbers of procs, and method #2 (keyed API call) for higher numbers.
3716 if (__kmp_adjust_gtid_mode) {
3717 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
3718 if (TCR_4(__kmp_gtid_mode) != 2) {
3719 TCW_4(__kmp_gtid_mode, 2);
3720 }
3721 } else {
3722 if (TCR_4(__kmp_gtid_mode) != 1) {
3723 TCW_4(__kmp_gtid_mode, 1);
3724 }
3725 }
3726 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003727
3728#ifdef KMP_ADJUST_BLOCKTIME
Jonathan Peyton30419822017-05-12 18:01:32 +00003729 /* Adjust blocktime to zero if necessary */
3730 /* Middle initialization might not have occurred yet */
3731 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
3732 if (__kmp_nth > __kmp_avail_proc) {
3733 __kmp_zero_bt = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003734 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003735 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003736#endif /* KMP_ADJUST_BLOCKTIME */
3737
Jonathan Peyton30419822017-05-12 18:01:32 +00003738 /* setup this new hierarchy */
3739 if (!(root = __kmp_root[gtid])) {
3740 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(sizeof(kmp_root_t));
3741 KMP_DEBUG_ASSERT(!root->r.r_root_team);
3742 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003743
Jonathan Peyton5375fe82016-11-14 21:13:44 +00003744#if KMP_STATS_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003745 // Initialize stats as soon as possible (right after gtid assignment).
3746 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid);
Jonathan Peytonf0682ac2018-07-30 17:41:08 +00003747 __kmp_stats_thread_ptr->startLife();
Jonathan Peyton30419822017-05-12 18:01:32 +00003748 KMP_SET_THREAD_STATE(SERIAL_REGION);
3749 KMP_INIT_PARTITIONED_TIMERS(OMP_serial);
Jonathan Peyton5375fe82016-11-14 21:13:44 +00003750#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003751 __kmp_initialize_root(root);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003752
Jonathan Peyton30419822017-05-12 18:01:32 +00003753 /* setup new root thread structure */
3754 if (root->r.r_uber_thread) {
3755 root_thread = root->r.r_uber_thread;
3756 } else {
3757 root_thread = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t));
3758 if (__kmp_storage_map) {
3759 __kmp_print_thread_storage_map(root_thread, gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003760 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003761 root_thread->th.th_info.ds.ds_gtid = gtid;
Joachim Protze82e94a52017-11-01 10:08:30 +00003762#if OMPT_SUPPORT
Jonathan Peyton3574f282018-10-04 14:57:04 +00003763 root_thread->th.ompt_thread_info.thread_data = ompt_data_none;
Joachim Protze82e94a52017-11-01 10:08:30 +00003764#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003765 root_thread->th.th_root = root;
3766 if (__kmp_env_consistency_check) {
3767 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid);
3768 }
3769#if USE_FAST_MEMORY
3770 __kmp_initialize_fast_memory(root_thread);
3771#endif /* USE_FAST_MEMORY */
Jim Cownie5e8470a2013-09-27 10:38:44 +00003772
Jonathan Peyton30419822017-05-12 18:01:32 +00003773#if KMP_USE_BGET
3774 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL);
3775 __kmp_initialize_bget(root_thread);
3776#endif
3777 __kmp_init_random(root_thread); // Initialize random number generator
3778 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003779
Jonathan Peyton30419822017-05-12 18:01:32 +00003780 /* setup the serial team held in reserve by the root thread */
3781 if (!root_thread->th.th_serial_team) {
3782 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3783 KF_TRACE(10, ("__kmp_register_root: before serial_team\n"));
3784 root_thread->th.th_serial_team =
3785 __kmp_allocate_team(root, 1, 1,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003786#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00003787 ompt_data_none, // root parallel id
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003788#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00003789#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003790 proc_bind_default,
Jim Cownie5e8470a2013-09-27 10:38:44 +00003791#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003792 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL));
3793 }
3794 KMP_ASSERT(root_thread->th.th_serial_team);
3795 KF_TRACE(10, ("__kmp_register_root: after serial_team = %p\n",
3796 root_thread->th.th_serial_team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003797
Jonathan Peyton30419822017-05-12 18:01:32 +00003798 /* drop root_thread into place */
3799 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003800
Jonathan Peyton30419822017-05-12 18:01:32 +00003801 root->r.r_root_team->t.t_threads[0] = root_thread;
3802 root->r.r_hot_team->t.t_threads[0] = root_thread;
3803 root_thread->th.th_serial_team->t.t_threads[0] = root_thread;
3804 // AC: the team created in reserve, not for execution (it is unused for now).
3805 root_thread->th.th_serial_team->t.t_serialized = 0;
3806 root->r.r_uber_thread = root_thread;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003807
Jonathan Peyton30419822017-05-12 18:01:32 +00003808 /* initialize the thread, get it ready to go */
3809 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid);
3810 TCW_4(__kmp_init_gtid, TRUE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003811
Jonathan Peyton30419822017-05-12 18:01:32 +00003812 /* prepare the master thread for get_gtid() */
3813 __kmp_gtid_set_specific(gtid);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003814
Jonathan Peyton7abf9d52016-05-26 18:19:10 +00003815#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +00003816 __kmp_itt_thread_name(gtid);
Jonathan Peyton7abf9d52016-05-26 18:19:10 +00003817#endif /* USE_ITT_BUILD */
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003818
Jonathan Peyton30419822017-05-12 18:01:32 +00003819#ifdef KMP_TDATA_GTID
3820 __kmp_gtid = gtid;
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00003821#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003822 __kmp_create_worker(gtid, root_thread, __kmp_stksize);
3823 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid);
3824
3825 KA_TRACE(20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
3826 "plain=%u\n",
3827 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team),
3828 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,
3829 KMP_INIT_BARRIER_STATE));
3830 { // Initialize barrier data.
3831 int b;
3832 for (b = 0; b < bs_last_barrier; ++b) {
3833 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE;
3834#if USE_DEBUGGER
3835 root_thread->th.th_bar[b].bb.b_worker_arrived = 0;
3836#endif
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003837 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003838 }
3839 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived ==
3840 KMP_INIT_BARRIER_STATE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003841
Alp Toker763b9392014-02-28 09:42:41 +00003842#if KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00003843#if OMP_40_ENABLED
3844 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED;
3845 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED;
3846 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED;
3847 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED;
3848#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003849 if (TCR_4(__kmp_init_middle)) {
3850 __kmp_affinity_set_init_mask(gtid, TRUE);
3851 }
Alp Toker763b9392014-02-28 09:42:41 +00003852#endif /* KMP_AFFINITY_SUPPORTED */
Jonathan Peyton92ca6182018-09-07 18:25:49 +00003853#if OMP_50_ENABLED
3854 root_thread->th.th_def_allocator = __kmp_def_allocator;
Jonathan Peyton6d88e042018-12-13 23:14:24 +00003855 root_thread->th.th_prev_level = 0;
3856 root_thread->th.th_prev_num_threads = 1;
Jonathan Peyton92ca6182018-09-07 18:25:49 +00003857#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00003858
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00003859 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(sizeof(kmp_cg_root_t));
3860 tmp->cg_root = root_thread;
3861 tmp->cg_thread_limit = __kmp_cg_max_nth;
3862 tmp->cg_nthreads = 1;
3863 KA_TRACE(100, ("__kmp_register_root: Thread %p created node %p with"
3864 " cg_nthreads init to 1\n",
3865 root_thread, tmp));
3866 tmp->up = NULL;
3867 root_thread->th.th_cg_roots = tmp;
3868
Jonathan Peyton30419822017-05-12 18:01:32 +00003869 __kmp_root_counter++;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003870
Joachim Protze82e94a52017-11-01 10:08:30 +00003871#if OMPT_SUPPORT
3872 if (!initial_thread && ompt_enabled.enabled) {
3873
Joachim Protze489cdb72018-09-10 14:34:54 +00003874 kmp_info_t *root_thread = ompt_get_thread();
Joachim Protze82e94a52017-11-01 10:08:30 +00003875
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00003876 ompt_set_thread_state(root_thread, ompt_state_overhead);
Joachim Protze82e94a52017-11-01 10:08:30 +00003877
3878 if (ompt_enabled.ompt_callback_thread_begin) {
3879 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)(
3880 ompt_thread_initial, __ompt_get_thread_data_internal());
3881 }
3882 ompt_data_t *task_data;
3883 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL);
3884 if (ompt_enabled.ompt_callback_task_create) {
3885 ompt_callbacks.ompt_callback(ompt_callback_task_create)(
3886 NULL, NULL, task_data, ompt_task_initial, 0, NULL);
3887 // initial task has nothing to return to
3888 }
3889
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00003890 ompt_set_thread_state(root_thread, ompt_state_work_serial);
Joachim Protze82e94a52017-11-01 10:08:30 +00003891 }
3892#endif
3893
Jonathan Peyton30419822017-05-12 18:01:32 +00003894 KMP_MB();
3895 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003896
Jonathan Peyton30419822017-05-12 18:01:32 +00003897 return gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003898}
3899
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003900#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00003901static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr, int level,
3902 const int max_level) {
3903 int i, n, nth;
3904 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams;
3905 if (!hot_teams || !hot_teams[level].hot_team) {
3906 return 0;
3907 }
3908 KMP_DEBUG_ASSERT(level < max_level);
3909 kmp_team_t *team = hot_teams[level].hot_team;
3910 nth = hot_teams[level].hot_team_nth;
3911 n = nth - 1; // master is not freed
3912 if (level < max_level - 1) {
3913 for (i = 0; i < nth; ++i) {
3914 kmp_info_t *th = team->t.t_threads[i];
3915 n += __kmp_free_hot_teams(root, th, level + 1, max_level);
3916 if (i > 0 && th->th.th_hot_teams) {
3917 __kmp_free(th->th.th_hot_teams);
3918 th->th.th_hot_teams = NULL;
3919 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003920 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003921 }
3922 __kmp_free_team(root, team, NULL);
3923 return n;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003924}
3925#endif
3926
Jonathan Peyton30419822017-05-12 18:01:32 +00003927// Resets a root thread and clear its root and hot teams.
3928// Returns the number of __kmp_threads entries directly and indirectly freed.
3929static int __kmp_reset_root(int gtid, kmp_root_t *root) {
3930 kmp_team_t *root_team = root->r.r_root_team;
3931 kmp_team_t *hot_team = root->r.r_hot_team;
3932 int n = hot_team->t.t_nproc;
3933 int i;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003934
Jonathan Peyton30419822017-05-12 18:01:32 +00003935 KMP_DEBUG_ASSERT(!root->r.r_active);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003936
Jonathan Peyton30419822017-05-12 18:01:32 +00003937 root->r.r_root_team = NULL;
3938 root->r.r_hot_team = NULL;
3939 // __kmp_free_team() does not free hot teams, so we have to clear r_hot_team
3940 // before call to __kmp_free_team().
3941 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003942#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00003943 if (__kmp_hot_teams_max_level >
3944 0) { // need to free nested hot teams and their threads if any
3945 for (i = 0; i < hot_team->t.t_nproc; ++i) {
3946 kmp_info_t *th = hot_team->t.t_threads[i];
3947 if (__kmp_hot_teams_max_level > 1) {
3948 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level);
3949 }
3950 if (th->th.th_hot_teams) {
3951 __kmp_free(th->th.th_hot_teams);
3952 th->th.th_hot_teams = NULL;
3953 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003954 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003955 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003956#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003957 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003958
Jonathan Peyton30419822017-05-12 18:01:32 +00003959 // Before we can reap the thread, we need to make certain that all other
3960 // threads in the teams that had this root as ancestor have stopped trying to
3961 // steal tasks.
3962 if (__kmp_tasking_mode != tskm_immediate_exec) {
3963 __kmp_wait_to_unref_task_teams();
3964 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003965
Jonathan Peyton30419822017-05-12 18:01:32 +00003966#if KMP_OS_WINDOWS
3967 /* Close Handle of root duplicated in __kmp_create_worker (tr #62919) */
3968 KA_TRACE(
3969 10, ("__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPEC
3970 "\n",
3971 (LPVOID) & (root->r.r_uber_thread->th),
3972 root->r.r_uber_thread->th.th_info.ds.ds_thread));
3973 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread);
3974#endif /* KMP_OS_WINDOWS */
Jim Cownie5e8470a2013-09-27 10:38:44 +00003975
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003976#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00003977 if (ompt_enabled.ompt_callback_thread_end) {
3978 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(
3979 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data));
Jonathan Peyton30419822017-05-12 18:01:32 +00003980 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003981#endif
3982
Jonathan Peyton30419822017-05-12 18:01:32 +00003983 TCW_4(__kmp_nth,
3984 __kmp_nth - 1); // __kmp_reap_thread will decrement __kmp_all_nth.
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00003985 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads--;
3986 KA_TRACE(100, ("__kmp_reset_root: Thread %p decrement cg_nthreads on node %p"
3987 " to %d\n",
3988 root->r.r_uber_thread, root->r.r_uber_thread->th.th_cg_roots,
3989 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads));
Jonathan Peytonf4392462017-07-27 20:58:41 +00003990
Jonathan Peyton30419822017-05-12 18:01:32 +00003991 __kmp_reap_thread(root->r.r_uber_thread, 1);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003992
Jonathan Peyton30419822017-05-12 18:01:32 +00003993 // We canot put root thread to __kmp_thread_pool, so we have to reap it istead
3994 // of freeing.
3995 root->r.r_uber_thread = NULL;
3996 /* mark root as no longer in use */
3997 root->r.r_begin = FALSE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003998
Jonathan Peyton30419822017-05-12 18:01:32 +00003999 return n;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004000}
4001
Jonathan Peyton30419822017-05-12 18:01:32 +00004002void __kmp_unregister_root_current_thread(int gtid) {
4003 KA_TRACE(1, ("__kmp_unregister_root_current_thread: enter T#%d\n", gtid));
4004 /* this lock should be ok, since unregister_root_current_thread is never
4005 called during an abort, only during a normal close. furthermore, if you
4006 have the forkjoin lock, you should never try to get the initz lock */
4007 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
4008 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
4009 KC_TRACE(10, ("__kmp_unregister_root_current_thread: already finished, "
4010 "exiting T#%d\n",
4011 gtid));
4012 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
4013 return;
4014 }
4015 kmp_root_t *root = __kmp_root[gtid];
Jim Cownie77c2a632014-09-03 11:34:33 +00004016
Jonathan Peyton30419822017-05-12 18:01:32 +00004017 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
4018 KMP_ASSERT(KMP_UBER_GTID(gtid));
4019 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
4020 KMP_ASSERT(root->r.r_active == FALSE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004021
Jonathan Peyton30419822017-05-12 18:01:32 +00004022 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00004023
Jonathan Peytondf6818b2016-06-14 17:57:47 +00004024#if OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00004025 kmp_info_t *thread = __kmp_threads[gtid];
4026 kmp_team_t *team = thread->th.th_team;
4027 kmp_task_team_t *task_team = thread->th.th_task_team;
Andrey Churbanov535b6fa2015-05-07 17:41:51 +00004028
Jonathan Peyton30419822017-05-12 18:01:32 +00004029 // we need to wait for the proxy tasks before finishing the thread
4030 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks) {
Jonathan Peyton6d247f72015-09-10 21:33:50 +00004031#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00004032 // the runtime is shutting down so we won't report any events
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00004033 thread->th.ompt_thread_info.state = ompt_state_undefined;
Jonathan Peyton6d247f72015-09-10 21:33:50 +00004034#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004035 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL));
4036 }
Andrey Churbanov535b6fa2015-05-07 17:41:51 +00004037#endif
4038
Jonathan Peyton30419822017-05-12 18:01:32 +00004039 __kmp_reset_root(gtid, root);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004040
Jonathan Peyton30419822017-05-12 18:01:32 +00004041 /* free up this thread slot */
4042 __kmp_gtid_set_specific(KMP_GTID_DNE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004043#ifdef KMP_TDATA_GTID
Jonathan Peyton30419822017-05-12 18:01:32 +00004044 __kmp_gtid = KMP_GTID_DNE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004045#endif
4046
Jonathan Peyton30419822017-05-12 18:01:32 +00004047 KMP_MB();
4048 KC_TRACE(10,
4049 ("__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004050
Jonathan Peyton30419822017-05-12 18:01:32 +00004051 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004052}
4053
Jonathan Peyton2321d572015-06-08 19:25:25 +00004054#if KMP_OS_WINDOWS
Jim Cownie5e8470a2013-09-27 10:38:44 +00004055/* __kmp_forkjoin_lock must be already held
Jonathan Peyton30419822017-05-12 18:01:32 +00004056 Unregisters a root thread that is not the current thread. Returns the number
4057 of __kmp_threads entries freed as a result. */
4058static int __kmp_unregister_root_other_thread(int gtid) {
4059 kmp_root_t *root = __kmp_root[gtid];
4060 int r;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004061
Jonathan Peyton30419822017-05-12 18:01:32 +00004062 KA_TRACE(1, ("__kmp_unregister_root_other_thread: enter T#%d\n", gtid));
4063 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
4064 KMP_ASSERT(KMP_UBER_GTID(gtid));
4065 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
4066 KMP_ASSERT(root->r.r_active == FALSE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004067
Jonathan Peyton30419822017-05-12 18:01:32 +00004068 r = __kmp_reset_root(gtid, root);
4069 KC_TRACE(10,
4070 ("__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid));
4071 return r;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004072}
Jonathan Peyton2321d572015-06-08 19:25:25 +00004073#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00004074
Jim Cownie5e8470a2013-09-27 10:38:44 +00004075#if KMP_DEBUG
4076void __kmp_task_info() {
4077
Jonathan Peyton30419822017-05-12 18:01:32 +00004078 kmp_int32 gtid = __kmp_entry_gtid();
4079 kmp_int32 tid = __kmp_tid_from_gtid(gtid);
4080 kmp_info_t *this_thr = __kmp_threads[gtid];
4081 kmp_team_t *steam = this_thr->th.th_serial_team;
4082 kmp_team_t *team = this_thr->th.th_team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004083
Jonathan Peytonbaad3f62018-08-09 22:04:30 +00004084 __kmp_printf(
4085 "__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p steam=%p curtask=%p "
4086 "ptask=%p\n",
4087 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task,
4088 team->t.t_implicit_task_taskdata[tid].td_parent);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004089}
4090#endif // KMP_DEBUG
4091
Jonathan Peyton30419822017-05-12 18:01:32 +00004092/* TODO optimize with one big memclr, take out what isn't needed, split
4093 responsibility to workers as much as possible, and delay initialization of
4094 features as much as possible */
4095static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team,
4096 int tid, int gtid) {
4097 /* this_thr->th.th_info.ds.ds_gtid is setup in
4098 kmp_allocate_thread/create_worker.
4099 this_thr->th.th_serial_team is setup in __kmp_allocate_thread */
4100 kmp_info_t *master = team->t.t_threads[0];
4101 KMP_DEBUG_ASSERT(this_thr != NULL);
4102 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team);
4103 KMP_DEBUG_ASSERT(team);
4104 KMP_DEBUG_ASSERT(team->t.t_threads);
4105 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4106 KMP_DEBUG_ASSERT(master);
4107 KMP_DEBUG_ASSERT(master->th.th_root);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004108
Jonathan Peyton30419822017-05-12 18:01:32 +00004109 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00004110
Jonathan Peyton30419822017-05-12 18:01:32 +00004111 TCW_SYNC_PTR(this_thr->th.th_team, team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004112
Jonathan Peyton30419822017-05-12 18:01:32 +00004113 this_thr->th.th_info.ds.ds_tid = tid;
4114 this_thr->th.th_set_nproc = 0;
4115 if (__kmp_tasking_mode != tskm_immediate_exec)
4116 // When tasking is possible, threads are not safe to reap until they are
4117 // done tasking; this will be set when tasking code is exited in wait
4118 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
4119 else // no tasking --> always safe to reap
4120 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004121#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00004122 this_thr->th.th_set_proc_bind = proc_bind_default;
4123#if KMP_AFFINITY_SUPPORTED
4124 this_thr->th.th_new_place = this_thr->th.th_current_place;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004125#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004126#endif
4127 this_thr->th.th_root = master->th.th_root;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004128
Jonathan Peyton30419822017-05-12 18:01:32 +00004129 /* setup the thread's cache of the team structure */
4130 this_thr->th.th_team_nproc = team->t.t_nproc;
4131 this_thr->th.th_team_master = master;
4132 this_thr->th.th_team_serialized = team->t.t_serialized;
4133 TCW_PTR(this_thr->th.th_sleep_loc, NULL);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004134
Jonathan Peyton30419822017-05-12 18:01:32 +00004135 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004136
Jonathan Peyton30419822017-05-12 18:01:32 +00004137 KF_TRACE(10, ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n",
4138 tid, gtid, this_thr, this_thr->th.th_current_task));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004139
Jonathan Peyton30419822017-05-12 18:01:32 +00004140 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr,
4141 team, tid, TRUE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004142
Jonathan Peyton30419822017-05-12 18:01:32 +00004143 KF_TRACE(10, ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n",
4144 tid, gtid, this_thr, this_thr->th.th_current_task));
4145 // TODO: Initialize ICVs from parent; GEH - isn't that already done in
4146 // __kmp_initialize_team()?
Jim Cownie5e8470a2013-09-27 10:38:44 +00004147
Jonathan Peyton30419822017-05-12 18:01:32 +00004148 /* TODO no worksharing in speculative threads */
4149 this_thr->th.th_dispatch = &team->t.t_dispatch[tid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00004150
Jonathan Peyton30419822017-05-12 18:01:32 +00004151 this_thr->th.th_local.this_construct = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004152
Jonathan Peyton30419822017-05-12 18:01:32 +00004153 if (!this_thr->th.th_pri_common) {
4154 this_thr->th.th_pri_common =
4155 (struct common_table *)__kmp_allocate(sizeof(struct common_table));
4156 if (__kmp_storage_map) {
4157 __kmp_print_storage_map_gtid(
4158 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1,
4159 sizeof(struct common_table), "th_%d.th_pri_common\n", gtid);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00004160 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004161 this_thr->th.th_pri_head = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00004162 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00004163
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00004164 if (this_thr != master && // Master's CG root is initialized elsewhere
4165 this_thr->th.th_cg_roots != master->th.th_cg_roots) { // CG root not set
4166 // Make new thread's CG root same as master's
4167 KMP_DEBUG_ASSERT(master->th.th_cg_roots);
4168 this_thr->th.th_cg_roots = master->th.th_cg_roots;
4169 // Increment new thread's CG root's counter to add the new thread
4170 this_thr->th.th_cg_roots->cg_nthreads++;
4171 KA_TRACE(100, ("__kmp_initialize_info: Thread %p increment cg_nthreads on"
4172 " node %p of thread %p to %d\n",
4173 this_thr, this_thr->th.th_cg_roots,
4174 this_thr->th.th_cg_roots->cg_root,
4175 this_thr->th.th_cg_roots->cg_nthreads));
4176 this_thr->th.th_current_task->td_icvs.thread_limit =
4177 this_thr->th.th_cg_roots->cg_thread_limit;
4178 }
4179
Jonathan Peyton30419822017-05-12 18:01:32 +00004180 /* Initialize dynamic dispatch */
4181 {
4182 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch;
4183 // Use team max_nproc since this will never change for the team.
4184 size_t disp_size =
4185 sizeof(dispatch_private_info_t) *
4186 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers);
4187 KD_TRACE(10, ("__kmp_initialize_info: T#%d max_nproc: %d\n", gtid,
4188 team->t.t_max_nproc));
4189 KMP_ASSERT(dispatch);
4190 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4191 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004192
Jonathan Peyton30419822017-05-12 18:01:32 +00004193 dispatch->th_disp_index = 0;
Jonathan Peytondf6818b2016-06-14 17:57:47 +00004194#if OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00004195 dispatch->th_doacross_buf_idx = 0;
Jonathan Peyton71909c52016-03-02 22:42:06 +00004196#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004197 if (!dispatch->th_disp_buffer) {
4198 dispatch->th_disp_buffer =
4199 (dispatch_private_info_t *)__kmp_allocate(disp_size);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004200
Jonathan Peyton30419822017-05-12 18:01:32 +00004201 if (__kmp_storage_map) {
4202 __kmp_print_storage_map_gtid(
4203 gtid, &dispatch->th_disp_buffer[0],
4204 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1
4205 ? 1
4206 : __kmp_dispatch_num_buffers],
4207 disp_size, "th_%d.th_dispatch.th_disp_buffer "
4208 "(team_%d.t_dispatch[%d].th_disp_buffer)",
4209 gtid, team->t.t_id, gtid);
4210 }
4211 } else {
4212 memset(&dispatch->th_disp_buffer[0], '\0', disp_size);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004213 }
4214
Jonathan Peyton30419822017-05-12 18:01:32 +00004215 dispatch->th_dispatch_pr_current = 0;
4216 dispatch->th_dispatch_sh_current = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004217
Jonathan Peyton30419822017-05-12 18:01:32 +00004218 dispatch->th_deo_fcn = 0; /* ORDERED */
4219 dispatch->th_dxo_fcn = 0; /* END ORDERED */
4220 }
Andrey Churbanov6d224db2015-02-10 18:37:43 +00004221
Jonathan Peyton30419822017-05-12 18:01:32 +00004222 this_thr->th.th_next_pool = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004223
Jonathan Peyton30419822017-05-12 18:01:32 +00004224 if (!this_thr->th.th_task_state_memo_stack) {
4225 size_t i;
4226 this_thr->th.th_task_state_memo_stack =
4227 (kmp_uint8 *)__kmp_allocate(4 * sizeof(kmp_uint8));
4228 this_thr->th.th_task_state_top = 0;
4229 this_thr->th.th_task_state_stack_sz = 4;
4230 for (i = 0; i < this_thr->th.th_task_state_stack_sz;
4231 ++i) // zero init the stack
4232 this_thr->th.th_task_state_memo_stack[i] = 0;
4233 }
4234
4235 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
4236 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
4237
4238 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00004239}
4240
Jonathan Peyton30419822017-05-12 18:01:32 +00004241/* allocate a new thread for the requesting team. this is only called from
4242 within a forkjoin critical section. we will first try to get an available
4243 thread from the thread pool. if none is available, we will fork a new one
4244 assuming we are able to create a new one. this should be assured, as the
4245 caller should check on this first. */
4246kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team,
4247 int new_tid) {
4248 kmp_team_t *serial_team;
4249 kmp_info_t *new_thr;
4250 int new_gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004251
Jonathan Peyton30419822017-05-12 18:01:32 +00004252 KA_TRACE(20, ("__kmp_allocate_thread: T#%d\n", __kmp_get_gtid()));
4253 KMP_DEBUG_ASSERT(root && team);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004254#if !KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004255 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid()));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004256#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004257 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00004258
Jonathan Peyton30419822017-05-12 18:01:32 +00004259 /* first, try to get one from the thread pool */
4260 if (__kmp_thread_pool) {
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00004261 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool);
Jonathan Peyton30419822017-05-12 18:01:32 +00004262 __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool;
4263 if (new_thr == __kmp_thread_pool_insert_pt) {
4264 __kmp_thread_pool_insert_pt = NULL;
4265 }
4266 TCW_4(new_thr->th.th_in_pool, FALSE);
Jonathan Peytonfeac33e2019-04-08 17:50:02 +00004267 __kmp_suspend_initialize_thread(new_thr);
4268 __kmp_lock_suspend_mx(new_thr);
4269 if (new_thr->th.th_active_in_pool == TRUE) {
4270 KMP_DEBUG_ASSERT(new_thr->th.th_active == TRUE);
4271 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
4272 new_thr->th.th_active_in_pool = FALSE;
4273 }
4274#if KMP_DEBUG
4275 else {
4276 KMP_DEBUG_ASSERT(new_thr->th.th_active == FALSE);
4277 }
4278#endif
4279 __kmp_unlock_suspend_mx(new_thr);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004280
Jonathan Peyton30419822017-05-12 18:01:32 +00004281 KA_TRACE(20, ("__kmp_allocate_thread: T#%d using thread T#%d\n",
4282 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid));
4283 KMP_ASSERT(!new_thr->th.th_team);
4284 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004285
Jonathan Peyton30419822017-05-12 18:01:32 +00004286 /* setup the thread structure */
4287 __kmp_initialize_info(new_thr, team, new_tid,
4288 new_thr->th.th_info.ds.ds_gtid);
4289 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004290
Jonathan Peyton30419822017-05-12 18:01:32 +00004291 TCW_4(__kmp_nth, __kmp_nth + 1);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004292
Jonathan Peyton30419822017-05-12 18:01:32 +00004293 new_thr->th.th_task_state = 0;
4294 new_thr->th.th_task_state_top = 0;
4295 new_thr->th.th_task_state_stack_sz = 4;
Andrey Churbanov6d224db2015-02-10 18:37:43 +00004296
Jim Cownie5e8470a2013-09-27 10:38:44 +00004297#ifdef KMP_ADJUST_BLOCKTIME
Jonathan Peyton30419822017-05-12 18:01:32 +00004298 /* Adjust blocktime back to zero if necessary */
4299 /* Middle initialization might not have occurred yet */
4300 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4301 if (__kmp_nth > __kmp_avail_proc) {
4302 __kmp_zero_bt = TRUE;
4303 }
4304 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00004305#endif /* KMP_ADJUST_BLOCKTIME */
4306
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004307#if KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00004308 // If thread entered pool via __kmp_free_thread, wait_flag should !=
4309 // KMP_BARRIER_PARENT_FLAG.
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004310 int b;
Jonathan Peyton30419822017-05-12 18:01:32 +00004311 kmp_balign_t *balign = new_thr->th.th_bar;
4312 for (b = 0; b < bs_last_barrier; ++b)
4313 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004314#endif
4315
Jonathan Peyton30419822017-05-12 18:01:32 +00004316 KF_TRACE(10, ("__kmp_allocate_thread: T#%d using thread %p T#%d\n",
4317 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004318
Jim Cownie5e8470a2013-09-27 10:38:44 +00004319 KMP_MB();
4320 return new_thr;
Jonathan Peyton30419822017-05-12 18:01:32 +00004321 }
4322
4323 /* no, well fork a new one */
4324 KMP_ASSERT(__kmp_nth == __kmp_all_nth);
4325 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity);
4326
4327#if KMP_USE_MONITOR
4328 // If this is the first worker thread the RTL is creating, then also
4329 // launch the monitor thread. We try to do this as early as possible.
4330 if (!TCR_4(__kmp_init_monitor)) {
4331 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
4332 if (!TCR_4(__kmp_init_monitor)) {
4333 KF_TRACE(10, ("before __kmp_create_monitor\n"));
4334 TCW_4(__kmp_init_monitor, 1);
4335 __kmp_create_monitor(&__kmp_monitor);
4336 KF_TRACE(10, ("after __kmp_create_monitor\n"));
4337#if KMP_OS_WINDOWS
4338 // AC: wait until monitor has started. This is a fix for CQ232808.
4339 // The reason is that if the library is loaded/unloaded in a loop with
4340 // small (parallel) work in between, then there is high probability that
4341 // monitor thread started after the library shutdown. At shutdown it is
4342 // too late to cope with the problem, because when the master is in
4343 // DllMain (process detach) the monitor has no chances to start (it is
4344 // blocked), and master has no means to inform the monitor that the
4345 // library has gone, because all the memory which the monitor can access
4346 // is going to be released/reset.
4347 while (TCR_4(__kmp_init_monitor) < 2) {
4348 KMP_YIELD(TRUE);
4349 }
4350 KF_TRACE(10, ("after monitor thread has started\n"));
4351#endif
4352 }
4353 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
4354 }
4355#endif
4356
4357 KMP_MB();
4358 for (new_gtid = 1; TCR_PTR(__kmp_threads[new_gtid]) != NULL; ++new_gtid) {
4359 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity);
4360 }
4361
4362 /* allocate space for it. */
4363 new_thr = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t));
4364
4365 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr);
4366
4367 if (__kmp_storage_map) {
4368 __kmp_print_thread_storage_map(new_thr, new_gtid);
4369 }
4370
4371 // add the reserve serialized team, initialized from the team's master thread
4372 {
4373 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team);
4374 KF_TRACE(10, ("__kmp_allocate_thread: before th_serial/serial_team\n"));
4375 new_thr->th.th_serial_team = serial_team =
4376 (kmp_team_t *)__kmp_allocate_team(root, 1, 1,
4377#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00004378 ompt_data_none, // root parallel id
Jonathan Peyton30419822017-05-12 18:01:32 +00004379#endif
4380#if OMP_40_ENABLED
4381 proc_bind_default,
4382#endif
4383 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL));
4384 }
4385 KMP_ASSERT(serial_team);
4386 serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for
4387 // execution (it is unused for now).
4388 serial_team->t.t_threads[0] = new_thr;
4389 KF_TRACE(10,
4390 ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n",
4391 new_thr));
4392
4393 /* setup the thread structures */
4394 __kmp_initialize_info(new_thr, team, new_tid, new_gtid);
4395
4396#if USE_FAST_MEMORY
4397 __kmp_initialize_fast_memory(new_thr);
4398#endif /* USE_FAST_MEMORY */
4399
4400#if KMP_USE_BGET
4401 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL);
4402 __kmp_initialize_bget(new_thr);
4403#endif
4404
4405 __kmp_init_random(new_thr); // Initialize random number generator
4406
4407 /* Initialize these only once when thread is grabbed for a team allocation */
4408 KA_TRACE(20,
4409 ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n",
4410 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
4411
4412 int b;
4413 kmp_balign_t *balign = new_thr->th.th_bar;
4414 for (b = 0; b < bs_last_barrier; ++b) {
4415 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE;
4416 balign[b].bb.team = NULL;
4417 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING;
4418 balign[b].bb.use_oncore_barrier = 0;
4419 }
4420
4421 new_thr->th.th_spin_here = FALSE;
4422 new_thr->th.th_next_waiting = 0;
Jonathan Peytona764af62018-07-19 19:17:00 +00004423#if KMP_OS_UNIX
4424 new_thr->th.th_blocking = false;
4425#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004426
4427#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
4428 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED;
4429 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED;
4430 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED;
4431 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED;
4432#endif
Jonathan Peyton92ca6182018-09-07 18:25:49 +00004433#if OMP_50_ENABLED
4434 new_thr->th.th_def_allocator = __kmp_def_allocator;
Jonathan Peyton6d88e042018-12-13 23:14:24 +00004435 new_thr->th.th_prev_level = 0;
4436 new_thr->th.th_prev_num_threads = 1;
Jonathan Peyton92ca6182018-09-07 18:25:49 +00004437#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004438
4439 TCW_4(new_thr->th.th_in_pool, FALSE);
4440 new_thr->th.th_active_in_pool = FALSE;
4441 TCW_4(new_thr->th.th_active, TRUE);
4442
4443 /* adjust the global counters */
4444 __kmp_all_nth++;
4445 __kmp_nth++;
4446
4447 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low
4448 // numbers of procs, and method #2 (keyed API call) for higher numbers.
4449 if (__kmp_adjust_gtid_mode) {
4450 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
4451 if (TCR_4(__kmp_gtid_mode) != 2) {
4452 TCW_4(__kmp_gtid_mode, 2);
4453 }
4454 } else {
4455 if (TCR_4(__kmp_gtid_mode) != 1) {
4456 TCW_4(__kmp_gtid_mode, 1);
4457 }
4458 }
4459 }
4460
4461#ifdef KMP_ADJUST_BLOCKTIME
4462 /* Adjust blocktime back to zero if necessary */
4463 /* Middle initialization might not have occurred yet */
4464 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4465 if (__kmp_nth > __kmp_avail_proc) {
4466 __kmp_zero_bt = TRUE;
4467 }
4468 }
4469#endif /* KMP_ADJUST_BLOCKTIME */
4470
4471 /* actually fork it and create the new worker thread */
4472 KF_TRACE(
4473 10, ("__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr));
4474 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize);
4475 KF_TRACE(10,
4476 ("__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr));
4477
4478 KA_TRACE(20, ("__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(),
4479 new_gtid));
4480 KMP_MB();
4481 return new_thr;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004482}
4483
Jonathan Peyton30419822017-05-12 18:01:32 +00004484/* Reinitialize team for reuse.
4485 The hot team code calls this case at every fork barrier, so EPCC barrier
4486 test are extremely sensitive to changes in it, esp. writes to the team
4487 struct, which cause a cache invalidation in all threads.
4488 IF YOU TOUCH THIS ROUTINE, RUN EPCC C SYNCBENCH ON A BIG-IRON MACHINE!!! */
4489static void __kmp_reinitialize_team(kmp_team_t *team,
4490 kmp_internal_control_t *new_icvs,
4491 ident_t *loc) {
4492 KF_TRACE(10, ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n",
4493 team->t.t_threads[0], team));
4494 KMP_DEBUG_ASSERT(team && new_icvs);
4495 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);
4496 KMP_CHECK_UPDATE(team->t.t_ident, loc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004497
Jonathan Peyton30419822017-05-12 18:01:32 +00004498 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID());
Jonathan Peyton30419822017-05-12 18:01:32 +00004499 // Copy ICVs to the master thread's implicit taskdata
4500 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE);
4501 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);
Jim Cownie181b4bb2013-12-23 17:28:57 +00004502
Jonathan Peyton30419822017-05-12 18:01:32 +00004503 KF_TRACE(10, ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n",
4504 team->t.t_threads[0], team));
Jim Cownie181b4bb2013-12-23 17:28:57 +00004505}
4506
Jonathan Peyton30419822017-05-12 18:01:32 +00004507/* Initialize the team data structure.
4508 This assumes the t_threads and t_max_nproc are already set.
4509 Also, we don't touch the arguments */
4510static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
4511 kmp_internal_control_t *new_icvs,
4512 ident_t *loc) {
4513 KF_TRACE(10, ("__kmp_initialize_team: enter: team=%p\n", team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004514
Jonathan Peyton30419822017-05-12 18:01:32 +00004515 /* verify */
4516 KMP_DEBUG_ASSERT(team);
4517 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc);
4518 KMP_DEBUG_ASSERT(team->t.t_threads);
4519 KMP_MB();
Jim Cownie181b4bb2013-12-23 17:28:57 +00004520
Jonathan Peyton30419822017-05-12 18:01:32 +00004521 team->t.t_master_tid = 0; /* not needed */
4522 /* team->t.t_master_bar; not needed */
4523 team->t.t_serialized = new_nproc > 1 ? 0 : 1;
4524 team->t.t_nproc = new_nproc;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004525
Jonathan Peyton30419822017-05-12 18:01:32 +00004526 /* team->t.t_parent = NULL; TODO not needed & would mess up hot team */
4527 team->t.t_next_pool = NULL;
4528 /* memset( team->t.t_threads, 0, sizeof(kmp_info_t*)*new_nproc ); would mess
4529 * up hot team */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004530
Jonathan Peyton30419822017-05-12 18:01:32 +00004531 TCW_SYNC_PTR(team->t.t_pkfn, NULL); /* not needed */
4532 team->t.t_invoke = NULL; /* not needed */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004533
Jonathan Peyton30419822017-05-12 18:01:32 +00004534 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00004535 team->t.t_sched.sched = new_icvs->sched.sched;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004536
4537#if KMP_ARCH_X86 || KMP_ARCH_X86_64
Jonathan Peyton30419822017-05-12 18:01:32 +00004538 team->t.t_fp_control_saved = FALSE; /* not needed */
4539 team->t.t_x87_fpu_control_word = 0; /* not needed */
4540 team->t.t_mxcsr = 0; /* not needed */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004541#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
4542
Jonathan Peyton30419822017-05-12 18:01:32 +00004543 team->t.t_construct = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004544
Jonathan Peyton30419822017-05-12 18:01:32 +00004545 team->t.t_ordered.dt.t_value = 0;
4546 team->t.t_master_active = FALSE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004547
Jim Cownie5e8470a2013-09-27 10:38:44 +00004548#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00004549 team->t.t_copypriv_data = NULL; /* not necessary, but nice for debugging */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004550#endif
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00004551#if KMP_OS_WINDOWS
Jonathan Peyton30419822017-05-12 18:01:32 +00004552 team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00004553#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00004554
Jonathan Peyton30419822017-05-12 18:01:32 +00004555 team->t.t_control_stack_top = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004556
Jonathan Peyton30419822017-05-12 18:01:32 +00004557 __kmp_reinitialize_team(team, new_icvs, loc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004558
Jonathan Peyton30419822017-05-12 18:01:32 +00004559 KMP_MB();
4560 KF_TRACE(10, ("__kmp_initialize_team: exit: team=%p\n", team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004561}
4562
Alp Toker98758b02014-03-02 04:12:06 +00004563#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
Jim Cownie5e8470a2013-09-27 10:38:44 +00004564/* Sets full mask for thread and returns old mask, no changes to structures. */
4565static void
Jonathan Peyton30419822017-05-12 18:01:32 +00004566__kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) {
4567 if (KMP_AFFINITY_CAPABLE()) {
4568 int status;
4569 if (old_mask != NULL) {
4570 status = __kmp_get_system_affinity(old_mask, TRUE);
4571 int error = errno;
4572 if (status != 0) {
Jonathan Peyton6a393f72017-09-05 15:43:58 +00004573 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError), KMP_ERR(error),
4574 __kmp_msg_null);
Jonathan Peyton30419822017-05-12 18:01:32 +00004575 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00004576 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004577 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE);
4578 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00004579}
4580#endif
4581
Alp Toker98758b02014-03-02 04:12:06 +00004582#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
Jim Cownie5e8470a2013-09-27 10:38:44 +00004583
Jim Cownie5e8470a2013-09-27 10:38:44 +00004584// __kmp_partition_places() is the heart of the OpenMP 4.0 affinity mechanism.
4585// It calculats the worker + master thread's partition based upon the parent
Alp Toker8f2d3f02014-02-24 10:40:15 +00004586// thread's partition, and binds each worker to a thread in their partition.
Jim Cownie5e8470a2013-09-27 10:38:44 +00004587// The master thread's partition should already include its current binding.
Jonathan Peyton30419822017-05-12 18:01:32 +00004588static void __kmp_partition_places(kmp_team_t *team, int update_master_only) {
4589 // Copy the master thread's place partion to the team struct
4590 kmp_info_t *master_th = team->t.t_threads[0];
4591 KMP_DEBUG_ASSERT(master_th != NULL);
4592 kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
4593 int first_place = master_th->th.th_first_place;
4594 int last_place = master_th->th.th_last_place;
4595 int masters_place = master_th->th.th_current_place;
4596 team->t.t_first_place = first_place;
4597 team->t.t_last_place = last_place;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004598
Jonathan Peyton30419822017-05-12 18:01:32 +00004599 KA_TRACE(20, ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
4600 "bound to place %d partition = [%d,%d]\n",
4601 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]),
4602 team->t.t_id, masters_place, first_place, last_place));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004603
Jonathan Peyton30419822017-05-12 18:01:32 +00004604 switch (proc_bind) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00004605
Jonathan Peyton30419822017-05-12 18:01:32 +00004606 case proc_bind_default:
4607 // serial teams might have the proc_bind policy set to proc_bind_default. It
4608 // doesn't matter, as we don't rebind master thread for any proc_bind policy
4609 KMP_DEBUG_ASSERT(team->t.t_nproc == 1);
4610 break;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004611
Jonathan Peyton30419822017-05-12 18:01:32 +00004612 case proc_bind_master: {
4613 int f;
4614 int n_th = team->t.t_nproc;
4615 for (f = 1; f < n_th; f++) {
4616 kmp_info_t *th = team->t.t_threads[f];
4617 KMP_DEBUG_ASSERT(th != NULL);
4618 th->th.th_first_place = first_place;
4619 th->th.th_last_place = last_place;
4620 th->th.th_new_place = masters_place;
Jonathan Peyton6d88e042018-12-13 23:14:24 +00004621#if OMP_50_ENABLED
4622 if (__kmp_display_affinity && masters_place != th->th.th_current_place &&
4623 team->t.t_display_affinity != 1) {
4624 team->t.t_display_affinity = 1;
4625 }
4626#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00004627
Jonathan Peyton30419822017-05-12 18:01:32 +00004628 KA_TRACE(100, ("__kmp_partition_places: master: T#%d(%d:%d) place %d "
4629 "partition = [%d,%d]\n",
4630 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
4631 f, masters_place, first_place, last_place));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004632 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004633 } break;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004634
Jonathan Peyton30419822017-05-12 18:01:32 +00004635 case proc_bind_close: {
4636 int f;
4637 int n_th = team->t.t_nproc;
4638 int n_places;
4639 if (first_place <= last_place) {
4640 n_places = last_place - first_place + 1;
4641 } else {
4642 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4643 }
4644 if (n_th <= n_places) {
4645 int place = masters_place;
4646 for (f = 1; f < n_th; f++) {
4647 kmp_info_t *th = team->t.t_threads[f];
4648 KMP_DEBUG_ASSERT(th != NULL);
4649
4650 if (place == last_place) {
4651 place = first_place;
4652 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4653 place = 0;
4654 } else {
4655 place++;
4656 }
4657 th->th.th_first_place = first_place;
4658 th->th.th_last_place = last_place;
4659 th->th.th_new_place = place;
Jonathan Peyton6d88e042018-12-13 23:14:24 +00004660#if OMP_50_ENABLED
4661 if (__kmp_display_affinity && place != th->th.th_current_place &&
4662 team->t.t_display_affinity != 1) {
4663 team->t.t_display_affinity = 1;
4664 }
4665#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004666
4667 KA_TRACE(100, ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
4668 "partition = [%d,%d]\n",
4669 __kmp_gtid_from_thread(team->t.t_threads[f]),
4670 team->t.t_id, f, place, first_place, last_place));
4671 }
4672 } else {
4673 int S, rem, gap, s_count;
4674 S = n_th / n_places;
4675 s_count = 0;
4676 rem = n_th - (S * n_places);
4677 gap = rem > 0 ? n_places / rem : n_places;
4678 int place = masters_place;
4679 int gap_ct = gap;
4680 for (f = 0; f < n_th; f++) {
4681 kmp_info_t *th = team->t.t_threads[f];
4682 KMP_DEBUG_ASSERT(th != NULL);
4683
4684 th->th.th_first_place = first_place;
4685 th->th.th_last_place = last_place;
4686 th->th.th_new_place = place;
Jonathan Peyton6d88e042018-12-13 23:14:24 +00004687#if OMP_50_ENABLED
4688 if (__kmp_display_affinity && place != th->th.th_current_place &&
4689 team->t.t_display_affinity != 1) {
4690 team->t.t_display_affinity = 1;
4691 }
4692#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004693 s_count++;
4694
4695 if ((s_count == S) && rem && (gap_ct == gap)) {
4696 // do nothing, add an extra thread to place on next iteration
4697 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4698 // we added an extra thread to this place; move to next place
4699 if (place == last_place) {
4700 place = first_place;
4701 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4702 place = 0;
4703 } else {
4704 place++;
4705 }
4706 s_count = 0;
4707 gap_ct = 1;
4708 rem--;
4709 } else if (s_count == S) { // place full; don't add extra
4710 if (place == last_place) {
4711 place = first_place;
4712 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4713 place = 0;
4714 } else {
4715 place++;
4716 }
4717 gap_ct++;
4718 s_count = 0;
4719 }
4720
4721 KA_TRACE(100,
4722 ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
4723 "partition = [%d,%d]\n",
4724 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f,
4725 th->th.th_new_place, first_place, last_place));
4726 }
4727 KMP_DEBUG_ASSERT(place == masters_place);
4728 }
4729 } break;
4730
4731 case proc_bind_spread: {
4732 int f;
4733 int n_th = team->t.t_nproc;
4734 int n_places;
4735 int thidx;
4736 if (first_place <= last_place) {
4737 n_places = last_place - first_place + 1;
4738 } else {
4739 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4740 }
4741 if (n_th <= n_places) {
Paul Osmialowskia0162792017-08-10 23:04:11 +00004742 int place = -1;
Jonathan Peyton30419822017-05-12 18:01:32 +00004743
Paul Osmialowskia0162792017-08-10 23:04:11 +00004744 if (n_places != static_cast<int>(__kmp_affinity_num_masks)) {
4745 int S = n_places / n_th;
4746 int s_count, rem, gap, gap_ct;
4747
4748 place = masters_place;
4749 rem = n_places - n_th * S;
4750 gap = rem ? n_th / rem : 1;
4751 gap_ct = gap;
4752 thidx = n_th;
4753 if (update_master_only == 1)
4754 thidx = 1;
4755 for (f = 0; f < thidx; f++) {
4756 kmp_info_t *th = team->t.t_threads[f];
4757 KMP_DEBUG_ASSERT(th != NULL);
4758
4759 th->th.th_first_place = place;
4760 th->th.th_new_place = place;
Jonathan Peyton6d88e042018-12-13 23:14:24 +00004761#if OMP_50_ENABLED
4762 if (__kmp_display_affinity && place != th->th.th_current_place &&
4763 team->t.t_display_affinity != 1) {
4764 team->t.t_display_affinity = 1;
4765 }
4766#endif
Paul Osmialowskia0162792017-08-10 23:04:11 +00004767 s_count = 1;
4768 while (s_count < S) {
4769 if (place == last_place) {
4770 place = first_place;
4771 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4772 place = 0;
4773 } else {
4774 place++;
4775 }
4776 s_count++;
4777 }
4778 if (rem && (gap_ct == gap)) {
4779 if (place == last_place) {
4780 place = first_place;
4781 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4782 place = 0;
4783 } else {
4784 place++;
4785 }
4786 rem--;
4787 gap_ct = 0;
4788 }
4789 th->th.th_last_place = place;
4790 gap_ct++;
4791
Jonathan Peyton30419822017-05-12 18:01:32 +00004792 if (place == last_place) {
4793 place = first_place;
4794 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4795 place = 0;
4796 } else {
4797 place++;
4798 }
Paul Osmialowskia0162792017-08-10 23:04:11 +00004799
Jonathan Peyton94a114f2017-10-20 19:30:57 +00004800 KA_TRACE(100,
4801 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
4802 "partition = [%d,%d], __kmp_affinity_num_masks: %u\n",
4803 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
4804 f, th->th.th_new_place, th->th.th_first_place,
4805 th->th.th_last_place, __kmp_affinity_num_masks));
Jonathan Peyton30419822017-05-12 18:01:32 +00004806 }
Paul Osmialowskia0162792017-08-10 23:04:11 +00004807 } else {
4808 /* Having uniform space of available computation places I can create
4809 T partitions of round(P/T) size and put threads into the first
4810 place of each partition. */
4811 double current = static_cast<double>(masters_place);
4812 double spacing =
Jonathan Peyton94a114f2017-10-20 19:30:57 +00004813 (static_cast<double>(n_places + 1) / static_cast<double>(n_th));
Paul Osmialowskia0162792017-08-10 23:04:11 +00004814 int first, last;
4815 kmp_info_t *th;
4816
4817 thidx = n_th + 1;
4818 if (update_master_only == 1)
4819 thidx = 1;
4820 for (f = 0; f < thidx; f++) {
4821 first = static_cast<int>(current);
4822 last = static_cast<int>(current + spacing) - 1;
4823 KMP_DEBUG_ASSERT(last >= first);
4824 if (first >= n_places) {
4825 if (masters_place) {
4826 first -= n_places;
4827 last -= n_places;
4828 if (first == (masters_place + 1)) {
4829 KMP_DEBUG_ASSERT(f == n_th);
4830 first--;
4831 }
4832 if (last == masters_place) {
4833 KMP_DEBUG_ASSERT(f == (n_th - 1));
4834 last--;
4835 }
4836 } else {
4837 KMP_DEBUG_ASSERT(f == n_th);
4838 first = 0;
4839 last = 0;
4840 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004841 }
Paul Osmialowskia0162792017-08-10 23:04:11 +00004842 if (last >= n_places) {
4843 last = (n_places - 1);
4844 }
4845 place = first;
4846 current += spacing;
4847 if (f < n_th) {
4848 KMP_DEBUG_ASSERT(0 <= first);
4849 KMP_DEBUG_ASSERT(n_places > first);
4850 KMP_DEBUG_ASSERT(0 <= last);
4851 KMP_DEBUG_ASSERT(n_places > last);
4852 KMP_DEBUG_ASSERT(last_place >= first_place);
4853 th = team->t.t_threads[f];
4854 KMP_DEBUG_ASSERT(th);
4855 th->th.th_first_place = first;
4856 th->th.th_new_place = place;
4857 th->th.th_last_place = last;
Jonathan Peyton6d88e042018-12-13 23:14:24 +00004858#if OMP_50_ENABLED
4859 if (__kmp_display_affinity && place != th->th.th_current_place &&
4860 team->t.t_display_affinity != 1) {
4861 team->t.t_display_affinity = 1;
4862 }
4863#endif
Jonathan Peyton94a114f2017-10-20 19:30:57 +00004864 KA_TRACE(100,
4865 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
4866 "partition = [%d,%d], spacing = %.4f\n",
4867 __kmp_gtid_from_thread(team->t.t_threads[f]),
4868 team->t.t_id, f, th->th.th_new_place,
4869 th->th.th_first_place, th->th.th_last_place, spacing));
Paul Osmialowskia0162792017-08-10 23:04:11 +00004870 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004871 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004872 }
4873 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
4874 } else {
4875 int S, rem, gap, s_count;
4876 S = n_th / n_places;
4877 s_count = 0;
4878 rem = n_th - (S * n_places);
4879 gap = rem > 0 ? n_places / rem : n_places;
4880 int place = masters_place;
4881 int gap_ct = gap;
4882 thidx = n_th;
4883 if (update_master_only == 1)
4884 thidx = 1;
4885 for (f = 0; f < thidx; f++) {
4886 kmp_info_t *th = team->t.t_threads[f];
4887 KMP_DEBUG_ASSERT(th != NULL);
4888
4889 th->th.th_first_place = place;
4890 th->th.th_last_place = place;
4891 th->th.th_new_place = place;
Jonathan Peyton6d88e042018-12-13 23:14:24 +00004892#if OMP_50_ENABLED
4893 if (__kmp_display_affinity && place != th->th.th_current_place &&
4894 team->t.t_display_affinity != 1) {
4895 team->t.t_display_affinity = 1;
4896 }
4897#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004898 s_count++;
4899
4900 if ((s_count == S) && rem && (gap_ct == gap)) {
4901 // do nothing, add an extra thread to place on next iteration
4902 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4903 // we added an extra thread to this place; move on to next place
4904 if (place == last_place) {
4905 place = first_place;
4906 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4907 place = 0;
4908 } else {
4909 place++;
4910 }
4911 s_count = 0;
4912 gap_ct = 1;
4913 rem--;
4914 } else if (s_count == S) { // place is full; don't add extra thread
4915 if (place == last_place) {
4916 place = first_place;
4917 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4918 place = 0;
4919 } else {
4920 place++;
4921 }
4922 gap_ct++;
4923 s_count = 0;
4924 }
4925
4926 KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
4927 "partition = [%d,%d]\n",
4928 __kmp_gtid_from_thread(team->t.t_threads[f]),
4929 team->t.t_id, f, th->th.th_new_place,
4930 th->th.th_first_place, th->th.th_last_place));
4931 }
4932 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
4933 }
4934 } break;
4935
4936 default:
4937 break;
4938 }
4939
4940 KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004941}
4942
Alp Toker98758b02014-03-02 04:12:06 +00004943#endif /* OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004944
Jonathan Peyton30419822017-05-12 18:01:32 +00004945/* allocate a new team data structure to use. take one off of the free pool if
4946 available */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004947kmp_team_t *
Jonathan Peyton30419822017-05-12 18:01:32 +00004948__kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00004949#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00004950 ompt_data_t ompt_parallel_data,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00004951#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00004952#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00004953 kmp_proc_bind_t new_proc_bind,
Jim Cownie5e8470a2013-09-27 10:38:44 +00004954#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004955 kmp_internal_control_t *new_icvs,
4956 int argc USE_NESTED_HOT_ARG(kmp_info_t *master)) {
4957 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team);
4958 int f;
4959 kmp_team_t *team;
4960 int use_hot_team = !root->r.r_active;
4961 int level = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004962
Jonathan Peyton30419822017-05-12 18:01:32 +00004963 KA_TRACE(20, ("__kmp_allocate_team: called\n"));
4964 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0);
4965 KMP_DEBUG_ASSERT(max_nproc >= new_nproc);
4966 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00004967
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004968#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004969 kmp_hot_team_ptr_t *hot_teams;
4970 if (master) {
4971 team = master->th.th_team;
4972 level = team->t.t_active_level;
4973 if (master->th.th_teams_microtask) { // in teams construct?
4974 if (master->th.th_teams_size.nteams > 1 &&
4975 ( // #teams > 1
4976 team->t.t_pkfn ==
4977 (microtask_t)__kmp_teams_master || // inner fork of the teams
4978 master->th.th_teams_level <
4979 team->t.t_level)) { // or nested parallel inside the teams
4980 ++level; // not increment if #teams==1, or for outer fork of the teams;
4981 // increment otherwise
4982 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004983 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004984 hot_teams = master->th.th_hot_teams;
4985 if (level < __kmp_hot_teams_max_level && hot_teams &&
4986 hot_teams[level]
4987 .hot_team) { // hot team has already been allocated for given level
4988 use_hot_team = 1;
4989 } else {
4990 use_hot_team = 0;
4991 }
4992 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004993#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004994 // Optimization to use a "hot" team
4995 if (use_hot_team && new_nproc > 1) {
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00004996 KMP_DEBUG_ASSERT(new_nproc <= max_nproc);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004997#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004998 team = hot_teams[level].hot_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004999#else
Jonathan Peyton30419822017-05-12 18:01:32 +00005000 team = root->r.r_hot_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005001#endif
5002#if KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00005003 if (__kmp_tasking_mode != tskm_immediate_exec) {
5004 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p "
5005 "task_team[1] = %p before reinit\n",
5006 team->t.t_task_team[0], team->t.t_task_team[1]));
5007 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005008#endif
5009
Jonathan Peyton30419822017-05-12 18:01:32 +00005010 // Has the number of threads changed?
5011 /* Let's assume the most common case is that the number of threads is
5012 unchanged, and put that case first. */
5013 if (team->t.t_nproc == new_nproc) { // Check changes in number of threads
5014 KA_TRACE(20, ("__kmp_allocate_team: reusing hot team\n"));
5015 // This case can mean that omp_set_num_threads() was called and the hot
Jonathan Peyton642688b2017-06-01 16:46:36 +00005016 // team size was already reduced, so we check the special flag
Jonathan Peyton30419822017-05-12 18:01:32 +00005017 if (team->t.t_size_changed == -1) {
5018 team->t.t_size_changed = 1;
5019 } else {
5020 KMP_CHECK_UPDATE(team->t.t_size_changed, 0);
5021 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005022
Jonathan Peyton30419822017-05-12 18:01:32 +00005023 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
5024 kmp_r_sched_t new_sched = new_icvs->sched;
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00005025 // set master's schedule as new run-time schedule
5026 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005027
Jonathan Peyton30419822017-05-12 18:01:32 +00005028 __kmp_reinitialize_team(team, new_icvs,
5029 root->r.r_uber_thread->th.th_ident);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005030
Jonathan Peyton30419822017-05-12 18:01:32 +00005031 KF_TRACE(10, ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0,
5032 team->t.t_threads[0], team));
5033 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005034
5035#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00005036#if KMP_AFFINITY_SUPPORTED
5037 if ((team->t.t_size_changed == 0) &&
5038 (team->t.t_proc_bind == new_proc_bind)) {
5039 if (new_proc_bind == proc_bind_spread) {
5040 __kmp_partition_places(
5041 team, 1); // add flag to update only master for spread
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005042 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005043 KA_TRACE(200, ("__kmp_allocate_team: reusing hot team #%d bindings: "
5044 "proc_bind = %d, partition = [%d,%d]\n",
5045 team->t.t_id, new_proc_bind, team->t.t_first_place,
5046 team->t.t_last_place));
5047 } else {
5048 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5049 __kmp_partition_places(team);
5050 }
5051#else
5052 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5053#endif /* KMP_AFFINITY_SUPPORTED */
5054#endif /* OMP_40_ENABLED */
5055 } else if (team->t.t_nproc > new_nproc) {
5056 KA_TRACE(20,
5057 ("__kmp_allocate_team: decreasing hot team thread count to %d\n",
5058 new_nproc));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005059
Jonathan Peyton30419822017-05-12 18:01:32 +00005060 team->t.t_size_changed = 1;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005061#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00005062 if (__kmp_hot_teams_mode == 0) {
5063 // AC: saved number of threads should correspond to team's value in this
5064 // mode, can be bigger in mode 1, when hot team has threads in reserve
5065 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc);
5066 hot_teams[level].hot_team_nth = new_nproc;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005067#endif // KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00005068 /* release the extra threads we don't need any more */
5069 for (f = new_nproc; f < team->t.t_nproc; f++) {
5070 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5071 if (__kmp_tasking_mode != tskm_immediate_exec) {
5072 // When decreasing team size, threads no longer in the team should
5073 // unref task team.
5074 team->t.t_threads[f]->th.th_task_team = NULL;
5075 }
5076 __kmp_free_thread(team->t.t_threads[f]);
5077 team->t.t_threads[f] = NULL;
5078 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005079#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00005080 } // (__kmp_hot_teams_mode == 0)
5081 else {
5082 // When keeping extra threads in team, switch threads to wait on own
5083 // b_go flag
5084 for (f = new_nproc; f < team->t.t_nproc; ++f) {
5085 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5086 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar;
5087 for (int b = 0; b < bs_last_barrier; ++b) {
5088 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) {
5089 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
Andrey Churbanovd6e1d7e2016-08-11 13:04:00 +00005090 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005091 KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0);
5092 }
5093 }
5094 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005095#endif // KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00005096 team->t.t_nproc = new_nproc;
5097 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00005098 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched);
Jonathan Peyton30419822017-05-12 18:01:32 +00005099 __kmp_reinitialize_team(team, new_icvs,
5100 root->r.r_uber_thread->th.th_ident);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005101
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00005102 // Update remaining threads
Jonathan Peyton30419822017-05-12 18:01:32 +00005103 for (f = 0; f < new_nproc; ++f) {
5104 team->t.t_threads[f]->th.th_team_nproc = new_nproc;
5105 }
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00005106
Jonathan Peyton30419822017-05-12 18:01:32 +00005107 // restore the current task state of the master thread: should be the
5108 // implicit task
5109 KF_TRACE(10, ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0,
5110 team->t.t_threads[0], team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005111
Jonathan Peyton30419822017-05-12 18:01:32 +00005112 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005113
5114#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00005115 for (f = 0; f < team->t.t_nproc; f++) {
5116 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
5117 team->t.t_threads[f]->th.th_team_nproc ==
5118 team->t.t_nproc);
5119 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005120#endif
5121
5122#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00005123 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5124#if KMP_AFFINITY_SUPPORTED
5125 __kmp_partition_places(team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005126#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00005127#endif
5128 } else { // team->t.t_nproc < new_nproc
Alp Toker98758b02014-03-02 04:12:06 +00005129#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00005130 kmp_affin_mask_t *old_mask;
5131 if (KMP_AFFINITY_CAPABLE()) {
5132 KMP_CPU_ALLOC(old_mask);
5133 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005134#endif
5135
Jonathan Peyton30419822017-05-12 18:01:32 +00005136 KA_TRACE(20,
5137 ("__kmp_allocate_team: increasing hot team thread count to %d\n",
5138 new_nproc));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005139
Jonathan Peyton30419822017-05-12 18:01:32 +00005140 team->t.t_size_changed = 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005141
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005142#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00005143 int avail_threads = hot_teams[level].hot_team_nth;
5144 if (new_nproc < avail_threads)
5145 avail_threads = new_nproc;
5146 kmp_info_t **other_threads = team->t.t_threads;
5147 for (f = team->t.t_nproc; f < avail_threads; ++f) {
5148 // Adjust barrier data of reserved threads (if any) of the team
5149 // Other data will be set in __kmp_initialize_info() below.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005150 int b;
Jonathan Peyton30419822017-05-12 18:01:32 +00005151 kmp_balign_t *balign = other_threads[f]->th.th_bar;
5152 for (b = 0; b < bs_last_barrier; ++b) {
5153 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5154 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00005155#if USE_DEBUGGER
Jonathan Peyton30419822017-05-12 18:01:32 +00005156 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00005157#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005158 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005159 }
5160 if (hot_teams[level].hot_team_nth >= new_nproc) {
5161 // we have all needed threads in reserve, no need to allocate any
5162 // this only possible in mode 1, cannot have reserved threads in mode 0
5163 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1);
5164 team->t.t_nproc = new_nproc; // just get reserved threads involved
5165 } else {
5166 // we may have some threads in reserve, but not enough
5167 team->t.t_nproc =
5168 hot_teams[level]
5169 .hot_team_nth; // get reserved threads involved if any
5170 hot_teams[level].hot_team_nth = new_nproc; // adjust hot team max size
5171#endif // KMP_NESTED_HOT_TEAMS
5172 if (team->t.t_max_nproc < new_nproc) {
5173 /* reallocate larger arrays */
5174 __kmp_reallocate_team_arrays(team, new_nproc);
5175 __kmp_reinitialize_team(team, new_icvs, NULL);
5176 }
5177
5178#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
5179 /* Temporarily set full mask for master thread before creation of
5180 workers. The reason is that workers inherit the affinity from master,
5181 so if a lot of workers are created on the single core quickly, they
5182 don't get a chance to set their own affinity for a long time. */
5183 __kmp_set_thread_affinity_mask_full_tmp(old_mask);
5184#endif
5185
5186 /* allocate new threads for the hot team */
5187 for (f = team->t.t_nproc; f < new_nproc; f++) {
5188 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f);
5189 KMP_DEBUG_ASSERT(new_worker);
5190 team->t.t_threads[f] = new_worker;
5191
5192 KA_TRACE(20,
5193 ("__kmp_allocate_team: team %d init T#%d arrived: "
5194 "join=%llu, plain=%llu\n",
5195 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f,
5196 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
5197 team->t.t_bar[bs_plain_barrier].b_arrived));
5198
5199 { // Initialize barrier data for new threads.
5200 int b;
5201 kmp_balign_t *balign = new_worker->th.th_bar;
5202 for (b = 0; b < bs_last_barrier; ++b) {
5203 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5204 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag !=
5205 KMP_BARRIER_PARENT_FLAG);
5206#if USE_DEBUGGER
5207 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5208#endif
5209 }
5210 }
5211 }
5212
5213#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
5214 if (KMP_AFFINITY_CAPABLE()) {
5215 /* Restore initial master thread's affinity mask */
5216 __kmp_set_system_affinity(old_mask, TRUE);
5217 KMP_CPU_FREE(old_mask);
5218 }
5219#endif
5220#if KMP_NESTED_HOT_TEAMS
5221 } // end of check of t_nproc vs. new_nproc vs. hot_team_nth
5222#endif // KMP_NESTED_HOT_TEAMS
5223 /* make sure everyone is syncronized */
5224 int old_nproc = team->t.t_nproc; // save old value and use to update only
5225 // new threads below
5226 __kmp_initialize_team(team, new_nproc, new_icvs,
5227 root->r.r_uber_thread->th.th_ident);
5228
5229 /* reinitialize the threads */
5230 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc);
5231 for (f = 0; f < team->t.t_nproc; ++f)
5232 __kmp_initialize_info(team->t.t_threads[f], team, f,
5233 __kmp_gtid_from_tid(f, team));
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00005234
Jonathan Peyton30419822017-05-12 18:01:32 +00005235 if (level) { // set th_task_state for new threads in nested hot team
5236 // __kmp_initialize_info() no longer zeroes th_task_state, so we should
5237 // only need to set the th_task_state for the new threads. th_task_state
5238 // for master thread will not be accurate until after this in
5239 // __kmp_fork_call(), so we look to the master's memo_stack to get the
5240 // correct value.
5241 for (f = old_nproc; f < team->t.t_nproc; ++f)
5242 team->t.t_threads[f]->th.th_task_state =
5243 team->t.t_threads[0]->th.th_task_state_memo_stack[level];
5244 } else { // set th_task_state for new threads in non-nested hot team
5245 int old_state =
5246 team->t.t_threads[0]->th.th_task_state; // copy master's state
5247 for (f = old_nproc; f < team->t.t_nproc; ++f)
5248 team->t.t_threads[f]->th.th_task_state = old_state;
5249 }
5250
5251#ifdef KMP_DEBUG
5252 for (f = 0; f < team->t.t_nproc; ++f) {
5253 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
5254 team->t.t_threads[f]->th.th_team_nproc ==
5255 team->t.t_nproc);
5256 }
5257#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005258
5259#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00005260 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5261#if KMP_AFFINITY_SUPPORTED
5262 __kmp_partition_places(team);
5263#endif
5264#endif
5265 } // Check changes in number of threads
5266
5267#if OMP_40_ENABLED
5268 kmp_info_t *master = team->t.t_threads[0];
5269 if (master->th.th_teams_microtask) {
5270 for (f = 1; f < new_nproc; ++f) {
5271 // propagate teams construct specific info to workers
5272 kmp_info_t *thr = team->t.t_threads[f];
5273 thr->th.th_teams_microtask = master->th.th_teams_microtask;
5274 thr->th.th_teams_level = master->th.th_teams_level;
5275 thr->th.th_teams_size = master->th.th_teams_size;
5276 }
5277 }
5278#endif /* OMP_40_ENABLED */
5279#if KMP_NESTED_HOT_TEAMS
5280 if (level) {
5281 // Sync barrier state for nested hot teams, not needed for outermost hot
5282 // team.
5283 for (f = 1; f < new_nproc; ++f) {
5284 kmp_info_t *thr = team->t.t_threads[f];
5285 int b;
5286 kmp_balign_t *balign = thr->th.th_bar;
5287 for (b = 0; b < bs_last_barrier; ++b) {
5288 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5289 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
5290#if USE_DEBUGGER
5291 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5292#endif
5293 }
5294 }
5295 }
5296#endif // KMP_NESTED_HOT_TEAMS
5297
5298 /* reallocate space for arguments if necessary */
5299 __kmp_alloc_argv_entries(argc, team, TRUE);
5300 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5301 // The hot team re-uses the previous task team,
5302 // if untouched during the previous release->gather phase.
5303
5304 KF_TRACE(10, (" hot_team = %p\n", team));
5305
5306#if KMP_DEBUG
5307 if (__kmp_tasking_mode != tskm_immediate_exec) {
5308 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p "
5309 "task_team[1] = %p after reinit\n",
5310 team->t.t_task_team[0], team->t.t_task_team[1]));
5311 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005312#endif
5313
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005314#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00005315 __ompt_team_assign_id(team, ompt_parallel_data);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005316#endif
5317
Jim Cownie5e8470a2013-09-27 10:38:44 +00005318 KMP_MB();
5319
Jim Cownie5e8470a2013-09-27 10:38:44 +00005320 return team;
Jonathan Peyton30419822017-05-12 18:01:32 +00005321 }
5322
5323 /* next, let's try to take one from the team pool */
5324 KMP_MB();
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00005325 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) {
Jonathan Peyton30419822017-05-12 18:01:32 +00005326 /* TODO: consider resizing undersized teams instead of reaping them, now
5327 that we have a resizing mechanism */
5328 if (team->t.t_max_nproc >= max_nproc) {
5329 /* take this team from the team pool */
5330 __kmp_team_pool = team->t.t_next_pool;
5331
5332 /* setup the team for fresh use */
5333 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5334
5335 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and "
5336 "task_team[1] %p to NULL\n",
5337 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5338 team->t.t_task_team[0] = NULL;
5339 team->t.t_task_team[1] = NULL;
5340
5341 /* reallocate space for arguments if necessary */
5342 __kmp_alloc_argv_entries(argc, team, TRUE);
5343 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5344
5345 KA_TRACE(
5346 20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5347 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5348 { // Initialize barrier data.
5349 int b;
5350 for (b = 0; b < bs_last_barrier; ++b) {
5351 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5352#if USE_DEBUGGER
5353 team->t.t_bar[b].b_master_arrived = 0;
5354 team->t.t_bar[b].b_team_arrived = 0;
5355#endif
5356 }
5357 }
5358
5359#if OMP_40_ENABLED
5360 team->t.t_proc_bind = new_proc_bind;
5361#endif
5362
5363 KA_TRACE(20, ("__kmp_allocate_team: using team from pool %d.\n",
5364 team->t.t_id));
5365
5366#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00005367 __ompt_team_assign_id(team, ompt_parallel_data);
Jonathan Peyton30419822017-05-12 18:01:32 +00005368#endif
5369
5370 KMP_MB();
5371
5372 return team;
5373 }
5374
Jonathan Peyton94a114f2017-10-20 19:30:57 +00005375 /* reap team if it is too small, then loop back and check the next one */
5376 // not sure if this is wise, but, will be redone during the hot-teams
5377 // rewrite.
5378 /* TODO: Use technique to find the right size hot-team, don't reap them */
Jonathan Peyton30419822017-05-12 18:01:32 +00005379 team = __kmp_reap_team(team);
5380 __kmp_team_pool = team;
5381 }
5382
5383 /* nothing available in the pool, no matter, make a new team! */
5384 KMP_MB();
5385 team = (kmp_team_t *)__kmp_allocate(sizeof(kmp_team_t));
5386
5387 /* and set it up */
5388 team->t.t_max_nproc = max_nproc;
5389 /* NOTE well, for some reason allocating one big buffer and dividing it up
5390 seems to really hurt performance a lot on the P4, so, let's not use this */
5391 __kmp_allocate_team_arrays(team, max_nproc);
5392
5393 KA_TRACE(20, ("__kmp_allocate_team: making a new team\n"));
5394 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5395
5396 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] "
5397 "%p to NULL\n",
5398 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5399 team->t.t_task_team[0] = NULL; // to be removed, as __kmp_allocate zeroes
5400 // memory, no need to duplicate
5401 team->t.t_task_team[1] = NULL; // to be removed, as __kmp_allocate zeroes
5402 // memory, no need to duplicate
5403
5404 if (__kmp_storage_map) {
5405 __kmp_print_team_storage_map("team", team, team->t.t_id, new_nproc);
5406 }
5407
5408 /* allocate space for arguments */
5409 __kmp_alloc_argv_entries(argc, team, FALSE);
5410 team->t.t_argc = argc;
5411
5412 KA_TRACE(20,
5413 ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5414 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5415 { // Initialize barrier data.
5416 int b;
5417 for (b = 0; b < bs_last_barrier; ++b) {
5418 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5419#if USE_DEBUGGER
5420 team->t.t_bar[b].b_master_arrived = 0;
5421 team->t.t_bar[b].b_team_arrived = 0;
5422#endif
5423 }
5424 }
5425
5426#if OMP_40_ENABLED
5427 team->t.t_proc_bind = new_proc_bind;
5428#endif
5429
5430#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00005431 __ompt_team_assign_id(team, ompt_parallel_data);
Jonathan Peyton30419822017-05-12 18:01:32 +00005432 team->t.ompt_serialized_team_info = NULL;
5433#endif
5434
5435 KMP_MB();
5436
5437 KA_TRACE(20, ("__kmp_allocate_team: done creating a new team %d.\n",
5438 team->t.t_id));
5439
5440 return team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005441}
5442
5443/* TODO implement hot-teams at all levels */
5444/* TODO implement lazy thread release on demand (disband request) */
5445
5446/* free the team. return it to the team pool. release all the threads
5447 * associated with it */
Jonathan Peyton30419822017-05-12 18:01:32 +00005448void __kmp_free_team(kmp_root_t *root,
5449 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) {
5450 int f;
5451 KA_TRACE(20, ("__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(),
5452 team->t.t_id));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005453
Jonathan Peyton30419822017-05-12 18:01:32 +00005454 /* verify state */
5455 KMP_DEBUG_ASSERT(root);
5456 KMP_DEBUG_ASSERT(team);
5457 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc);
5458 KMP_DEBUG_ASSERT(team->t.t_threads);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005459
Jonathan Peyton30419822017-05-12 18:01:32 +00005460 int use_hot_team = team == root->r.r_hot_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005461#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00005462 int level;
5463 kmp_hot_team_ptr_t *hot_teams;
5464 if (master) {
5465 level = team->t.t_active_level - 1;
5466 if (master->th.th_teams_microtask) { // in teams construct?
5467 if (master->th.th_teams_size.nteams > 1) {
5468 ++level; // level was not increased in teams construct for
5469 // team_of_masters
5470 }
5471 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
5472 master->th.th_teams_level == team->t.t_level) {
5473 ++level; // level was not increased in teams construct for
5474 // team_of_workers before the parallel
5475 } // team->t.t_level will be increased inside parallel
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005476 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005477 hot_teams = master->th.th_hot_teams;
5478 if (level < __kmp_hot_teams_max_level) {
5479 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team);
5480 use_hot_team = 1;
5481 }
5482 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005483#endif // KMP_NESTED_HOT_TEAMS
5484
Jonathan Peyton30419822017-05-12 18:01:32 +00005485 /* team is done working */
5486 TCW_SYNC_PTR(team->t.t_pkfn,
5487 NULL); // Important for Debugging Support Library.
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00005488#if KMP_OS_WINDOWS
Jonathan Peyton30419822017-05-12 18:01:32 +00005489 team->t.t_copyin_counter = 0; // init counter for possible reuse
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00005490#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00005491 // Do not reset pointer to parent team to NULL for hot teams.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005492
Jonathan Peyton30419822017-05-12 18:01:32 +00005493 /* if we are non-hot team, release our threads */
5494 if (!use_hot_team) {
5495 if (__kmp_tasking_mode != tskm_immediate_exec) {
5496 // Wait for threads to reach reapable state
5497 for (f = 1; f < team->t.t_nproc; ++f) {
5498 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5499 kmp_info_t *th = team->t.t_threads[f];
5500 volatile kmp_uint32 *state = &th->th.th_reap_state;
5501 while (*state != KMP_SAFE_TO_REAP) {
Andrey Churbanov581490e2017-02-06 18:53:32 +00005502#if KMP_OS_WINDOWS
Jonathan Peyton30419822017-05-12 18:01:32 +00005503 // On Windows a thread can be killed at any time, check this
5504 DWORD ecode;
5505 if (!__kmp_is_thread_alive(th, &ecode)) {
5506 *state = KMP_SAFE_TO_REAP; // reset the flag for dead thread
5507 break;
5508 }
Andrey Churbanov581490e2017-02-06 18:53:32 +00005509#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00005510 // first check if thread is sleeping
5511 kmp_flag_64 fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th);
5512 if (fl.is_sleeping())
5513 fl.resume(__kmp_gtid_from_thread(th));
5514 KMP_CPU_PAUSE();
5515 }
5516 }
Andrey Churbanov581490e2017-02-06 18:53:32 +00005517
Jonathan Peyton30419822017-05-12 18:01:32 +00005518 // Delete task teams
5519 int tt_idx;
5520 for (tt_idx = 0; tt_idx < 2; ++tt_idx) {
5521 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
5522 if (task_team != NULL) {
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00005523 for (f = 0; f < team->t.t_nproc; ++f) { // threads unref task teams
5524 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
Jonathan Peyton30419822017-05-12 18:01:32 +00005525 team->t.t_threads[f]->th.th_task_team = NULL;
5526 }
5527 KA_TRACE(
5528 20,
5529 ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n",
5530 __kmp_get_gtid(), task_team, team->t.t_id));
Jonathan Peytonbaaccfa2015-11-16 22:48:41 +00005531#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00005532 __kmp_free_task_team(master, task_team);
Jonathan Peytonbaaccfa2015-11-16 22:48:41 +00005533#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00005534 team->t.t_task_team[tt_idx] = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005535 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005536 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005537 }
5538
Jonathan Peyton30419822017-05-12 18:01:32 +00005539 // Reset pointer to parent team only for non-hot teams.
5540 team->t.t_parent = NULL;
5541 team->t.t_level = 0;
5542 team->t.t_active_level = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005543
Jonathan Peyton30419822017-05-12 18:01:32 +00005544 /* free the worker threads */
5545 for (f = 1; f < team->t.t_nproc; ++f) {
5546 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5547 __kmp_free_thread(team->t.t_threads[f]);
5548 team->t.t_threads[f] = NULL;
5549 }
5550
5551 /* put the team back in the team pool */
5552 /* TODO limit size of team pool, call reap_team if pool too large */
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00005553 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool);
Jonathan Peyton30419822017-05-12 18:01:32 +00005554 __kmp_team_pool = (volatile kmp_team_t *)team;
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00005555 } else { // Check if team was created for the masters in a teams construct
5556 // See if first worker is a CG root
5557 KMP_DEBUG_ASSERT(team->t.t_threads[1] &&
5558 team->t.t_threads[1]->th.th_cg_roots);
5559 if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) {
5560 // Clean up the CG root nodes on workers so that this team can be re-used
5561 for (f = 1; f < team->t.t_nproc; ++f) {
5562 kmp_info_t *thr = team->t.t_threads[f];
5563 KMP_DEBUG_ASSERT(thr && thr->th.th_cg_roots &&
5564 thr->th.th_cg_roots->cg_root == thr);
5565 // Pop current CG root off list
5566 kmp_cg_root_t *tmp = thr->th.th_cg_roots;
5567 thr->th.th_cg_roots = tmp->up;
5568 KA_TRACE(100, ("__kmp_free_team: Thread %p popping node %p and moving"
5569 " up to node %p. cg_nthreads was %d\n",
5570 thr, tmp, thr->th.th_cg_roots, tmp->cg_nthreads));
5571 __kmp_free(tmp);
5572 // Restore current task's thread_limit from CG root
5573 if (thr->th.th_cg_roots)
5574 thr->th.th_current_task->td_icvs.thread_limit =
5575 thr->th.th_cg_roots->cg_thread_limit;
5576 }
5577 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005578 }
5579
5580 KMP_MB();
5581}
Jim Cownie5e8470a2013-09-27 10:38:44 +00005582
5583/* reap the team. destroy it, reclaim all its resources and free its memory */
Jonathan Peyton30419822017-05-12 18:01:32 +00005584kmp_team_t *__kmp_reap_team(kmp_team_t *team) {
5585 kmp_team_t *next_pool = team->t.t_next_pool;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005586
Jonathan Peyton30419822017-05-12 18:01:32 +00005587 KMP_DEBUG_ASSERT(team);
5588 KMP_DEBUG_ASSERT(team->t.t_dispatch);
5589 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
5590 KMP_DEBUG_ASSERT(team->t.t_threads);
5591 KMP_DEBUG_ASSERT(team->t.t_argv);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005592
Jonathan Peyton30419822017-05-12 18:01:32 +00005593 /* TODO clean the threads that are a part of this? */
Jim Cownie5e8470a2013-09-27 10:38:44 +00005594
Jonathan Peyton30419822017-05-12 18:01:32 +00005595 /* free stuff */
5596 __kmp_free_team_arrays(team);
5597 if (team->t.t_argv != &team->t.t_inline_argv[0])
5598 __kmp_free((void *)team->t.t_argv);
5599 __kmp_free(team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005600
Jonathan Peyton30419822017-05-12 18:01:32 +00005601 KMP_MB();
5602 return next_pool;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005603}
5604
Jim Cownie5e8470a2013-09-27 10:38:44 +00005605// Free the thread. Don't reap it, just place it on the pool of available
5606// threads.
5607//
5608// Changes for Quad issue 527845: We need a predictable OMP tid <-> gtid
5609// binding for the affinity mechanism to be useful.
5610//
5611// Now, we always keep the free list (__kmp_thread_pool) sorted by gtid.
5612// However, we want to avoid a potential performance problem by always
5613// scanning through the list to find the correct point at which to insert
5614// the thread (potential N**2 behavior). To do this we keep track of the
5615// last place a thread struct was inserted (__kmp_thread_pool_insert_pt).
5616// With single-level parallelism, threads will always be added to the tail
5617// of the list, kept track of by __kmp_thread_pool_insert_pt. With nested
5618// parallelism, all bets are off and we may need to scan through the entire
5619// free list.
5620//
5621// This change also has a potentially large performance benefit, for some
5622// applications. Previously, as threads were freed from the hot team, they
5623// would be placed back on the free list in inverse order. If the hot team
5624// grew back to it's original size, then the freed thread would be placed
5625// back on the hot team in reverse order. This could cause bad cache
5626// locality problems on programs where the size of the hot team regularly
5627// grew and shrunk.
5628//
5629// Now, for single-level parallelism, the OMP tid is alway == gtid.
Jonathan Peyton30419822017-05-12 18:01:32 +00005630void __kmp_free_thread(kmp_info_t *this_th) {
5631 int gtid;
5632 kmp_info_t **scan;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005633
Jonathan Peyton30419822017-05-12 18:01:32 +00005634 KA_TRACE(20, ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n",
5635 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005636
Jonathan Peyton30419822017-05-12 18:01:32 +00005637 KMP_DEBUG_ASSERT(this_th);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005638
Jonathan Peyton30419822017-05-12 18:01:32 +00005639 // When moving thread to pool, switch thread to wait on own b_go flag, and
5640 // uninitialized (NULL team).
5641 int b;
5642 kmp_balign_t *balign = this_th->th.th_bar;
5643 for (b = 0; b < bs_last_barrier; ++b) {
5644 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG)
5645 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
5646 balign[b].bb.team = NULL;
5647 balign[b].bb.leaf_kids = 0;
5648 }
5649 this_th->th.th_task_state = 0;
Andrey Churbanov3336aa02018-03-19 18:05:15 +00005650 this_th->th.th_reap_state = KMP_SAFE_TO_REAP;
Jonathan Peyton30419822017-05-12 18:01:32 +00005651
5652 /* put thread back on the free pool */
5653 TCW_PTR(this_th->th.th_team, NULL);
5654 TCW_PTR(this_th->th.th_root, NULL);
5655 TCW_PTR(this_th->th.th_dispatch, NULL); /* NOT NEEDED */
5656
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00005657 while (this_th->th.th_cg_roots) {
5658 this_th->th.th_cg_roots->cg_nthreads--;
5659 KA_TRACE(100, ("__kmp_free_thread: Thread %p decrement cg_nthreads on node"
5660 " %p of thread %p to %d\n",
5661 this_th, this_th->th.th_cg_roots,
5662 this_th->th.th_cg_roots->cg_root,
5663 this_th->th.th_cg_roots->cg_nthreads));
5664 kmp_cg_root_t *tmp = this_th->th.th_cg_roots;
5665 if (tmp->cg_root == this_th) { // Thread is a cg_root
5666 KMP_DEBUG_ASSERT(tmp->cg_nthreads == 0);
5667 KA_TRACE(
5668 5, ("__kmp_free_thread: Thread %p freeing node %p\n", this_th, tmp));
5669 this_th->th.th_cg_roots = tmp->up;
5670 __kmp_free(tmp);
5671 } else { // Worker thread
5672 this_th->th.th_cg_roots = NULL;
5673 break;
5674 }
5675 }
5676
Jonathan Peytonbff8ded2018-01-10 18:24:09 +00005677 /* If the implicit task assigned to this thread can be used by other threads
5678 * -> multiple threads can share the data and try to free the task at
5679 * __kmp_reap_thread at exit. This duplicate use of the task data can happen
5680 * with higher probability when hot team is disabled but can occurs even when
5681 * the hot team is enabled */
5682 __kmp_free_implicit_task(this_th);
5683 this_th->th.th_current_task = NULL;
5684
Jonathan Peyton30419822017-05-12 18:01:32 +00005685 // If the __kmp_thread_pool_insert_pt is already past the new insert
5686 // point, then we need to re-scan the entire list.
5687 gtid = this_th->th.th_info.ds.ds_gtid;
5688 if (__kmp_thread_pool_insert_pt != NULL) {
5689 KMP_DEBUG_ASSERT(__kmp_thread_pool != NULL);
5690 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) {
5691 __kmp_thread_pool_insert_pt = NULL;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005692 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005693 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005694
Jonathan Peyton30419822017-05-12 18:01:32 +00005695 // Scan down the list to find the place to insert the thread.
5696 // scan is the address of a link in the list, possibly the address of
5697 // __kmp_thread_pool itself.
5698 //
5699 // In the absence of nested parallism, the for loop will have 0 iterations.
5700 if (__kmp_thread_pool_insert_pt != NULL) {
5701 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool);
5702 } else {
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00005703 scan = CCAST(kmp_info_t **, &__kmp_thread_pool);
Jonathan Peyton30419822017-05-12 18:01:32 +00005704 }
5705 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid);
5706 scan = &((*scan)->th.th_next_pool))
5707 ;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005708
Jonathan Peyton30419822017-05-12 18:01:32 +00005709 // Insert the new element on the list, and set __kmp_thread_pool_insert_pt
5710 // to its address.
5711 TCW_PTR(this_th->th.th_next_pool, *scan);
5712 __kmp_thread_pool_insert_pt = *scan = this_th;
5713 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) ||
5714 (this_th->th.th_info.ds.ds_gtid <
5715 this_th->th.th_next_pool->th.th_info.ds.ds_gtid));
5716 TCW_4(this_th->th.th_in_pool, TRUE);
Jonathan Peytonfeac33e2019-04-08 17:50:02 +00005717 __kmp_suspend_initialize_thread(this_th);
5718 __kmp_lock_suspend_mx(this_th);
5719 if (this_th->th.th_active == TRUE) {
5720 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
5721 this_th->th.th_active_in_pool = TRUE;
5722 }
5723#if KMP_DEBUG
5724 else {
5725 KMP_DEBUG_ASSERT(this_th->th.th_active_in_pool == FALSE);
5726 }
5727#endif
5728 __kmp_unlock_suspend_mx(this_th);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005729
Jonathan Peyton30419822017-05-12 18:01:32 +00005730 TCW_4(__kmp_nth, __kmp_nth - 1);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005731
5732#ifdef KMP_ADJUST_BLOCKTIME
Jonathan Peyton30419822017-05-12 18:01:32 +00005733 /* Adjust blocktime back to user setting or default if necessary */
5734 /* Middle initialization might never have occurred */
5735 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5736 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
5737 if (__kmp_nth <= __kmp_avail_proc) {
5738 __kmp_zero_bt = FALSE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005739 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005740 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005741#endif /* KMP_ADJUST_BLOCKTIME */
5742
Jonathan Peyton30419822017-05-12 18:01:32 +00005743 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00005744}
5745
Jim Cownie5e8470a2013-09-27 10:38:44 +00005746/* ------------------------------------------------------------------------ */
5747
Jonathan Peyton30419822017-05-12 18:01:32 +00005748void *__kmp_launch_thread(kmp_info_t *this_thr) {
5749 int gtid = this_thr->th.th_info.ds.ds_gtid;
5750 /* void *stack_data;*/
5751 kmp_team_t *(*volatile pteam);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005752
Jonathan Peyton30419822017-05-12 18:01:32 +00005753 KMP_MB();
5754 KA_TRACE(10, ("__kmp_launch_thread: T#%d start\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005755
Jonathan Peyton30419822017-05-12 18:01:32 +00005756 if (__kmp_env_consistency_check) {
5757 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid); // ATT: Memory leak?
5758 }
5759
5760#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00005761 ompt_data_t *thread_data;
5762 if (ompt_enabled.enabled) {
5763 thread_data = &(this_thr->th.ompt_thread_info.thread_data);
Jonathan Peyton3574f282018-10-04 14:57:04 +00005764 *thread_data = ompt_data_none;
Joachim Protze82e94a52017-11-01 10:08:30 +00005765
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00005766 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00005767 this_thr->th.ompt_thread_info.wait_id = 0;
Joachim Protze82e94a52017-11-01 10:08:30 +00005768 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0);
5769 if (ompt_enabled.ompt_callback_thread_begin) {
5770 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)(
5771 ompt_thread_worker, thread_data);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005772 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005773 }
5774#endif
5775
Joachim Protze82e94a52017-11-01 10:08:30 +00005776#if OMPT_SUPPORT
5777 if (ompt_enabled.enabled) {
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00005778 this_thr->th.ompt_thread_info.state = ompt_state_idle;
Joachim Protze82e94a52017-11-01 10:08:30 +00005779 }
5780#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00005781 /* This is the place where threads wait for work */
5782 while (!TCR_4(__kmp_global.g.g_done)) {
5783 KMP_DEBUG_ASSERT(this_thr == __kmp_threads[gtid]);
5784 KMP_MB();
5785
5786 /* wait for work to do */
5787 KA_TRACE(20, ("__kmp_launch_thread: T#%d waiting for work\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005788
Jonathan Peyton30419822017-05-12 18:01:32 +00005789 /* No tid yet since not part of a team */
5790 __kmp_fork_barrier(gtid, KMP_GTID_DNE);
5791
5792#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00005793 if (ompt_enabled.enabled) {
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00005794 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00005795 }
5796#endif
5797
5798 pteam = (kmp_team_t * (*))(&this_thr->th.th_team);
5799
5800 /* have we been allocated? */
5801 if (TCR_SYNC_PTR(*pteam) && !TCR_4(__kmp_global.g.g_done)) {
Jonathan Peyton30419822017-05-12 18:01:32 +00005802 /* we were just woken up, so run our new task */
5803 if (TCR_SYNC_PTR((*pteam)->t.t_pkfn) != NULL) {
5804 int rc;
5805 KA_TRACE(20,
5806 ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n",
5807 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5808 (*pteam)->t.t_pkfn));
5809
5810 updateHWFPControl(*pteam);
5811
5812#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00005813 if (ompt_enabled.enabled) {
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00005814 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
Jonathan Peyton30419822017-05-12 18:01:32 +00005815 }
5816#endif
5817
Jonathan Peytonf0682ac2018-07-30 17:41:08 +00005818 rc = (*pteam)->t.t_invoke(gtid);
Jonathan Peyton30419822017-05-12 18:01:32 +00005819 KMP_ASSERT(rc);
5820
Jim Cownie5e8470a2013-09-27 10:38:44 +00005821 KMP_MB();
Jonathan Peyton30419822017-05-12 18:01:32 +00005822 KA_TRACE(20, ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n",
5823 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5824 (*pteam)->t.t_pkfn));
5825 }
Joachim Protze82e94a52017-11-01 10:08:30 +00005826#if OMPT_SUPPORT
5827 if (ompt_enabled.enabled) {
5828 /* no frame set while outside task */
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00005829 __ompt_get_task_info_object(0)->frame.exit_frame = ompt_data_none;
Joachim Protze82e94a52017-11-01 10:08:30 +00005830
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00005831 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00005832 }
5833#endif
Joachim Protze82e94a52017-11-01 10:08:30 +00005834 /* join barrier after parallel region */
5835 __kmp_join_barrier(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005836 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005837 }
5838 TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005839
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005840#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00005841 if (ompt_enabled.ompt_callback_thread_end) {
5842 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(thread_data);
Jonathan Peyton30419822017-05-12 18:01:32 +00005843 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005844#endif
5845
Jonathan Peyton30419822017-05-12 18:01:32 +00005846 this_thr->th.th_task_team = NULL;
5847 /* run the destructors for the threadprivate data for this thread */
5848 __kmp_common_destroy_gtid(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005849
Jonathan Peyton30419822017-05-12 18:01:32 +00005850 KA_TRACE(10, ("__kmp_launch_thread: T#%d done\n", gtid));
5851 KMP_MB();
5852 return this_thr;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005853}
5854
5855/* ------------------------------------------------------------------------ */
Jim Cownie5e8470a2013-09-27 10:38:44 +00005856
Jonathan Peyton30419822017-05-12 18:01:32 +00005857void __kmp_internal_end_dest(void *specific_gtid) {
5858#if KMP_COMPILER_ICC
5859#pragma warning(push)
5860#pragma warning(disable : 810) // conversion from "void *" to "int" may lose
5861// significant bits
5862#endif
5863 // Make sure no significant bits are lost
5864 int gtid = (kmp_intptr_t)specific_gtid - 1;
5865#if KMP_COMPILER_ICC
5866#pragma warning(pop)
5867#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005868
Jonathan Peyton30419822017-05-12 18:01:32 +00005869 KA_TRACE(30, ("__kmp_internal_end_dest: T#%d\n", gtid));
5870 /* NOTE: the gtid is stored as gitd+1 in the thread-local-storage
5871 * this is because 0 is reserved for the nothing-stored case */
Jim Cownie5e8470a2013-09-27 10:38:44 +00005872
Jonathan Peyton30419822017-05-12 18:01:32 +00005873 /* josh: One reason for setting the gtid specific data even when it is being
5874 destroyed by pthread is to allow gtid lookup through thread specific data
5875 (__kmp_gtid_get_specific). Some of the code, especially stat code,
5876 that gets executed in the call to __kmp_internal_end_thread, actually
5877 gets the gtid through the thread specific data. Setting it here seems
5878 rather inelegant and perhaps wrong, but allows __kmp_internal_end_thread
5879 to run smoothly.
5880 todo: get rid of this after we remove the dependence on
5881 __kmp_gtid_get_specific */
5882 if (gtid >= 0 && KMP_UBER_GTID(gtid))
5883 __kmp_gtid_set_specific(gtid);
5884#ifdef KMP_TDATA_GTID
5885 __kmp_gtid = gtid;
5886#endif
5887 __kmp_internal_end_thread(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005888}
5889
Jonathan Peyton99016992015-05-26 17:32:53 +00005890#if KMP_OS_UNIX && KMP_DYNAMIC_LIB
Jim Cownie5e8470a2013-09-27 10:38:44 +00005891
Jonathan Peyton30419822017-05-12 18:01:32 +00005892// 2009-09-08 (lev): It looks the destructor does not work. In simple test cases
5893// destructors work perfectly, but in real libomp.so I have no evidence it is
5894// ever called. However, -fini linker option in makefile.mk works fine.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005895
Jonathan Peyton30419822017-05-12 18:01:32 +00005896__attribute__((destructor)) void __kmp_internal_end_dtor(void) {
5897 __kmp_internal_end_atexit();
Jim Cownie5e8470a2013-09-27 10:38:44 +00005898}
5899
Jonathan Peyton30419822017-05-12 18:01:32 +00005900void __kmp_internal_end_fini(void) { __kmp_internal_end_atexit(); }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005901
5902#endif
5903
Jonathan Peyton30419822017-05-12 18:01:32 +00005904/* [Windows] josh: when the atexit handler is called, there may still be more
5905 than one thread alive */
5906void __kmp_internal_end_atexit(void) {
5907 KA_TRACE(30, ("__kmp_internal_end_atexit\n"));
5908 /* [Windows]
5909 josh: ideally, we want to completely shutdown the library in this atexit
5910 handler, but stat code that depends on thread specific data for gtid fails
5911 because that data becomes unavailable at some point during the shutdown, so
5912 we call __kmp_internal_end_thread instead. We should eventually remove the
5913 dependency on __kmp_get_specific_gtid in the stat code and use
5914 __kmp_internal_end_library to cleanly shutdown the library.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005915
Jonathan Peyton30419822017-05-12 18:01:32 +00005916 // TODO: Can some of this comment about GVS be removed?
5917 I suspect that the offending stat code is executed when the calling thread
5918 tries to clean up a dead root thread's data structures, resulting in GVS
5919 code trying to close the GVS structures for that thread, but since the stat
5920 code uses __kmp_get_specific_gtid to get the gtid with the assumption that
5921 the calling thread is cleaning up itself instead of another thread, it get
5922 confused. This happens because allowing a thread to unregister and cleanup
5923 another thread is a recent modification for addressing an issue.
5924 Based on the current design (20050722), a thread may end up
5925 trying to unregister another thread only if thread death does not trigger
5926 the calling of __kmp_internal_end_thread. For Linux* OS, there is the
5927 thread specific data destructor function to detect thread death. For
5928 Windows dynamic, there is DllMain(THREAD_DETACH). For Windows static, there
5929 is nothing. Thus, the workaround is applicable only for Windows static
5930 stat library. */
5931 __kmp_internal_end_library(-1);
5932#if KMP_OS_WINDOWS
5933 __kmp_close_console();
5934#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005935}
5936
Jonathan Peyton30419822017-05-12 18:01:32 +00005937static void __kmp_reap_thread(kmp_info_t *thread, int is_root) {
5938 // It is assumed __kmp_forkjoin_lock is acquired.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005939
Jonathan Peyton30419822017-05-12 18:01:32 +00005940 int gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005941
Jonathan Peyton30419822017-05-12 18:01:32 +00005942 KMP_DEBUG_ASSERT(thread != NULL);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005943
Jonathan Peyton30419822017-05-12 18:01:32 +00005944 gtid = thread->th.th_info.ds.ds_gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005945
Jonathan Peyton30419822017-05-12 18:01:32 +00005946 if (!is_root) {
Jonathan Peyton30419822017-05-12 18:01:32 +00005947 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
5948 /* Assume the threads are at the fork barrier here */
5949 KA_TRACE(
5950 20, ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n",
5951 gtid));
5952 /* Need release fence here to prevent seg faults for tree forkjoin barrier
5953 * (GEH) */
5954 ANNOTATE_HAPPENS_BEFORE(thread);
5955 kmp_flag_64 flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread);
5956 __kmp_release_64(&flag);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005957 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005958
Jonathan Peyton30419822017-05-12 18:01:32 +00005959 // Terminate OS thread.
5960 __kmp_reap_worker(thread);
Jonathan Peyton7ca7ef02016-11-21 16:18:57 +00005961
Jonathan Peyton30419822017-05-12 18:01:32 +00005962 // The thread was killed asynchronously. If it was actively
5963 // spinning in the thread pool, decrement the global count.
5964 //
5965 // There is a small timing hole here - if the worker thread was just waking
5966 // up after sleeping in the pool, had reset it's th_active_in_pool flag but
5967 // not decremented the global counter __kmp_thread_pool_active_nth yet, then
5968 // the global counter might not get updated.
5969 //
5970 // Currently, this can only happen as the library is unloaded,
5971 // so there are no harmful side effects.
5972 if (thread->th.th_active_in_pool) {
5973 thread->th.th_active_in_pool = FALSE;
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00005974 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
5975 KMP_DEBUG_ASSERT(__kmp_thread_pool_active_nth >= 0);
Jonathan Peyton30419822017-05-12 18:01:32 +00005976 }
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005977 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005978
Jonathan Peyton30419822017-05-12 18:01:32 +00005979 __kmp_free_implicit_task(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005980
Jonathan Peyton30419822017-05-12 18:01:32 +00005981// Free the fast memory for tasking
5982#if USE_FAST_MEMORY
5983 __kmp_free_fast_memory(thread);
5984#endif /* USE_FAST_MEMORY */
5985
5986 __kmp_suspend_uninitialize_thread(thread);
5987
5988 KMP_DEBUG_ASSERT(__kmp_threads[gtid] == thread);
5989 TCW_SYNC_PTR(__kmp_threads[gtid], NULL);
5990
5991 --__kmp_all_nth;
5992// __kmp_nth was decremented when thread is added to the pool.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005993
5994#ifdef KMP_ADJUST_BLOCKTIME
Jonathan Peyton30419822017-05-12 18:01:32 +00005995 /* Adjust blocktime back to user setting or default if necessary */
5996 /* Middle initialization might never have occurred */
5997 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5998 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
5999 if (__kmp_nth <= __kmp_avail_proc) {
6000 __kmp_zero_bt = FALSE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006001 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006002 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006003#endif /* KMP_ADJUST_BLOCKTIME */
6004
Jonathan Peyton30419822017-05-12 18:01:32 +00006005 /* free the memory being used */
6006 if (__kmp_env_consistency_check) {
6007 if (thread->th.th_cons) {
6008 __kmp_free_cons_stack(thread->th.th_cons);
6009 thread->th.th_cons = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006010 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006011 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006012
Jonathan Peyton30419822017-05-12 18:01:32 +00006013 if (thread->th.th_pri_common != NULL) {
6014 __kmp_free(thread->th.th_pri_common);
6015 thread->th.th_pri_common = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006016 }
Andrey Churbanov6d224db2015-02-10 18:37:43 +00006017
Jonathan Peyton30419822017-05-12 18:01:32 +00006018 if (thread->th.th_task_state_memo_stack != NULL) {
6019 __kmp_free(thread->th.th_task_state_memo_stack);
6020 thread->th.th_task_state_memo_stack = NULL;
6021 }
6022
6023#if KMP_USE_BGET
6024 if (thread->th.th_local.bget_data != NULL) {
6025 __kmp_finalize_bget(thread);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006026 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006027#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006028
Alp Toker98758b02014-03-02 04:12:06 +00006029#if KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00006030 if (thread->th.th_affin_mask != NULL) {
6031 KMP_CPU_FREE(thread->th.th_affin_mask);
6032 thread->th.th_affin_mask = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006033 }
Alp Toker98758b02014-03-02 04:12:06 +00006034#endif /* KMP_AFFINITY_SUPPORTED */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006035
Jonathan Peytonf6399362018-07-09 17:51:13 +00006036#if KMP_USE_HIER_SCHED
6037 if (thread->th.th_hier_bar_data != NULL) {
6038 __kmp_free(thread->th.th_hier_bar_data);
6039 thread->th.th_hier_bar_data = NULL;
6040 }
6041#endif
6042
Jonathan Peyton30419822017-05-12 18:01:32 +00006043 __kmp_reap_team(thread->th.th_serial_team);
6044 thread->th.th_serial_team = NULL;
6045 __kmp_free(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006046
Jonathan Peyton30419822017-05-12 18:01:32 +00006047 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006048
6049} // __kmp_reap_thread
6050
Jonathan Peyton30419822017-05-12 18:01:32 +00006051static void __kmp_internal_end(void) {
6052 int i;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006053
Jonathan Peyton30419822017-05-12 18:01:32 +00006054 /* First, unregister the library */
6055 __kmp_unregister_library();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006056
Jonathan Peyton30419822017-05-12 18:01:32 +00006057#if KMP_OS_WINDOWS
6058 /* In Win static library, we can't tell when a root actually dies, so we
6059 reclaim the data structures for any root threads that have died but not
6060 unregistered themselves, in order to shut down cleanly.
6061 In Win dynamic library we also can't tell when a thread dies. */
6062 __kmp_reclaim_dead_roots(); // AC: moved here to always clean resources of
6063// dead roots
Jonathan Peytonb66d1aa2016-09-27 17:11:17 +00006064#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006065
Jonathan Peyton30419822017-05-12 18:01:32 +00006066 for (i = 0; i < __kmp_threads_capacity; i++)
6067 if (__kmp_root[i])
6068 if (__kmp_root[i]->r.r_active)
6069 break;
6070 KMP_MB(); /* Flush all pending memory write invalidates. */
6071 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6072
6073 if (i < __kmp_threads_capacity) {
6074#if KMP_USE_MONITOR
6075 // 2009-09-08 (lev): Other alive roots found. Why do we kill the monitor??
6076 KMP_MB(); /* Flush all pending memory write invalidates. */
6077
Jonathan Peyton94a114f2017-10-20 19:30:57 +00006078 // Need to check that monitor was initialized before reaping it. If we are
6079 // called form __kmp_atfork_child (which sets __kmp_init_parallel = 0), then
6080 // __kmp_monitor will appear to contain valid data, but it is only valid in
6081 // the parent process, not the child.
Jonathan Peyton30419822017-05-12 18:01:32 +00006082 // New behavior (201008): instead of keying off of the flag
6083 // __kmp_init_parallel, the monitor thread creation is keyed off
6084 // of the new flag __kmp_init_monitor.
6085 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
6086 if (TCR_4(__kmp_init_monitor)) {
6087 __kmp_reap_monitor(&__kmp_monitor);
6088 TCW_4(__kmp_init_monitor, 0);
6089 }
6090 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
6091 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n"));
6092#endif // KMP_USE_MONITOR
6093 } else {
6094/* TODO move this to cleanup code */
6095#ifdef KMP_DEBUG
6096 /* make sure that everything has properly ended */
6097 for (i = 0; i < __kmp_threads_capacity; i++) {
6098 if (__kmp_root[i]) {
6099 // KMP_ASSERT( ! KMP_UBER_GTID( i ) ); // AC:
6100 // there can be uber threads alive here
6101 KMP_ASSERT(!__kmp_root[i]->r.r_active); // TODO: can they be active?
6102 }
6103 }
6104#endif
6105
6106 KMP_MB();
6107
6108 // Reap the worker threads.
6109 // This is valid for now, but be careful if threads are reaped sooner.
6110 while (__kmp_thread_pool != NULL) { // Loop thru all the thread in the pool.
6111 // Get the next thread from the pool.
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00006112 kmp_info_t *thread = CCAST(kmp_info_t *, __kmp_thread_pool);
Jonathan Peyton30419822017-05-12 18:01:32 +00006113 __kmp_thread_pool = thread->th.th_next_pool;
6114 // Reap it.
6115 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP);
6116 thread->th.th_next_pool = NULL;
6117 thread->th.th_in_pool = FALSE;
Jonathan Peytonfeac33e2019-04-08 17:50:02 +00006118 thread->th.th_active_in_pool = FALSE;
6119 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
Jonathan Peyton30419822017-05-12 18:01:32 +00006120 __kmp_reap_thread(thread, 0);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006121 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006122 __kmp_thread_pool_insert_pt = NULL;
6123
6124 // Reap teams.
6125 while (__kmp_team_pool != NULL) { // Loop thru all the teams in the pool.
6126 // Get the next team from the pool.
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00006127 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool);
Jonathan Peyton30419822017-05-12 18:01:32 +00006128 __kmp_team_pool = team->t.t_next_pool;
6129 // Reap it.
6130 team->t.t_next_pool = NULL;
6131 __kmp_reap_team(team);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006132 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006133
6134 __kmp_reap_task_teams();
6135
Jonathan Peytona764af62018-07-19 19:17:00 +00006136#if KMP_OS_UNIX
6137 // Threads that are not reaped should not access any resources since they
6138 // are going to be deallocated soon, so the shutdown sequence should wait
6139 // until all threads either exit the final spin-waiting loop or begin
6140 // sleeping after the given blocktime.
6141 for (i = 0; i < __kmp_threads_capacity; i++) {
6142 kmp_info_t *thr = __kmp_threads[i];
6143 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking))
6144 KMP_CPU_PAUSE();
6145 }
6146#endif
6147
Jonathan Peyton30419822017-05-12 18:01:32 +00006148 for (i = 0; i < __kmp_threads_capacity; ++i) {
6149 // TBD: Add some checking...
6150 // Something like KMP_DEBUG_ASSERT( __kmp_thread[ i ] == NULL );
6151 }
6152
6153 /* Make sure all threadprivate destructors get run by joining with all
6154 worker threads before resetting this flag */
6155 TCW_SYNC_4(__kmp_init_common, FALSE);
6156
6157 KA_TRACE(10, ("__kmp_internal_end: all workers reaped\n"));
6158 KMP_MB();
6159
6160#if KMP_USE_MONITOR
6161 // See note above: One of the possible fixes for CQ138434 / CQ140126
6162 //
6163 // FIXME: push both code fragments down and CSE them?
6164 // push them into __kmp_cleanup() ?
6165 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
6166 if (TCR_4(__kmp_init_monitor)) {
6167 __kmp_reap_monitor(&__kmp_monitor);
6168 TCW_4(__kmp_init_monitor, 0);
6169 }
6170 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
6171 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n"));
6172#endif
6173 } /* else !__kmp_global.t_active */
6174 TCW_4(__kmp_init_gtid, FALSE);
6175 KMP_MB(); /* Flush all pending memory write invalidates. */
6176
6177 __kmp_cleanup();
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006178#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00006179 ompt_fini();
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006180#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006181}
6182
Jonathan Peyton30419822017-05-12 18:01:32 +00006183void __kmp_internal_end_library(int gtid_req) {
6184 /* if we have already cleaned up, don't try again, it wouldn't be pretty */
6185 /* this shouldn't be a race condition because __kmp_internal_end() is the
6186 only place to clear __kmp_serial_init */
6187 /* we'll check this later too, after we get the lock */
6188 // 2009-09-06: We do not set g_abort without setting g_done. This check looks
6189 // redundaant, because the next check will work in any case.
6190 if (__kmp_global.g.g_abort) {
6191 KA_TRACE(11, ("__kmp_internal_end_library: abort, exiting\n"));
6192 /* TODO abort? */
6193 return;
6194 }
6195 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6196 KA_TRACE(10, ("__kmp_internal_end_library: already finished\n"));
6197 return;
6198 }
6199
6200 KMP_MB(); /* Flush all pending memory write invalidates. */
6201
6202 /* find out who we are and what we should do */
6203 {
6204 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6205 KA_TRACE(
6206 10, ("__kmp_internal_end_library: enter T#%d (%d)\n", gtid, gtid_req));
6207 if (gtid == KMP_GTID_SHUTDOWN) {
6208 KA_TRACE(10, ("__kmp_internal_end_library: !__kmp_init_runtime, system "
6209 "already shutdown\n"));
6210 return;
6211 } else if (gtid == KMP_GTID_MONITOR) {
6212 KA_TRACE(10, ("__kmp_internal_end_library: monitor thread, gtid not "
6213 "registered, or system shutdown\n"));
6214 return;
6215 } else if (gtid == KMP_GTID_DNE) {
6216 KA_TRACE(10, ("__kmp_internal_end_library: gtid not registered or system "
6217 "shutdown\n"));
6218 /* we don't know who we are, but we may still shutdown the library */
6219 } else if (KMP_UBER_GTID(gtid)) {
6220 /* unregister ourselves as an uber thread. gtid is no longer valid */
6221 if (__kmp_root[gtid]->r.r_active) {
6222 __kmp_global.g.g_abort = -1;
6223 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6224 KA_TRACE(10,
6225 ("__kmp_internal_end_library: root still active, abort T#%d\n",
6226 gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006227 return;
Jonathan Peyton30419822017-05-12 18:01:32 +00006228 } else {
6229 KA_TRACE(
6230 10,
6231 ("__kmp_internal_end_library: unregistering sibling T#%d\n", gtid));
6232 __kmp_unregister_root_current_thread(gtid);
6233 }
6234 } else {
6235/* worker threads may call this function through the atexit handler, if they
6236 * call exit() */
6237/* For now, skip the usual subsequent processing and just dump the debug buffer.
6238 TODO: do a thorough shutdown instead */
6239#ifdef DUMP_DEBUG_ON_EXIT
6240 if (__kmp_debug_buf)
6241 __kmp_dump_debug_buffer();
6242#endif
6243 return;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006244 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006245 }
6246 /* synchronize the termination process */
6247 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006248
Jonathan Peyton30419822017-05-12 18:01:32 +00006249 /* have we already finished */
6250 if (__kmp_global.g.g_abort) {
6251 KA_TRACE(10, ("__kmp_internal_end_library: abort, exiting\n"));
6252 /* TODO abort? */
6253 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6254 return;
6255 }
6256 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6257 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6258 return;
6259 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006260
Jonathan Peyton30419822017-05-12 18:01:32 +00006261 /* We need this lock to enforce mutex between this reading of
6262 __kmp_threads_capacity and the writing by __kmp_register_root.
6263 Alternatively, we can use a counter of roots that is atomically updated by
6264 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and
6265 __kmp_internal_end_*. */
6266 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006267
Jonathan Peyton30419822017-05-12 18:01:32 +00006268 /* now we can safely conduct the actual termination */
6269 __kmp_internal_end();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006270
Jonathan Peyton30419822017-05-12 18:01:32 +00006271 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6272 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006273
Jonathan Peyton30419822017-05-12 18:01:32 +00006274 KA_TRACE(10, ("__kmp_internal_end_library: exit\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006275
Jonathan Peyton30419822017-05-12 18:01:32 +00006276#ifdef DUMP_DEBUG_ON_EXIT
6277 if (__kmp_debug_buf)
6278 __kmp_dump_debug_buffer();
6279#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006280
Jonathan Peyton30419822017-05-12 18:01:32 +00006281#if KMP_OS_WINDOWS
6282 __kmp_close_console();
6283#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006284
Jonathan Peyton30419822017-05-12 18:01:32 +00006285 __kmp_fini_allocator();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006286
6287} // __kmp_internal_end_library
6288
Jonathan Peyton30419822017-05-12 18:01:32 +00006289void __kmp_internal_end_thread(int gtid_req) {
6290 int i;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006291
Jonathan Peyton30419822017-05-12 18:01:32 +00006292 /* if we have already cleaned up, don't try again, it wouldn't be pretty */
6293 /* this shouldn't be a race condition because __kmp_internal_end() is the
6294 * only place to clear __kmp_serial_init */
6295 /* we'll check this later too, after we get the lock */
6296 // 2009-09-06: We do not set g_abort without setting g_done. This check looks
6297 // redundant, because the next check will work in any case.
6298 if (__kmp_global.g.g_abort) {
6299 KA_TRACE(11, ("__kmp_internal_end_thread: abort, exiting\n"));
6300 /* TODO abort? */
6301 return;
6302 }
6303 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6304 KA_TRACE(10, ("__kmp_internal_end_thread: already finished\n"));
6305 return;
6306 }
6307
6308 KMP_MB(); /* Flush all pending memory write invalidates. */
6309
6310 /* find out who we are and what we should do */
6311 {
6312 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6313 KA_TRACE(10,
6314 ("__kmp_internal_end_thread: enter T#%d (%d)\n", gtid, gtid_req));
6315 if (gtid == KMP_GTID_SHUTDOWN) {
6316 KA_TRACE(10, ("__kmp_internal_end_thread: !__kmp_init_runtime, system "
6317 "already shutdown\n"));
6318 return;
6319 } else if (gtid == KMP_GTID_MONITOR) {
6320 KA_TRACE(10, ("__kmp_internal_end_thread: monitor thread, gtid not "
6321 "registered, or system shutdown\n"));
6322 return;
6323 } else if (gtid == KMP_GTID_DNE) {
6324 KA_TRACE(10, ("__kmp_internal_end_thread: gtid not registered or system "
6325 "shutdown\n"));
6326 return;
6327 /* we don't know who we are */
6328 } else if (KMP_UBER_GTID(gtid)) {
6329 /* unregister ourselves as an uber thread. gtid is no longer valid */
6330 if (__kmp_root[gtid]->r.r_active) {
6331 __kmp_global.g.g_abort = -1;
6332 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6333 KA_TRACE(10,
6334 ("__kmp_internal_end_thread: root still active, abort T#%d\n",
6335 gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006336 return;
Jonathan Peyton30419822017-05-12 18:01:32 +00006337 } else {
6338 KA_TRACE(10, ("__kmp_internal_end_thread: unregistering sibling T#%d\n",
6339 gtid));
6340 __kmp_unregister_root_current_thread(gtid);
6341 }
6342 } else {
6343 /* just a worker thread, let's leave */
6344 KA_TRACE(10, ("__kmp_internal_end_thread: worker thread T#%d\n", gtid));
6345
6346 if (gtid >= 0) {
6347 __kmp_threads[gtid]->th.th_task_team = NULL;
6348 }
6349
6350 KA_TRACE(10,
6351 ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n",
6352 gtid));
6353 return;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006354 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006355 }
Jonathan Peyton8b3842f2018-10-05 17:59:39 +00006356#if KMP_DYNAMIC_LIB
Jonathan Peyton61708b12019-03-25 18:17:55 +00006357#if OMP_50_ENABLED
6358 if (__kmp_pause_status != kmp_hard_paused)
6359#endif
6360 // AC: lets not shutdown the dynamic library at the exit of uber thread,
6361 // because we will better shutdown later in the library destructor.
6362 {
Jonathan Peyton9b8bb322019-01-16 20:07:39 +00006363 KA_TRACE(10, ("__kmp_internal_end_thread: exiting T#%d\n", gtid_req));
6364 return;
6365 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006366#endif
6367 /* synchronize the termination process */
6368 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006369
Jonathan Peyton30419822017-05-12 18:01:32 +00006370 /* have we already finished */
6371 if (__kmp_global.g.g_abort) {
6372 KA_TRACE(10, ("__kmp_internal_end_thread: abort, exiting\n"));
6373 /* TODO abort? */
6374 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6375 return;
6376 }
6377 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6378 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6379 return;
6380 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006381
Jonathan Peyton30419822017-05-12 18:01:32 +00006382 /* We need this lock to enforce mutex between this reading of
6383 __kmp_threads_capacity and the writing by __kmp_register_root.
6384 Alternatively, we can use a counter of roots that is atomically updated by
6385 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and
6386 __kmp_internal_end_*. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006387
Jonathan Peyton30419822017-05-12 18:01:32 +00006388 /* should we finish the run-time? are all siblings done? */
6389 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006390
Jonathan Peyton30419822017-05-12 18:01:32 +00006391 for (i = 0; i < __kmp_threads_capacity; ++i) {
6392 if (KMP_UBER_GTID(i)) {
6393 KA_TRACE(
6394 10,
6395 ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i));
6396 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6397 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6398 return;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006399 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006400 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006401
Jonathan Peyton30419822017-05-12 18:01:32 +00006402 /* now we can safely conduct the actual termination */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006403
Jonathan Peyton30419822017-05-12 18:01:32 +00006404 __kmp_internal_end();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006405
Jonathan Peyton30419822017-05-12 18:01:32 +00006406 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6407 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006408
Jonathan Peyton30419822017-05-12 18:01:32 +00006409 KA_TRACE(10, ("__kmp_internal_end_thread: exit T#%d\n", gtid_req));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006410
Jonathan Peyton30419822017-05-12 18:01:32 +00006411#ifdef DUMP_DEBUG_ON_EXIT
6412 if (__kmp_debug_buf)
6413 __kmp_dump_debug_buffer();
6414#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006415} // __kmp_internal_end_thread
6416
Jonathan Peyton30419822017-05-12 18:01:32 +00006417// -----------------------------------------------------------------------------
Jim Cownie5e8470a2013-09-27 10:38:44 +00006418// Library registration stuff.
6419
Jonathan Peyton30419822017-05-12 18:01:32 +00006420static long __kmp_registration_flag = 0;
6421// Random value used to indicate library initialization.
6422static char *__kmp_registration_str = NULL;
6423// Value to be saved in env var __KMP_REGISTERED_LIB_<pid>.
Jim Cownie5e8470a2013-09-27 10:38:44 +00006424
Jonathan Peyton30419822017-05-12 18:01:32 +00006425static inline char *__kmp_reg_status_name() {
6426 /* On RHEL 3u5 if linked statically, getpid() returns different values in
6427 each thread. If registration and unregistration go in different threads
6428 (omp_misc_other_root_exit.cpp test case), the name of registered_lib_env
6429 env var can not be found, because the name will contain different pid. */
6430 return __kmp_str_format("__KMP_REGISTERED_LIB_%d", (int)getpid());
Jim Cownie5e8470a2013-09-27 10:38:44 +00006431} // __kmp_reg_status_get
6432
Jonathan Peyton30419822017-05-12 18:01:32 +00006433void __kmp_register_library_startup(void) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006434
Jonathan Peyton30419822017-05-12 18:01:32 +00006435 char *name = __kmp_reg_status_name(); // Name of the environment variable.
6436 int done = 0;
6437 union {
6438 double dtime;
6439 long ltime;
6440 } time;
6441#if KMP_ARCH_X86 || KMP_ARCH_X86_64
6442 __kmp_initialize_system_tick();
6443#endif
6444 __kmp_read_system_time(&time.dtime);
6445 __kmp_registration_flag = 0xCAFE0000L | (time.ltime & 0x0000FFFFL);
6446 __kmp_registration_str =
6447 __kmp_str_format("%p-%lx-%s", &__kmp_registration_flag,
6448 __kmp_registration_flag, KMP_LIBRARY_FILE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006449
Jonathan Peyton30419822017-05-12 18:01:32 +00006450 KA_TRACE(50, ("__kmp_register_library_startup: %s=\"%s\"\n", name,
6451 __kmp_registration_str));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006452
Jonathan Peyton30419822017-05-12 18:01:32 +00006453 while (!done) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006454
Jonathan Peyton30419822017-05-12 18:01:32 +00006455 char *value = NULL; // Actual value of the environment variable.
Jim Cownie5e8470a2013-09-27 10:38:44 +00006456
Jonathan Peyton30419822017-05-12 18:01:32 +00006457 // Set environment variable, but do not overwrite if it is exist.
6458 __kmp_env_set(name, __kmp_registration_str, 0);
6459 // Check the variable is written.
6460 value = __kmp_env_get(name);
6461 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006462
Jonathan Peyton30419822017-05-12 18:01:32 +00006463 done = 1; // Ok, environment variable set successfully, exit the loop.
Jim Cownie5e8470a2013-09-27 10:38:44 +00006464
Jonathan Peyton30419822017-05-12 18:01:32 +00006465 } else {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006466
Jonathan Peyton30419822017-05-12 18:01:32 +00006467 // Oops. Write failed. Another copy of OpenMP RTL is in memory.
6468 // Check whether it alive or dead.
6469 int neighbor = 0; // 0 -- unknown status, 1 -- alive, 2 -- dead.
6470 char *tail = value;
6471 char *flag_addr_str = NULL;
6472 char *flag_val_str = NULL;
6473 char const *file_name = NULL;
6474 __kmp_str_split(tail, '-', &flag_addr_str, &tail);
6475 __kmp_str_split(tail, '-', &flag_val_str, &tail);
6476 file_name = tail;
6477 if (tail != NULL) {
6478 long *flag_addr = 0;
6479 long flag_val = 0;
Jonathan Peytonbaad3f62018-08-09 22:04:30 +00006480 KMP_SSCANF(flag_addr_str, "%p", RCAST(void**, &flag_addr));
Jonathan Peyton30419822017-05-12 18:01:32 +00006481 KMP_SSCANF(flag_val_str, "%lx", &flag_val);
6482 if (flag_addr != 0 && flag_val != 0 && strcmp(file_name, "") != 0) {
6483 // First, check whether environment-encoded address is mapped into
6484 // addr space.
6485 // If so, dereference it to see if it still has the right value.
6486 if (__kmp_is_address_mapped(flag_addr) && *flag_addr == flag_val) {
6487 neighbor = 1;
6488 } else {
6489 // If not, then we know the other copy of the library is no longer
6490 // running.
6491 neighbor = 2;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006492 }
6493 }
6494 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006495 switch (neighbor) {
6496 case 0: // Cannot parse environment variable -- neighbor status unknown.
6497 // Assume it is the incompatible format of future version of the
6498 // library. Assume the other library is alive.
6499 // WARN( ... ); // TODO: Issue a warning.
6500 file_name = "unknown library";
Joachim Protze0c599c32019-02-04 15:59:42 +00006501 KMP_FALLTHROUGH();
Jonathan Peyton30419822017-05-12 18:01:32 +00006502 // Attention! Falling to the next case. That's intentional.
6503 case 1: { // Neighbor is alive.
6504 // Check it is allowed.
6505 char *duplicate_ok = __kmp_env_get("KMP_DUPLICATE_LIB_OK");
6506 if (!__kmp_str_match_true(duplicate_ok)) {
6507 // That's not allowed. Issue fatal error.
Jonathan Peyton6a393f72017-09-05 15:43:58 +00006508 __kmp_fatal(KMP_MSG(DuplicateLibrary, KMP_LIBRARY_FILE, file_name),
6509 KMP_HNT(DuplicateLibrary), __kmp_msg_null);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006510 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006511 KMP_INTERNAL_FREE(duplicate_ok);
6512 __kmp_duplicate_library_ok = 1;
6513 done = 1; // Exit the loop.
6514 } break;
6515 case 2: { // Neighbor is dead.
6516 // Clear the variable and try to register library again.
6517 __kmp_env_unset(name);
6518 } break;
6519 default: { KMP_DEBUG_ASSERT(0); } break;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006520 }
6521 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006522 KMP_INTERNAL_FREE((void *)value);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006523 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006524 KMP_INTERNAL_FREE((void *)name);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006525
6526} // func __kmp_register_library_startup
6527
Jonathan Peyton30419822017-05-12 18:01:32 +00006528void __kmp_unregister_library(void) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006529
Jonathan Peyton30419822017-05-12 18:01:32 +00006530 char *name = __kmp_reg_status_name();
6531 char *value = __kmp_env_get(name);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006532
Jonathan Peyton30419822017-05-12 18:01:32 +00006533 KMP_DEBUG_ASSERT(__kmp_registration_flag != 0);
6534 KMP_DEBUG_ASSERT(__kmp_registration_str != NULL);
6535 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
6536 // Ok, this is our variable. Delete it.
6537 __kmp_env_unset(name);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006538 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006539
Jonathan Peyton30419822017-05-12 18:01:32 +00006540 KMP_INTERNAL_FREE(__kmp_registration_str);
6541 KMP_INTERNAL_FREE(value);
6542 KMP_INTERNAL_FREE(name);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006543
Jonathan Peyton30419822017-05-12 18:01:32 +00006544 __kmp_registration_flag = 0;
6545 __kmp_registration_str = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006546
6547} // __kmp_unregister_library
6548
Jim Cownie5e8470a2013-09-27 10:38:44 +00006549// End of Library registration stuff.
Jonathan Peyton30419822017-05-12 18:01:32 +00006550// -----------------------------------------------------------------------------
Jim Cownie5e8470a2013-09-27 10:38:44 +00006551
Jonathan Peyton492e0a32017-06-13 17:17:26 +00006552#if KMP_MIC_SUPPORTED
Andrey Churbanov613edeb2015-02-20 18:14:43 +00006553
Jonathan Peyton30419822017-05-12 18:01:32 +00006554static void __kmp_check_mic_type() {
6555 kmp_cpuid_t cpuid_state = {0};
6556 kmp_cpuid_t *cs_p = &cpuid_state;
6557 __kmp_x86_cpuid(1, 0, cs_p);
6558 // We don't support mic1 at the moment
6559 if ((cs_p->eax & 0xff0) == 0xB10) {
6560 __kmp_mic_type = mic2;
6561 } else if ((cs_p->eax & 0xf0ff0) == 0x50670) {
6562 __kmp_mic_type = mic3;
6563 } else {
6564 __kmp_mic_type = non_mic;
6565 }
Andrey Churbanov613edeb2015-02-20 18:14:43 +00006566}
6567
Jonathan Peyton492e0a32017-06-13 17:17:26 +00006568#endif /* KMP_MIC_SUPPORTED */
Andrey Churbanov613edeb2015-02-20 18:14:43 +00006569
Jonathan Peyton30419822017-05-12 18:01:32 +00006570static void __kmp_do_serial_initialize(void) {
6571 int i, gtid;
6572 int size;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006573
Jonathan Peyton30419822017-05-12 18:01:32 +00006574 KA_TRACE(10, ("__kmp_do_serial_initialize: enter\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006575
Jonathan Peyton30419822017-05-12 18:01:32 +00006576 KMP_DEBUG_ASSERT(sizeof(kmp_int32) == 4);
6577 KMP_DEBUG_ASSERT(sizeof(kmp_uint32) == 4);
6578 KMP_DEBUG_ASSERT(sizeof(kmp_int64) == 8);
6579 KMP_DEBUG_ASSERT(sizeof(kmp_uint64) == 8);
6580 KMP_DEBUG_ASSERT(sizeof(kmp_intptr_t) == sizeof(void *));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006581
Jonathan Peyton82a13bf2015-09-21 18:01:02 +00006582#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00006583 ompt_pre_init();
Jonathan Peyton82a13bf2015-09-21 18:01:02 +00006584#endif
6585
Jonathan Peyton30419822017-05-12 18:01:32 +00006586 __kmp_validate_locks();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006587
Jonathan Peyton30419822017-05-12 18:01:32 +00006588 /* Initialize internal memory allocator */
6589 __kmp_init_allocator();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006590
Jonathan Peyton30419822017-05-12 18:01:32 +00006591 /* Register the library startup via an environment variable and check to see
6592 whether another copy of the library is already registered. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006593
Jonathan Peyton30419822017-05-12 18:01:32 +00006594 __kmp_register_library_startup();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006595
Jonathan Peyton30419822017-05-12 18:01:32 +00006596 /* TODO reinitialization of library */
6597 if (TCR_4(__kmp_global.g.g_done)) {
6598 KA_TRACE(10, ("__kmp_do_serial_initialize: reinitialization of library\n"));
6599 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006600
Jonathan Peyton30419822017-05-12 18:01:32 +00006601 __kmp_global.g.g_abort = 0;
6602 TCW_SYNC_4(__kmp_global.g.g_done, FALSE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006603
Jonathan Peyton30419822017-05-12 18:01:32 +00006604/* initialize the locks */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006605#if KMP_USE_ADAPTIVE_LOCKS
6606#if KMP_DEBUG_ADAPTIVE_LOCKS
Jonathan Peyton30419822017-05-12 18:01:32 +00006607 __kmp_init_speculative_stats();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006608#endif
6609#endif
Jonathan Peytonad579922015-12-17 16:19:05 +00006610#if KMP_STATS_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00006611 __kmp_stats_init();
Jonathan Peytonad579922015-12-17 16:19:05 +00006612#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006613 __kmp_init_lock(&__kmp_global_lock);
6614 __kmp_init_queuing_lock(&__kmp_dispatch_lock);
6615 __kmp_init_lock(&__kmp_debug_lock);
6616 __kmp_init_atomic_lock(&__kmp_atomic_lock);
6617 __kmp_init_atomic_lock(&__kmp_atomic_lock_1i);
6618 __kmp_init_atomic_lock(&__kmp_atomic_lock_2i);
6619 __kmp_init_atomic_lock(&__kmp_atomic_lock_4i);
6620 __kmp_init_atomic_lock(&__kmp_atomic_lock_4r);
6621 __kmp_init_atomic_lock(&__kmp_atomic_lock_8i);
6622 __kmp_init_atomic_lock(&__kmp_atomic_lock_8r);
6623 __kmp_init_atomic_lock(&__kmp_atomic_lock_8c);
6624 __kmp_init_atomic_lock(&__kmp_atomic_lock_10r);
6625 __kmp_init_atomic_lock(&__kmp_atomic_lock_16r);
6626 __kmp_init_atomic_lock(&__kmp_atomic_lock_16c);
6627 __kmp_init_atomic_lock(&__kmp_atomic_lock_20c);
6628 __kmp_init_atomic_lock(&__kmp_atomic_lock_32c);
6629 __kmp_init_bootstrap_lock(&__kmp_forkjoin_lock);
6630 __kmp_init_bootstrap_lock(&__kmp_exit_lock);
Jonathan Peytonb66d1aa2016-09-27 17:11:17 +00006631#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00006632 __kmp_init_bootstrap_lock(&__kmp_monitor_lock);
Jonathan Peytonb66d1aa2016-09-27 17:11:17 +00006633#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006634 __kmp_init_bootstrap_lock(&__kmp_tp_cached_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006635
Jonathan Peyton30419822017-05-12 18:01:32 +00006636 /* conduct initialization and initial setup of configuration */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006637
Jonathan Peyton30419822017-05-12 18:01:32 +00006638 __kmp_runtime_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006639
Jonathan Peyton492e0a32017-06-13 17:17:26 +00006640#if KMP_MIC_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00006641 __kmp_check_mic_type();
Andrey Churbanov613edeb2015-02-20 18:14:43 +00006642#endif
6643
Jonathan Peyton30419822017-05-12 18:01:32 +00006644// Some global variable initialization moved here from kmp_env_initialize()
Jim Cownie5e8470a2013-09-27 10:38:44 +00006645#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00006646 kmp_diag = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006647#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006648 __kmp_abort_delay = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006649
Jonathan Peyton30419822017-05-12 18:01:32 +00006650 // From __kmp_init_dflt_team_nth()
6651 /* assume the entire machine will be used */
6652 __kmp_dflt_team_nth_ub = __kmp_xproc;
6653 if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH) {
6654 __kmp_dflt_team_nth_ub = KMP_MIN_NTH;
6655 }
6656 if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) {
6657 __kmp_dflt_team_nth_ub = __kmp_sys_max_nth;
6658 }
6659 __kmp_max_nth = __kmp_sys_max_nth;
Jonathan Peytonf4392462017-07-27 20:58:41 +00006660 __kmp_cg_max_nth = __kmp_sys_max_nth;
Jonathan Peyton4f90c822017-08-02 20:04:45 +00006661 __kmp_teams_max_nth = __kmp_xproc; // set a "reasonable" default
6662 if (__kmp_teams_max_nth > __kmp_sys_max_nth) {
6663 __kmp_teams_max_nth = __kmp_sys_max_nth;
6664 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006665
Jonathan Peyton30419822017-05-12 18:01:32 +00006666 // Three vars below moved here from __kmp_env_initialize() "KMP_BLOCKTIME"
6667 // part
6668 __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME;
Jonathan Peytone1c7c132016-10-07 18:12:19 +00006669#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00006670 __kmp_monitor_wakeups =
6671 KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
6672 __kmp_bt_intervals =
6673 KMP_INTERVALS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
Jonathan Peytone1c7c132016-10-07 18:12:19 +00006674#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006675 // From "KMP_LIBRARY" part of __kmp_env_initialize()
6676 __kmp_library = library_throughput;
6677 // From KMP_SCHEDULE initialization
6678 __kmp_static = kmp_sch_static_balanced;
6679// AC: do not use analytical here, because it is non-monotonous
6680//__kmp_guided = kmp_sch_guided_iterative_chunked;
6681//__kmp_auto = kmp_sch_guided_analytical_chunked; // AC: it is the default, no
6682// need to repeat assignment
6683// Barrier initialization. Moved here from __kmp_env_initialize() Barrier branch
6684// bit control and barrier method control parts
Jim Cownie4cc4bb42014-10-07 16:25:50 +00006685#if KMP_FAST_REDUCTION_BARRIER
Jonathan Peyton30419822017-05-12 18:01:32 +00006686#define kmp_reduction_barrier_gather_bb ((int)1)
6687#define kmp_reduction_barrier_release_bb ((int)1)
6688#define kmp_reduction_barrier_gather_pat bp_hyper_bar
6689#define kmp_reduction_barrier_release_pat bp_hyper_bar
6690#endif // KMP_FAST_REDUCTION_BARRIER
6691 for (i = bs_plain_barrier; i < bs_last_barrier; i++) {
6692 __kmp_barrier_gather_branch_bits[i] = __kmp_barrier_gather_bb_dflt;
6693 __kmp_barrier_release_branch_bits[i] = __kmp_barrier_release_bb_dflt;
6694 __kmp_barrier_gather_pattern[i] = __kmp_barrier_gather_pat_dflt;
6695 __kmp_barrier_release_pattern[i] = __kmp_barrier_release_pat_dflt;
6696#if KMP_FAST_REDUCTION_BARRIER
6697 if (i == bs_reduction_barrier) { // tested and confirmed on ALTIX only (
6698 // lin_64 ): hyper,1
6699 __kmp_barrier_gather_branch_bits[i] = kmp_reduction_barrier_gather_bb;
6700 __kmp_barrier_release_branch_bits[i] = kmp_reduction_barrier_release_bb;
6701 __kmp_barrier_gather_pattern[i] = kmp_reduction_barrier_gather_pat;
6702 __kmp_barrier_release_pattern[i] = kmp_reduction_barrier_release_pat;
Andrey Churbanov613edeb2015-02-20 18:14:43 +00006703 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006704#endif // KMP_FAST_REDUCTION_BARRIER
6705 }
6706#if KMP_FAST_REDUCTION_BARRIER
6707#undef kmp_reduction_barrier_release_pat
6708#undef kmp_reduction_barrier_gather_pat
6709#undef kmp_reduction_barrier_release_bb
6710#undef kmp_reduction_barrier_gather_bb
6711#endif // KMP_FAST_REDUCTION_BARRIER
Jonathan Peyton492e0a32017-06-13 17:17:26 +00006712#if KMP_MIC_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00006713 if (__kmp_mic_type == mic2) { // KNC
6714 // AC: plane=3,2, forkjoin=2,1 are optimal for 240 threads on KNC
6715 __kmp_barrier_gather_branch_bits[bs_plain_barrier] = 3; // plain gather
6716 __kmp_barrier_release_branch_bits[bs_forkjoin_barrier] =
6717 1; // forkjoin release
6718 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6719 __kmp_barrier_release_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6720 }
6721#if KMP_FAST_REDUCTION_BARRIER
6722 if (__kmp_mic_type == mic2) { // KNC
6723 __kmp_barrier_gather_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6724 __kmp_barrier_release_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6725 }
Jonathan Peyton492e0a32017-06-13 17:17:26 +00006726#endif // KMP_FAST_REDUCTION_BARRIER
6727#endif // KMP_MIC_SUPPORTED
Jim Cownie5e8470a2013-09-27 10:38:44 +00006728
Jonathan Peyton30419822017-05-12 18:01:32 +00006729// From KMP_CHECKS initialization
Jim Cownie5e8470a2013-09-27 10:38:44 +00006730#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00006731 __kmp_env_checks = TRUE; /* development versions have the extra checks */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006732#else
Jonathan Peyton30419822017-05-12 18:01:32 +00006733 __kmp_env_checks = FALSE; /* port versions do not have the extra checks */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006734#endif
6735
Jonathan Peyton30419822017-05-12 18:01:32 +00006736 // From "KMP_FOREIGN_THREADS_THREADPRIVATE" initialization
6737 __kmp_foreign_tp = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006738
Jonathan Peyton30419822017-05-12 18:01:32 +00006739 __kmp_global.g.g_dynamic = FALSE;
6740 __kmp_global.g.g_dynamic_mode = dynamic_default;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006741
Jonathan Peyton30419822017-05-12 18:01:32 +00006742 __kmp_env_initialize(NULL);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00006743
Jonathan Peyton30419822017-05-12 18:01:32 +00006744// Print all messages in message catalog for testing purposes.
6745#ifdef KMP_DEBUG
6746 char const *val = __kmp_env_get("KMP_DUMP_CATALOG");
6747 if (__kmp_str_match_true(val)) {
6748 kmp_str_buf_t buffer;
6749 __kmp_str_buf_init(&buffer);
6750 __kmp_i18n_dump_catalog(&buffer);
6751 __kmp_printf("%s", buffer.str);
6752 __kmp_str_buf_free(&buffer);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006753 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006754 __kmp_env_free(&val);
6755#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006756
Jonathan Peyton30419822017-05-12 18:01:32 +00006757 __kmp_threads_capacity =
6758 __kmp_initial_threads_capacity(__kmp_dflt_team_nth_ub);
6759 // Moved here from __kmp_env_initialize() "KMP_ALL_THREADPRIVATE" part
6760 __kmp_tp_capacity = __kmp_default_tp_capacity(
6761 __kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006762
Jonathan Peyton30419822017-05-12 18:01:32 +00006763 // If the library is shut down properly, both pools must be NULL. Just in
6764 // case, set them to NULL -- some memory may leak, but subsequent code will
6765 // work even if pools are not freed.
6766 KMP_DEBUG_ASSERT(__kmp_thread_pool == NULL);
6767 KMP_DEBUG_ASSERT(__kmp_thread_pool_insert_pt == NULL);
6768 KMP_DEBUG_ASSERT(__kmp_team_pool == NULL);
6769 __kmp_thread_pool = NULL;
6770 __kmp_thread_pool_insert_pt = NULL;
6771 __kmp_team_pool = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006772
Jonathan Peyton30419822017-05-12 18:01:32 +00006773 /* Allocate all of the variable sized records */
6774 /* NOTE: __kmp_threads_capacity entries are allocated, but the arrays are
6775 * expandable */
6776 /* Since allocation is cache-aligned, just add extra padding at the end */
6777 size =
6778 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * __kmp_threads_capacity +
6779 CACHE_LINE;
6780 __kmp_threads = (kmp_info_t **)__kmp_allocate(size);
6781 __kmp_root = (kmp_root_t **)((char *)__kmp_threads +
6782 sizeof(kmp_info_t *) * __kmp_threads_capacity);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006783
Jonathan Peyton30419822017-05-12 18:01:32 +00006784 /* init thread counts */
6785 KMP_DEBUG_ASSERT(__kmp_all_nth ==
6786 0); // Asserts fail if the library is reinitializing and
6787 KMP_DEBUG_ASSERT(__kmp_nth == 0); // something was wrong in termination.
6788 __kmp_all_nth = 0;
6789 __kmp_nth = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006790
Jonathan Peyton30419822017-05-12 18:01:32 +00006791 /* setup the uber master thread and hierarchy */
6792 gtid = __kmp_register_root(TRUE);
6793 KA_TRACE(10, ("__kmp_do_serial_initialize T#%d\n", gtid));
6794 KMP_ASSERT(KMP_UBER_GTID(gtid));
6795 KMP_ASSERT(KMP_INITIAL_GTID(gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006796
Jonathan Peyton30419822017-05-12 18:01:32 +00006797 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006798
Jonathan Peyton30419822017-05-12 18:01:32 +00006799 __kmp_common_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006800
Jonathan Peyton30419822017-05-12 18:01:32 +00006801#if KMP_OS_UNIX
6802 /* invoke the child fork handler */
6803 __kmp_register_atfork();
6804#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006805
Jonathan Peyton8b3842f2018-10-05 17:59:39 +00006806#if !KMP_DYNAMIC_LIB
Jonathan Peyton30419822017-05-12 18:01:32 +00006807 {
6808 /* Invoke the exit handler when the program finishes, only for static
6809 library. For dynamic library, we already have _fini and DllMain. */
6810 int rc = atexit(__kmp_internal_end_atexit);
6811 if (rc != 0) {
Jonathan Peyton6a393f72017-09-05 15:43:58 +00006812 __kmp_fatal(KMP_MSG(FunctionError, "atexit()"), KMP_ERR(rc),
6813 __kmp_msg_null);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006814 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006815 }
6816#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006817
Jonathan Peyton30419822017-05-12 18:01:32 +00006818#if KMP_HANDLE_SIGNALS
6819#if KMP_OS_UNIX
6820 /* NOTE: make sure that this is called before the user installs their own
6821 signal handlers so that the user handlers are called first. this way they
6822 can return false, not call our handler, avoid terminating the library, and
6823 continue execution where they left off. */
6824 __kmp_install_signals(FALSE);
6825#endif /* KMP_OS_UNIX */
6826#if KMP_OS_WINDOWS
6827 __kmp_install_signals(TRUE);
6828#endif /* KMP_OS_WINDOWS */
6829#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006830
Jonathan Peyton30419822017-05-12 18:01:32 +00006831 /* we have finished the serial initialization */
6832 __kmp_init_counter++;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006833
Jonathan Peyton30419822017-05-12 18:01:32 +00006834 __kmp_init_serial = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006835
Jonathan Peyton30419822017-05-12 18:01:32 +00006836 if (__kmp_settings) {
6837 __kmp_env_print();
6838 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006839
6840#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00006841 if (__kmp_display_env || __kmp_display_env_verbose) {
6842 __kmp_env_print_2();
6843 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006844#endif // OMP_40_ENABLED
6845
Jonathan Peyton82a13bf2015-09-21 18:01:02 +00006846#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00006847 ompt_post_init();
Jonathan Peyton82a13bf2015-09-21 18:01:02 +00006848#endif
6849
Jonathan Peyton30419822017-05-12 18:01:32 +00006850 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006851
Jonathan Peyton30419822017-05-12 18:01:32 +00006852 KA_TRACE(10, ("__kmp_do_serial_initialize: exit\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006853}
6854
Jonathan Peyton30419822017-05-12 18:01:32 +00006855void __kmp_serial_initialize(void) {
6856 if (__kmp_init_serial) {
6857 return;
6858 }
6859 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6860 if (__kmp_init_serial) {
6861 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6862 return;
6863 }
6864 __kmp_do_serial_initialize();
6865 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6866}
6867
6868static void __kmp_do_middle_initialize(void) {
6869 int i, j;
6870 int prev_dflt_team_nth;
6871
6872 if (!__kmp_init_serial) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006873 __kmp_do_serial_initialize();
Jonathan Peyton30419822017-05-12 18:01:32 +00006874 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006875
Jonathan Peyton30419822017-05-12 18:01:32 +00006876 KA_TRACE(10, ("__kmp_middle_initialize: enter\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006877
Jonathan Peyton30419822017-05-12 18:01:32 +00006878 // Save the previous value for the __kmp_dflt_team_nth so that
6879 // we can avoid some reinitialization if it hasn't changed.
6880 prev_dflt_team_nth = __kmp_dflt_team_nth;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006881
Alp Toker98758b02014-03-02 04:12:06 +00006882#if KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00006883 // __kmp_affinity_initialize() will try to set __kmp_ncores to the
6884 // number of cores on the machine.
6885 __kmp_affinity_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006886
Jonathan Peyton30419822017-05-12 18:01:32 +00006887 // Run through the __kmp_threads array and set the affinity mask
6888 // for each root thread that is currently registered with the RTL.
6889 for (i = 0; i < __kmp_threads_capacity; i++) {
6890 if (TCR_PTR(__kmp_threads[i]) != NULL) {
6891 __kmp_affinity_set_init_mask(i, TRUE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006892 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006893 }
Alp Toker98758b02014-03-02 04:12:06 +00006894#endif /* KMP_AFFINITY_SUPPORTED */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006895
Jonathan Peyton30419822017-05-12 18:01:32 +00006896 KMP_ASSERT(__kmp_xproc > 0);
6897 if (__kmp_avail_proc == 0) {
6898 __kmp_avail_proc = __kmp_xproc;
6899 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006900
Jonathan Peyton30419822017-05-12 18:01:32 +00006901 // If there were empty places in num_threads list (OMP_NUM_THREADS=,,2,3),
6902 // correct them now
6903 j = 0;
6904 while ((j < __kmp_nested_nth.used) && !__kmp_nested_nth.nth[j]) {
6905 __kmp_nested_nth.nth[j] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub =
6906 __kmp_avail_proc;
6907 j++;
6908 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006909
Jonathan Peyton30419822017-05-12 18:01:32 +00006910 if (__kmp_dflt_team_nth == 0) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006911#ifdef KMP_DFLT_NTH_CORES
Jonathan Peyton30419822017-05-12 18:01:32 +00006912 // Default #threads = #cores
6913 __kmp_dflt_team_nth = __kmp_ncores;
6914 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
6915 "__kmp_ncores (%d)\n",
6916 __kmp_dflt_team_nth));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006917#else
Jonathan Peyton30419822017-05-12 18:01:32 +00006918 // Default #threads = #available OS procs
6919 __kmp_dflt_team_nth = __kmp_avail_proc;
6920 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
6921 "__kmp_avail_proc(%d)\n",
6922 __kmp_dflt_team_nth));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006923#endif /* KMP_DFLT_NTH_CORES */
Jonathan Peyton30419822017-05-12 18:01:32 +00006924 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006925
Jonathan Peyton30419822017-05-12 18:01:32 +00006926 if (__kmp_dflt_team_nth < KMP_MIN_NTH) {
6927 __kmp_dflt_team_nth = KMP_MIN_NTH;
6928 }
6929 if (__kmp_dflt_team_nth > __kmp_sys_max_nth) {
6930 __kmp_dflt_team_nth = __kmp_sys_max_nth;
6931 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006932
Jonathan Peyton30419822017-05-12 18:01:32 +00006933 // There's no harm in continuing if the following check fails,
6934 // but it indicates an error in the previous logic.
6935 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006936
Jonathan Peyton30419822017-05-12 18:01:32 +00006937 if (__kmp_dflt_team_nth != prev_dflt_team_nth) {
6938 // Run through the __kmp_threads array and set the num threads icv for each
6939 // root thread that is currently registered with the RTL (which has not
6940 // already explicitly set its nthreads-var with a call to
6941 // omp_set_num_threads()).
6942 for (i = 0; i < __kmp_threads_capacity; i++) {
6943 kmp_info_t *thread = __kmp_threads[i];
6944 if (thread == NULL)
6945 continue;
6946 if (thread->th.th_current_task->td_icvs.nproc != 0)
6947 continue;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006948
Jonathan Peyton30419822017-05-12 18:01:32 +00006949 set__nproc(__kmp_threads[i], __kmp_dflt_team_nth);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006950 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006951 }
6952 KA_TRACE(
6953 20,
6954 ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n",
6955 __kmp_dflt_team_nth));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006956
6957#ifdef KMP_ADJUST_BLOCKTIME
Jonathan Peyton30419822017-05-12 18:01:32 +00006958 /* Adjust blocktime to zero if necessary now that __kmp_avail_proc is set */
6959 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
6960 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
6961 if (__kmp_nth > __kmp_avail_proc) {
6962 __kmp_zero_bt = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006963 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006964 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006965#endif /* KMP_ADJUST_BLOCKTIME */
6966
Jonathan Peyton30419822017-05-12 18:01:32 +00006967 /* we have finished middle initialization */
6968 TCW_SYNC_4(__kmp_init_middle, TRUE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006969
Jonathan Peyton30419822017-05-12 18:01:32 +00006970 KA_TRACE(10, ("__kmp_do_middle_initialize: exit\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006971}
6972
Jonathan Peyton30419822017-05-12 18:01:32 +00006973void __kmp_middle_initialize(void) {
6974 if (__kmp_init_middle) {
6975 return;
6976 }
6977 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6978 if (__kmp_init_middle) {
6979 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6980 return;
6981 }
6982 __kmp_do_middle_initialize();
6983 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6984}
6985
6986void __kmp_parallel_initialize(void) {
6987 int gtid = __kmp_entry_gtid(); // this might be a new root
6988
6989 /* synchronize parallel initialization (for sibling) */
6990 if (TCR_4(__kmp_init_parallel))
6991 return;
6992 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6993 if (TCR_4(__kmp_init_parallel)) {
6994 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6995 return;
6996 }
6997
6998 /* TODO reinitialization after we have already shut down */
6999 if (TCR_4(__kmp_global.g.g_done)) {
7000 KA_TRACE(
7001 10,
7002 ("__kmp_parallel_initialize: attempt to init while shutting down\n"));
7003 __kmp_infinite_loop();
7004 }
7005
7006 /* jc: The lock __kmp_initz_lock is already held, so calling
7007 __kmp_serial_initialize would cause a deadlock. So we call
7008 __kmp_do_serial_initialize directly. */
7009 if (!__kmp_init_middle) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00007010 __kmp_do_middle_initialize();
Jonathan Peyton30419822017-05-12 18:01:32 +00007011 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007012
Jonathan Peyton9b8bb322019-01-16 20:07:39 +00007013#if OMP_50_ENABLED
7014 __kmp_resume_if_hard_paused();
7015#endif
7016
Jonathan Peyton30419822017-05-12 18:01:32 +00007017 /* begin initialization */
7018 KA_TRACE(10, ("__kmp_parallel_initialize: enter\n"));
7019 KMP_ASSERT(KMP_UBER_GTID(gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007020
7021#if KMP_ARCH_X86 || KMP_ARCH_X86_64
Jonathan Peyton30419822017-05-12 18:01:32 +00007022 // Save the FP control regs.
7023 // Worker threads will set theirs to these values at thread startup.
7024 __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
7025 __kmp_store_mxcsr(&__kmp_init_mxcsr);
7026 __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007027#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
7028
7029#if KMP_OS_UNIX
Jonathan Peyton30419822017-05-12 18:01:32 +00007030#if KMP_HANDLE_SIGNALS
7031 /* must be after __kmp_serial_initialize */
7032 __kmp_install_signals(TRUE);
7033#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00007034#endif
7035
Jonathan Peyton30419822017-05-12 18:01:32 +00007036 __kmp_suspend_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00007037
Jonathan Peyton749b4d52016-01-27 21:02:04 +00007038#if defined(USE_LOAD_BALANCE)
Jonathan Peyton30419822017-05-12 18:01:32 +00007039 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
7040 __kmp_global.g.g_dynamic_mode = dynamic_load_balance;
7041 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007042#else
Jonathan Peyton30419822017-05-12 18:01:32 +00007043 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
7044 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
7045 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007046#endif
7047
Jonathan Peyton30419822017-05-12 18:01:32 +00007048 if (__kmp_version) {
7049 __kmp_print_version_2();
7050 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007051
Jonathan Peyton30419822017-05-12 18:01:32 +00007052 /* we have finished parallel initialization */
7053 TCW_SYNC_4(__kmp_init_parallel, TRUE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007054
Jonathan Peyton30419822017-05-12 18:01:32 +00007055 KMP_MB();
7056 KA_TRACE(10, ("__kmp_parallel_initialize: exit\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007057
Jonathan Peyton30419822017-05-12 18:01:32 +00007058 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007059}
7060
Jim Cownie5e8470a2013-09-27 10:38:44 +00007061/* ------------------------------------------------------------------------ */
7062
Jonathan Peyton30419822017-05-12 18:01:32 +00007063void __kmp_run_before_invoked_task(int gtid, int tid, kmp_info_t *this_thr,
7064 kmp_team_t *team) {
7065 kmp_disp_t *dispatch;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007066
Jonathan Peyton30419822017-05-12 18:01:32 +00007067 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00007068
Jonathan Peyton30419822017-05-12 18:01:32 +00007069 /* none of the threads have encountered any constructs, yet. */
7070 this_thr->th.th_local.this_construct = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007071#if KMP_CACHE_MANAGE
Jonathan Peyton30419822017-05-12 18:01:32 +00007072 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007073#endif /* KMP_CACHE_MANAGE */
Jonathan Peyton30419822017-05-12 18:01:32 +00007074 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch);
7075 KMP_DEBUG_ASSERT(dispatch);
7076 KMP_DEBUG_ASSERT(team->t.t_dispatch);
7077 // KMP_DEBUG_ASSERT( this_thr->th.th_dispatch == &team->t.t_dispatch[
7078 // this_thr->th.th_info.ds.ds_tid ] );
Jim Cownie5e8470a2013-09-27 10:38:44 +00007079
Jonathan Peyton30419822017-05-12 18:01:32 +00007080 dispatch->th_disp_index = 0; /* reset the dispatch buffer counter */
Jonathan Peytondf6818b2016-06-14 17:57:47 +00007081#if OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00007082 dispatch->th_doacross_buf_idx =
7083 0; /* reset the doacross dispatch buffer counter */
Jonathan Peyton71909c52016-03-02 22:42:06 +00007084#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00007085 if (__kmp_env_consistency_check)
7086 __kmp_push_parallel(gtid, team->t.t_ident);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007087
Jonathan Peyton30419822017-05-12 18:01:32 +00007088 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007089}
7090
Jonathan Peyton30419822017-05-12 18:01:32 +00007091void __kmp_run_after_invoked_task(int gtid, int tid, kmp_info_t *this_thr,
7092 kmp_team_t *team) {
7093 if (__kmp_env_consistency_check)
7094 __kmp_pop_parallel(gtid, team->t.t_ident);
Andrey Churbanovdf0d75e2016-10-27 11:43:07 +00007095
Jonathan Peyton30419822017-05-12 18:01:32 +00007096 __kmp_finish_implicit_task(this_thr);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007097}
7098
Jonathan Peyton30419822017-05-12 18:01:32 +00007099int __kmp_invoke_task_func(int gtid) {
7100 int rc;
7101 int tid = __kmp_tid_from_gtid(gtid);
7102 kmp_info_t *this_thr = __kmp_threads[gtid];
7103 kmp_team_t *team = this_thr->th.th_team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007104
Jonathan Peyton30419822017-05-12 18:01:32 +00007105 __kmp_run_before_invoked_task(gtid, tid, this_thr, team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007106#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +00007107 if (__itt_stack_caller_create_ptr) {
7108 __kmp_itt_stack_callee_enter(
7109 (__itt_caller)
7110 team->t.t_stack_id); // inform ittnotify about entering user's code
7111 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007112#endif /* USE_ITT_BUILD */
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007113#if INCLUDE_SSC_MARKS
Jonathan Peyton30419822017-05-12 18:01:32 +00007114 SSC_MARK_INVOKING();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007115#endif
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00007116
7117#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00007118 void *dummy;
7119 void **exit_runtime_p;
Joachim Protze82e94a52017-11-01 10:08:30 +00007120 ompt_data_t *my_task_data;
7121 ompt_data_t *my_parallel_data;
7122 int ompt_team_size;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00007123
Joachim Protze82e94a52017-11-01 10:08:30 +00007124 if (ompt_enabled.enabled) {
Joachim Protzec255ca72017-11-05 14:11:10 +00007125 exit_runtime_p = &(
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00007126 team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame.exit_frame.ptr);
Jonathan Peyton30419822017-05-12 18:01:32 +00007127 } else {
7128 exit_runtime_p = &dummy;
7129 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00007130
Joachim Protze82e94a52017-11-01 10:08:30 +00007131 my_task_data =
7132 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data);
7133 my_parallel_data = &(team->t.ompt_team_info.parallel_data);
7134 if (ompt_enabled.ompt_callback_implicit_task) {
7135 ompt_team_size = team->t.t_nproc;
7136 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
7137 ompt_scope_begin, my_parallel_data, my_task_data, ompt_team_size,
Joachim Protze2b46d302019-01-15 15:36:53 +00007138 __kmp_tid_from_gtid(gtid), ompt_task_implicit); // TODO: Can this be ompt_task_initial?
Joachim Protze9be9cf22018-05-07 12:42:21 +00007139 OMPT_CUR_TASK_INFO(this_thr)->thread_num = __kmp_tid_from_gtid(gtid);
Jonathan Peyton30419822017-05-12 18:01:32 +00007140 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00007141#endif
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00007142
Jonathan Peytond2b53ca2019-04-03 18:53:26 +00007143#if KMP_STATS_ENABLED
7144 stats_state_e previous_state = KMP_GET_THREAD_STATE();
7145 if (previous_state == stats_state_e::TEAMS_REGION) {
7146 KMP_PUSH_PARTITIONED_TIMER(OMP_teams);
7147 } else {
7148 KMP_PUSH_PARTITIONED_TIMER(OMP_parallel);
Jonathan Peyton30419822017-05-12 18:01:32 +00007149 }
Jonathan Peytond2b53ca2019-04-03 18:53:26 +00007150 KMP_SET_THREAD_STATE(IMPLICIT_TASK);
7151#endif
7152
7153 rc = __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid,
7154 tid, (int)team->t.t_argc, (void **)team->t.t_argv
7155#if OMPT_SUPPORT
7156 ,
7157 exit_runtime_p
7158#endif
7159 );
7160#if OMPT_SUPPORT
7161 *exit_runtime_p = NULL;
7162#endif
7163
7164#if KMP_STATS_ENABLED
7165 if (previous_state == stats_state_e::TEAMS_REGION) {
7166 KMP_SET_THREAD_STATE(previous_state);
7167 }
7168 KMP_POP_PARTITIONED_TIMER();
7169#endif
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00007170
Jim Cownie5e8470a2013-09-27 10:38:44 +00007171#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +00007172 if (__itt_stack_caller_create_ptr) {
7173 __kmp_itt_stack_callee_leave(
7174 (__itt_caller)
7175 team->t.t_stack_id); // inform ittnotify about leaving user's code
7176 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007177#endif /* USE_ITT_BUILD */
Jonathan Peyton30419822017-05-12 18:01:32 +00007178 __kmp_run_after_invoked_task(gtid, tid, this_thr, team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007179
Jonathan Peyton30419822017-05-12 18:01:32 +00007180 return rc;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007181}
7182
7183#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00007184void __kmp_teams_master(int gtid) {
7185 // This routine is called by all master threads in teams construct
7186 kmp_info_t *thr = __kmp_threads[gtid];
7187 kmp_team_t *team = thr->th.th_team;
7188 ident_t *loc = team->t.t_ident;
7189 thr->th.th_set_nproc = thr->th.th_teams_size.nth;
7190 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask);
7191 KMP_DEBUG_ASSERT(thr->th.th_set_nproc);
7192 KA_TRACE(20, ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid,
7193 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask));
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00007194
7195 // This thread is a new CG root. Set up the proper variables.
7196 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(sizeof(kmp_cg_root_t));
7197 tmp->cg_root = thr; // Make thr the CG root
7198 // Init to thread limit that was stored when league masters were forked
7199 tmp->cg_thread_limit = thr->th.th_current_task->td_icvs.thread_limit;
7200 tmp->cg_nthreads = 1; // Init counter to one active thread, this one
7201 KA_TRACE(100, ("__kmp_teams_master: Thread %p created node %p and init"
7202 " cg_threads to 1\n",
7203 thr, tmp));
7204 tmp->up = thr->th.th_cg_roots;
7205 thr->th.th_cg_roots = tmp;
7206
Jonathan Peyton30419822017-05-12 18:01:32 +00007207// Launch league of teams now, but not let workers execute
7208// (they hang on fork barrier until next parallel)
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007209#if INCLUDE_SSC_MARKS
Jonathan Peyton30419822017-05-12 18:01:32 +00007210 SSC_MARK_FORKING();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007211#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00007212 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc,
Jonathan Peyton30419822017-05-12 18:01:32 +00007213 (microtask_t)thr->th.th_teams_microtask, // "wrapped" task
7214 VOLATILE_CAST(launch_t) __kmp_invoke_task_func, NULL);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007215#if INCLUDE_SSC_MARKS
Jonathan Peyton30419822017-05-12 18:01:32 +00007216 SSC_MARK_JOINING();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007217#endif
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00007218 // If the team size was reduced from the limit, set it to the new size
7219 if (thr->th.th_team_nproc < thr->th.th_teams_size.nth)
7220 thr->th.th_teams_size.nth = thr->th.th_team_nproc;
Jonathan Peyton30419822017-05-12 18:01:32 +00007221 // AC: last parameter "1" eliminates join barrier which won't work because
7222 // worker threads are in a fork barrier waiting for more parallel regions
7223 __kmp_join_call(loc, gtid
Jonathan Peytonf89fbbb2015-08-31 18:15:00 +00007224#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00007225 ,
7226 fork_context_intel
Jonathan Peytonf89fbbb2015-08-31 18:15:00 +00007227#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00007228 ,
7229 1);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007230}
7231
Jonathan Peyton30419822017-05-12 18:01:32 +00007232int __kmp_invoke_teams_master(int gtid) {
7233 kmp_info_t *this_thr = __kmp_threads[gtid];
7234 kmp_team_t *team = this_thr->th.th_team;
7235#if KMP_DEBUG
7236 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized)
7237 KMP_DEBUG_ASSERT((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn ==
7238 (void *)__kmp_teams_master);
7239#endif
7240 __kmp_run_before_invoked_task(gtid, 0, this_thr, team);
7241 __kmp_teams_master(gtid);
7242 __kmp_run_after_invoked_task(gtid, 0, this_thr, team);
7243 return 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007244}
7245#endif /* OMP_40_ENABLED */
7246
7247/* this sets the requested number of threads for the next parallel region
Jonathan Peyton30419822017-05-12 18:01:32 +00007248 encountered by this team. since this should be enclosed in the forkjoin
7249 critical section it should avoid race conditions with assymmetrical nested
7250 parallelism */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007251
Jonathan Peyton30419822017-05-12 18:01:32 +00007252void __kmp_push_num_threads(ident_t *id, int gtid, int num_threads) {
7253 kmp_info_t *thr = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00007254
Jonathan Peyton30419822017-05-12 18:01:32 +00007255 if (num_threads > 0)
7256 thr->th.th_set_nproc = num_threads;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007257}
7258
7259#if OMP_40_ENABLED
7260
7261/* this sets the requested number of teams for the teams region and/or
Jonathan Peyton30419822017-05-12 18:01:32 +00007262 the number of threads for the next parallel region encountered */
7263void __kmp_push_num_teams(ident_t *id, int gtid, int num_teams,
7264 int num_threads) {
7265 kmp_info_t *thr = __kmp_threads[gtid];
7266 KMP_DEBUG_ASSERT(num_teams >= 0);
7267 KMP_DEBUG_ASSERT(num_threads >= 0);
Jonathan Peyton1be692e2015-11-30 20:14:05 +00007268
Jonathan Peyton30419822017-05-12 18:01:32 +00007269 if (num_teams == 0)
7270 num_teams = 1; // default number of teams is 1.
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007271 if (num_teams > __kmp_teams_max_nth) { // if too many teams requested?
Jonathan Peyton30419822017-05-12 18:01:32 +00007272 if (!__kmp_reserve_warn) {
7273 __kmp_reserve_warn = 1;
7274 __kmp_msg(kmp_ms_warning,
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007275 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth),
Jonathan Peyton30419822017-05-12 18:01:32 +00007276 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007277 }
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007278 num_teams = __kmp_teams_max_nth;
Jonathan Peyton30419822017-05-12 18:01:32 +00007279 }
7280 // Set number of teams (number of threads in the outer "parallel" of the
7281 // teams)
7282 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007283
Jonathan Peyton30419822017-05-12 18:01:32 +00007284 // Remember the number of threads for inner parallel regions
7285 if (num_threads == 0) {
7286 if (!TCR_4(__kmp_init_middle))
7287 __kmp_middle_initialize(); // get __kmp_avail_proc calculated
7288 num_threads = __kmp_avail_proc / num_teams;
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007289 if (num_teams * num_threads > __kmp_teams_max_nth) {
Jonathan Peyton30419822017-05-12 18:01:32 +00007290 // adjust num_threads w/o warning as it is not user setting
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007291 num_threads = __kmp_teams_max_nth / num_teams;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007292 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007293 } else {
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00007294 // This thread will be the master of the league masters
7295 // Store new thread limit; old limit is saved in th_cg_roots list
7296 thr->th.th_current_task->td_icvs.thread_limit = num_threads;
7297
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007298 if (num_teams * num_threads > __kmp_teams_max_nth) {
7299 int new_threads = __kmp_teams_max_nth / num_teams;
Jonathan Peyton30419822017-05-12 18:01:32 +00007300 if (!__kmp_reserve_warn) { // user asked for too many threads
Jonathan Peyton65ebfee2019-02-11 21:04:23 +00007301 __kmp_reserve_warn = 1; // conflicts with KMP_TEAMS_THREAD_LIMIT
Jonathan Peyton30419822017-05-12 18:01:32 +00007302 __kmp_msg(kmp_ms_warning,
7303 KMP_MSG(CantFormThrTeam, num_threads, new_threads),
7304 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
7305 }
7306 num_threads = new_threads;
7307 }
7308 }
7309 thr->th.th_teams_size.nth = num_threads;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007310}
7311
Jim Cownie5e8470a2013-09-27 10:38:44 +00007312// Set the proc_bind var to use in the following parallel region.
Jonathan Peyton30419822017-05-12 18:01:32 +00007313void __kmp_push_proc_bind(ident_t *id, int gtid, kmp_proc_bind_t proc_bind) {
7314 kmp_info_t *thr = __kmp_threads[gtid];
7315 thr->th.th_set_proc_bind = proc_bind;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007316}
7317
7318#endif /* OMP_40_ENABLED */
7319
7320/* Launch the worker threads into the microtask. */
7321
Jonathan Peyton30419822017-05-12 18:01:32 +00007322void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team) {
7323 kmp_info_t *this_thr = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00007324
7325#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00007326 int f;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007327#endif /* KMP_DEBUG */
7328
Jonathan Peyton30419822017-05-12 18:01:32 +00007329 KMP_DEBUG_ASSERT(team);
7330 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7331 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7332 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007333
Jonathan Peyton30419822017-05-12 18:01:32 +00007334 team->t.t_construct = 0; /* no single directives seen yet */
7335 team->t.t_ordered.dt.t_value =
7336 0; /* thread 0 enters the ordered section first */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007337
Jonathan Peyton30419822017-05-12 18:01:32 +00007338 /* Reset the identifiers on the dispatch buffer */
7339 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
7340 if (team->t.t_max_nproc > 1) {
7341 int i;
7342 for (i = 0; i < __kmp_dispatch_num_buffers; ++i) {
7343 team->t.t_disp_buffer[i].buffer_index = i;
Jonathan Peytondf6818b2016-06-14 17:57:47 +00007344#if OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00007345 team->t.t_disp_buffer[i].doacross_buf_idx = i;
Jonathan Peyton71909c52016-03-02 22:42:06 +00007346#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00007347 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007348 } else {
7349 team->t.t_disp_buffer[0].buffer_index = 0;
7350#if OMP_45_ENABLED
7351 team->t.t_disp_buffer[0].doacross_buf_idx = 0;
7352#endif
7353 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007354
Jonathan Peyton30419822017-05-12 18:01:32 +00007355 KMP_MB(); /* Flush all pending memory write invalidates. */
7356 KMP_ASSERT(this_thr->th.th_team == team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007357
7358#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00007359 for (f = 0; f < team->t.t_nproc; f++) {
7360 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
7361 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc);
7362 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007363#endif /* KMP_DEBUG */
7364
Jonathan Peyton30419822017-05-12 18:01:32 +00007365 /* release the worker threads so they may begin working */
7366 __kmp_fork_barrier(gtid, 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007367}
7368
Jonathan Peyton30419822017-05-12 18:01:32 +00007369void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team) {
7370 kmp_info_t *this_thr = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00007371
Jonathan Peyton30419822017-05-12 18:01:32 +00007372 KMP_DEBUG_ASSERT(team);
7373 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7374 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7375 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007376
Jonathan Peyton30419822017-05-12 18:01:32 +00007377/* Join barrier after fork */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007378
7379#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00007380 if (__kmp_threads[gtid] &&
7381 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) {
7382 __kmp_printf("GTID: %d, __kmp_threads[%d]=%p\n", gtid, gtid,
7383 __kmp_threads[gtid]);
7384 __kmp_printf("__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, "
7385 "team->t.t_nproc=%d\n",
7386 gtid, __kmp_threads[gtid]->th.th_team_nproc, team,
7387 team->t.t_nproc);
7388 __kmp_print_structure();
7389 }
7390 KMP_DEBUG_ASSERT(__kmp_threads[gtid] &&
7391 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007392#endif /* KMP_DEBUG */
7393
Jonathan Peyton30419822017-05-12 18:01:32 +00007394 __kmp_join_barrier(gtid); /* wait for everyone */
Joachim Protze82e94a52017-11-01 10:08:30 +00007395#if OMPT_SUPPORT
Jonas Hahnfeld82768d02018-02-23 16:46:25 +00007396 if (ompt_enabled.enabled &&
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00007397 this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) {
Jonas Hahnfeld82768d02018-02-23 16:46:25 +00007398 int ds_tid = this_thr->th.th_info.ds.ds_tid;
7399 ompt_data_t *task_data = OMPT_CUR_TASK_DATA(this_thr);
Joachim Protze0e0d6cd2018-12-18 08:52:30 +00007400 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
Joachim Protze82e94a52017-11-01 10:08:30 +00007401#if OMPT_OPTIONAL
7402 void *codeptr = NULL;
7403 if (KMP_MASTER_TID(ds_tid) &&
7404 (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait) ||
7405 ompt_callbacks.ompt_callback(ompt_callback_sync_region)))
7406 codeptr = OMPT_CUR_TEAM_INFO(this_thr)->master_return_address;
7407
7408 if (ompt_enabled.ompt_callback_sync_region_wait) {
7409 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
Jonathan Peytonad1ad7a2019-02-28 20:55:39 +00007410 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data,
7411 codeptr);
Joachim Protze82e94a52017-11-01 10:08:30 +00007412 }
7413 if (ompt_enabled.ompt_callback_sync_region) {
7414 ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
Jonathan Peytonad1ad7a2019-02-28 20:55:39 +00007415 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data,
7416 codeptr);
Joachim Protze82e94a52017-11-01 10:08:30 +00007417 }
7418#endif
7419 if (!KMP_MASTER_TID(ds_tid) && ompt_enabled.ompt_callback_implicit_task) {
7420 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
Joachim Protze2b46d302019-01-15 15:36:53 +00007421 ompt_scope_end, NULL, task_data, 0, ds_tid, ompt_task_implicit); // TODO: Can this be ompt_task_initial?
Joachim Protze82e94a52017-11-01 10:08:30 +00007422 }
Joachim Protze82e94a52017-11-01 10:08:30 +00007423 }
7424#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00007425
Jonathan Peyton30419822017-05-12 18:01:32 +00007426 KMP_MB(); /* Flush all pending memory write invalidates. */
7427 KMP_ASSERT(this_thr->th.th_team == team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007428}
7429
Jim Cownie5e8470a2013-09-27 10:38:44 +00007430/* ------------------------------------------------------------------------ */
7431
7432#ifdef USE_LOAD_BALANCE
7433
Jim Cownie5e8470a2013-09-27 10:38:44 +00007434// Return the worker threads actively spinning in the hot team, if we
7435// are at the outermost level of parallelism. Otherwise, return 0.
Jonathan Peyton30419822017-05-12 18:01:32 +00007436static int __kmp_active_hot_team_nproc(kmp_root_t *root) {
7437 int i;
7438 int retval;
7439 kmp_team_t *hot_team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007440
Jonathan Peyton30419822017-05-12 18:01:32 +00007441 if (root->r.r_active) {
7442 return 0;
7443 }
7444 hot_team = root->r.r_hot_team;
7445 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
7446 return hot_team->t.t_nproc - 1; // Don't count master thread
7447 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007448
Jonathan Peyton30419822017-05-12 18:01:32 +00007449 // Skip the master thread - it is accounted for elsewhere.
7450 retval = 0;
7451 for (i = 1; i < hot_team->t.t_nproc; i++) {
7452 if (hot_team->t.t_threads[i]->th.th_active) {
7453 retval++;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007454 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007455 }
7456 return retval;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007457}
7458
Jim Cownie5e8470a2013-09-27 10:38:44 +00007459// Perform an automatic adjustment to the number of
7460// threads used by the next parallel region.
Jonathan Peyton30419822017-05-12 18:01:32 +00007461static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc) {
7462 int retval;
7463 int pool_active;
7464 int hot_team_active;
7465 int team_curr_active;
7466 int system_active;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007467
Jonathan Peyton30419822017-05-12 18:01:32 +00007468 KB_TRACE(20, ("__kmp_load_balance_nproc: called root:%p set_nproc:%d\n", root,
7469 set_nproc));
7470 KMP_DEBUG_ASSERT(root);
7471 KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0]
7472 ->th.th_current_task->td_icvs.dynamic == TRUE);
7473 KMP_DEBUG_ASSERT(set_nproc > 1);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007474
Jonathan Peyton30419822017-05-12 18:01:32 +00007475 if (set_nproc == 1) {
7476 KB_TRACE(20, ("__kmp_load_balance_nproc: serial execution.\n"));
7477 return 1;
7478 }
7479
7480 // Threads that are active in the thread pool, active in the hot team for this
7481 // particular root (if we are at the outer par level), and the currently
7482 // executing thread (to become the master) are available to add to the new
7483 // team, but are currently contributing to the system load, and must be
7484 // accounted for.
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00007485 pool_active = __kmp_thread_pool_active_nth;
Jonathan Peyton30419822017-05-12 18:01:32 +00007486 hot_team_active = __kmp_active_hot_team_nproc(root);
7487 team_curr_active = pool_active + hot_team_active + 1;
7488
7489 // Check the system load.
7490 system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active);
7491 KB_TRACE(30, ("__kmp_load_balance_nproc: system active = %d pool active = %d "
7492 "hot team active = %d\n",
7493 system_active, pool_active, hot_team_active));
7494
7495 if (system_active < 0) {
7496 // There was an error reading the necessary info from /proc, so use the
7497 // thread limit algorithm instead. Once we set __kmp_global.g.g_dynamic_mode
7498 // = dynamic_thread_limit, we shouldn't wind up getting back here.
7499 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
7500 KMP_WARNING(CantLoadBalUsing, "KMP_DYNAMIC_MODE=thread limit");
7501
7502 // Make this call behave like the thread limit algorithm.
7503 retval = __kmp_avail_proc - __kmp_nth +
7504 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
7505 if (retval > set_nproc) {
7506 retval = set_nproc;
7507 }
7508 if (retval < KMP_MIN_NTH) {
7509 retval = KMP_MIN_NTH;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007510 }
7511
Jonathan Peyton30419822017-05-12 18:01:32 +00007512 KB_TRACE(20, ("__kmp_load_balance_nproc: thread limit exit. retval:%d\n",
7513 retval));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007514 return retval;
Jonathan Peyton30419822017-05-12 18:01:32 +00007515 }
7516
7517 // There is a slight delay in the load balance algorithm in detecting new
7518 // running procs. The real system load at this instant should be at least as
7519 // large as the #active omp thread that are available to add to the team.
7520 if (system_active < team_curr_active) {
7521 system_active = team_curr_active;
7522 }
7523 retval = __kmp_avail_proc - system_active + team_curr_active;
7524 if (retval > set_nproc) {
7525 retval = set_nproc;
7526 }
7527 if (retval < KMP_MIN_NTH) {
7528 retval = KMP_MIN_NTH;
7529 }
7530
7531 KB_TRACE(20, ("__kmp_load_balance_nproc: exit. retval:%d\n", retval));
7532 return retval;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007533} // __kmp_load_balance_nproc()
7534
7535#endif /* USE_LOAD_BALANCE */
7536
Jim Cownie5e8470a2013-09-27 10:38:44 +00007537/* ------------------------------------------------------------------------ */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007538
7539/* NOTE: this is called with the __kmp_init_lock held */
Jonathan Peyton30419822017-05-12 18:01:32 +00007540void __kmp_cleanup(void) {
7541 int f;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007542
Jonathan Peyton30419822017-05-12 18:01:32 +00007543 KA_TRACE(10, ("__kmp_cleanup: enter\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007544
Jonathan Peyton30419822017-05-12 18:01:32 +00007545 if (TCR_4(__kmp_init_parallel)) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00007546#if KMP_HANDLE_SIGNALS
Jonathan Peyton30419822017-05-12 18:01:32 +00007547 __kmp_remove_signals();
Jim Cownie5e8470a2013-09-27 10:38:44 +00007548#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00007549 TCW_4(__kmp_init_parallel, FALSE);
7550 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007551
Jonathan Peyton30419822017-05-12 18:01:32 +00007552 if (TCR_4(__kmp_init_middle)) {
Alp Toker763b9392014-02-28 09:42:41 +00007553#if KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00007554 __kmp_affinity_uninitialize();
Alp Toker763b9392014-02-28 09:42:41 +00007555#endif /* KMP_AFFINITY_SUPPORTED */
Jonathan Peyton30419822017-05-12 18:01:32 +00007556 __kmp_cleanup_hierarchy();
7557 TCW_4(__kmp_init_middle, FALSE);
7558 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007559
Jonathan Peyton30419822017-05-12 18:01:32 +00007560 KA_TRACE(10, ("__kmp_cleanup: go serial cleanup\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007561
Jonathan Peyton30419822017-05-12 18:01:32 +00007562 if (__kmp_init_serial) {
7563 __kmp_runtime_destroy();
7564 __kmp_init_serial = FALSE;
7565 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007566
Andrey Churbanov9e9333a2018-03-05 18:42:01 +00007567 __kmp_cleanup_threadprivate_caches();
7568
Jonathan Peyton30419822017-05-12 18:01:32 +00007569 for (f = 0; f < __kmp_threads_capacity; f++) {
7570 if (__kmp_root[f] != NULL) {
7571 __kmp_free(__kmp_root[f]);
7572 __kmp_root[f] = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007573 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007574 }
7575 __kmp_free(__kmp_threads);
7576 // __kmp_threads and __kmp_root were allocated at once, as single block, so
7577 // there is no need in freeing __kmp_root.
7578 __kmp_threads = NULL;
7579 __kmp_root = NULL;
7580 __kmp_threads_capacity = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007581
Andrey Churbanov5c56fb52015-02-20 18:05:17 +00007582#if KMP_USE_DYNAMIC_LOCK
Jonathan Peyton30419822017-05-12 18:01:32 +00007583 __kmp_cleanup_indirect_user_locks();
Andrey Churbanov5c56fb52015-02-20 18:05:17 +00007584#else
Jonathan Peyton30419822017-05-12 18:01:32 +00007585 __kmp_cleanup_user_locks();
Andrey Churbanov5c56fb52015-02-20 18:05:17 +00007586#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00007587
Jonathan Peyton30419822017-05-12 18:01:32 +00007588#if KMP_AFFINITY_SUPPORTED
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00007589 KMP_INTERNAL_FREE(CCAST(char *, __kmp_cpuinfo_file));
Jonathan Peyton30419822017-05-12 18:01:32 +00007590 __kmp_cpuinfo_file = NULL;
7591#endif /* KMP_AFFINITY_SUPPORTED */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007592
Jonathan Peyton30419822017-05-12 18:01:32 +00007593#if KMP_USE_ADAPTIVE_LOCKS
7594#if KMP_DEBUG_ADAPTIVE_LOCKS
7595 __kmp_print_speculative_stats();
7596#endif
7597#endif
7598 KMP_INTERNAL_FREE(__kmp_nested_nth.nth);
7599 __kmp_nested_nth.nth = NULL;
7600 __kmp_nested_nth.size = 0;
7601 __kmp_nested_nth.used = 0;
7602 KMP_INTERNAL_FREE(__kmp_nested_proc_bind.bind_types);
7603 __kmp_nested_proc_bind.bind_types = NULL;
7604 __kmp_nested_proc_bind.size = 0;
7605 __kmp_nested_proc_bind.used = 0;
Jonathan Peyton6d88e042018-12-13 23:14:24 +00007606#if OMP_50_ENABLED
7607 if (__kmp_affinity_format) {
7608 KMP_INTERNAL_FREE(__kmp_affinity_format);
7609 __kmp_affinity_format = NULL;
7610 }
7611#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00007612
Jonathan Peyton30419822017-05-12 18:01:32 +00007613 __kmp_i18n_catclose();
Jim Cownie5e8470a2013-09-27 10:38:44 +00007614
Jonathan Peytonf6399362018-07-09 17:51:13 +00007615#if KMP_USE_HIER_SCHED
7616 __kmp_hier_scheds.deallocate();
7617#endif
7618
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007619#if KMP_STATS_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00007620 __kmp_stats_fini();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007621#endif
7622
Jonathan Peyton30419822017-05-12 18:01:32 +00007623 KA_TRACE(10, ("__kmp_cleanup: exit\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007624}
7625
7626/* ------------------------------------------------------------------------ */
Jonathan Peyton30419822017-05-12 18:01:32 +00007627
7628int __kmp_ignore_mppbeg(void) {
7629 char *env;
7630
7631 if ((env = getenv("KMP_IGNORE_MPPBEG")) != NULL) {
7632 if (__kmp_str_match_false(env))
7633 return FALSE;
7634 }
7635 // By default __kmpc_begin() is no-op.
7636 return TRUE;
7637}
7638
7639int __kmp_ignore_mppend(void) {
7640 char *env;
7641
7642 if ((env = getenv("KMP_IGNORE_MPPEND")) != NULL) {
7643 if (__kmp_str_match_false(env))
7644 return FALSE;
7645 }
7646 // By default __kmpc_end() is no-op.
7647 return TRUE;
7648}
7649
7650void __kmp_internal_begin(void) {
7651 int gtid;
7652 kmp_root_t *root;
7653
7654 /* this is a very important step as it will register new sibling threads
7655 and assign these new uber threads a new gtid */
7656 gtid = __kmp_entry_gtid();
7657 root = __kmp_threads[gtid]->th.th_root;
7658 KMP_ASSERT(KMP_UBER_GTID(gtid));
7659
7660 if (root->r.r_begin)
7661 return;
7662 __kmp_acquire_lock(&root->r.r_begin_lock, gtid);
7663 if (root->r.r_begin) {
7664 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7665 return;
7666 }
7667
7668 root->r.r_begin = TRUE;
7669
7670 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7671}
7672
Jim Cownie5e8470a2013-09-27 10:38:44 +00007673/* ------------------------------------------------------------------------ */
7674
Jonathan Peyton30419822017-05-12 18:01:32 +00007675void __kmp_user_set_library(enum library_type arg) {
7676 int gtid;
7677 kmp_root_t *root;
7678 kmp_info_t *thread;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007679
Jonathan Peyton30419822017-05-12 18:01:32 +00007680 /* first, make sure we are initialized so we can get our gtid */
7681
7682 gtid = __kmp_entry_gtid();
7683 thread = __kmp_threads[gtid];
7684
7685 root = thread->th.th_root;
7686
7687 KA_TRACE(20, ("__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg,
7688 library_serial));
7689 if (root->r.r_in_parallel) { /* Must be called in serial section of top-level
7690 thread */
7691 KMP_WARNING(SetLibraryIncorrectCall);
7692 return;
7693 }
7694
7695 switch (arg) {
7696 case library_serial:
7697 thread->th.th_set_nproc = 0;
7698 set__nproc(thread, 1);
7699 break;
7700 case library_turnaround:
7701 thread->th.th_set_nproc = 0;
7702 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
7703 : __kmp_dflt_team_nth_ub);
7704 break;
7705 case library_throughput:
7706 thread->th.th_set_nproc = 0;
7707 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
7708 : __kmp_dflt_team_nth_ub);
7709 break;
7710 default:
7711 KMP_FATAL(UnknownLibraryType, arg);
7712 }
7713
7714 __kmp_aux_set_library(arg);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007715}
7716
Jonathan Peyton30419822017-05-12 18:01:32 +00007717void __kmp_aux_set_stacksize(size_t arg) {
7718 if (!__kmp_init_serial)
7719 __kmp_serial_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00007720
7721#if KMP_OS_DARWIN
Jonathan Peyton30419822017-05-12 18:01:32 +00007722 if (arg & (0x1000 - 1)) {
7723 arg &= ~(0x1000 - 1);
7724 if (arg + 0x1000) /* check for overflow if we round up */
7725 arg += 0x1000;
7726 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007727#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00007728 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007729
Jonathan Peyton30419822017-05-12 18:01:32 +00007730 /* only change the default stacksize before the first parallel region */
7731 if (!TCR_4(__kmp_init_parallel)) {
7732 size_t value = arg; /* argument is in bytes */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007733
Jonathan Peyton30419822017-05-12 18:01:32 +00007734 if (value < __kmp_sys_min_stksize)
7735 value = __kmp_sys_min_stksize;
7736 else if (value > KMP_MAX_STKSIZE)
7737 value = KMP_MAX_STKSIZE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007738
Jonathan Peyton30419822017-05-12 18:01:32 +00007739 __kmp_stksize = value;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007740
Jonathan Peyton30419822017-05-12 18:01:32 +00007741 __kmp_env_stksize = TRUE; /* was KMP_STACKSIZE specified? */
7742 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007743
Jonathan Peyton30419822017-05-12 18:01:32 +00007744 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007745}
7746
7747/* set the behaviour of the runtime library */
7748/* TODO this can cause some odd behaviour with sibling parallelism... */
Jonathan Peyton30419822017-05-12 18:01:32 +00007749void __kmp_aux_set_library(enum library_type arg) {
7750 __kmp_library = arg;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007751
Jonathan Peyton30419822017-05-12 18:01:32 +00007752 switch (__kmp_library) {
7753 case library_serial: {
7754 KMP_INFORM(LibraryIsSerial);
Jonathan Peyton30419822017-05-12 18:01:32 +00007755 } break;
7756 case library_turnaround:
Jonathan Peytone47d32f2019-02-28 19:11:29 +00007757 if (__kmp_use_yield == 1 && !__kmp_use_yield_exp_set)
7758 __kmp_use_yield = 2; // only yield when oversubscribed
Jonathan Peyton30419822017-05-12 18:01:32 +00007759 break;
7760 case library_throughput:
Jonathan Peytone47d32f2019-02-28 19:11:29 +00007761 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME)
7762 __kmp_dflt_blocktime = 200;
Jonathan Peyton30419822017-05-12 18:01:32 +00007763 break;
7764 default:
7765 KMP_FATAL(UnknownLibraryType, arg);
7766 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007767}
7768
Jonathan Peyton6d88e042018-12-13 23:14:24 +00007769/* Getting team information common for all team API */
7770// Returns NULL if not in teams construct
7771static kmp_team_t *__kmp_aux_get_team_info(int &teams_serialized) {
7772 kmp_info_t *thr = __kmp_entry_thread();
7773 teams_serialized = 0;
7774 if (thr->th.th_teams_microtask) {
7775 kmp_team_t *team = thr->th.th_team;
7776 int tlevel = thr->th.th_teams_level; // the level of the teams construct
7777 int ii = team->t.t_level;
7778 teams_serialized = team->t.t_serialized;
7779 int level = tlevel + 1;
7780 KMP_DEBUG_ASSERT(ii >= tlevel);
7781 while (ii > level) {
7782 for (teams_serialized = team->t.t_serialized;
7783 (teams_serialized > 0) && (ii > level); teams_serialized--, ii--) {
7784 }
7785 if (team->t.t_serialized && (!teams_serialized)) {
7786 team = team->t.t_parent;
7787 continue;
7788 }
7789 if (ii > level) {
7790 team = team->t.t_parent;
7791 ii--;
7792 }
7793 }
7794 return team;
7795 }
7796 return NULL;
7797}
7798
7799int __kmp_aux_get_team_num() {
7800 int serialized;
7801 kmp_team_t *team = __kmp_aux_get_team_info(serialized);
7802 if (team) {
7803 if (serialized > 1) {
7804 return 0; // teams region is serialized ( 1 team of 1 thread ).
7805 } else {
7806 return team->t.t_master_tid;
7807 }
7808 }
7809 return 0;
7810}
7811
7812int __kmp_aux_get_num_teams() {
7813 int serialized;
7814 kmp_team_t *team = __kmp_aux_get_team_info(serialized);
7815 if (team) {
7816 if (serialized > 1) {
7817 return 1;
7818 } else {
7819 return team->t.t_parent->t.t_nproc;
7820 }
7821 }
7822 return 1;
7823}
7824
7825/* ------------------------------------------------------------------------ */
7826
7827#if OMP_50_ENABLED
7828/*
7829 * Affinity Format Parser
7830 *
7831 * Field is in form of: %[[[0].]size]type
7832 * % and type are required (%% means print a literal '%')
7833 * type is either single char or long name surrounded by {},
7834 * e.g., N or {num_threads}
7835 * 0 => leading zeros
7836 * . => right justified when size is specified
7837 * by default output is left justified
7838 * size is the *minimum* field length
7839 * All other characters are printed as is
7840 *
7841 * Available field types:
7842 * L {thread_level} - omp_get_level()
7843 * n {thread_num} - omp_get_thread_num()
7844 * h {host} - name of host machine
7845 * P {process_id} - process id (integer)
7846 * T {thread_identifier} - native thread identifier (integer)
7847 * N {num_threads} - omp_get_num_threads()
7848 * A {ancestor_tnum} - omp_get_ancestor_thread_num(omp_get_level()-1)
7849 * a {thread_affinity} - comma separated list of integers or integer ranges
7850 * (values of affinity mask)
7851 *
7852 * Implementation-specific field types can be added
7853 * If a type is unknown, print "undefined"
7854*/
7855
7856// Structure holding the short name, long name, and corresponding data type
7857// for snprintf. A table of these will represent the entire valid keyword
7858// field types.
7859typedef struct kmp_affinity_format_field_t {
7860 char short_name; // from spec e.g., L -> thread level
7861 const char *long_name; // from spec thread_level -> thread level
7862 char field_format; // data type for snprintf (typically 'd' or 's'
7863 // for integer or string)
7864} kmp_affinity_format_field_t;
7865
7866static const kmp_affinity_format_field_t __kmp_affinity_format_table[] = {
7867#if KMP_AFFINITY_SUPPORTED
7868 {'A', "thread_affinity", 's'},
7869#endif
7870 {'t', "team_num", 'd'},
7871 {'T', "num_teams", 'd'},
7872 {'L', "nesting_level", 'd'},
7873 {'n', "thread_num", 'd'},
7874 {'N', "num_threads", 'd'},
7875 {'a', "ancestor_tnum", 'd'},
7876 {'H', "host", 's'},
7877 {'P', "process_id", 'd'},
7878 {'i', "native_thread_id", 'd'}};
7879
7880// Return the number of characters it takes to hold field
7881static int __kmp_aux_capture_affinity_field(int gtid, const kmp_info_t *th,
7882 const char **ptr,
7883 kmp_str_buf_t *field_buffer) {
7884 int rc, format_index, field_value;
7885 const char *width_left, *width_right;
7886 bool pad_zeros, right_justify, parse_long_name, found_valid_name;
7887 static const int FORMAT_SIZE = 20;
7888 char format[FORMAT_SIZE] = {0};
7889 char absolute_short_name = 0;
7890
7891 KMP_DEBUG_ASSERT(gtid >= 0);
7892 KMP_DEBUG_ASSERT(th);
7893 KMP_DEBUG_ASSERT(**ptr == '%');
7894 KMP_DEBUG_ASSERT(field_buffer);
7895
7896 __kmp_str_buf_clear(field_buffer);
7897
7898 // Skip the initial %
7899 (*ptr)++;
7900
7901 // Check for %% first
7902 if (**ptr == '%') {
7903 __kmp_str_buf_cat(field_buffer, "%", 1);
7904 (*ptr)++; // skip over the second %
7905 return 1;
7906 }
7907
7908 // Parse field modifiers if they are present
7909 pad_zeros = false;
7910 if (**ptr == '0') {
7911 pad_zeros = true;
7912 (*ptr)++; // skip over 0
7913 }
7914 right_justify = false;
7915 if (**ptr == '.') {
7916 right_justify = true;
7917 (*ptr)++; // skip over .
7918 }
7919 // Parse width of field: [width_left, width_right)
7920 width_left = width_right = NULL;
7921 if (**ptr >= '0' && **ptr <= '9') {
7922 width_left = *ptr;
7923 SKIP_DIGITS(*ptr);
7924 width_right = *ptr;
7925 }
7926
7927 // Create the format for KMP_SNPRINTF based on flags parsed above
7928 format_index = 0;
7929 format[format_index++] = '%';
7930 if (!right_justify)
7931 format[format_index++] = '-';
7932 if (pad_zeros)
7933 format[format_index++] = '0';
7934 if (width_left && width_right) {
7935 int i = 0;
7936 // Only allow 8 digit number widths.
7937 // This also prevents overflowing format variable
7938 while (i < 8 && width_left < width_right) {
7939 format[format_index++] = *width_left;
7940 width_left++;
7941 i++;
7942 }
7943 }
7944
7945 // Parse a name (long or short)
7946 // Canonicalize the name into absolute_short_name
7947 found_valid_name = false;
7948 parse_long_name = (**ptr == '{');
7949 if (parse_long_name)
7950 (*ptr)++; // skip initial left brace
7951 for (size_t i = 0; i < sizeof(__kmp_affinity_format_table) /
7952 sizeof(__kmp_affinity_format_table[0]);
7953 ++i) {
7954 char short_name = __kmp_affinity_format_table[i].short_name;
7955 const char *long_name = __kmp_affinity_format_table[i].long_name;
7956 char field_format = __kmp_affinity_format_table[i].field_format;
7957 if (parse_long_name) {
7958 int length = KMP_STRLEN(long_name);
7959 if (strncmp(*ptr, long_name, length) == 0) {
7960 found_valid_name = true;
7961 (*ptr) += length; // skip the long name
7962 }
7963 } else if (**ptr == short_name) {
7964 found_valid_name = true;
7965 (*ptr)++; // skip the short name
7966 }
7967 if (found_valid_name) {
7968 format[format_index++] = field_format;
7969 format[format_index++] = '\0';
7970 absolute_short_name = short_name;
7971 break;
7972 }
7973 }
7974 if (parse_long_name) {
7975 if (**ptr != '}') {
7976 absolute_short_name = 0;
7977 } else {
7978 (*ptr)++; // skip over the right brace
7979 }
7980 }
7981
7982 // Attempt to fill the buffer with the requested
7983 // value using snprintf within __kmp_str_buf_print()
7984 switch (absolute_short_name) {
7985 case 't':
7986 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_team_num());
7987 break;
7988 case 'T':
7989 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_num_teams());
7990 break;
7991 case 'L':
7992 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_level);
7993 break;
7994 case 'n':
7995 rc = __kmp_str_buf_print(field_buffer, format, __kmp_tid_from_gtid(gtid));
7996 break;
7997 case 'H': {
7998 static const int BUFFER_SIZE = 256;
7999 char buf[BUFFER_SIZE];
8000 __kmp_expand_host_name(buf, BUFFER_SIZE);
8001 rc = __kmp_str_buf_print(field_buffer, format, buf);
8002 } break;
8003 case 'P':
8004 rc = __kmp_str_buf_print(field_buffer, format, getpid());
8005 break;
8006 case 'i':
8007 rc = __kmp_str_buf_print(field_buffer, format, __kmp_gettid());
8008 break;
8009 case 'N':
8010 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_nproc);
8011 break;
8012 case 'a':
8013 field_value =
8014 __kmp_get_ancestor_thread_num(gtid, th->th.th_team->t.t_level - 1);
8015 rc = __kmp_str_buf_print(field_buffer, format, field_value);
8016 break;
8017#if KMP_AFFINITY_SUPPORTED
8018 case 'A': {
8019 kmp_str_buf_t buf;
8020 __kmp_str_buf_init(&buf);
8021 __kmp_affinity_str_buf_mask(&buf, th->th.th_affin_mask);
8022 rc = __kmp_str_buf_print(field_buffer, format, buf.str);
8023 __kmp_str_buf_free(&buf);
8024 } break;
8025#endif
8026 default:
8027 // According to spec, If an implementation does not have info for field
8028 // type, then "undefined" is printed
8029 rc = __kmp_str_buf_print(field_buffer, "%s", "undefined");
8030 // Skip the field
8031 if (parse_long_name) {
8032 SKIP_TOKEN(*ptr);
8033 if (**ptr == '}')
8034 (*ptr)++;
8035 } else {
8036 (*ptr)++;
8037 }
8038 }
8039
8040 KMP_ASSERT(format_index <= FORMAT_SIZE);
8041 return rc;
8042}
8043
8044/*
8045 * Return number of characters needed to hold the affinity string
8046 * (not including null byte character)
8047 * The resultant string is printed to buffer, which the caller can then
8048 * handle afterwards
8049*/
8050size_t __kmp_aux_capture_affinity(int gtid, const char *format,
8051 kmp_str_buf_t *buffer) {
8052 const char *parse_ptr;
8053 size_t retval;
8054 const kmp_info_t *th;
8055 kmp_str_buf_t field;
8056
8057 KMP_DEBUG_ASSERT(buffer);
8058 KMP_DEBUG_ASSERT(gtid >= 0);
8059
8060 __kmp_str_buf_init(&field);
8061 __kmp_str_buf_clear(buffer);
8062
8063 th = __kmp_threads[gtid];
8064 retval = 0;
8065
8066 // If format is NULL or zero-length string, then we use
8067 // affinity-format-var ICV
8068 parse_ptr = format;
8069 if (parse_ptr == NULL || *parse_ptr == '\0') {
8070 parse_ptr = __kmp_affinity_format;
8071 }
8072 KMP_DEBUG_ASSERT(parse_ptr);
8073
8074 while (*parse_ptr != '\0') {
8075 // Parse a field
8076 if (*parse_ptr == '%') {
8077 // Put field in the buffer
8078 int rc = __kmp_aux_capture_affinity_field(gtid, th, &parse_ptr, &field);
8079 __kmp_str_buf_catbuf(buffer, &field);
8080 retval += rc;
8081 } else {
8082 // Put literal character in buffer
8083 __kmp_str_buf_cat(buffer, parse_ptr, 1);
8084 retval++;
8085 parse_ptr++;
8086 }
8087 }
8088 __kmp_str_buf_free(&field);
8089 return retval;
8090}
8091
8092// Displays the affinity string to stdout
8093void __kmp_aux_display_affinity(int gtid, const char *format) {
8094 kmp_str_buf_t buf;
8095 __kmp_str_buf_init(&buf);
8096 __kmp_aux_capture_affinity(gtid, format, &buf);
8097 __kmp_fprintf(kmp_out, "%s" KMP_END_OF_LINE, buf.str);
8098 __kmp_str_buf_free(&buf);
8099}
8100#endif // OMP_50_ENABLED
8101
Jim Cownie5e8470a2013-09-27 10:38:44 +00008102/* ------------------------------------------------------------------------ */
Jim Cownie5e8470a2013-09-27 10:38:44 +00008103
Jonathan Peyton30419822017-05-12 18:01:32 +00008104void __kmp_aux_set_blocktime(int arg, kmp_info_t *thread, int tid) {
8105 int blocktime = arg; /* argument is in milliseconds */
Jonathan Peytone1c7c132016-10-07 18:12:19 +00008106#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00008107 int bt_intervals;
Jonathan Peytone1c7c132016-10-07 18:12:19 +00008108#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00008109 int bt_set;
Jim Cownie5e8470a2013-09-27 10:38:44 +00008110
Jonathan Peyton30419822017-05-12 18:01:32 +00008111 __kmp_save_internal_controls(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00008112
Jonathan Peyton30419822017-05-12 18:01:32 +00008113 /* Normalize and set blocktime for the teams */
8114 if (blocktime < KMP_MIN_BLOCKTIME)
8115 blocktime = KMP_MIN_BLOCKTIME;
8116 else if (blocktime > KMP_MAX_BLOCKTIME)
8117 blocktime = KMP_MAX_BLOCKTIME;
Jim Cownie5e8470a2013-09-27 10:38:44 +00008118
Jonathan Peyton30419822017-05-12 18:01:32 +00008119 set__blocktime_team(thread->th.th_team, tid, blocktime);
8120 set__blocktime_team(thread->th.th_serial_team, 0, blocktime);
Jim Cownie5e8470a2013-09-27 10:38:44 +00008121
Jonathan Peytone1c7c132016-10-07 18:12:19 +00008122#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00008123 /* Calculate and set blocktime intervals for the teams */
8124 bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups);
Jim Cownie5e8470a2013-09-27 10:38:44 +00008125
Jonathan Peyton30419822017-05-12 18:01:32 +00008126 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals);
8127 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals);
Jonathan Peytone1c7c132016-10-07 18:12:19 +00008128#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00008129
Jonathan Peyton30419822017-05-12 18:01:32 +00008130 /* Set whether blocktime has been set to "TRUE" */
8131 bt_set = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00008132
Jonathan Peyton30419822017-05-12 18:01:32 +00008133 set__bt_set_team(thread->th.th_team, tid, bt_set);
8134 set__bt_set_team(thread->th.th_serial_team, 0, bt_set);
Jonathan Peytone1c7c132016-10-07 18:12:19 +00008135#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00008136 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
8137 "bt_intervals=%d, monitor_updates=%d\n",
8138 __kmp_gtid_from_tid(tid, thread->th.th_team),
8139 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals,
8140 __kmp_monitor_wakeups));
Samuel Antao33515192016-10-20 13:20:17 +00008141#else
Jonathan Peyton30419822017-05-12 18:01:32 +00008142 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n",
8143 __kmp_gtid_from_tid(tid, thread->th.th_team),
8144 thread->th.th_team->t.t_id, tid, blocktime));
Jonathan Peytone1c7c132016-10-07 18:12:19 +00008145#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00008146}
8147
Jonathan Peyton30419822017-05-12 18:01:32 +00008148void __kmp_aux_set_defaults(char const *str, int len) {
8149 if (!__kmp_init_serial) {
8150 __kmp_serial_initialize();
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00008151 }
Jonathan Peyton30419822017-05-12 18:01:32 +00008152 __kmp_env_initialize(str);
Jim Cownie5e8470a2013-09-27 10:38:44 +00008153
Jonathan Peyton30419822017-05-12 18:01:32 +00008154 if (__kmp_settings
Jim Cownie5e8470a2013-09-27 10:38:44 +00008155#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00008156 || __kmp_display_env || __kmp_display_env_verbose
Jim Cownie5e8470a2013-09-27 10:38:44 +00008157#endif // OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00008158 ) {
8159 __kmp_env_print();
8160 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00008161} // __kmp_aux_set_defaults
8162
8163/* ------------------------------------------------------------------------ */
Jonathan Peyton30419822017-05-12 18:01:32 +00008164/* internal fast reduction routines */
Jim Cownie5e8470a2013-09-27 10:38:44 +00008165
Jim Cownie5e8470a2013-09-27 10:38:44 +00008166PACKED_REDUCTION_METHOD_T
Jonathan Peyton30419822017-05-12 18:01:32 +00008167__kmp_determine_reduction_method(
8168 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size,
8169 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),
8170 kmp_critical_name *lck) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00008171
Jonathan Peyton30419822017-05-12 18:01:32 +00008172 // Default reduction method: critical construct ( lck != NULL, like in current
8173 // PAROPT )
8174 // If ( reduce_data!=NULL && reduce_func!=NULL ): the tree-reduction method
8175 // can be selected by RTL
8176 // If loc->flags contains KMP_IDENT_ATOMIC_REDUCE, the atomic reduce method
8177 // can be selected by RTL
8178 // Finally, it's up to OpenMP RTL to make a decision on which method to select
8179 // among generated by PAROPT.
Jim Cownie5e8470a2013-09-27 10:38:44 +00008180
Jonathan Peyton30419822017-05-12 18:01:32 +00008181 PACKED_REDUCTION_METHOD_T retval;
Jim Cownie5e8470a2013-09-27 10:38:44 +00008182
Jonathan Peyton30419822017-05-12 18:01:32 +00008183 int team_size;
Jim Cownie5e8470a2013-09-27 10:38:44 +00008184
Jonathan Peyton30419822017-05-12 18:01:32 +00008185 KMP_DEBUG_ASSERT(loc); // it would be nice to test ( loc != 0 )
8186 KMP_DEBUG_ASSERT(lck); // it would be nice to test ( lck != 0 )
Jim Cownie5e8470a2013-09-27 10:38:44 +00008187
Jonathan Peyton30419822017-05-12 18:01:32 +00008188#define FAST_REDUCTION_ATOMIC_METHOD_GENERATED \
8189 ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE)) == (KMP_IDENT_ATOMIC_REDUCE))
8190#define FAST_REDUCTION_TREE_METHOD_GENERATED ((reduce_data) && (reduce_func))
Jim Cownie5e8470a2013-09-27 10:38:44 +00008191
Jonathan Peyton30419822017-05-12 18:01:32 +00008192 retval = critical_reduce_block;
Jim Cownie5e8470a2013-09-27 10:38:44 +00008193
Jonathan Peyton30419822017-05-12 18:01:32 +00008194 // another choice of getting a team size (with 1 dynamic deference) is slower
8195 team_size = __kmp_get_team_num_threads(global_tid);
8196 if (team_size == 1) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00008197
Jonathan Peyton30419822017-05-12 18:01:32 +00008198 retval = empty_reduce_block;
Jim Cownie5e8470a2013-09-27 10:38:44 +00008199
Jonathan Peyton30419822017-05-12 18:01:32 +00008200 } else {
Jim Cownie5e8470a2013-09-27 10:38:44 +00008201
Jonathan Peyton30419822017-05-12 18:01:32 +00008202 int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
Jim Cownie5e8470a2013-09-27 10:38:44 +00008203
Jonathan Peyton30419822017-05-12 18:01:32 +00008204#if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64
Jim Cownie5e8470a2013-09-27 10:38:44 +00008205
Kamil Rytarowskia56ac942018-12-09 16:40:33 +00008206#if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
Kamil Rytarowski7e1ea992018-12-09 16:46:48 +00008207 KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD
Jim Cownie5e8470a2013-09-27 10:38:44 +00008208
Jonathan Peyton30419822017-05-12 18:01:32 +00008209 int teamsize_cutoff = 4;
Jonathan Peyton91b78702015-06-08 19:39:07 +00008210
Jonathan Peyton492e0a32017-06-13 17:17:26 +00008211#if KMP_MIC_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00008212 if (__kmp_mic_type != non_mic) {
8213 teamsize_cutoff = 8;
8214 }
Andrey Churbanov613edeb2015-02-20 18:14:43 +00008215#endif
Jonathan Peytonbaad3f62018-08-09 22:04:30 +00008216 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
Jonathan Peyton30419822017-05-12 18:01:32 +00008217 if (tree_available) {
8218 if (team_size <= teamsize_cutoff) {
8219 if (atomic_available) {
8220 retval = atomic_reduce_block;
Jim Cownie5e8470a2013-09-27 10:38:44 +00008221 }
Jonathan Peyton30419822017-05-12 18:01:32 +00008222 } else {
8223 retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
8224 }
8225 } else if (atomic_available) {
8226 retval = atomic_reduce_block;
8227 }
8228#else
8229#error "Unknown or unsupported OS"
Jonathan Peyton17e53b92018-12-10 18:26:50 +00008230#endif // KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||
8231 // KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD
Jim Cownie5e8470a2013-09-27 10:38:44 +00008232
Jonathan Peyton30419822017-05-12 18:01:32 +00008233#elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS
8234
Andrey Churbanov855d0982018-11-07 12:27:38 +00008235#if KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_HURD
Jonathan Peyton30419822017-05-12 18:01:32 +00008236
8237 // basic tuning
8238
8239 if (atomic_available) {
8240 if (num_vars <= 2) { // && ( team_size <= 8 ) due to false-sharing ???
8241 retval = atomic_reduce_block;
8242 }
8243 } // otherwise: use critical section
8244
8245#elif KMP_OS_DARWIN
8246
Jonathan Peytonbaad3f62018-08-09 22:04:30 +00008247 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
Jonathan Peyton30419822017-05-12 18:01:32 +00008248 if (atomic_available && (num_vars <= 3)) {
8249 retval = atomic_reduce_block;
8250 } else if (tree_available) {
8251 if ((reduce_size > (9 * sizeof(kmp_real64))) &&
8252 (reduce_size < (2000 * sizeof(kmp_real64)))) {
8253 retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER;
8254 }
8255 } // otherwise: use critical section
8256
8257#else
8258#error "Unknown or unsupported OS"
8259#endif
8260
8261#else
8262#error "Unknown or unsupported architecture"
8263#endif
8264 }
8265
8266 // KMP_FORCE_REDUCTION
8267
8268 // If the team is serialized (team_size == 1), ignore the forced reduction
8269 // method and stay with the unsynchronized method (empty_reduce_block)
8270 if (__kmp_force_reduction_method != reduction_method_not_defined &&
8271 team_size != 1) {
8272
8273 PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block;
8274
8275 int atomic_available, tree_available;
8276
8277 switch ((forced_retval = __kmp_force_reduction_method)) {
8278 case critical_reduce_block:
8279 KMP_ASSERT(lck); // lck should be != 0
8280 break;
8281
8282 case atomic_reduce_block:
8283 atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
8284 if (!atomic_available) {
8285 KMP_WARNING(RedMethodNotSupported, "atomic");
8286 forced_retval = critical_reduce_block;
8287 }
8288 break;
8289
8290 case tree_reduce_block:
8291 tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8292 if (!tree_available) {
8293 KMP_WARNING(RedMethodNotSupported, "tree");
8294 forced_retval = critical_reduce_block;
8295 } else {
8296#if KMP_FAST_REDUCTION_BARRIER
8297 forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
8298#endif
8299 }
8300 break;
8301
8302 default:
8303 KMP_ASSERT(0); // "unsupported method specified"
Jim Cownie5e8470a2013-09-27 10:38:44 +00008304 }
8305
Jonathan Peyton30419822017-05-12 18:01:32 +00008306 retval = forced_retval;
8307 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00008308
Jonathan Peyton30419822017-05-12 18:01:32 +00008309 KA_TRACE(10, ("reduction method selected=%08x\n", retval));
Jim Cownie5e8470a2013-09-27 10:38:44 +00008310
Jonathan Peyton30419822017-05-12 18:01:32 +00008311#undef FAST_REDUCTION_TREE_METHOD_GENERATED
8312#undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED
8313
8314 return (retval);
Jim Cownie5e8470a2013-09-27 10:38:44 +00008315}
8316
8317// this function is for testing set/get/determine reduce method
Jonathan Peyton30419822017-05-12 18:01:32 +00008318kmp_int32 __kmp_get_reduce_method(void) {
8319 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8);
Jim Cownie5e8470a2013-09-27 10:38:44 +00008320}
Jonathan Peyton9b8bb322019-01-16 20:07:39 +00008321
8322#if OMP_50_ENABLED
8323
8324// Soft pause sets up threads to ignore blocktime and just go to sleep.
8325// Spin-wait code checks __kmp_pause_status and reacts accordingly.
8326void __kmp_soft_pause() { __kmp_pause_status = kmp_soft_paused; }
8327
8328// Hard pause shuts down the runtime completely. Resume happens naturally when
8329// OpenMP is used subsequently.
8330void __kmp_hard_pause() {
8331 __kmp_pause_status = kmp_hard_paused;
8332 __kmp_internal_end_thread(-1);
8333}
8334
8335// Soft resume sets __kmp_pause_status, and wakes up all threads.
8336void __kmp_resume_if_soft_paused() {
8337 if (__kmp_pause_status == kmp_soft_paused) {
8338 __kmp_pause_status = kmp_not_paused;
8339
8340 for (int gtid = 1; gtid < __kmp_threads_capacity; ++gtid) {
8341 kmp_info_t *thread = __kmp_threads[gtid];
8342 if (thread) { // Wake it if sleeping
8343 kmp_flag_64 fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread);
8344 if (fl.is_sleeping())
8345 fl.resume(gtid);
8346 else if (__kmp_try_suspend_mx(thread)) { // got suspend lock
8347 __kmp_unlock_suspend_mx(thread); // unlock it; it won't sleep
8348 } else { // thread holds the lock and may sleep soon
8349 do { // until either the thread sleeps, or we can get the lock
8350 if (fl.is_sleeping()) {
8351 fl.resume(gtid);
8352 break;
8353 } else if (__kmp_try_suspend_mx(thread)) {
8354 __kmp_unlock_suspend_mx(thread);
8355 break;
8356 }
8357 } while (1);
8358 }
8359 }
8360 }
8361 }
8362}
8363
8364// This function is called via __kmpc_pause_resource. Returns 0 if successful.
8365// TODO: add warning messages
8366int __kmp_pause_resource(kmp_pause_status_t level) {
8367 if (level == kmp_not_paused) { // requesting resume
8368 if (__kmp_pause_status == kmp_not_paused) {
8369 // error message about runtime not being paused, so can't resume
8370 return 1;
8371 } else {
8372 KMP_DEBUG_ASSERT(__kmp_pause_status == kmp_soft_paused ||
8373 __kmp_pause_status == kmp_hard_paused);
8374 __kmp_pause_status = kmp_not_paused;
8375 return 0;
8376 }
8377 } else if (level == kmp_soft_paused) { // requesting soft pause
8378 if (__kmp_pause_status != kmp_not_paused) {
8379 // error message about already being paused
8380 return 1;
8381 } else {
8382 __kmp_soft_pause();
8383 return 0;
8384 }
8385 } else if (level == kmp_hard_paused) { // requesting hard pause
8386 if (__kmp_pause_status != kmp_not_paused) {
8387 // error message about already being paused
8388 return 1;
8389 } else {
8390 __kmp_hard_pause();
8391 return 0;
8392 }
8393 } else {
8394 // error message about invalid level
8395 return 1;
8396 }
8397}
8398
8399#endif // OMP_50_ENABLED