blob: 5fb6e2623ee54c795b218289a2a9e47266366bb0 [file] [log] [blame]
Jim Cownie5e8470a2013-09-27 10:38:44 +00001/*
Jonathan Peytonde4749b2016-12-14 23:01:24 +00002 * kmp_runtime.cpp -- KPTS runtime support library
Jim Cownie5e8470a2013-09-27 10:38:44 +00003 */
4
Jim Cownie5e8470a2013-09-27 10:38:44 +00005//===----------------------------------------------------------------------===//
6//
7// The LLVM Compiler Infrastructure
8//
9// This file is dual licensed under the MIT and the University of Illinois Open
10// Source Licenses. See LICENSE.txt for details.
11//
12//===----------------------------------------------------------------------===//
13
Jim Cownie5e8470a2013-09-27 10:38:44 +000014#include "kmp.h"
Jonathan Peyton30419822017-05-12 18:01:32 +000015#include "kmp_affinity.h"
Jim Cownie5e8470a2013-09-27 10:38:44 +000016#include "kmp_atomic.h"
Jim Cownie5e8470a2013-09-27 10:38:44 +000017#include "kmp_environment.h"
Jonathan Peyton30419822017-05-12 18:01:32 +000018#include "kmp_error.h"
Jim Cownie5e8470a2013-09-27 10:38:44 +000019#include "kmp_i18n.h"
20#include "kmp_io.h"
Jonathan Peyton30419822017-05-12 18:01:32 +000021#include "kmp_itt.h"
22#include "kmp_settings.h"
Jim Cownie4cc4bb42014-10-07 16:25:50 +000023#include "kmp_stats.h"
Jonathan Peyton30419822017-05-12 18:01:32 +000024#include "kmp_str.h"
Jim Cownie4cc4bb42014-10-07 16:25:50 +000025#include "kmp_wait_release.h"
Jonathan Peyton30419822017-05-12 18:01:32 +000026#include "kmp_wrapper_getpid.h"
Jonathan Peytonf6399362018-07-09 17:51:13 +000027#include "kmp_dispatch.h"
28#if KMP_USE_HIER_SCHED
29#include "kmp_dispatch_hier.h"
30#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +000031
Andrey Churbanovd7d088f2015-04-29 16:42:24 +000032#if OMPT_SUPPORT
33#include "ompt-specific.h"
34#endif
35
Jim Cownie5e8470a2013-09-27 10:38:44 +000036/* these are temporary issues to be dealt with */
37#define KMP_USE_PRCTL 0
Jim Cownie5e8470a2013-09-27 10:38:44 +000038
Jim Cownie5e8470a2013-09-27 10:38:44 +000039#if KMP_OS_WINDOWS
40#include <process.h>
41#endif
42
Jonas Hahnfeld50fed042016-11-07 15:58:36 +000043#include "tsan_annotations.h"
Jim Cownie5e8470a2013-09-27 10:38:44 +000044
45#if defined(KMP_GOMP_COMPAT)
Jonathan Peyton30419822017-05-12 18:01:32 +000046char const __kmp_version_alt_comp[] =
47 KMP_VERSION_PREFIX "alternative compiler support: yes";
Jim Cownie5e8470a2013-09-27 10:38:44 +000048#endif /* defined(KMP_GOMP_COMPAT) */
49
50char const __kmp_version_omp_api[] = KMP_VERSION_PREFIX "API version: "
Jonathan Peytone844a542017-03-06 22:07:40 +000051#if OMP_50_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +000052 "5.0 (201611)";
Jonathan Peytone844a542017-03-06 22:07:40 +000053#elif OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +000054 "4.5 (201511)";
Jonathan Peyton74f3ffc2016-09-30 15:50:14 +000055#elif OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +000056 "4.0 (201307)";
Jim Cownie5e8470a2013-09-27 10:38:44 +000057#else
Jonathan Peyton30419822017-05-12 18:01:32 +000058 "3.1 (201107)";
Jim Cownie5e8470a2013-09-27 10:38:44 +000059#endif
60
61#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +000062char const __kmp_version_lock[] =
63 KMP_VERSION_PREFIX "lock type: run time selectable";
Jim Cownie5e8470a2013-09-27 10:38:44 +000064#endif /* KMP_DEBUG */
65
Jonathan Peyton30419822017-05-12 18:01:32 +000066#define KMP_MIN(x, y) ((x) < (y) ? (x) : (y))
Jim Cownie181b4bb2013-12-23 17:28:57 +000067
Jim Cownie5e8470a2013-09-27 10:38:44 +000068/* ------------------------------------------------------------------------ */
Jim Cownie5e8470a2013-09-27 10:38:44 +000069
Jonathan Peyton37e2ef52018-07-09 17:36:22 +000070#if KMP_USE_MONITOR
Jim Cownie5e8470a2013-09-27 10:38:44 +000071kmp_info_t __kmp_monitor;
Jonathan Peyton37e2ef52018-07-09 17:36:22 +000072#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +000073
Jim Cownie5e8470a2013-09-27 10:38:44 +000074/* Forward declarations */
75
Jonathan Peyton30419822017-05-12 18:01:32 +000076void __kmp_cleanup(void);
Jim Cownie5e8470a2013-09-27 10:38:44 +000077
Jonathan Peyton30419822017-05-12 18:01:32 +000078static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *, int tid,
79 int gtid);
80static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
81 kmp_internal_control_t *new_icvs,
82 ident_t *loc);
Jonathan Peyton2321d572015-06-08 19:25:25 +000083#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +000084static void __kmp_partition_places(kmp_team_t *team,
85 int update_master_only = 0);
Jonathan Peyton2321d572015-06-08 19:25:25 +000086#endif
Jonathan Peyton30419822017-05-12 18:01:32 +000087static void __kmp_do_serial_initialize(void);
88void __kmp_fork_barrier(int gtid, int tid);
89void __kmp_join_barrier(int gtid);
90void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc,
91 kmp_internal_control_t *new_icvs, ident_t *loc);
Jim Cownie5e8470a2013-09-27 10:38:44 +000092
Jim Cownie5e8470a2013-09-27 10:38:44 +000093#ifdef USE_LOAD_BALANCE
Jonathan Peyton30419822017-05-12 18:01:32 +000094static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc);
Jim Cownie5e8470a2013-09-27 10:38:44 +000095#endif
96
Jonathan Peyton1800ece2018-01-10 18:27:01 +000097static int __kmp_expand_threads(int nNeed);
Jonathan Peyton2321d572015-06-08 19:25:25 +000098#if KMP_OS_WINDOWS
Jonathan Peyton30419822017-05-12 18:01:32 +000099static int __kmp_unregister_root_other_thread(int gtid);
Jonathan Peyton2321d572015-06-08 19:25:25 +0000100#endif
Jonathan Peyton30419822017-05-12 18:01:32 +0000101static void __kmp_unregister_library(void); // called by __kmp_internal_end()
102static void __kmp_reap_thread(kmp_info_t *thread, int is_root);
Jonathan Peytoneaa9e402018-01-10 18:21:48 +0000103kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000104
Jim Cownie5e8470a2013-09-27 10:38:44 +0000105/* Calculate the identifier of the current thread */
Jonathan Peyton30419822017-05-12 18:01:32 +0000106/* fast (and somewhat portable) way to get unique identifier of executing
107 thread. Returns KMP_GTID_DNE if we haven't been assigned a gtid. */
Jonathan Peyton30419822017-05-12 18:01:32 +0000108int __kmp_get_global_thread_id() {
109 int i;
110 kmp_info_t **other_threads;
111 size_t stack_data;
112 char *stack_addr;
113 size_t stack_size;
114 char *stack_base;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000115
Jonathan Peyton30419822017-05-12 18:01:32 +0000116 KA_TRACE(
117 1000,
118 ("*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n",
119 __kmp_nth, __kmp_all_nth));
Jim Cownie5e8470a2013-09-27 10:38:44 +0000120
Jonathan Peyton30419822017-05-12 18:01:32 +0000121 /* JPH - to handle the case where __kmpc_end(0) is called immediately prior to
122 a parallel region, made it return KMP_GTID_DNE to force serial_initialize
123 by caller. Had to handle KMP_GTID_DNE at all call-sites, or else guarantee
124 __kmp_init_gtid for this to work. */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000125
Jonathan Peyton30419822017-05-12 18:01:32 +0000126 if (!TCR_4(__kmp_init_gtid))
127 return KMP_GTID_DNE;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000128
129#ifdef KMP_TDATA_GTID
Jonathan Peyton30419822017-05-12 18:01:32 +0000130 if (TCR_4(__kmp_gtid_mode) >= 3) {
131 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using TDATA\n"));
132 return __kmp_gtid;
133 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000134#endif
Jonathan Peyton30419822017-05-12 18:01:32 +0000135 if (TCR_4(__kmp_gtid_mode) >= 2) {
136 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using keyed TLS\n"));
137 return __kmp_gtid_get_specific();
138 }
139 KA_TRACE(1000, ("*** __kmp_get_global_thread_id: using internal alg.\n"));
140
141 stack_addr = (char *)&stack_data;
142 other_threads = __kmp_threads;
143
144 /* ATT: The code below is a source of potential bugs due to unsynchronized
145 access to __kmp_threads array. For example:
146 1. Current thread loads other_threads[i] to thr and checks it, it is
147 non-NULL.
148 2. Current thread is suspended by OS.
149 3. Another thread unregisters and finishes (debug versions of free()
150 may fill memory with something like 0xEF).
151 4. Current thread is resumed.
152 5. Current thread reads junk from *thr.
153 TODO: Fix it. --ln */
154
155 for (i = 0; i < __kmp_threads_capacity; i++) {
156
157 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i]);
158 if (!thr)
159 continue;
160
161 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize);
162 stack_base = (char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase);
163
164 /* stack grows down -- search through all of the active threads */
165
166 if (stack_addr <= stack_base) {
167 size_t stack_diff = stack_base - stack_addr;
168
169 if (stack_diff <= stack_size) {
170 /* The only way we can be closer than the allocated */
171 /* stack size is if we are running on this thread. */
172 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i);
173 return i;
174 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000175 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000176 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000177
Jonathan Peyton30419822017-05-12 18:01:32 +0000178 /* get specific to try and determine our gtid */
179 KA_TRACE(1000,
180 ("*** __kmp_get_global_thread_id: internal alg. failed to find "
181 "thread, using TLS\n"));
182 i = __kmp_gtid_get_specific();
Jim Cownie5e8470a2013-09-27 10:38:44 +0000183
Jonathan Peyton30419822017-05-12 18:01:32 +0000184 /*fprintf( stderr, "=== %d\n", i ); */ /* GROO */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000185
Jonathan Peyton30419822017-05-12 18:01:32 +0000186 /* if we havn't been assigned a gtid, then return code */
187 if (i < 0)
Jim Cownie5e8470a2013-09-27 10:38:44 +0000188 return i;
Jonathan Peyton30419822017-05-12 18:01:32 +0000189
190 /* dynamically updated stack window for uber threads to avoid get_specific
191 call */
192 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) {
193 KMP_FATAL(StackOverflow, i);
194 }
195
196 stack_base = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
197 if (stack_addr > stack_base) {
198 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr);
199 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
200 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr -
201 stack_base);
202 } else {
203 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
204 stack_base - stack_addr);
205 }
206
207 /* Reprint stack bounds for ubermaster since they have been refined */
208 if (__kmp_storage_map) {
209 char *stack_end = (char *)other_threads[i]->th.th_info.ds.ds_stackbase;
210 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize;
211 __kmp_print_storage_map_gtid(i, stack_beg, stack_end,
212 other_threads[i]->th.th_info.ds.ds_stacksize,
213 "th_%d stack (refinement)", i);
214 }
215 return i;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000216}
217
Jonathan Peyton30419822017-05-12 18:01:32 +0000218int __kmp_get_global_thread_id_reg() {
219 int gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000220
Jonathan Peyton30419822017-05-12 18:01:32 +0000221 if (!__kmp_init_serial) {
222 gtid = KMP_GTID_DNE;
223 } else
Jim Cownie5e8470a2013-09-27 10:38:44 +0000224#ifdef KMP_TDATA_GTID
Jonathan Peyton30419822017-05-12 18:01:32 +0000225 if (TCR_4(__kmp_gtid_mode) >= 3) {
226 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using TDATA\n"));
227 gtid = __kmp_gtid;
228 } else
Jim Cownie5e8470a2013-09-27 10:38:44 +0000229#endif
Jonathan Peyton30419822017-05-12 18:01:32 +0000230 if (TCR_4(__kmp_gtid_mode) >= 2) {
231 KA_TRACE(1000, ("*** __kmp_get_global_thread_id_reg: using keyed TLS\n"));
232 gtid = __kmp_gtid_get_specific();
233 } else {
234 KA_TRACE(1000,
235 ("*** __kmp_get_global_thread_id_reg: using internal alg.\n"));
236 gtid = __kmp_get_global_thread_id();
237 }
238
239 /* we must be a new uber master sibling thread */
240 if (gtid == KMP_GTID_DNE) {
241 KA_TRACE(10,
242 ("__kmp_get_global_thread_id_reg: Encountered new root thread. "
243 "Registering a new gtid.\n"));
244 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
245 if (!__kmp_init_serial) {
246 __kmp_do_serial_initialize();
247 gtid = __kmp_gtid_get_specific();
Jim Cownie5e8470a2013-09-27 10:38:44 +0000248 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +0000249 gtid = __kmp_register_root(FALSE);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000250 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000251 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
252 /*__kmp_printf( "+++ %d\n", gtid ); */ /* GROO */
253 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000254
Jonathan Peyton30419822017-05-12 18:01:32 +0000255 KMP_DEBUG_ASSERT(gtid >= 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000256
Jonathan Peyton30419822017-05-12 18:01:32 +0000257 return gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000258}
259
260/* caller must hold forkjoin_lock */
Jonathan Peyton30419822017-05-12 18:01:32 +0000261void __kmp_check_stack_overlap(kmp_info_t *th) {
262 int f;
263 char *stack_beg = NULL;
264 char *stack_end = NULL;
265 int gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000266
Jonathan Peyton30419822017-05-12 18:01:32 +0000267 KA_TRACE(10, ("__kmp_check_stack_overlap: called\n"));
268 if (__kmp_storage_map) {
269 stack_end = (char *)th->th.th_info.ds.ds_stackbase;
270 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000271
Jonathan Peyton30419822017-05-12 18:01:32 +0000272 gtid = __kmp_gtid_from_thread(th);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000273
Jonathan Peyton30419822017-05-12 18:01:32 +0000274 if (gtid == KMP_GTID_MONITOR) {
275 __kmp_print_storage_map_gtid(
276 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
277 "th_%s stack (%s)", "mon",
278 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
Jim Cownie5e8470a2013-09-27 10:38:44 +0000279 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +0000280 __kmp_print_storage_map_gtid(
281 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
282 "th_%d stack (%s)", gtid,
283 (th->th.th_info.ds.ds_stackgrow) ? "initial" : "actual");
284 }
285 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000286
Jonathan Peyton30419822017-05-12 18:01:32 +0000287 /* No point in checking ubermaster threads since they use refinement and
288 * cannot overlap */
289 gtid = __kmp_gtid_from_thread(th);
290 if (__kmp_env_checks == TRUE && !KMP_UBER_GTID(gtid)) {
291 KA_TRACE(10,
292 ("__kmp_check_stack_overlap: performing extensive checking\n"));
293 if (stack_beg == NULL) {
294 stack_end = (char *)th->th.th_info.ds.ds_stackbase;
295 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
296 }
297
298 for (f = 0; f < __kmp_threads_capacity; f++) {
299 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f]);
300
301 if (f_th && f_th != th) {
302 char *other_stack_end =
303 (char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase);
304 char *other_stack_beg =
305 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize);
306 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) ||
307 (stack_end > other_stack_beg && stack_end < other_stack_end)) {
308
309 /* Print the other stack values before the abort */
310 if (__kmp_storage_map)
311 __kmp_print_storage_map_gtid(
312 -1, other_stack_beg, other_stack_end,
313 (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize),
314 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th));
315
Jonathan Peyton6a393f72017-09-05 15:43:58 +0000316 __kmp_fatal(KMP_MSG(StackOverlap), KMP_HNT(ChangeStackLimit),
317 __kmp_msg_null);
Jonathan Peyton30419822017-05-12 18:01:32 +0000318 }
319 }
320 }
321 }
322 KA_TRACE(10, ("__kmp_check_stack_overlap: returning\n"));
323}
324
325/* ------------------------------------------------------------------------ */
326
327void __kmp_infinite_loop(void) {
328 static int done = FALSE;
329
330 while (!done) {
331 KMP_YIELD(1);
332 }
333}
334
335#define MAX_MESSAGE 512
336
337void __kmp_print_storage_map_gtid(int gtid, void *p1, void *p2, size_t size,
338 char const *format, ...) {
339 char buffer[MAX_MESSAGE];
340 va_list ap;
341
342 va_start(ap, format);
343 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP storage map: %p %p%8lu %s\n", p1,
344 p2, (unsigned long)size, format);
345 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
346 __kmp_vprintf(kmp_err, buffer, ap);
347#if KMP_PRINT_DATA_PLACEMENT
348 int node;
349 if (gtid >= 0) {
350 if (p1 <= p2 && (char *)p2 - (char *)p1 == size) {
351 if (__kmp_storage_map_verbose) {
352 node = __kmp_get_host_node(p1);
353 if (node < 0) /* doesn't work, so don't try this next time */
354 __kmp_storage_map_verbose = FALSE;
355 else {
356 char *last;
357 int lastNode;
358 int localProc = __kmp_get_cpu_from_gtid(gtid);
359
360 const int page_size = KMP_GET_PAGE_SIZE();
361
362 p1 = (void *)((size_t)p1 & ~((size_t)page_size - 1));
363 p2 = (void *)(((size_t)p2 - 1) & ~((size_t)page_size - 1));
364 if (localProc >= 0)
365 __kmp_printf_no_lock(" GTID %d localNode %d\n", gtid,
366 localProc >> 1);
367 else
368 __kmp_printf_no_lock(" GTID %d\n", gtid);
369#if KMP_USE_PRCTL
370 /* The more elaborate format is disabled for now because of the prctl
371 * hanging bug. */
372 do {
373 last = p1;
374 lastNode = node;
375 /* This loop collates adjacent pages with the same host node. */
376 do {
377 (char *)p1 += page_size;
378 } while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode);
379 __kmp_printf_no_lock(" %p-%p memNode %d\n", last, (char *)p1 - 1,
380 lastNode);
381 } while (p1 <= p2);
382#else
383 __kmp_printf_no_lock(" %p-%p memNode %d\n", p1,
384 (char *)p1 + (page_size - 1),
385 __kmp_get_host_node(p1));
386 if (p1 < p2) {
387 __kmp_printf_no_lock(" %p-%p memNode %d\n", p2,
388 (char *)p2 + (page_size - 1),
389 __kmp_get_host_node(p2));
390 }
391#endif
392 }
393 }
394 } else
395 __kmp_printf_no_lock(" %s\n", KMP_I18N_STR(StorageMapWarning));
396 }
397#endif /* KMP_PRINT_DATA_PLACEMENT */
398 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
399}
400
401void __kmp_warn(char const *format, ...) {
402 char buffer[MAX_MESSAGE];
403 va_list ap;
404
405 if (__kmp_generate_warnings == kmp_warnings_off) {
406 return;
407 }
408
409 va_start(ap, format);
410
411 KMP_SNPRINTF(buffer, sizeof(buffer), "OMP warning: %s\n", format);
412 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
413 __kmp_vprintf(kmp_err, buffer, ap);
414 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
415
416 va_end(ap);
417}
418
419void __kmp_abort_process() {
420 // Later threads may stall here, but that's ok because abort() will kill them.
421 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock);
422
423 if (__kmp_debug_buf) {
424 __kmp_dump_debug_buffer();
Jonathan Peytonbd3a7632017-09-27 20:36:27 +0000425 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000426
427 if (KMP_OS_WINDOWS) {
428 // Let other threads know of abnormal termination and prevent deadlock
429 // if abort happened during library initialization or shutdown
430 __kmp_global.g.g_abort = SIGABRT;
431
432 /* On Windows* OS by default abort() causes pop-up error box, which stalls
433 nightly testing. Unfortunately, we cannot reliably suppress pop-up error
434 boxes. _set_abort_behavior() works well, but this function is not
435 available in VS7 (this is not problem for DLL, but it is a problem for
436 static OpenMP RTL). SetErrorMode (and so, timelimit utility) does not
437 help, at least in some versions of MS C RTL.
438
439 It seems following sequence is the only way to simulate abort() and
440 avoid pop-up error box. */
441 raise(SIGABRT);
442 _exit(3); // Just in case, if signal ignored, exit anyway.
443 } else {
444 abort();
Jonathan Peytonbd3a7632017-09-27 20:36:27 +0000445 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000446
447 __kmp_infinite_loop();
448 __kmp_release_bootstrap_lock(&__kmp_exit_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000449
450} // __kmp_abort_process
451
Jonathan Peyton30419822017-05-12 18:01:32 +0000452void __kmp_abort_thread(void) {
453 // TODO: Eliminate g_abort global variable and this function.
454 // In case of abort just call abort(), it will kill all the threads.
455 __kmp_infinite_loop();
Jim Cownie5e8470a2013-09-27 10:38:44 +0000456} // __kmp_abort_thread
457
Jonathan Peyton30419822017-05-12 18:01:32 +0000458/* Print out the storage map for the major kmp_info_t thread data structures
459 that are allocated together. */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000460
Jonathan Peyton30419822017-05-12 18:01:32 +0000461static void __kmp_print_thread_storage_map(kmp_info_t *thr, int gtid) {
462 __kmp_print_storage_map_gtid(gtid, thr, thr + 1, sizeof(kmp_info_t), "th_%d",
463 gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000464
Jonathan Peyton30419822017-05-12 18:01:32 +0000465 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team,
466 sizeof(kmp_desc_t), "th_%d.th_info", gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000467
Jonathan Peyton30419822017-05-12 18:01:32 +0000468 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head,
469 sizeof(kmp_local_t), "th_%d.th_local", gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000470
Jonathan Peyton30419822017-05-12 18:01:32 +0000471 __kmp_print_storage_map_gtid(
472 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier],
473 sizeof(kmp_balign_t) * bs_last_barrier, "th_%d.th_bar", gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000474
Jonathan Peyton30419822017-05-12 18:01:32 +0000475 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier],
476 &thr->th.th_bar[bs_plain_barrier + 1],
477 sizeof(kmp_balign_t), "th_%d.th_bar[plain]",
478 gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000479
Jonathan Peyton30419822017-05-12 18:01:32 +0000480 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier],
481 &thr->th.th_bar[bs_forkjoin_barrier + 1],
482 sizeof(kmp_balign_t), "th_%d.th_bar[forkjoin]",
483 gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000484
Jonathan Peyton30419822017-05-12 18:01:32 +0000485#if KMP_FAST_REDUCTION_BARRIER
486 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier],
487 &thr->th.th_bar[bs_reduction_barrier + 1],
488 sizeof(kmp_balign_t), "th_%d.th_bar[reduction]",
489 gtid);
490#endif // KMP_FAST_REDUCTION_BARRIER
Jim Cownie5e8470a2013-09-27 10:38:44 +0000491}
492
Jonathan Peyton30419822017-05-12 18:01:32 +0000493/* Print out the storage map for the major kmp_team_t team data structures
494 that are allocated together. */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000495
Jonathan Peyton30419822017-05-12 18:01:32 +0000496static void __kmp_print_team_storage_map(const char *header, kmp_team_t *team,
497 int team_id, int num_thr) {
498 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;
499 __kmp_print_storage_map_gtid(-1, team, team + 1, sizeof(kmp_team_t), "%s_%d",
500 header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000501
Jonathan Peyton30419822017-05-12 18:01:32 +0000502 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0],
503 &team->t.t_bar[bs_last_barrier],
504 sizeof(kmp_balign_team_t) * bs_last_barrier,
505 "%s_%d.t_bar", header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000506
Jonathan Peyton30419822017-05-12 18:01:32 +0000507 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier],
508 &team->t.t_bar[bs_plain_barrier + 1],
509 sizeof(kmp_balign_team_t), "%s_%d.t_bar[plain]",
510 header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000511
Jonathan Peyton30419822017-05-12 18:01:32 +0000512 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier],
513 &team->t.t_bar[bs_forkjoin_barrier + 1],
514 sizeof(kmp_balign_team_t),
515 "%s_%d.t_bar[forkjoin]", header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000516
Jonathan Peyton30419822017-05-12 18:01:32 +0000517#if KMP_FAST_REDUCTION_BARRIER
518 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier],
519 &team->t.t_bar[bs_reduction_barrier + 1],
520 sizeof(kmp_balign_team_t),
521 "%s_%d.t_bar[reduction]", header, team_id);
522#endif // KMP_FAST_REDUCTION_BARRIER
Jim Cownie5e8470a2013-09-27 10:38:44 +0000523
Jonathan Peyton30419822017-05-12 18:01:32 +0000524 __kmp_print_storage_map_gtid(
525 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr],
526 sizeof(kmp_disp_t) * num_thr, "%s_%d.t_dispatch", header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000527
Jonathan Peyton30419822017-05-12 18:01:32 +0000528 __kmp_print_storage_map_gtid(
529 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr],
530 sizeof(kmp_info_t *) * num_thr, "%s_%d.t_threads", header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000531
Jonathan Peyton30419822017-05-12 18:01:32 +0000532 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0],
533 &team->t.t_disp_buffer[num_disp_buff],
534 sizeof(dispatch_shared_info_t) * num_disp_buff,
535 "%s_%d.t_disp_buffer", header, team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000536
Jonathan Peyton30419822017-05-12 18:01:32 +0000537 __kmp_print_storage_map_gtid(-1, &team->t.t_taskq, &team->t.t_copypriv_data,
538 sizeof(kmp_taskq_t), "%s_%d.t_taskq", header,
539 team_id);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000540}
541
Jonathan Peyton92ca6182018-09-07 18:25:49 +0000542static void __kmp_init_allocator() {
543#if OMP_50_ENABLED
544 __kmp_init_memkind();
545#endif
546}
547static void __kmp_fini_allocator() {
548#if OMP_50_ENABLED
549 __kmp_fini_memkind();
550#endif
551}
Jim Cownie5e8470a2013-09-27 10:38:44 +0000552
553/* ------------------------------------------------------------------------ */
554
Jonathan Peyton8b3842f2018-10-05 17:59:39 +0000555#if KMP_DYNAMIC_LIB
Jonathan Peyton30419822017-05-12 18:01:32 +0000556#if KMP_OS_WINDOWS
Jim Cownie5e8470a2013-09-27 10:38:44 +0000557
Jonathan Peyton30419822017-05-12 18:01:32 +0000558static void __kmp_reset_lock(kmp_bootstrap_lock_t *lck) {
559 // TODO: Change to __kmp_break_bootstrap_lock().
560 __kmp_init_bootstrap_lock(lck); // make the lock released
Jim Cownie5e8470a2013-09-27 10:38:44 +0000561}
562
Jonathan Peyton30419822017-05-12 18:01:32 +0000563static void __kmp_reset_locks_on_process_detach(int gtid_req) {
564 int i;
565 int thread_count;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000566
Jonathan Peyton30419822017-05-12 18:01:32 +0000567 // PROCESS_DETACH is expected to be called by a thread that executes
568 // ProcessExit() or FreeLibrary(). OS terminates other threads (except the one
569 // calling ProcessExit or FreeLibrary). So, it might be safe to access the
570 // __kmp_threads[] without taking the forkjoin_lock. However, in fact, some
571 // threads can be still alive here, although being about to be terminated. The
572 // threads in the array with ds_thread==0 are most suspicious. Actually, it
573 // can be not safe to access the __kmp_threads[].
Jim Cownie5e8470a2013-09-27 10:38:44 +0000574
Jonathan Peyton30419822017-05-12 18:01:32 +0000575 // TODO: does it make sense to check __kmp_roots[] ?
Jim Cownie5e8470a2013-09-27 10:38:44 +0000576
Jonathan Peyton30419822017-05-12 18:01:32 +0000577 // Let's check that there are no other alive threads registered with the OMP
578 // lib.
579 while (1) {
580 thread_count = 0;
581 for (i = 0; i < __kmp_threads_capacity; ++i) {
582 if (!__kmp_threads)
583 continue;
584 kmp_info_t *th = __kmp_threads[i];
585 if (th == NULL)
586 continue;
587 int gtid = th->th.th_info.ds.ds_gtid;
588 if (gtid == gtid_req)
589 continue;
590 if (gtid < 0)
591 continue;
592 DWORD exit_val;
593 int alive = __kmp_is_thread_alive(th, &exit_val);
594 if (alive) {
595 ++thread_count;
596 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000597 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000598 if (thread_count == 0)
599 break; // success
600 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000601
Jonathan Peyton30419822017-05-12 18:01:32 +0000602 // Assume that I'm alone. Now it might be safe to check and reset locks.
603 // __kmp_forkjoin_lock and __kmp_stdio_lock are expected to be reset.
604 __kmp_reset_lock(&__kmp_forkjoin_lock);
605#ifdef KMP_DEBUG
606 __kmp_reset_lock(&__kmp_stdio_lock);
607#endif // KMP_DEBUG
Jim Cownie5e8470a2013-09-27 10:38:44 +0000608}
609
Jonathan Peyton30419822017-05-12 18:01:32 +0000610BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) {
611 //__kmp_acquire_bootstrap_lock( &__kmp_initz_lock );
Jim Cownie5e8470a2013-09-27 10:38:44 +0000612
Jonathan Peyton30419822017-05-12 18:01:32 +0000613 switch (fdwReason) {
Jim Cownie5e8470a2013-09-27 10:38:44 +0000614
Jonathan Peyton30419822017-05-12 18:01:32 +0000615 case DLL_PROCESS_ATTACH:
616 KA_TRACE(10, ("DllMain: PROCESS_ATTACH\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +0000617
618 return TRUE;
Jonathan Peyton30419822017-05-12 18:01:32 +0000619
620 case DLL_PROCESS_DETACH:
621 KA_TRACE(10, ("DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific()));
622
623 if (lpReserved != NULL) {
624 // lpReserved is used for telling the difference:
625 // lpReserved == NULL when FreeLibrary() was called,
626 // lpReserved != NULL when the process terminates.
627 // When FreeLibrary() is called, worker threads remain alive. So they will
628 // release the forkjoin lock by themselves. When the process terminates,
629 // worker threads disappear triggering the problem of unreleased forkjoin
630 // lock as described below.
631
632 // A worker thread can take the forkjoin lock. The problem comes up if
633 // that worker thread becomes dead before it releases the forkjoin lock.
634 // The forkjoin lock remains taken, while the thread executing
635 // DllMain()->PROCESS_DETACH->__kmp_internal_end_library() below will try
636 // to take the forkjoin lock and will always fail, so that the application
637 // will never finish [normally]. This scenario is possible if
638 // __kmpc_end() has not been executed. It looks like it's not a corner
639 // case, but common cases:
640 // - the main function was compiled by an alternative compiler;
641 // - the main function was compiled by icl but without /Qopenmp
642 // (application with plugins);
643 // - application terminates by calling C exit(), Fortran CALL EXIT() or
644 // Fortran STOP.
645 // - alive foreign thread prevented __kmpc_end from doing cleanup.
646 //
647 // This is a hack to work around the problem.
648 // TODO: !!! figure out something better.
649 __kmp_reset_locks_on_process_detach(__kmp_gtid_get_specific());
650 }
651
652 __kmp_internal_end_library(__kmp_gtid_get_specific());
653
654 return TRUE;
655
656 case DLL_THREAD_ATTACH:
657 KA_TRACE(10, ("DllMain: THREAD_ATTACH\n"));
658
659 /* if we want to register new siblings all the time here call
660 * __kmp_get_gtid(); */
661 return TRUE;
662
663 case DLL_THREAD_DETACH:
664 KA_TRACE(10, ("DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific()));
665
666 __kmp_internal_end_thread(__kmp_gtid_get_specific());
667 return TRUE;
668 }
669
670 return TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000671}
672
Jonathan Peyton30419822017-05-12 18:01:32 +0000673#endif /* KMP_OS_WINDOWS */
Jonathan Peyton99016992015-05-26 17:32:53 +0000674#endif /* KMP_DYNAMIC_LIB */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000675
Jim Cownie5e8470a2013-09-27 10:38:44 +0000676/* Change the library type to "status" and return the old type */
677/* called from within initialization routines where __kmp_initz_lock is held */
Jonathan Peyton30419822017-05-12 18:01:32 +0000678int __kmp_change_library(int status) {
679 int old_status;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000680
Jonathan Peyton30419822017-05-12 18:01:32 +0000681 old_status = __kmp_yield_init &
682 1; // check whether KMP_LIBRARY=throughput (even init count)
Jim Cownie5e8470a2013-09-27 10:38:44 +0000683
Jonathan Peyton30419822017-05-12 18:01:32 +0000684 if (status) {
685 __kmp_yield_init |= 1; // throughput => turnaround (odd init count)
686 } else {
687 __kmp_yield_init &= ~1; // turnaround => throughput (even init count)
688 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000689
Jonathan Peyton30419822017-05-12 18:01:32 +0000690 return old_status; // return previous setting of whether
691 // KMP_LIBRARY=throughput
Jim Cownie5e8470a2013-09-27 10:38:44 +0000692}
693
Jonathan Peyton30419822017-05-12 18:01:32 +0000694/* __kmp_parallel_deo -- Wait until it's our turn. */
695void __kmp_parallel_deo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
696 int gtid = *gtid_ref;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000697#ifdef BUILD_PARALLEL_ORDERED
Jonathan Peyton30419822017-05-12 18:01:32 +0000698 kmp_team_t *team = __kmp_team_from_gtid(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000699#endif /* BUILD_PARALLEL_ORDERED */
700
Jonathan Peyton30419822017-05-12 18:01:32 +0000701 if (__kmp_env_consistency_check) {
702 if (__kmp_threads[gtid]->th.th_root->r.r_active)
Andrey Churbanov5c56fb52015-02-20 18:05:17 +0000703#if KMP_USE_DYNAMIC_LOCK
Jonathan Peyton30419822017-05-12 18:01:32 +0000704 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL, 0);
Andrey Churbanov5c56fb52015-02-20 18:05:17 +0000705#else
Jonathan Peyton30419822017-05-12 18:01:32 +0000706 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL);
Andrey Churbanov5c56fb52015-02-20 18:05:17 +0000707#endif
Jonathan Peyton30419822017-05-12 18:01:32 +0000708 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000709#ifdef BUILD_PARALLEL_ORDERED
Jonathan Peyton30419822017-05-12 18:01:32 +0000710 if (!team->t.t_serialized) {
711 KMP_MB();
712 KMP_WAIT_YIELD(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid),
713 KMP_EQ, NULL);
714 KMP_MB();
715 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000716#endif /* BUILD_PARALLEL_ORDERED */
717}
718
Jonathan Peyton30419822017-05-12 18:01:32 +0000719/* __kmp_parallel_dxo -- Signal the next task. */
720void __kmp_parallel_dxo(int *gtid_ref, int *cid_ref, ident_t *loc_ref) {
721 int gtid = *gtid_ref;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000722#ifdef BUILD_PARALLEL_ORDERED
Jonathan Peyton30419822017-05-12 18:01:32 +0000723 int tid = __kmp_tid_from_gtid(gtid);
724 kmp_team_t *team = __kmp_team_from_gtid(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000725#endif /* BUILD_PARALLEL_ORDERED */
726
Jonathan Peyton30419822017-05-12 18:01:32 +0000727 if (__kmp_env_consistency_check) {
728 if (__kmp_threads[gtid]->th.th_root->r.r_active)
729 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref);
730 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000731#ifdef BUILD_PARALLEL_ORDERED
Jonathan Peyton30419822017-05-12 18:01:32 +0000732 if (!team->t.t_serialized) {
733 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000734
Jonathan Peyton30419822017-05-12 18:01:32 +0000735 /* use the tid of the next thread in this team */
736 /* TODO replace with general release procedure */
737 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000738
Jonathan Peyton30419822017-05-12 18:01:32 +0000739 KMP_MB(); /* Flush all pending memory write invalidates. */
740 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000741#endif /* BUILD_PARALLEL_ORDERED */
742}
743
744/* ------------------------------------------------------------------------ */
Jim Cownie5e8470a2013-09-27 10:38:44 +0000745/* The BARRIER for a SINGLE process section is always explicit */
746
Jonathan Peyton30419822017-05-12 18:01:32 +0000747int __kmp_enter_single(int gtid, ident_t *id_ref, int push_ws) {
748 int status;
749 kmp_info_t *th;
750 kmp_team_t *team;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000751
Jonathan Peyton30419822017-05-12 18:01:32 +0000752 if (!TCR_4(__kmp_init_parallel))
753 __kmp_parallel_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +0000754
Jonathan Peyton30419822017-05-12 18:01:32 +0000755 th = __kmp_threads[gtid];
756 team = th->th.th_team;
757 status = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000758
Jonathan Peyton30419822017-05-12 18:01:32 +0000759 th->th.th_ident = id_ref;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000760
Jonathan Peyton30419822017-05-12 18:01:32 +0000761 if (team->t.t_serialized) {
762 status = 1;
763 } else {
764 kmp_int32 old_this = th->th.th_local.this_construct;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000765
Jonathan Peyton30419822017-05-12 18:01:32 +0000766 ++th->th.th_local.this_construct;
767 /* try to set team count to thread count--success means thread got the
768 single block */
769 /* TODO: Should this be acquire or release? */
770 if (team->t.t_construct == old_this) {
Jonathan Peyton37e2ef52018-07-09 17:36:22 +0000771 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this,
772 th->th.th_local.this_construct);
Jonathan Peyton30419822017-05-12 18:01:32 +0000773 }
Andrey Churbanov51aecb82015-05-06 19:22:36 +0000774#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +0000775 if (__itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 &&
776 KMP_MASTER_GTID(gtid) &&
Andrey Churbanov51aecb82015-05-06 19:22:36 +0000777#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +0000778 th->th.th_teams_microtask == NULL &&
Andrey Churbanov51aecb82015-05-06 19:22:36 +0000779#endif
Jonathan Peyton30419822017-05-12 18:01:32 +0000780 team->t.t_active_level ==
781 1) { // Only report metadata by master of active team at level 1
782 __kmp_itt_metadata_single(id_ref);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000783 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000784#endif /* USE_ITT_BUILD */
785 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000786
Jonathan Peyton30419822017-05-12 18:01:32 +0000787 if (__kmp_env_consistency_check) {
788 if (status && push_ws) {
789 __kmp_push_workshare(gtid, ct_psingle, id_ref);
790 } else {
791 __kmp_check_workshare(gtid, ct_psingle, id_ref);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000792 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000793 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000794#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +0000795 if (status) {
796 __kmp_itt_single_start(gtid);
797 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000798#endif /* USE_ITT_BUILD */
Jonathan Peyton30419822017-05-12 18:01:32 +0000799 return status;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000800}
801
Jonathan Peyton30419822017-05-12 18:01:32 +0000802void __kmp_exit_single(int gtid) {
Jim Cownie5e8470a2013-09-27 10:38:44 +0000803#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +0000804 __kmp_itt_single_end(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000805#endif /* USE_ITT_BUILD */
Jonathan Peyton30419822017-05-12 18:01:32 +0000806 if (__kmp_env_consistency_check)
807 __kmp_pop_workshare(gtid, ct_psingle, NULL);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000808}
809
Jonathan Peyton30419822017-05-12 18:01:32 +0000810/* determine if we can go parallel or must use a serialized parallel region and
Jim Cownie5e8470a2013-09-27 10:38:44 +0000811 * how many threads we can use
812 * set_nproc is the number of threads requested for the team
813 * returns 0 if we should serialize or only use one thread,
814 * otherwise the number of threads to use
Jonathan Peyton30419822017-05-12 18:01:32 +0000815 * The forkjoin lock is held by the caller. */
816static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team,
817 int master_tid, int set_nthreads
Jim Cownie5e8470a2013-09-27 10:38:44 +0000818#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +0000819 ,
820 int enter_teams
Jim Cownie5e8470a2013-09-27 10:38:44 +0000821#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +0000822 ) {
823 int capacity;
824 int new_nthreads;
825 KMP_DEBUG_ASSERT(__kmp_init_serial);
826 KMP_DEBUG_ASSERT(root && parent_team);
Jim Cownie5e8470a2013-09-27 10:38:44 +0000827
Jonathan Peyton30419822017-05-12 18:01:32 +0000828 // If dyn-var is set, dynamically adjust the number of desired threads,
829 // according to the method specified by dynamic_mode.
830 new_nthreads = set_nthreads;
831 if (!get__dynamic_2(parent_team, master_tid)) {
832 ;
833 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000834#ifdef USE_LOAD_BALANCE
Jonathan Peyton30419822017-05-12 18:01:32 +0000835 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) {
836 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads);
837 if (new_nthreads == 1) {
838 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "
839 "reservation to 1 thread\n",
840 master_tid));
841 return 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000842 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000843 if (new_nthreads < set_nthreads) {
844 KC_TRACE(10, ("__kmp_reserve_threads: T#%d load balance reduced "
845 "reservation to %d threads\n",
846 master_tid, new_nthreads));
847 }
848 }
Jim Cownie5e8470a2013-09-27 10:38:44 +0000849#endif /* USE_LOAD_BALANCE */
Jonathan Peyton30419822017-05-12 18:01:32 +0000850 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) {
851 new_nthreads = __kmp_avail_proc - __kmp_nth +
852 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
853 if (new_nthreads <= 1) {
854 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "
855 "reservation to 1 thread\n",
856 master_tid));
857 return 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000858 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000859 if (new_nthreads < set_nthreads) {
860 KC_TRACE(10, ("__kmp_reserve_threads: T#%d thread limit reduced "
861 "reservation to %d threads\n",
862 master_tid, new_nthreads));
863 } else {
864 new_nthreads = set_nthreads;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000865 }
Jonathan Peyton30419822017-05-12 18:01:32 +0000866 } else if (__kmp_global.g.g_dynamic_mode == dynamic_random) {
867 if (set_nthreads > 2) {
868 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]);
869 new_nthreads = (new_nthreads % set_nthreads) + 1;
870 if (new_nthreads == 1) {
871 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "
872 "reservation to 1 thread\n",
873 master_tid));
Jim Cownie5e8470a2013-09-27 10:38:44 +0000874 return 1;
Jonathan Peyton30419822017-05-12 18:01:32 +0000875 }
876 if (new_nthreads < set_nthreads) {
877 KC_TRACE(10, ("__kmp_reserve_threads: T#%d dynamic random reduced "
878 "reservation to %d threads\n",
879 master_tid, new_nthreads));
880 }
881 }
882 } else {
883 KMP_ASSERT(0);
884 }
885
Jonathan Peytonf4392462017-07-27 20:58:41 +0000886 // Respect KMP_ALL_THREADS/KMP_DEVICE_THREAD_LIMIT.
Jonathan Peyton30419822017-05-12 18:01:32 +0000887 if (__kmp_nth + new_nthreads -
888 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
889 __kmp_max_nth) {
890 int tl_nthreads = __kmp_max_nth - __kmp_nth +
891 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
892 if (tl_nthreads <= 0) {
893 tl_nthreads = 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000894 }
895
Jonathan Peyton30419822017-05-12 18:01:32 +0000896 // If dyn-var is false, emit a 1-time warning.
897 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
898 __kmp_reserve_warn = 1;
899 __kmp_msg(kmp_ms_warning,
900 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
901 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
902 }
903 if (tl_nthreads == 1) {
Jonathan Peytonf4392462017-07-27 20:58:41 +0000904 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
905 "reduced reservation to 1 thread\n",
Jonathan Peyton30419822017-05-12 18:01:32 +0000906 master_tid));
907 return 1;
908 }
Jonathan Peytonf4392462017-07-27 20:58:41 +0000909 KC_TRACE(10, ("__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
910 "reservation to %d threads\n",
911 master_tid, tl_nthreads));
912 new_nthreads = tl_nthreads;
913 }
914
915 // Respect OMP_THREAD_LIMIT
916 if (root->r.r_cg_nthreads + new_nthreads -
917 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
918 __kmp_cg_max_nth) {
919 int tl_nthreads = __kmp_cg_max_nth - root->r.r_cg_nthreads +
920 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
921 if (tl_nthreads <= 0) {
922 tl_nthreads = 1;
923 }
924
925 // If dyn-var is false, emit a 1-time warning.
926 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
927 __kmp_reserve_warn = 1;
928 __kmp_msg(kmp_ms_warning,
929 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
930 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
931 }
932 if (tl_nthreads == 1) {
933 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
934 "reduced reservation to 1 thread\n",
935 master_tid));
936 return 1;
937 }
938 KC_TRACE(10, ("__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
Jonathan Peyton30419822017-05-12 18:01:32 +0000939 "reservation to %d threads\n",
940 master_tid, tl_nthreads));
941 new_nthreads = tl_nthreads;
942 }
943
944 // Check if the threads array is large enough, or needs expanding.
Jonathan Peyton30419822017-05-12 18:01:32 +0000945 // See comment in __kmp_register_root() about the adjustment if
946 // __kmp_threads[0] == NULL.
947 capacity = __kmp_threads_capacity;
948 if (TCR_PTR(__kmp_threads[0]) == NULL) {
949 --capacity;
950 }
951 if (__kmp_nth + new_nthreads -
952 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
953 capacity) {
954 // Expand the threads array.
955 int slotsRequired = __kmp_nth + new_nthreads -
956 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) -
957 capacity;
Jonathan Peyton1800ece2018-01-10 18:27:01 +0000958 int slotsAdded = __kmp_expand_threads(slotsRequired);
Jonathan Peyton30419822017-05-12 18:01:32 +0000959 if (slotsAdded < slotsRequired) {
960 // The threads array was not expanded enough.
961 new_nthreads -= (slotsRequired - slotsAdded);
962 KMP_ASSERT(new_nthreads >= 1);
963
964 // If dyn-var is false, emit a 1-time warning.
965 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
966 __kmp_reserve_warn = 1;
967 if (__kmp_tp_cached) {
968 __kmp_msg(kmp_ms_warning,
969 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
970 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
971 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
972 } else {
973 __kmp_msg(kmp_ms_warning,
974 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
975 KMP_HNT(SystemLimitOnThreads), __kmp_msg_null);
976 }
977 }
978 }
979 }
980
Jonathan Peyton642688b2017-06-01 16:46:36 +0000981#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +0000982 if (new_nthreads == 1) {
983 KC_TRACE(10,
984 ("__kmp_reserve_threads: T#%d serializing team after reclaiming "
985 "dead roots and rechecking; requested %d threads\n",
986 __kmp_get_gtid(), set_nthreads));
Jonathan Peyton642688b2017-06-01 16:46:36 +0000987 } else {
988 KC_TRACE(10, ("__kmp_reserve_threads: T#%d allocating %d threads; requested"
989 " %d threads\n",
990 __kmp_get_gtid(), new_nthreads, set_nthreads));
Jonathan Peyton30419822017-05-12 18:01:32 +0000991 }
Jonathan Peyton642688b2017-06-01 16:46:36 +0000992#endif // KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +0000993 return new_nthreads;
Jim Cownie5e8470a2013-09-27 10:38:44 +0000994}
995
Jonathan Peyton30419822017-05-12 18:01:32 +0000996/* Allocate threads from the thread pool and assign them to the new team. We are
997 assured that there are enough threads available, because we checked on that
998 earlier within critical section forkjoin */
999static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
1000 kmp_info_t *master_th, int master_gtid) {
1001 int i;
1002 int use_hot_team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001003
Jonathan Peyton30419822017-05-12 18:01:32 +00001004 KA_TRACE(10, ("__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc));
1005 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid());
1006 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00001007
Jonathan Peyton30419822017-05-12 18:01:32 +00001008 /* first, let's setup the master thread */
1009 master_th->th.th_info.ds.ds_tid = 0;
1010 master_th->th.th_team = team;
1011 master_th->th.th_team_nproc = team->t.t_nproc;
1012 master_th->th.th_team_master = master_th;
1013 master_th->th.th_team_serialized = FALSE;
1014 master_th->th.th_dispatch = &team->t.t_dispatch[0];
Jim Cownie5e8470a2013-09-27 10:38:44 +00001015
Jonathan Peyton30419822017-05-12 18:01:32 +00001016/* make sure we are not the optimized hot team */
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001017#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00001018 use_hot_team = 0;
1019 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams;
1020 if (hot_teams) { // hot teams array is not allocated if
1021 // KMP_HOT_TEAMS_MAX_LEVEL=0
1022 int level = team->t.t_active_level - 1; // index in array of hot teams
1023 if (master_th->th.th_teams_microtask) { // are we inside the teams?
1024 if (master_th->th.th_teams_size.nteams > 1) {
1025 ++level; // level was not increased in teams construct for
1026 // team_of_masters
1027 }
1028 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
1029 master_th->th.th_teams_level == team->t.t_level) {
1030 ++level; // level was not increased in teams construct for
1031 // team_of_workers before the parallel
1032 } // team->t.t_level will be increased inside parallel
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001033 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001034 if (level < __kmp_hot_teams_max_level) {
1035 if (hot_teams[level].hot_team) {
1036 // hot team has already been allocated for given level
1037 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team);
1038 use_hot_team = 1; // the team is ready to use
1039 } else {
1040 use_hot_team = 0; // AC: threads are not allocated yet
1041 hot_teams[level].hot_team = team; // remember new hot team
1042 hot_teams[level].hot_team_nth = team->t.t_nproc;
1043 }
1044 } else {
1045 use_hot_team = 0;
1046 }
1047 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001048#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001049 use_hot_team = team == root->r.r_hot_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001050#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001051 if (!use_hot_team) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00001052
Jonathan Peyton30419822017-05-12 18:01:32 +00001053 /* install the master thread */
1054 team->t.t_threads[0] = master_th;
1055 __kmp_initialize_info(master_th, team, 0, master_gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001056
Jonathan Peyton30419822017-05-12 18:01:32 +00001057 /* now, install the worker threads */
1058 for (i = 1; i < team->t.t_nproc; i++) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00001059
Jonathan Peyton30419822017-05-12 18:01:32 +00001060 /* fork or reallocate a new thread and install it in team */
1061 kmp_info_t *thr = __kmp_allocate_thread(root, team, i);
1062 team->t.t_threads[i] = thr;
1063 KMP_DEBUG_ASSERT(thr);
1064 KMP_DEBUG_ASSERT(thr->th.th_team == team);
1065 /* align team and thread arrived states */
1066 KA_TRACE(20, ("__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
1067 "T#%d(%d:%d) join =%llu, plain=%llu\n",
1068 __kmp_gtid_from_tid(0, team), team->t.t_id, 0,
1069 __kmp_gtid_from_tid(i, team), team->t.t_id, i,
1070 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
1071 team->t.t_bar[bs_plain_barrier].b_arrived));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001072#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001073 thr->th.th_teams_microtask = master_th->th.th_teams_microtask;
1074 thr->th.th_teams_level = master_th->th.th_teams_level;
1075 thr->th.th_teams_size = master_th->th.th_teams_size;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001076#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001077 { // Initialize threads' barrier data.
1078 int b;
1079 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar;
1080 for (b = 0; b < bs_last_barrier; ++b) {
1081 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
1082 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00001083#if USE_DEBUGGER
Jonathan Peyton30419822017-05-12 18:01:32 +00001084 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00001085#endif
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00001086 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001087 }
1088 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001089
Alp Toker98758b02014-03-02 04:12:06 +00001090#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00001091 __kmp_partition_places(team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001092#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001093 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001094
Jonathan Peyton30419822017-05-12 18:01:32 +00001095 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00001096}
1097
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001098#if KMP_ARCH_X86 || KMP_ARCH_X86_64
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001099// Propagate any changes to the floating point control registers out to the team
Jonathan Peyton30419822017-05-12 18:01:32 +00001100// We try to avoid unnecessary writes to the relevant cache line in the team
1101// structure, so we don't make changes unless they are needed.
1102inline static void propagateFPControl(kmp_team_t *team) {
1103 if (__kmp_inherit_fp_control) {
1104 kmp_int16 x87_fpu_control_word;
1105 kmp_uint32 mxcsr;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001106
Jonathan Peyton30419822017-05-12 18:01:32 +00001107 // Get master values of FPU control flags (both X87 and vector)
1108 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1109 __kmp_store_mxcsr(&mxcsr);
1110 mxcsr &= KMP_X86_MXCSR_MASK;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001111
Jonathan Peyton94a114f2017-10-20 19:30:57 +00001112 // There is no point looking at t_fp_control_saved here.
1113 // If it is TRUE, we still have to update the values if they are different
Jonas Hahnfeldf0a1c652017-11-03 18:28:19 +00001114 // from those we now have. If it is FALSE we didn't save anything yet, but
1115 // our objective is the same. We have to ensure that the values in the team
1116 // are the same as those we have.
Jonathan Peyton94a114f2017-10-20 19:30:57 +00001117 // So, this code achieves what we need whether or not t_fp_control_saved is
1118 // true. By checking whether the value needs updating we avoid unnecessary
1119 // writes that would put the cache-line into a written state, causing all
1120 // threads in the team to have to read it again.
Jonathan Peyton30419822017-05-12 18:01:32 +00001121 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word);
1122 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr);
1123 // Although we don't use this value, other code in the runtime wants to know
1124 // whether it should restore them. So we must ensure it is correct.
1125 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE);
1126 } else {
1127 // Similarly here. Don't write to this cache-line in the team structure
1128 // unless we have to.
1129 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE);
1130 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001131}
1132
Jonathan Peyton30419822017-05-12 18:01:32 +00001133// Do the opposite, setting the hardware registers to the updated values from
1134// the team.
1135inline static void updateHWFPControl(kmp_team_t *team) {
1136 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) {
1137 // Only reset the fp control regs if they have been changed in the team.
1138 // the parallel region that we are exiting.
1139 kmp_int16 x87_fpu_control_word;
1140 kmp_uint32 mxcsr;
1141 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1142 __kmp_store_mxcsr(&mxcsr);
1143 mxcsr &= KMP_X86_MXCSR_MASK;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001144
Jonathan Peyton30419822017-05-12 18:01:32 +00001145 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) {
1146 __kmp_clear_x87_fpu_status_word();
1147 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001148 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001149
1150 if (team->t.t_mxcsr != mxcsr) {
1151 __kmp_load_mxcsr(&team->t.t_mxcsr);
1152 }
1153 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001154}
1155#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001156#define propagateFPControl(x) ((void)0)
1157#define updateHWFPControl(x) ((void)0)
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001158#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1159
Jonathan Peyton30419822017-05-12 18:01:32 +00001160static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team,
1161 int realloc); // forward declaration
Jim Cownie5e8470a2013-09-27 10:38:44 +00001162
Jonathan Peyton30419822017-05-12 18:01:32 +00001163/* Run a parallel region that has been serialized, so runs only in a team of the
1164 single master thread. */
1165void __kmp_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
1166 kmp_info_t *this_thr;
1167 kmp_team_t *serial_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001168
Jonathan Peyton30419822017-05-12 18:01:32 +00001169 KC_TRACE(10, ("__kmpc_serialized_parallel: called by T#%d\n", global_tid));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001170
Jonathan Peyton30419822017-05-12 18:01:32 +00001171 /* Skip all this code for autopar serialized loops since it results in
1172 unacceptable overhead */
1173 if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR))
1174 return;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001175
Jonathan Peyton30419822017-05-12 18:01:32 +00001176 if (!TCR_4(__kmp_init_parallel))
1177 __kmp_parallel_initialize();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001178
Jonathan Peyton30419822017-05-12 18:01:32 +00001179 this_thr = __kmp_threads[global_tid];
1180 serial_team = this_thr->th.th_serial_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001181
Jonathan Peyton30419822017-05-12 18:01:32 +00001182 /* utilize the serialized team held by this thread */
1183 KMP_DEBUG_ASSERT(serial_team);
1184 KMP_MB();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001185
Jonathan Peyton30419822017-05-12 18:01:32 +00001186 if (__kmp_tasking_mode != tskm_immediate_exec) {
1187 KMP_DEBUG_ASSERT(
1188 this_thr->th.th_task_team ==
1189 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]);
1190 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] ==
1191 NULL);
1192 KA_TRACE(20, ("__kmpc_serialized_parallel: T#%d pushing task_team %p / "
1193 "team %p, new task_team = NULL\n",
1194 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
1195 this_thr->th.th_task_team = NULL;
1196 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001197
1198#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001199 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
1200 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1201 proc_bind = proc_bind_false;
1202 } else if (proc_bind == proc_bind_default) {
1203 // No proc_bind clause was specified, so use the current value
1204 // of proc-bind-var for this parallel region.
1205 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
1206 }
1207 // Reset for next parallel region
1208 this_thr->th.th_set_proc_bind = proc_bind_default;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001209#endif /* OMP_40_ENABLED */
1210
Joachim Protze82e94a52017-11-01 10:08:30 +00001211#if OMPT_SUPPORT
Jonathan Peyton3574f282018-10-04 14:57:04 +00001212 ompt_data_t ompt_parallel_data = ompt_data_none;
Joachim Protze82e94a52017-11-01 10:08:30 +00001213 ompt_data_t *implicit_task_data;
1214 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
1215 if (ompt_enabled.enabled &&
1216 this_thr->th.ompt_thread_info.state != omp_state_overhead) {
1217
1218 ompt_task_info_t *parent_task_info;
1219 parent_task_info = OMPT_CUR_TASK_INFO(this_thr);
1220
Joachim Protzec255ca72017-11-05 14:11:10 +00001221 parent_task_info->frame.enter_frame = OMPT_GET_FRAME_ADDRESS(1);
Joachim Protze82e94a52017-11-01 10:08:30 +00001222 if (ompt_enabled.ompt_callback_parallel_begin) {
1223 int team_size = 1;
1224
1225 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
1226 &(parent_task_info->task_data), &(parent_task_info->frame),
Joachim Protze489cdb72018-09-10 14:34:54 +00001227 &ompt_parallel_data, team_size, ompt_parallel_invoker_program,
1228 codeptr);
Joachim Protze82e94a52017-11-01 10:08:30 +00001229 }
1230 }
1231#endif // OMPT_SUPPORT
1232
Jonathan Peyton30419822017-05-12 18:01:32 +00001233 if (this_thr->th.th_team != serial_team) {
1234 // Nested level will be an index in the nested nthreads array
1235 int level = this_thr->th.th_team->t.t_level;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001236
Jonathan Peyton30419822017-05-12 18:01:32 +00001237 if (serial_team->t.t_serialized) {
1238 /* this serial team was already used
1239 TODO increase performance by making this locks more specific */
1240 kmp_team_t *new_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001241
Jonathan Peyton30419822017-05-12 18:01:32 +00001242 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001243
Jonathan Peyton30419822017-05-12 18:01:32 +00001244 new_team = __kmp_allocate_team(this_thr->th.th_root, 1, 1,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001245#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00001246 ompt_parallel_data,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001247#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001248#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001249 proc_bind,
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001250#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001251 &this_thr->th.th_current_task->td_icvs,
1252 0 USE_NESTED_HOT_ARG(NULL));
1253 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1254 KMP_ASSERT(new_team);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001255
Jonathan Peyton30419822017-05-12 18:01:32 +00001256 /* setup new serialized team and install it */
1257 new_team->t.t_threads[0] = this_thr;
1258 new_team->t.t_parent = this_thr->th.th_team;
1259 serial_team = new_team;
1260 this_thr->th.th_serial_team = serial_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001261
Jonathan Peyton30419822017-05-12 18:01:32 +00001262 KF_TRACE(
1263 10,
1264 ("__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",
1265 global_tid, serial_team));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001266
Jonathan Peyton30419822017-05-12 18:01:32 +00001267 /* TODO the above breaks the requirement that if we run out of resources,
1268 then we can still guarantee that serialized teams are ok, since we may
1269 need to allocate a new one */
1270 } else {
1271 KF_TRACE(
1272 10,
1273 ("__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",
1274 global_tid, serial_team));
1275 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001276
Jonathan Peyton30419822017-05-12 18:01:32 +00001277 /* we have to initialize this serial team */
1278 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1279 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1280 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team);
1281 serial_team->t.t_ident = loc;
1282 serial_team->t.t_serialized = 1;
1283 serial_team->t.t_nproc = 1;
1284 serial_team->t.t_parent = this_thr->th.th_team;
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00001285 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched;
Jonathan Peyton30419822017-05-12 18:01:32 +00001286 this_thr->th.th_team = serial_team;
1287 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001288
Jonathan Peyton30419822017-05-12 18:01:32 +00001289 KF_TRACE(10, ("__kmpc_serialized_parallel: T#d curtask=%p\n", global_tid,
1290 this_thr->th.th_current_task));
1291 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1);
1292 this_thr->th.th_current_task->td_flags.executing = 0;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001293
Jonathan Peyton30419822017-05-12 18:01:32 +00001294 __kmp_push_current_task_to_thread(this_thr, serial_team, 0);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001295
Jonathan Peyton30419822017-05-12 18:01:32 +00001296 /* TODO: GEH: do ICVs work for nested serialized teams? Don't we need an
1297 implicit task for each serialized task represented by
1298 team->t.t_serialized? */
1299 copy_icvs(&this_thr->th.th_current_task->td_icvs,
1300 &this_thr->th.th_current_task->td_parent->td_icvs);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001301
Jonathan Peyton30419822017-05-12 18:01:32 +00001302 // Thread value exists in the nested nthreads array for the next nested
1303 // level
1304 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1305 this_thr->th.th_current_task->td_icvs.nproc =
1306 __kmp_nested_nth.nth[level + 1];
1307 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001308
1309#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001310 if (__kmp_nested_proc_bind.used &&
1311 (level + 1 < __kmp_nested_proc_bind.used)) {
1312 this_thr->th.th_current_task->td_icvs.proc_bind =
1313 __kmp_nested_proc_bind.bind_types[level + 1];
1314 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001315#endif /* OMP_40_ENABLED */
1316
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00001317#if USE_DEBUGGER
Jonathan Peyton30419822017-05-12 18:01:32 +00001318 serial_team->t.t_pkfn = (microtask_t)(~0); // For the debugger.
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00001319#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001320 this_thr->th.th_info.ds.ds_tid = 0;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001321
Jonathan Peyton30419822017-05-12 18:01:32 +00001322 /* set thread cache values */
1323 this_thr->th.th_team_nproc = 1;
1324 this_thr->th.th_team_master = this_thr;
1325 this_thr->th.th_team_serialized = 1;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001326
Jonathan Peyton30419822017-05-12 18:01:32 +00001327 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1;
1328 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level;
Jonathan Peyton92ca6182018-09-07 18:25:49 +00001329#if OMP_50_ENABLED
1330 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator; // save
1331#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001332
Jonathan Peyton30419822017-05-12 18:01:32 +00001333 propagateFPControl(serial_team);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001334
Jonathan Peyton30419822017-05-12 18:01:32 +00001335 /* check if we need to allocate dispatch buffers stack */
1336 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1337 if (!serial_team->t.t_dispatch->th_disp_buffer) {
1338 serial_team->t.t_dispatch->th_disp_buffer =
1339 (dispatch_private_info_t *)__kmp_allocate(
1340 sizeof(dispatch_private_info_t));
1341 }
1342 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001343
Jonathan Peyton30419822017-05-12 18:01:32 +00001344 KMP_MB();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001345
Jonathan Peyton30419822017-05-12 18:01:32 +00001346 } else {
1347 /* this serialized team is already being used,
1348 * that's fine, just add another nested level */
1349 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
1350 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1351 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1352 ++serial_team->t.t_serialized;
1353 this_thr->th.th_team_serialized = serial_team->t.t_serialized;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001354
Jonathan Peyton30419822017-05-12 18:01:32 +00001355 // Nested level will be an index in the nested nthreads array
1356 int level = this_thr->th.th_team->t.t_level;
1357 // Thread value exists in the nested nthreads array for the next nested
1358 // level
1359 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1360 this_thr->th.th_current_task->td_icvs.nproc =
1361 __kmp_nested_nth.nth[level + 1];
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001362 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001363 serial_team->t.t_level++;
1364 KF_TRACE(10, ("__kmpc_serialized_parallel: T#%d increasing nesting level "
1365 "of serial team %p to %d\n",
1366 global_tid, serial_team, serial_team->t.t_level));
1367
1368 /* allocate/push dispatch buffers stack */
1369 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1370 {
1371 dispatch_private_info_t *disp_buffer =
1372 (dispatch_private_info_t *)__kmp_allocate(
1373 sizeof(dispatch_private_info_t));
1374 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;
1375 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;
1376 }
1377 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1378
1379 KMP_MB();
1380 }
Olga Malyshevadbdcfa12017-04-04 13:56:50 +00001381#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001382 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq);
Olga Malyshevadbdcfa12017-04-04 13:56:50 +00001383#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001384
Jonathan Peyton30419822017-05-12 18:01:32 +00001385 if (__kmp_env_consistency_check)
1386 __kmp_push_parallel(global_tid, NULL);
Joachim Protze82e94a52017-11-01 10:08:30 +00001387#if OMPT_SUPPORT
1388 serial_team->t.ompt_team_info.master_return_address = codeptr;
1389 if (ompt_enabled.enabled &&
1390 this_thr->th.ompt_thread_info.state != omp_state_overhead) {
Joachim Protzec255ca72017-11-05 14:11:10 +00001391 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame = OMPT_GET_FRAME_ADDRESS(1);
Joachim Protze82e94a52017-11-01 10:08:30 +00001392
1393 ompt_lw_taskteam_t lw_taskteam;
1394 __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid,
1395 &ompt_parallel_data, codeptr);
1396
1397 __ompt_lw_taskteam_link(&lw_taskteam, this_thr, 1);
1398 // don't use lw_taskteam after linking. content was swaped
1399
1400 /* OMPT implicit task begin */
1401 implicit_task_data = OMPT_CUR_TASK_DATA(this_thr);
1402 if (ompt_enabled.ompt_callback_implicit_task) {
1403 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1404 ompt_scope_begin, OMPT_CUR_TEAM_DATA(this_thr),
1405 OMPT_CUR_TASK_DATA(this_thr), 1, __kmp_tid_from_gtid(global_tid));
Joachim Protze9be9cf22018-05-07 12:42:21 +00001406 OMPT_CUR_TASK_INFO(this_thr)
1407 ->thread_num = __kmp_tid_from_gtid(global_tid);
Joachim Protze82e94a52017-11-01 10:08:30 +00001408 }
1409
1410 /* OMPT state */
1411 this_thr->th.ompt_thread_info.state = omp_state_work_parallel;
Joachim Protzec255ca72017-11-05 14:11:10 +00001412 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame = OMPT_GET_FRAME_ADDRESS(1);
Joachim Protze82e94a52017-11-01 10:08:30 +00001413 }
1414#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001415}
Jim Cownie181b4bb2013-12-23 17:28:57 +00001416
Jim Cownie5e8470a2013-09-27 10:38:44 +00001417/* most of the work for a fork */
1418/* return true if we really went parallel, false if serialized */
Jonathan Peyton30419822017-05-12 18:01:32 +00001419int __kmp_fork_call(ident_t *loc, int gtid,
1420 enum fork_context_e call_context, // Intel, GNU, ...
Joachim Protze82e94a52017-11-01 10:08:30 +00001421 kmp_int32 argc, microtask_t microtask, launch_t invoker,
Jim Cownie5e8470a2013-09-27 10:38:44 +00001422/* TODO: revert workaround for Intel(R) 64 tracker #96 */
Andrey Churbanovcbda8682015-01-13 14:43:35 +00001423#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
Jonathan Peyton30419822017-05-12 18:01:32 +00001424 va_list *ap
Jim Cownie5e8470a2013-09-27 10:38:44 +00001425#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001426 va_list ap
Jim Cownie5e8470a2013-09-27 10:38:44 +00001427#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001428 ) {
1429 void **argv;
1430 int i;
1431 int master_tid;
1432 int master_this_cons;
1433 kmp_team_t *team;
1434 kmp_team_t *parent_team;
1435 kmp_info_t *master_th;
1436 kmp_root_t *root;
1437 int nthreads;
1438 int master_active;
1439 int master_set_numthreads;
1440 int level;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001441#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001442 int active_level;
1443 int teams_level;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001444#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001445#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00001446 kmp_hot_team_ptr_t **p_hot_teams;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001447#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001448 { // KMP_TIME_BLOCK
Jonathan Peyton5375fe82016-11-14 21:13:44 +00001449 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call);
Jonathan Peyton45be4502015-08-11 21:36:41 +00001450 KMP_COUNT_VALUE(OMP_PARALLEL_args, argc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001451
Jonathan Peyton30419822017-05-12 18:01:32 +00001452 KA_TRACE(20, ("__kmp_fork_call: enter T#%d\n", gtid));
1453 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL) {
1454 /* Some systems prefer the stack for the root thread(s) to start with */
1455 /* some gap from the parent stack to prevent false sharing. */
1456 void *dummy = KMP_ALLOCA(__kmp_stkpadding);
1457 /* These 2 lines below are so this does not get optimized out */
1458 if (__kmp_stkpadding > KMP_MAX_STKPADDING)
1459 __kmp_stkpadding += (short)((kmp_int64)dummy);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001460 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001461
1462 /* initialize if needed */
Jonathan Peyton30419822017-05-12 18:01:32 +00001463 KMP_DEBUG_ASSERT(
1464 __kmp_init_serial); // AC: potentially unsafe, not in sync with shutdown
1465 if (!TCR_4(__kmp_init_parallel))
1466 __kmp_parallel_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00001467
1468 /* setup current data */
Jonathan Peyton30419822017-05-12 18:01:32 +00001469 master_th = __kmp_threads[gtid]; // AC: potentially unsafe, not in sync with
1470 // shutdown
1471 parent_team = master_th->th.th_team;
1472 master_tid = master_th->th.th_info.ds.ds_tid;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001473 master_this_cons = master_th->th.th_local.this_construct;
Jonathan Peyton30419822017-05-12 18:01:32 +00001474 root = master_th->th.th_root;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001475 master_active = root->r.r_active;
1476 master_set_numthreads = master_th->th.th_set_nproc;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001477
1478#if OMPT_SUPPORT
Jonathan Peyton3574f282018-10-04 14:57:04 +00001479 ompt_data_t ompt_parallel_data = ompt_data_none;
Joachim Protze82e94a52017-11-01 10:08:30 +00001480 ompt_data_t *parent_task_data;
Joachim Protzec5836064b2018-05-28 08:14:58 +00001481 omp_frame_t *ompt_frame;
Joachim Protze82e94a52017-11-01 10:08:30 +00001482 ompt_data_t *implicit_task_data;
1483 void *return_address = NULL;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001484
Joachim Protze82e94a52017-11-01 10:08:30 +00001485 if (ompt_enabled.enabled) {
1486 __ompt_get_task_info_internal(0, NULL, &parent_task_data, &ompt_frame,
1487 NULL, NULL);
1488 return_address = OMPT_LOAD_RETURN_ADDRESS(gtid);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001489 }
1490#endif
1491
Jim Cownie5e8470a2013-09-27 10:38:44 +00001492 // Nested level will be an index in the nested nthreads array
Jonathan Peyton30419822017-05-12 18:01:32 +00001493 level = parent_team->t.t_level;
1494 // used to launch non-serial teams even if nested is not allowed
1495 active_level = parent_team->t.t_active_level;
Jonathan Peytonc76f9f02016-06-21 19:12:07 +00001496#if OMP_40_ENABLED
Jonathan Peyton642688b2017-06-01 16:46:36 +00001497 // needed to check nesting inside the teams
1498 teams_level = master_th->th.th_teams_level;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001499#endif
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001500#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00001501 p_hot_teams = &master_th->th.th_hot_teams;
1502 if (*p_hot_teams == NULL && __kmp_hot_teams_max_level > 0) {
1503 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate(
1504 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level);
1505 (*p_hot_teams)[0].hot_team = root->r.r_hot_team;
Jonathan Peyton642688b2017-06-01 16:46:36 +00001506 // it is either actual or not needed (when active_level > 0)
1507 (*p_hot_teams)[0].hot_team_nth = 1;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001508 }
1509#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001510
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001511#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00001512 if (ompt_enabled.enabled) {
1513 if (ompt_enabled.ompt_callback_parallel_begin) {
1514 int team_size = master_set_numthreads
1515 ? master_set_numthreads
1516 : get__nproc_2(parent_team, master_tid);
1517 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
1518 parent_task_data, ompt_frame, &ompt_parallel_data, team_size,
1519 OMPT_INVOKER(call_context), return_address);
1520 }
1521 master_th->th.ompt_thread_info.state = omp_state_overhead;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001522 }
1523#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001524
Jim Cownie5e8470a2013-09-27 10:38:44 +00001525 master_th->th.th_ident = loc;
1526
1527#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001528 if (master_th->th.th_teams_microtask && ap &&
1529 microtask != (microtask_t)__kmp_teams_master && level == teams_level) {
1530 // AC: This is start of parallel that is nested inside teams construct.
1531 // The team is actual (hot), all workers are ready at the fork barrier.
1532 // No lock needed to initialize the team a bit, then free workers.
1533 parent_team->t.t_ident = loc;
1534 __kmp_alloc_argv_entries(argc, parent_team, TRUE);
1535 parent_team->t.t_argc = argc;
1536 argv = (void **)parent_team->t.t_argv;
1537 for (i = argc - 1; i >= 0; --i)
Jim Cownie5e8470a2013-09-27 10:38:44 +00001538/* TODO: revert workaround for Intel(R) 64 tracker #96 */
Andrey Churbanovcbda8682015-01-13 14:43:35 +00001539#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
Jonathan Peyton30419822017-05-12 18:01:32 +00001540 *argv++ = va_arg(*ap, void *);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001541#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001542 *argv++ = va_arg(ap, void *);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001543#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001544 // Increment our nested depth levels, but not increase the serialization
1545 if (parent_team == master_th->th.th_serial_team) {
1546 // AC: we are in serialized parallel
1547 __kmpc_serialized_parallel(loc, gtid);
1548 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1);
1549 // AC: need this in order enquiry functions work
1550 // correctly, will restore at join time
1551 parent_team->t.t_serialized--;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001552#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001553 void *dummy;
1554 void **exit_runtime_p;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001555
Jonathan Peyton30419822017-05-12 18:01:32 +00001556 ompt_lw_taskteam_t lw_taskteam;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001557
Joachim Protze82e94a52017-11-01 10:08:30 +00001558 if (ompt_enabled.enabled) {
1559 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1560 &ompt_parallel_data, return_address);
Joachim Protzec255ca72017-11-05 14:11:10 +00001561 exit_runtime_p = &(lw_taskteam.ompt_task_info.frame.exit_frame);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001562
Joachim Protze82e94a52017-11-01 10:08:30 +00001563 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1564 // don't use lw_taskteam after linking. content was swaped
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001565
Jonathan Peyton30419822017-05-12 18:01:32 +00001566 /* OMPT implicit task begin */
Joachim Protze82e94a52017-11-01 10:08:30 +00001567 implicit_task_data = OMPT_CUR_TASK_DATA(master_th);
1568 if (ompt_enabled.ompt_callback_implicit_task) {
1569 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1570 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1571 implicit_task_data, 1, __kmp_tid_from_gtid(gtid));
Joachim Protze9be9cf22018-05-07 12:42:21 +00001572 OMPT_CUR_TASK_INFO(master_th)
1573 ->thread_num = __kmp_tid_from_gtid(gtid);
Jonathan Peyton30419822017-05-12 18:01:32 +00001574 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001575
Jonathan Peyton30419822017-05-12 18:01:32 +00001576 /* OMPT state */
Joachim Protze82e94a52017-11-01 10:08:30 +00001577 master_th->th.ompt_thread_info.state = omp_state_work_parallel;
Jonathan Peyton30419822017-05-12 18:01:32 +00001578 } else {
1579 exit_runtime_p = &dummy;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001580 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001581#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001582
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001583 {
Jonathan Peyton30419822017-05-12 18:01:32 +00001584 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1585 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1586 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv
1587#if OMPT_SUPPORT
1588 ,
1589 exit_runtime_p
1590#endif
1591 );
Jim Cownie5e8470a2013-09-27 10:38:44 +00001592 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001593
Jonathan Peyton30419822017-05-12 18:01:32 +00001594#if OMPT_SUPPORT
1595 *exit_runtime_p = NULL;
Joachim Protze82e94a52017-11-01 10:08:30 +00001596 if (ompt_enabled.enabled) {
Joachim Protzec255ca72017-11-05 14:11:10 +00001597 OMPT_CUR_TASK_INFO(master_th)->frame.exit_frame = NULL;
Joachim Protze82e94a52017-11-01 10:08:30 +00001598 if (ompt_enabled.ompt_callback_implicit_task) {
1599 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1600 ompt_scope_end, NULL, implicit_task_data, 1,
Joachim Protze9be9cf22018-05-07 12:42:21 +00001601 OMPT_CUR_TASK_INFO(master_th)->thread_num);
Jonathan Peyton30419822017-05-12 18:01:32 +00001602 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001603 __ompt_lw_taskteam_unlink(master_th);
Jonathan Peyton30419822017-05-12 18:01:32 +00001604
Joachim Protze82e94a52017-11-01 10:08:30 +00001605 if (ompt_enabled.ompt_callback_parallel_end) {
1606 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1607 OMPT_CUR_TEAM_DATA(master_th), OMPT_CUR_TASK_DATA(master_th),
1608 OMPT_INVOKER(call_context), return_address);
Jonathan Peyton30419822017-05-12 18:01:32 +00001609 }
Joachim Protze82e94a52017-11-01 10:08:30 +00001610 master_th->th.ompt_thread_info.state = omp_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00001611 }
1612#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001613 return TRUE;
Jonathan Peyton30419822017-05-12 18:01:32 +00001614 }
1615
1616 parent_team->t.t_pkfn = microtask;
Jonathan Peyton30419822017-05-12 18:01:32 +00001617 parent_team->t.t_invoke = invoker;
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00001618 KMP_ATOMIC_INC(&root->r.r_in_parallel);
Jonathan Peyton30419822017-05-12 18:01:32 +00001619 parent_team->t.t_active_level++;
1620 parent_team->t.t_level++;
Jonathan Peyton92ca6182018-09-07 18:25:49 +00001621#if OMP_50_ENABLED
1622 parent_team->t.t_def_allocator = master_th->th.th_def_allocator; // save
1623#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001624
1625 /* Change number of threads in the team if requested */
1626 if (master_set_numthreads) { // The parallel has num_threads clause
1627 if (master_set_numthreads < master_th->th.th_teams_size.nth) {
1628 // AC: only can reduce number of threads dynamically, can't increase
1629 kmp_info_t **other_threads = parent_team->t.t_threads;
1630 parent_team->t.t_nproc = master_set_numthreads;
1631 for (i = 0; i < master_set_numthreads; ++i) {
1632 other_threads[i]->th.th_team_nproc = master_set_numthreads;
1633 }
1634 // Keep extra threads hot in the team for possible next parallels
1635 }
1636 master_th->th.th_set_nproc = 0;
1637 }
1638
1639#if USE_DEBUGGER
1640 if (__kmp_debugging) { // Let debugger override number of threads.
1641 int nth = __kmp_omp_num_threads(loc);
Jonathan Peyton642688b2017-06-01 16:46:36 +00001642 if (nth > 0) { // 0 means debugger doesn't want to change num threads
Jonathan Peyton30419822017-05-12 18:01:32 +00001643 master_set_numthreads = nth;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00001644 }
1645 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001646#endif
1647
1648 KF_TRACE(10, ("__kmp_fork_call: before internal fork: root=%p, team=%p, "
1649 "master_th=%p, gtid=%d\n",
1650 root, parent_team, master_th, gtid));
1651 __kmp_internal_fork(loc, gtid, parent_team);
1652 KF_TRACE(10, ("__kmp_fork_call: after internal fork: root=%p, team=%p, "
1653 "master_th=%p, gtid=%d\n",
1654 root, parent_team, master_th, gtid));
1655
1656 /* Invoke microtask for MASTER thread */
1657 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
1658 parent_team->t.t_id, parent_team->t.t_pkfn));
1659
Jonathan Peytonf0682ac2018-07-30 17:41:08 +00001660 if (!parent_team->t.t_invoke(gtid)) {
1661 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread");
Jonathan Peyton30419822017-05-12 18:01:32 +00001662 }
1663 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
1664 parent_team->t.t_id, parent_team->t.t_pkfn));
1665 KMP_MB(); /* Flush all pending memory write invalidates. */
1666
1667 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid));
1668
1669 return TRUE;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001670 } // Parallel closely nested in teams construct
Jim Cownie5e8470a2013-09-27 10:38:44 +00001671#endif /* OMP_40_ENABLED */
1672
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001673#if KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00001674 if (__kmp_tasking_mode != tskm_immediate_exec) {
1675 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
1676 parent_team->t.t_task_team[master_th->th.th_task_state]);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001677 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001678#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001679
Jonathan Peyton30419822017-05-12 18:01:32 +00001680 if (parent_team->t.t_active_level >=
1681 master_th->th.th_current_task->td_icvs.max_active_levels) {
1682 nthreads = 1;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001683 } else {
Andrey Churbanov92effc42015-08-18 10:08:27 +00001684#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001685 int enter_teams = ((ap == NULL && active_level == 0) ||
1686 (ap && teams_level > 0 && teams_level == level));
Andrey Churbanov92effc42015-08-18 10:08:27 +00001687#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001688 nthreads =
1689 master_set_numthreads
1690 ? master_set_numthreads
1691 : get__nproc_2(
1692 parent_team,
1693 master_tid); // TODO: get nproc directly from current task
Andrey Churbanov92effc42015-08-18 10:08:27 +00001694
Jonathan Peyton30419822017-05-12 18:01:32 +00001695 // Check if we need to take forkjoin lock? (no need for serialized
1696 // parallel out of teams construct). This code moved here from
1697 // __kmp_reserve_threads() to speedup nested serialized parallels.
1698 if (nthreads > 1) {
1699 if ((!get__nested(master_th) && (root->r.r_in_parallel
Andrey Churbanov92effc42015-08-18 10:08:27 +00001700#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001701 && !enter_teams
Andrey Churbanov92effc42015-08-18 10:08:27 +00001702#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00001703 )) ||
1704 (__kmp_library == library_serial)) {
Jonathan Peyton642688b2017-06-01 16:46:36 +00001705 KC_TRACE(10, ("__kmp_fork_call: T#%d serializing team; requested %d"
1706 " threads\n",
1707 gtid, nthreads));
Jonathan Peyton30419822017-05-12 18:01:32 +00001708 nthreads = 1;
Andrey Churbanov92effc42015-08-18 10:08:27 +00001709 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001710 }
1711 if (nthreads > 1) {
1712 /* determine how many new threads we can use */
1713 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
Jonathan Peyton30419822017-05-12 18:01:32 +00001714 nthreads = __kmp_reserve_threads(
1715 root, parent_team, master_tid, nthreads
Jim Cownie5e8470a2013-09-27 10:38:44 +00001716#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001717 /* AC: If we execute teams from parallel region (on host), then
1718 teams should be created but each can only have 1 thread if
1719 nesting is disabled. If teams called from serial region, then
1720 teams and their threads should be created regardless of the
1721 nesting setting. */
1722 ,
1723 enter_teams
Jim Cownie5e8470a2013-09-27 10:38:44 +00001724#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00001725 );
1726 if (nthreads == 1) {
1727 // Free lock for single thread execution here; for multi-thread
1728 // execution it will be freed later after team of threads created
1729 // and initialized
1730 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
Andrey Churbanov92effc42015-08-18 10:08:27 +00001731 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001732 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00001733 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001734 KMP_DEBUG_ASSERT(nthreads > 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001735
Jonathan Peyton30419822017-05-12 18:01:32 +00001736 // If we temporarily changed the set number of threads then restore it now
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001737 master_th->th.th_set_nproc = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001738
Jim Cownie5e8470a2013-09-27 10:38:44 +00001739 /* create a serialized parallel region? */
Jonathan Peyton30419822017-05-12 18:01:32 +00001740 if (nthreads == 1) {
1741/* josh todo: hypothetical question: what do we do for OS X*? */
1742#if KMP_OS_LINUX && \
1743 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1744 void *args[argc];
Jim Cownie5e8470a2013-09-27 10:38:44 +00001745#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001746 void **args = (void **)KMP_ALLOCA(argc * sizeof(void *));
1747#endif /* KMP_OS_LINUX && ( KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || \
1748 KMP_ARCH_AARCH64) */
Jim Cownie5e8470a2013-09-27 10:38:44 +00001749
Jonathan Peyton30419822017-05-12 18:01:32 +00001750 KA_TRACE(20,
1751 ("__kmp_fork_call: T#%d serializing parallel region\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00001752
Jonathan Peyton30419822017-05-12 18:01:32 +00001753 __kmpc_serialized_parallel(loc, gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001754
Jonathan Peyton30419822017-05-12 18:01:32 +00001755 if (call_context == fork_context_intel) {
1756 /* TODO this sucks, use the compiler itself to pass args! :) */
1757 master_th->th.th_serial_team->t.t_ident = loc;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001758#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001759 if (!ap) {
1760 // revert change made in __kmpc_serialized_parallel()
1761 master_th->th.th_serial_team->t.t_level--;
1762// Get args from parent team for teams construct
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001763
1764#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001765 void *dummy;
1766 void **exit_runtime_p;
Joachim Protze82e94a52017-11-01 10:08:30 +00001767 ompt_task_info_t *task_info;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001768
Jonathan Peyton30419822017-05-12 18:01:32 +00001769 ompt_lw_taskteam_t lw_taskteam;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001770
Joachim Protze82e94a52017-11-01 10:08:30 +00001771 if (ompt_enabled.enabled) {
Jonathan Peyton30419822017-05-12 18:01:32 +00001772 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
Joachim Protze82e94a52017-11-01 10:08:30 +00001773 &ompt_parallel_data, return_address);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001774
Joachim Protze82e94a52017-11-01 10:08:30 +00001775 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1776 // don't use lw_taskteam after linking. content was swaped
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001777
Joachim Protze82e94a52017-11-01 10:08:30 +00001778 task_info = OMPT_CUR_TASK_INFO(master_th);
Joachim Protzec255ca72017-11-05 14:11:10 +00001779 exit_runtime_p = &(task_info->frame.exit_frame);
Joachim Protze82e94a52017-11-01 10:08:30 +00001780 if (ompt_enabled.ompt_callback_implicit_task) {
1781 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1782 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1783 &(task_info->task_data), 1, __kmp_tid_from_gtid(gtid));
Joachim Protze9be9cf22018-05-07 12:42:21 +00001784 OMPT_CUR_TASK_INFO(master_th)
1785 ->thread_num = __kmp_tid_from_gtid(gtid);
Jonathan Peyton30419822017-05-12 18:01:32 +00001786 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001787
Jonathan Peyton30419822017-05-12 18:01:32 +00001788 /* OMPT state */
Joachim Protze82e94a52017-11-01 10:08:30 +00001789 master_th->th.ompt_thread_info.state = omp_state_work_parallel;
Jonathan Peyton30419822017-05-12 18:01:32 +00001790 } else {
1791 exit_runtime_p = &dummy;
1792 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001793#endif
1794
Jonathan Peyton30419822017-05-12 18:01:32 +00001795 {
1796 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1797 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1798 __kmp_invoke_microtask(microtask, gtid, 0, argc,
1799 parent_team->t.t_argv
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001800#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001801 ,
1802 exit_runtime_p
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001803#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001804 );
1805 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001806
1807#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00001808 if (ompt_enabled.enabled) {
1809 exit_runtime_p = NULL;
1810 if (ompt_enabled.ompt_callback_implicit_task) {
1811 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1812 ompt_scope_end, NULL, &(task_info->task_data), 1,
Joachim Protze9be9cf22018-05-07 12:42:21 +00001813 OMPT_CUR_TASK_INFO(master_th)->thread_num);
Jonathan Peyton30419822017-05-12 18:01:32 +00001814 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001815
Jonathan Peyton30419822017-05-12 18:01:32 +00001816 __ompt_lw_taskteam_unlink(master_th);
Joachim Protze82e94a52017-11-01 10:08:30 +00001817 if (ompt_enabled.ompt_callback_parallel_end) {
1818 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1819 OMPT_CUR_TEAM_DATA(master_th), parent_task_data,
1820 OMPT_INVOKER(call_context), return_address);
Jonathan Peyton30419822017-05-12 18:01:32 +00001821 }
Joachim Protze82e94a52017-11-01 10:08:30 +00001822 master_th->th.ompt_thread_info.state = omp_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00001823 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001824#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001825 } else if (microtask == (microtask_t)__kmp_teams_master) {
1826 KMP_DEBUG_ASSERT(master_th->th.th_team ==
1827 master_th->th.th_serial_team);
1828 team = master_th->th.th_team;
1829 // team->t.t_pkfn = microtask;
1830 team->t.t_invoke = invoker;
1831 __kmp_alloc_argv_entries(argc, team, TRUE);
1832 team->t.t_argc = argc;
1833 argv = (void **)team->t.t_argv;
1834 if (ap) {
1835 for (i = argc - 1; i >= 0; --i)
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001836// TODO: revert workaround for Intel(R) 64 tracker #96
Andrey Churbanovcbda8682015-01-13 14:43:35 +00001837#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
Jonathan Peyton30419822017-05-12 18:01:32 +00001838 *argv++ = va_arg(*ap, void *);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001839#else
Jonathan Peyton30419822017-05-12 18:01:32 +00001840 *argv++ = va_arg(ap, void *);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00001841#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001842 } else {
1843 for (i = 0; i < argc; ++i)
1844 // Get args from parent team for teams construct
1845 argv[i] = parent_team->t.t_argv[i];
1846 }
1847 // AC: revert change made in __kmpc_serialized_parallel()
1848 // because initial code in teams should have level=0
1849 team->t.t_level--;
1850 // AC: call special invoker for outer "parallel" of teams construct
Jonathan Peytonf0682ac2018-07-30 17:41:08 +00001851 invoker(gtid);
Jonathan Peyton30419822017-05-12 18:01:32 +00001852 } else {
1853#endif /* OMP_40_ENABLED */
1854 argv = args;
1855 for (i = argc - 1; i >= 0; --i)
1856// TODO: revert workaround for Intel(R) 64 tracker #96
1857#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
1858 *argv++ = va_arg(*ap, void *);
1859#else
1860 *argv++ = va_arg(ap, void *);
1861#endif
1862 KMP_MB();
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001863
1864#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001865 void *dummy;
1866 void **exit_runtime_p;
Joachim Protze82e94a52017-11-01 10:08:30 +00001867 ompt_task_info_t *task_info;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001868
Jonathan Peyton30419822017-05-12 18:01:32 +00001869 ompt_lw_taskteam_t lw_taskteam;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001870
Joachim Protze82e94a52017-11-01 10:08:30 +00001871 if (ompt_enabled.enabled) {
Jonathan Peyton30419822017-05-12 18:01:32 +00001872 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
Joachim Protze82e94a52017-11-01 10:08:30 +00001873 &ompt_parallel_data, return_address);
1874 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1875 // don't use lw_taskteam after linking. content was swaped
1876 task_info = OMPT_CUR_TASK_INFO(master_th);
Joachim Protzec255ca72017-11-05 14:11:10 +00001877 exit_runtime_p = &(task_info->frame.exit_frame);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001878
Jonathan Peyton30419822017-05-12 18:01:32 +00001879 /* OMPT implicit task begin */
Joachim Protze82e94a52017-11-01 10:08:30 +00001880 implicit_task_data = OMPT_CUR_TASK_DATA(master_th);
1881 if (ompt_enabled.ompt_callback_implicit_task) {
1882 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1883 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1884 implicit_task_data, 1, __kmp_tid_from_gtid(gtid));
Joachim Protze9be9cf22018-05-07 12:42:21 +00001885 OMPT_CUR_TASK_INFO(master_th)
1886 ->thread_num = __kmp_tid_from_gtid(gtid);
Jonathan Peyton30419822017-05-12 18:01:32 +00001887 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001888
Jonathan Peyton30419822017-05-12 18:01:32 +00001889 /* OMPT state */
Joachim Protze82e94a52017-11-01 10:08:30 +00001890 master_th->th.ompt_thread_info.state = omp_state_work_parallel;
Jonathan Peyton30419822017-05-12 18:01:32 +00001891 } else {
1892 exit_runtime_p = &dummy;
1893 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001894#endif
1895
Jonathan Peyton30419822017-05-12 18:01:32 +00001896 {
1897 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1898 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1899 __kmp_invoke_microtask(microtask, gtid, 0, argc, args
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001900#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00001901 ,
1902 exit_runtime_p
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001903#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00001904 );
1905 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001906
1907#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00001908 if (ompt_enabled.enabled) {
1909 *exit_runtime_p = NULL;
1910 if (ompt_enabled.ompt_callback_implicit_task) {
1911 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1912 ompt_scope_end, NULL, &(task_info->task_data), 1,
Joachim Protze9be9cf22018-05-07 12:42:21 +00001913 OMPT_CUR_TASK_INFO(master_th)->thread_num);
Jonathan Peyton30419822017-05-12 18:01:32 +00001914 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001915
Joachim Protze82e94a52017-11-01 10:08:30 +00001916 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
Jonathan Peyton30419822017-05-12 18:01:32 +00001917 __ompt_lw_taskteam_unlink(master_th);
Joachim Protze82e94a52017-11-01 10:08:30 +00001918 if (ompt_enabled.ompt_callback_parallel_end) {
1919 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1920 &ompt_parallel_data, parent_task_data,
1921 OMPT_INVOKER(call_context), return_address);
Jonathan Peyton30419822017-05-12 18:01:32 +00001922 }
Joachim Protze82e94a52017-11-01 10:08:30 +00001923 master_th->th.ompt_thread_info.state = omp_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00001924 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001925#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00001926#if OMP_40_ENABLED
Jim Cownie5e8470a2013-09-27 10:38:44 +00001927 }
Jonathan Peyton30419822017-05-12 18:01:32 +00001928#endif /* OMP_40_ENABLED */
1929 } else if (call_context == fork_context_gnu) {
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001930#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00001931 ompt_lw_taskteam_t lwt;
1932 __ompt_lw_taskteam_init(&lwt, master_th, gtid, &ompt_parallel_data,
1933 return_address);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001934
Joachim Protzec255ca72017-11-05 14:11:10 +00001935 lwt.ompt_task_info.frame.exit_frame = NULL;
Joachim Protze82e94a52017-11-01 10:08:30 +00001936 __ompt_lw_taskteam_link(&lwt, master_th, 1);
1937// don't use lw_taskteam after linking. content was swaped
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00001938#endif
1939
Jonathan Peyton30419822017-05-12 18:01:32 +00001940 // we were called from GNU native code
1941 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00001942 return FALSE;
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00001943 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00001944 KMP_ASSERT2(call_context < fork_context_last,
1945 "__kmp_fork_call: unknown fork_context parameter");
1946 }
1947
1948 KA_TRACE(20, ("__kmp_fork_call: T#%d serial exit\n", gtid));
1949 KMP_MB();
1950 return FALSE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001951 }
1952
Jim Cownie5e8470a2013-09-27 10:38:44 +00001953 // GEH: only modify the executing flag in the case when not serialized
1954 // serialized case is handled in kmpc_serialized_parallel
Jonathan Peyton30419822017-05-12 18:01:32 +00001955 KF_TRACE(10, ("__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
1956 "curtask=%p, curtask_max_aclevel=%d\n",
1957 parent_team->t.t_active_level, master_th,
1958 master_th->th.th_current_task,
1959 master_th->th.th_current_task->td_icvs.max_active_levels));
1960 // TODO: GEH - cannot do this assertion because root thread not set up as
1961 // executing
Jim Cownie5e8470a2013-09-27 10:38:44 +00001962 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 1 );
1963 master_th->th.th_current_task->td_flags.executing = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00001964
1965#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00001966 if (!master_th->th.th_teams_microtask || level > teams_level)
Jim Cownie5e8470a2013-09-27 10:38:44 +00001967#endif /* OMP_40_ENABLED */
1968 {
Jonathan Peyton30419822017-05-12 18:01:32 +00001969 /* Increment our nested depth level */
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00001970 KMP_ATOMIC_INC(&root->r.r_in_parallel);
Jim Cownie5e8470a2013-09-27 10:38:44 +00001971 }
1972
Jim Cownie5e8470a2013-09-27 10:38:44 +00001973 // See if we need to make a copy of the ICVs.
Jim Cownie5e8470a2013-09-27 10:38:44 +00001974 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc;
Jonathan Peyton30419822017-05-12 18:01:32 +00001975 if ((level + 1 < __kmp_nested_nth.used) &&
1976 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) {
1977 nthreads_icv = __kmp_nested_nth.nth[level + 1];
1978 } else {
1979 nthreads_icv = 0; // don't update
Jim Cownie5e8470a2013-09-27 10:38:44 +00001980 }
1981
1982#if OMP_40_ENABLED
Jim Cownie5e8470a2013-09-27 10:38:44 +00001983 // Figure out the proc_bind_policy for the new team.
Jim Cownie5e8470a2013-09-27 10:38:44 +00001984 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;
Jonathan Peyton30419822017-05-12 18:01:32 +00001985 kmp_proc_bind_t proc_bind_icv =
1986 proc_bind_default; // proc_bind_default means don't update
1987 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1988 proc_bind = proc_bind_false;
1989 } else {
1990 if (proc_bind == proc_bind_default) {
1991 // No proc_bind clause specified; use current proc-bind-var for this
1992 // parallel region
1993 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;
1994 }
1995 /* else: The proc_bind policy was specified explicitly on parallel clause.
1996 This overrides proc-bind-var for this parallel region, but does not
1997 change proc-bind-var. */
1998 // Figure the value of proc-bind-var for the child threads.
1999 if ((level + 1 < __kmp_nested_proc_bind.used) &&
2000 (__kmp_nested_proc_bind.bind_types[level + 1] !=
2001 master_th->th.th_current_task->td_icvs.proc_bind)) {
2002 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1];
2003 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002004 }
2005
Jim Cownie5e8470a2013-09-27 10:38:44 +00002006 // Reset for next parallel region
Jim Cownie5e8470a2013-09-27 10:38:44 +00002007 master_th->th.th_set_proc_bind = proc_bind_default;
2008#endif /* OMP_40_ENABLED */
2009
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002010 if ((nthreads_icv > 0)
Jim Cownie5e8470a2013-09-27 10:38:44 +00002011#if OMP_40_ENABLED
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002012 || (proc_bind_icv != proc_bind_default)
Jim Cownie5e8470a2013-09-27 10:38:44 +00002013#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00002014 ) {
2015 kmp_internal_control_t new_icvs;
2016 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs);
2017 new_icvs.next = NULL;
2018 if (nthreads_icv > 0) {
2019 new_icvs.nproc = nthreads_icv;
2020 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002021
2022#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002023 if (proc_bind_icv != proc_bind_default) {
2024 new_icvs.proc_bind = proc_bind_icv;
2025 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002026#endif /* OMP_40_ENABLED */
2027
Jonathan Peyton30419822017-05-12 18:01:32 +00002028 /* allocate a new parallel team */
2029 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n"));
2030 team = __kmp_allocate_team(root, nthreads, nthreads,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002031#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002032 ompt_parallel_data,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002033#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002034#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002035 proc_bind,
Jim Cownie5e8470a2013-09-27 10:38:44 +00002036#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002037 &new_icvs, argc USE_NESTED_HOT_ARG(master_th));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002038 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00002039 /* allocate a new parallel team */
2040 KF_TRACE(10, ("__kmp_fork_call: before __kmp_allocate_team\n"));
2041 team = __kmp_allocate_team(root, nthreads, nthreads,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002042#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002043 ompt_parallel_data,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002044#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002045#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002046 proc_bind,
Jim Cownie5e8470a2013-09-27 10:38:44 +00002047#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002048 &master_th->th.th_current_task->td_icvs,
2049 argc USE_NESTED_HOT_ARG(master_th));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002050 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002051 KF_TRACE(
2052 10, ("__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002053
2054 /* setup the new team */
Jonathan Peytonb044e4f2016-05-23 18:01:19 +00002055 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid);
2056 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons);
2057 KMP_CHECK_UPDATE(team->t.t_ident, loc);
2058 KMP_CHECK_UPDATE(team->t.t_parent, parent_team);
2059 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002060#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002061 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address,
2062 return_address);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002063#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002064 KMP_CHECK_UPDATE(team->t.t_invoke, invoker); // TODO move to root, maybe
2065// TODO: parent_team->t.t_level == INT_MAX ???
Jim Cownie5e8470a2013-09-27 10:38:44 +00002066#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002067 if (!master_th->th.th_teams_microtask || level > teams_level) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002068#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00002069 int new_level = parent_team->t.t_level + 1;
2070 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2071 new_level = parent_team->t.t_active_level + 1;
2072 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002073#if OMP_40_ENABLED
2074 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00002075 // AC: Do not increase parallel level at start of the teams construct
2076 int new_level = parent_team->t.t_level;
2077 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2078 new_level = parent_team->t.t_active_level;
2079 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002080 }
2081#endif /* OMP_40_ENABLED */
Jonathan Peytonb044e4f2016-05-23 18:01:19 +00002082 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid);
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00002083 // set master's schedule as new run-time schedule
2084 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002085
Jonathan Peyton45ca5da2015-10-19 19:33:38 +00002086#if OMP_40_ENABLED
Jonathan Peytonb044e4f2016-05-23 18:01:19 +00002087 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq);
Jonathan Peyton45ca5da2015-10-19 19:33:38 +00002088#endif
Jonathan Peyton92ca6182018-09-07 18:25:49 +00002089#if OMP_50_ENABLED
2090 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator);
2091#endif
Jonathan Peyton45ca5da2015-10-19 19:33:38 +00002092
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002093 // Update the floating point rounding in the team if required.
2094 propagateFPControl(team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002095
Jonathan Peyton30419822017-05-12 18:01:32 +00002096 if (__kmp_tasking_mode != tskm_immediate_exec) {
2097 // Set master's task team to team's task team. Unless this is hot team, it
2098 // should be NULL.
Jonathan Peyton30419822017-05-12 18:01:32 +00002099 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2100 parent_team->t.t_task_team[master_th->th.th_task_state]);
Jonathan Peyton30419822017-05-12 18:01:32 +00002101 KA_TRACE(20, ("__kmp_fork_call: Master T#%d pushing task_team %p / team "
2102 "%p, new task_team %p / team %p\n",
2103 __kmp_gtid_from_thread(master_th),
2104 master_th->th.th_task_team, parent_team,
2105 team->t.t_task_team[master_th->th.th_task_state], team));
Jonathan Peytond3f2b942016-02-09 22:32:41 +00002106
Jonathan Peyton30419822017-05-12 18:01:32 +00002107 if (active_level || master_th->th.th_task_team) {
2108 // Take a memo of master's task_state
2109 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2110 if (master_th->th.th_task_state_top >=
2111 master_th->th.th_task_state_stack_sz) { // increase size
2112 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz;
2113 kmp_uint8 *old_stack, *new_stack;
2114 kmp_uint32 i;
2115 new_stack = (kmp_uint8 *)__kmp_allocate(new_size);
2116 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) {
2117 new_stack[i] = master_th->th.th_task_state_memo_stack[i];
2118 }
2119 for (i = master_th->th.th_task_state_stack_sz; i < new_size;
2120 ++i) { // zero-init rest of stack
2121 new_stack[i] = 0;
2122 }
2123 old_stack = master_th->th.th_task_state_memo_stack;
2124 master_th->th.th_task_state_memo_stack = new_stack;
2125 master_th->th.th_task_state_stack_sz = new_size;
2126 __kmp_free(old_stack);
Andrey Churbanov6d224db2015-02-10 18:37:43 +00002127 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002128 // Store master's task_state on stack
2129 master_th->th
2130 .th_task_state_memo_stack[master_th->th.th_task_state_top] =
2131 master_th->th.th_task_state;
2132 master_th->th.th_task_state_top++;
2133#if KMP_NESTED_HOT_TEAMS
Jonathan Peytonca10a762018-08-24 18:05:00 +00002134 if (master_th->th.th_hot_teams &&
Jonathan Peytonf4c07202018-11-28 20:15:11 +00002135 active_level < __kmp_hot_teams_max_level &&
Jonathan Peytonca10a762018-08-24 18:05:00 +00002136 team == master_th->th.th_hot_teams[active_level].hot_team) {
Jonathan Peyton642688b2017-06-01 16:46:36 +00002137 // Restore master's nested state if nested hot team
Jonathan Peyton30419822017-05-12 18:01:32 +00002138 master_th->th.th_task_state =
2139 master_th->th
2140 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2141 } else {
2142#endif
2143 master_th->th.th_task_state = 0;
2144#if KMP_NESTED_HOT_TEAMS
2145 }
2146#endif
2147 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002148#if !KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00002149 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) ||
2150 (team == root->r.r_hot_team));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002151#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002152 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002153
Jonathan Peyton30419822017-05-12 18:01:32 +00002154 KA_TRACE(
2155 20,
2156 ("__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",
2157 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id,
2158 team->t.t_nproc));
2159 KMP_DEBUG_ASSERT(team != root->r.r_hot_team ||
2160 (team->t.t_master_tid == 0 &&
2161 (team->t.t_parent == root->r.r_root_team ||
2162 team->t.t_parent->t.t_serialized)));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002163 KMP_MB();
2164
2165 /* now, setup the arguments */
Jonathan Peyton30419822017-05-12 18:01:32 +00002166 argv = (void **)team->t.t_argv;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002167#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002168 if (ap) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002169#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00002170 for (i = argc - 1; i >= 0; --i) {
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002171// TODO: revert workaround for Intel(R) 64 tracker #96
Andrey Churbanovcbda8682015-01-13 14:43:35 +00002172#if (KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) && KMP_OS_LINUX
Jonathan Peyton30419822017-05-12 18:01:32 +00002173 void *new_argv = va_arg(*ap, void *);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002174#else
Jonathan Peyton30419822017-05-12 18:01:32 +00002175 void *new_argv = va_arg(ap, void *);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002176#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002177 KMP_CHECK_UPDATE(*argv, new_argv);
2178 argv++;
2179 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002180#if OMP_40_ENABLED
2181 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00002182 for (i = 0; i < argc; ++i) {
2183 // Get args from parent team for teams construct
2184 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]);
2185 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002186 }
2187#endif /* OMP_40_ENABLED */
2188
2189 /* now actually fork the threads */
Jonathan Peytonb044e4f2016-05-23 18:01:19 +00002190 KMP_CHECK_UPDATE(team->t.t_master_active, master_active);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002191 if (!root->r.r_active) // Only do assignment if it prevents cache ping-pong
Jonathan Peyton30419822017-05-12 18:01:32 +00002192 root->r.r_active = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002193
Jonathan Peyton30419822017-05-12 18:01:32 +00002194 __kmp_fork_team_threads(root, team, master_th, gtid);
2195 __kmp_setup_icv_copy(team, nthreads,
2196 &master_th->th.th_current_task->td_icvs, loc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002197
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002198#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002199 master_th->th.ompt_thread_info.state = omp_state_work_parallel;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002200#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002201
Jonathan Peyton30419822017-05-12 18:01:32 +00002202 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002203
Jim Cownie5e8470a2013-09-27 10:38:44 +00002204#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +00002205 if (team->t.t_active_level == 1 // only report frames at level 1
2206#if OMP_40_ENABLED
Andrey Churbanov51aecb82015-05-06 19:22:36 +00002207 && !master_th->th.th_teams_microtask // not in teams construct
Jonathan Peyton30419822017-05-12 18:01:32 +00002208#endif /* OMP_40_ENABLED */
2209 ) {
Andrey Churbanov51aecb82015-05-06 19:22:36 +00002210#if USE_ITT_NOTIFY
Jonathan Peyton30419822017-05-12 18:01:32 +00002211 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2212 (__kmp_forkjoin_frames_mode == 3 ||
2213 __kmp_forkjoin_frames_mode == 1)) {
2214 kmp_uint64 tmp_time = 0;
2215 if (__itt_get_timestamp_ptr)
2216 tmp_time = __itt_get_timestamp();
2217 // Internal fork - report frame begin
2218 master_th->th.th_frame_time = tmp_time;
2219 if (__kmp_forkjoin_frames_mode == 3)
2220 team->t.t_region_time = tmp_time;
Jonathan Peyton642688b2017-06-01 16:46:36 +00002221 } else
2222// only one notification scheme (either "submit" or "forking/joined", not both)
Andrey Churbanov51aecb82015-05-06 19:22:36 +00002223#endif /* USE_ITT_NOTIFY */
Jonathan Peyton30419822017-05-12 18:01:32 +00002224 if ((__itt_frame_begin_v3_ptr || KMP_ITT_DEBUG) &&
2225 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) {
Jonathan Peyton8c432f22018-01-04 22:56:47 +00002226 // Mark start of "parallel" region for Intel(R) VTune(TM) analyzer.
Jonathan Peyton30419822017-05-12 18:01:32 +00002227 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
2228 }
Andrey Churbanovf6451d92015-01-16 15:58:03 +00002229 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002230#endif /* USE_ITT_BUILD */
2231
2232 /* now go on and do the work */
Jonathan Peyton30419822017-05-12 18:01:32 +00002233 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002234 KMP_MB();
Jonathan Peyton30419822017-05-12 18:01:32 +00002235 KF_TRACE(10,
2236 ("__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n",
2237 root, team, master_th, gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002238
2239#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +00002240 if (__itt_stack_caller_create_ptr) {
2241 team->t.t_stack_id =
2242 __kmp_itt_stack_caller_create(); // create new stack stitching id
2243 // before entering fork barrier
Jim Cownie5e8470a2013-09-27 10:38:44 +00002244 }
2245#endif /* USE_ITT_BUILD */
2246
2247#if OMP_40_ENABLED
Jonathan Peyton642688b2017-06-01 16:46:36 +00002248 // AC: skip __kmp_internal_fork at teams construct, let only master
2249 // threads execute
2250 if (ap)
Jim Cownie5e8470a2013-09-27 10:38:44 +00002251#endif /* OMP_40_ENABLED */
2252 {
Jonathan Peyton30419822017-05-12 18:01:32 +00002253 __kmp_internal_fork(loc, gtid, team);
2254 KF_TRACE(10, ("__kmp_internal_fork : after : root=%p, team=%p, "
2255 "master_th=%p, gtid=%d\n",
2256 root, team, master_th, gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002257 }
2258
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002259 if (call_context == fork_context_gnu) {
Jonathan Peyton30419822017-05-12 18:01:32 +00002260 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid));
2261 return TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002262 }
2263
2264 /* Invoke microtask for MASTER thread */
Jonathan Peyton30419822017-05-12 18:01:32 +00002265 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
2266 team->t.t_id, team->t.t_pkfn));
2267 } // END of timer KMP_fork_call block
Jim Cownie5e8470a2013-09-27 10:38:44 +00002268
Jonathan Peytonf0682ac2018-07-30 17:41:08 +00002269 if (!team->t.t_invoke(gtid)) {
2270 KMP_ASSERT2(0, "cannot invoke microtask for MASTER thread");
Jonathan Peyton30419822017-05-12 18:01:32 +00002271 }
2272 KA_TRACE(20, ("__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
2273 team->t.t_id, team->t.t_pkfn));
2274 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00002275
Jonathan Peyton30419822017-05-12 18:01:32 +00002276 KA_TRACE(20, ("__kmp_fork_call: parallel exit T#%d\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002277
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002278#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002279 if (ompt_enabled.enabled) {
2280 master_th->th.ompt_thread_info.state = omp_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00002281 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002282#endif
2283
Jonathan Peyton30419822017-05-12 18:01:32 +00002284 return TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002285}
2286
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002287#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00002288static inline void __kmp_join_restore_state(kmp_info_t *thread,
2289 kmp_team_t *team) {
2290 // restore state outside the region
2291 thread->th.ompt_thread_info.state =
Joachim Protze82e94a52017-11-01 10:08:30 +00002292 ((team->t.t_serialized) ? omp_state_work_serial
2293 : omp_state_work_parallel);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002294}
2295
Joachim Protze82e94a52017-11-01 10:08:30 +00002296static inline void __kmp_join_ompt(int gtid, kmp_info_t *thread,
2297 kmp_team_t *team, ompt_data_t *parallel_data,
2298 fork_context_e fork_context, void *codeptr) {
2299 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2300 if (ompt_enabled.ompt_callback_parallel_end) {
2301 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
2302 parallel_data, &(task_info->task_data), OMPT_INVOKER(fork_context),
2303 codeptr);
Jonathan Peyton30419822017-05-12 18:01:32 +00002304 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002305
Joachim Protzec255ca72017-11-05 14:11:10 +00002306 task_info->frame.enter_frame = NULL;
Jonathan Peyton30419822017-05-12 18:01:32 +00002307 __kmp_join_restore_state(thread, team);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002308}
2309#endif
2310
Jonathan Peyton30419822017-05-12 18:01:32 +00002311void __kmp_join_call(ident_t *loc, int gtid
Jonathan Peytonf89fbbb2015-08-31 18:15:00 +00002312#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00002313 ,
2314 enum fork_context_e fork_context
Jonathan Peytonf89fbbb2015-08-31 18:15:00 +00002315#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002316#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002317 ,
2318 int exit_teams
Jim Cownie5e8470a2013-09-27 10:38:44 +00002319#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00002320 ) {
2321 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call);
2322 kmp_team_t *team;
2323 kmp_team_t *parent_team;
2324 kmp_info_t *master_th;
2325 kmp_root_t *root;
2326 int master_active;
2327 int i;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002328
Jonathan Peyton30419822017-05-12 18:01:32 +00002329 KA_TRACE(20, ("__kmp_join_call: enter T#%d\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002330
Jonathan Peyton30419822017-05-12 18:01:32 +00002331 /* setup current data */
2332 master_th = __kmp_threads[gtid];
2333 root = master_th->th.th_root;
2334 team = master_th->th.th_team;
2335 parent_team = team->t.t_parent;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002336
Jonathan Peyton30419822017-05-12 18:01:32 +00002337 master_th->th.th_ident = loc;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002338
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002339#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002340 if (ompt_enabled.enabled) {
2341 master_th->th.ompt_thread_info.state = omp_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00002342 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002343#endif
2344
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002345#if KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00002346 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) {
2347 KA_TRACE(20, ("__kmp_join_call: T#%d, old team = %p old task_team = %p, "
2348 "th_task_team = %p\n",
2349 __kmp_gtid_from_thread(master_th), team,
2350 team->t.t_task_team[master_th->th.th_task_state],
2351 master_th->th.th_task_team));
2352 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2353 team->t.t_task_team[master_th->th.th_task_state]);
2354 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002355#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002356
Jonathan Peyton30419822017-05-12 18:01:32 +00002357 if (team->t.t_serialized) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002358#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002359 if (master_th->th.th_teams_microtask) {
2360 // We are in teams construct
2361 int level = team->t.t_level;
2362 int tlevel = master_th->th.th_teams_level;
2363 if (level == tlevel) {
2364 // AC: we haven't incremented it earlier at start of teams construct,
2365 // so do it here - at the end of teams construct
2366 team->t.t_level++;
2367 } else if (level == tlevel + 1) {
2368 // AC: we are exiting parallel inside teams, need to increment
2369 // serialization in order to restore it in the next call to
2370 // __kmpc_end_serialized_parallel
2371 team->t.t_serialized++;
2372 }
Andrey Churbanov6d224db2015-02-10 18:37:43 +00002373 }
Jonathan Peyton441f3372015-09-21 17:24:46 +00002374#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00002375 __kmpc_end_serialized_parallel(loc, gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002376
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002377#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002378 if (ompt_enabled.enabled) {
Jonathan Peyton30419822017-05-12 18:01:32 +00002379 __kmp_join_restore_state(master_th, parent_team);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00002380 }
2381#endif
2382
Jonathan Peyton30419822017-05-12 18:01:32 +00002383 return;
2384 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002385
Jonathan Peyton30419822017-05-12 18:01:32 +00002386 master_active = team->t.t_master_active;
2387
2388#if OMP_40_ENABLED
2389 if (!exit_teams)
2390#endif /* OMP_40_ENABLED */
2391 {
2392 // AC: No barrier for internal teams at exit from teams construct.
2393 // But there is barrier for external team (league).
2394 __kmp_internal_join(loc, gtid, team);
2395 }
2396#if OMP_40_ENABLED
2397 else {
2398 master_th->th.th_task_state =
2399 0; // AC: no tasking in teams (out of any parallel)
2400 }
2401#endif /* OMP_40_ENABLED */
2402
2403 KMP_MB();
2404
2405#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002406 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data);
2407 void *codeptr = team->t.ompt_team_info.master_return_address;
Jonathan Peyton30419822017-05-12 18:01:32 +00002408#endif
2409
2410#if USE_ITT_BUILD
2411 if (__itt_stack_caller_create_ptr) {
2412 __kmp_itt_stack_caller_destroy(
2413 (__itt_caller)team->t
2414 .t_stack_id); // destroy the stack stitching id after join barrier
2415 }
2416
Jonathan Peyton8c432f22018-01-04 22:56:47 +00002417 // Mark end of "parallel" region for Intel(R) VTune(TM) analyzer.
Jonathan Peyton30419822017-05-12 18:01:32 +00002418 if (team->t.t_active_level == 1
2419#if OMP_40_ENABLED
2420 && !master_th->th.th_teams_microtask /* not in teams construct */
2421#endif /* OMP_40_ENABLED */
2422 ) {
2423 master_th->th.th_ident = loc;
2424 // only one notification scheme (either "submit" or "forking/joined", not
2425 // both)
2426 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2427 __kmp_forkjoin_frames_mode == 3)
2428 __kmp_itt_frame_submit(gtid, team->t.t_region_time,
2429 master_th->th.th_frame_time, 0, loc,
2430 master_th->th.th_team_nproc, 1);
2431 else if ((__itt_frame_end_v3_ptr || KMP_ITT_DEBUG) &&
2432 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames)
2433 __kmp_itt_region_joined(gtid);
2434 } // active_level == 1
2435#endif /* USE_ITT_BUILD */
2436
2437#if OMP_40_ENABLED
2438 if (master_th->th.th_teams_microtask && !exit_teams &&
2439 team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
2440 team->t.t_level == master_th->th.th_teams_level + 1) {
2441 // AC: We need to leave the team structure intact at the end of parallel
2442 // inside the teams construct, so that at the next parallel same (hot) team
2443 // works, only adjust nesting levels
2444
2445 /* Decrement our nested depth level */
2446 team->t.t_level--;
2447 team->t.t_active_level--;
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00002448 KMP_ATOMIC_DEC(&root->r.r_in_parallel);
Jonathan Peyton30419822017-05-12 18:01:32 +00002449
2450 /* Restore number of threads in the team if needed */
2451 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) {
2452 int old_num = master_th->th.th_team_nproc;
2453 int new_num = master_th->th.th_teams_size.nth;
2454 kmp_info_t **other_threads = team->t.t_threads;
2455 team->t.t_nproc = new_num;
2456 for (i = 0; i < old_num; ++i) {
2457 other_threads[i]->th.th_team_nproc = new_num;
2458 }
2459 // Adjust states of non-used threads of the team
2460 for (i = old_num; i < new_num; ++i) {
2461 // Re-initialize thread's barrier data.
2462 int b;
2463 kmp_balign_t *balign = other_threads[i]->th.th_bar;
2464 for (b = 0; b < bs_last_barrier; ++b) {
2465 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
2466 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
2467#if USE_DEBUGGER
2468 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
2469#endif
2470 }
2471 if (__kmp_tasking_mode != tskm_immediate_exec) {
2472 // Synchronize thread's task state
2473 other_threads[i]->th.th_task_state = master_th->th.th_task_state;
2474 }
2475 }
2476 }
2477
2478#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002479 if (ompt_enabled.enabled) {
2480 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context,
2481 codeptr);
Jonathan Peyton30419822017-05-12 18:01:32 +00002482 }
2483#endif
2484
2485 return;
2486 }
2487#endif /* OMP_40_ENABLED */
2488
2489 /* do cleanup and restore the parent team */
2490 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid;
2491 master_th->th.th_local.this_construct = team->t.t_master_this_cons;
2492
2493 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid];
2494
2495 /* jc: The following lock has instructions with REL and ACQ semantics,
2496 separating the parallel user code called in this parallel region
2497 from the serial user code called after this function returns. */
2498 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2499
2500#if OMP_40_ENABLED
2501 if (!master_th->th.th_teams_microtask ||
2502 team->t.t_level > master_th->th.th_teams_level)
2503#endif /* OMP_40_ENABLED */
2504 {
2505 /* Decrement our nested depth level */
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00002506 KMP_ATOMIC_DEC(&root->r.r_in_parallel);
Jonathan Peyton30419822017-05-12 18:01:32 +00002507 }
2508 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0);
2509
Joachim Protze82e94a52017-11-01 10:08:30 +00002510#if OMPT_SUPPORT
2511 if (ompt_enabled.enabled) {
2512 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2513 if (ompt_enabled.ompt_callback_implicit_task) {
2514 int ompt_team_size = team->t.t_nproc;
2515 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
2516 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size,
Joachim Protze9be9cf22018-05-07 12:42:21 +00002517 OMPT_CUR_TASK_INFO(master_th)->thread_num);
Jonathan Peyton30419822017-05-12 18:01:32 +00002518 }
Joachim Protze82e94a52017-11-01 10:08:30 +00002519
Joachim Protzec255ca72017-11-05 14:11:10 +00002520 task_info->frame.exit_frame = NULL;
Joachim Protze82e94a52017-11-01 10:08:30 +00002521 task_info->task_data = ompt_data_none;
Jonathan Peyton30419822017-05-12 18:01:32 +00002522 }
2523#endif
2524
2525 KF_TRACE(10, ("__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0,
2526 master_th, team));
2527 __kmp_pop_current_task_from_thread(master_th);
2528
2529#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
2530 // Restore master thread's partition.
2531 master_th->th.th_first_place = team->t.t_first_place;
2532 master_th->th.th_last_place = team->t.t_last_place;
2533#endif /* OMP_40_ENABLED */
Jonathan Peyton92ca6182018-09-07 18:25:49 +00002534#if OMP_50_ENABLED
2535 master_th->th.th_def_allocator = team->t.t_def_allocator;
2536#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002537
2538 updateHWFPControl(team);
2539
2540 if (root->r.r_active != master_active)
2541 root->r.r_active = master_active;
2542
2543 __kmp_free_team(root, team USE_NESTED_HOT_ARG(
2544 master_th)); // this will free worker threads
2545
2546 /* this race was fun to find. make sure the following is in the critical
2547 region otherwise assertions may fail occasionally since the old team may be
2548 reallocated and the hierarchy appears inconsistent. it is actually safe to
2549 run and won't cause any bugs, but will cause those assertion failures. it's
2550 only one deref&assign so might as well put this in the critical region */
2551 master_th->th.th_team = parent_team;
2552 master_th->th.th_team_nproc = parent_team->t.t_nproc;
2553 master_th->th.th_team_master = parent_team->t.t_threads[0];
2554 master_th->th.th_team_serialized = parent_team->t.t_serialized;
2555
2556 /* restore serialized team, if need be */
2557 if (parent_team->t.t_serialized &&
2558 parent_team != master_th->th.th_serial_team &&
2559 parent_team != root->r.r_root_team) {
2560 __kmp_free_team(root,
2561 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL));
2562 master_th->th.th_serial_team = parent_team;
2563 }
2564
2565 if (__kmp_tasking_mode != tskm_immediate_exec) {
2566 if (master_th->th.th_task_state_top >
2567 0) { // Restore task state from memo stack
2568 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2569 // Remember master's state if we re-use this nested hot team
2570 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] =
2571 master_th->th.th_task_state;
2572 --master_th->th.th_task_state_top; // pop
2573 // Now restore state at this level
2574 master_th->th.th_task_state =
2575 master_th->th
2576 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2577 }
2578 // Copy the task team from the parent team to the master thread
2579 master_th->th.th_task_team =
2580 parent_team->t.t_task_team[master_th->th.th_task_state];
2581 KA_TRACE(20,
2582 ("__kmp_join_call: Master T#%d restoring task_team %p / team %p\n",
2583 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,
2584 parent_team));
2585 }
2586
2587 // TODO: GEH - cannot do this assertion because root thread not set up as
2588 // executing
2589 // KMP_ASSERT( master_th->th.th_current_task->td_flags.executing == 0 );
2590 master_th->th.th_current_task->td_flags.executing = 1;
2591
2592 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2593
2594#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00002595 if (ompt_enabled.enabled) {
2596 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, fork_context,
2597 codeptr);
Jonathan Peyton30419822017-05-12 18:01:32 +00002598 }
2599#endif
2600
2601 KMP_MB();
2602 KA_TRACE(20, ("__kmp_join_call: exit T#%d\n", gtid));
2603}
Jim Cownie5e8470a2013-09-27 10:38:44 +00002604
2605/* Check whether we should push an internal control record onto the
2606 serial team stack. If so, do it. */
Jonathan Peyton30419822017-05-12 18:01:32 +00002607void __kmp_save_internal_controls(kmp_info_t *thread) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002608
Jonathan Peyton30419822017-05-12 18:01:32 +00002609 if (thread->th.th_team != thread->th.th_serial_team) {
2610 return;
2611 }
2612 if (thread->th.th_team->t.t_serialized > 1) {
2613 int push = 0;
2614
2615 if (thread->th.th_team->t.t_control_stack_top == NULL) {
2616 push = 1;
2617 } else {
2618 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level !=
2619 thread->th.th_team->t.t_serialized) {
2620 push = 1;
2621 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002622 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002623 if (push) { /* push a record on the serial team's stack */
2624 kmp_internal_control_t *control =
2625 (kmp_internal_control_t *)__kmp_allocate(
2626 sizeof(kmp_internal_control_t));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002627
Jonathan Peyton30419822017-05-12 18:01:32 +00002628 copy_icvs(control, &thread->th.th_current_task->td_icvs);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002629
Jonathan Peyton30419822017-05-12 18:01:32 +00002630 control->serial_nesting_level = thread->th.th_team->t.t_serialized;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002631
Jonathan Peyton30419822017-05-12 18:01:32 +00002632 control->next = thread->th.th_team->t.t_control_stack_top;
2633 thread->th.th_team->t.t_control_stack_top = control;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002634 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002635 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002636}
2637
2638/* Changes set_nproc */
Jonathan Peyton30419822017-05-12 18:01:32 +00002639void __kmp_set_num_threads(int new_nth, int gtid) {
2640 kmp_info_t *thread;
2641 kmp_root_t *root;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002642
Jonathan Peyton30419822017-05-12 18:01:32 +00002643 KF_TRACE(10, ("__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth));
2644 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002645
Jonathan Peyton30419822017-05-12 18:01:32 +00002646 if (new_nth < 1)
2647 new_nth = 1;
2648 else if (new_nth > __kmp_max_nth)
2649 new_nth = __kmp_max_nth;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002650
Jonathan Peyton30419822017-05-12 18:01:32 +00002651 KMP_COUNT_VALUE(OMP_set_numthreads, new_nth);
2652 thread = __kmp_threads[gtid];
Andrey Churbanov82318c62018-11-14 13:49:41 +00002653 if (thread->th.th_current_task->td_icvs.nproc == new_nth)
2654 return; // nothing to do
Jim Cownie5e8470a2013-09-27 10:38:44 +00002655
Jonathan Peyton30419822017-05-12 18:01:32 +00002656 __kmp_save_internal_controls(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002657
Jonathan Peyton30419822017-05-12 18:01:32 +00002658 set__nproc(thread, new_nth);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002659
Jonathan Peyton30419822017-05-12 18:01:32 +00002660 // If this omp_set_num_threads() call will cause the hot team size to be
2661 // reduced (in the absence of a num_threads clause), then reduce it now,
2662 // rather than waiting for the next parallel region.
2663 root = thread->th.th_root;
2664 if (__kmp_init_parallel && (!root->r.r_active) &&
2665 (root->r.r_hot_team->t.t_nproc > new_nth)
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002666#if KMP_NESTED_HOT_TEAMS
2667 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode
2668#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002669 ) {
2670 kmp_team_t *hot_team = root->r.r_hot_team;
2671 int f;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002672
Jonathan Peyton30419822017-05-12 18:01:32 +00002673 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002674
Jonathan Peyton30419822017-05-12 18:01:32 +00002675 // Release the extra threads we don't need any more.
2676 for (f = new_nth; f < hot_team->t.t_nproc; f++) {
2677 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2678 if (__kmp_tasking_mode != tskm_immediate_exec) {
2679 // When decreasing team size, threads no longer in the team should unref
2680 // task team.
2681 hot_team->t.t_threads[f]->th.th_task_team = NULL;
2682 }
2683 __kmp_free_thread(hot_team->t.t_threads[f]);
2684 hot_team->t.t_threads[f] = NULL;
2685 }
2686 hot_team->t.t_nproc = new_nth;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002687#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00002688 if (thread->th.th_hot_teams) {
2689 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team);
2690 thread->th.th_hot_teams[0].hot_team_nth = new_nth;
2691 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00002692#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00002693
Jonathan Peyton30419822017-05-12 18:01:32 +00002694 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002695
Jonathan Peyton30419822017-05-12 18:01:32 +00002696 // Update the t_nproc field in the threads that are still active.
2697 for (f = 0; f < new_nth; f++) {
2698 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2699 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002700 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002701 // Special flag in case omp_set_num_threads() call
2702 hot_team->t.t_size_changed = -1;
2703 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002704}
2705
Jim Cownie5e8470a2013-09-27 10:38:44 +00002706/* Changes max_active_levels */
Jonathan Peyton30419822017-05-12 18:01:32 +00002707void __kmp_set_max_active_levels(int gtid, int max_active_levels) {
2708 kmp_info_t *thread;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002709
Jonathan Peyton30419822017-05-12 18:01:32 +00002710 KF_TRACE(10, ("__kmp_set_max_active_levels: new max_active_levels for thread "
2711 "%d = (%d)\n",
2712 gtid, max_active_levels));
2713 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002714
Jonathan Peyton30419822017-05-12 18:01:32 +00002715 // validate max_active_levels
2716 if (max_active_levels < 0) {
2717 KMP_WARNING(ActiveLevelsNegative, max_active_levels);
2718 // We ignore this call if the user has specified a negative value.
2719 // The current setting won't be changed. The last valid setting will be
2720 // used. A warning will be issued (if warnings are allowed as controlled by
2721 // the KMP_WARNINGS env var).
2722 KF_TRACE(10, ("__kmp_set_max_active_levels: the call is ignored: new "
2723 "max_active_levels for thread %d = (%d)\n",
2724 gtid, max_active_levels));
2725 return;
2726 }
2727 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT) {
2728 // it's OK, the max_active_levels is within the valid range: [ 0;
2729 // KMP_MAX_ACTIVE_LEVELS_LIMIT ]
2730 // We allow a zero value. (implementation defined behavior)
2731 } else {
2732 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels,
2733 KMP_MAX_ACTIVE_LEVELS_LIMIT);
2734 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT;
2735 // Current upper limit is MAX_INT. (implementation defined behavior)
2736 // If the input exceeds the upper limit, we correct the input to be the
2737 // upper limit. (implementation defined behavior)
2738 // Actually, the flow should never get here until we use MAX_INT limit.
2739 }
2740 KF_TRACE(10, ("__kmp_set_max_active_levels: after validation: new "
2741 "max_active_levels for thread %d = (%d)\n",
2742 gtid, max_active_levels));
Jim Cownie5e8470a2013-09-27 10:38:44 +00002743
Jonathan Peyton30419822017-05-12 18:01:32 +00002744 thread = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00002745
Jonathan Peyton30419822017-05-12 18:01:32 +00002746 __kmp_save_internal_controls(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002747
Jonathan Peyton30419822017-05-12 18:01:32 +00002748 set__max_active_levels(thread, max_active_levels);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002749}
2750
2751/* Gets max_active_levels */
Jonathan Peyton30419822017-05-12 18:01:32 +00002752int __kmp_get_max_active_levels(int gtid) {
2753 kmp_info_t *thread;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002754
Jonathan Peyton30419822017-05-12 18:01:32 +00002755 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d\n", gtid));
2756 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002757
Jonathan Peyton30419822017-05-12 18:01:32 +00002758 thread = __kmp_threads[gtid];
2759 KMP_DEBUG_ASSERT(thread->th.th_current_task);
2760 KF_TRACE(10, ("__kmp_get_max_active_levels: thread %d, curtask=%p, "
2761 "curtask_maxaclevel=%d\n",
2762 gtid, thread->th.th_current_task,
2763 thread->th.th_current_task->td_icvs.max_active_levels));
2764 return thread->th.th_current_task->td_icvs.max_active_levels;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002765}
2766
2767/* Changes def_sched_var ICV values (run-time schedule kind and chunk) */
Jonathan Peyton30419822017-05-12 18:01:32 +00002768void __kmp_set_schedule(int gtid, kmp_sched_t kind, int chunk) {
2769 kmp_info_t *thread;
2770 // kmp_team_t *team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002771
Jonathan Peyton30419822017-05-12 18:01:32 +00002772 KF_TRACE(10, ("__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n",
2773 gtid, (int)kind, chunk));
2774 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002775
Jonathan Peyton30419822017-05-12 18:01:32 +00002776 // Check if the kind parameter is valid, correct if needed.
2777 // Valid parameters should fit in one of two intervals - standard or extended:
2778 // <lower>, <valid>, <upper_std>, <lower_ext>, <valid>, <upper>
2779 // 2008-01-25: 0, 1 - 4, 5, 100, 101 - 102, 103
2780 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper ||
2781 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) {
2782 // TODO: Hint needs attention in case we change the default schedule.
2783 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind),
2784 KMP_HNT(DefaultScheduleKindUsed, "static, no chunk"),
2785 __kmp_msg_null);
2786 kind = kmp_sched_default;
2787 chunk = 0; // ignore chunk value in case of bad kind
2788 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002789
Jonathan Peyton30419822017-05-12 18:01:32 +00002790 thread = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00002791
Jonathan Peyton30419822017-05-12 18:01:32 +00002792 __kmp_save_internal_controls(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002793
Jonathan Peyton30419822017-05-12 18:01:32 +00002794 if (kind < kmp_sched_upper_std) {
2795 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK) {
2796 // differ static chunked vs. unchunked: chunk should be invalid to
2797 // indicate unchunked schedule (which is the default)
2798 thread->th.th_current_task->td_icvs.sched.r_sched_type = kmp_sch_static;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002799 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00002800 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2801 __kmp_sch_map[kind - kmp_sched_lower - 1];
Jim Cownie5e8470a2013-09-27 10:38:44 +00002802 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002803 } else {
2804 // __kmp_sch_map[ kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2805 // kmp_sched_lower - 2 ];
2806 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2807 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2808 kmp_sched_lower - 2];
2809 }
Andrey Churbanovd454c732017-06-05 17:17:33 +00002810 if (kind == kmp_sched_auto || chunk < 1) {
Jonathan Peyton30419822017-05-12 18:01:32 +00002811 // ignore parameter chunk for schedule auto
2812 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK;
2813 } else {
2814 thread->th.th_current_task->td_icvs.sched.chunk = chunk;
2815 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002816}
2817
2818/* Gets def_sched_var ICV values */
Jonathan Peyton30419822017-05-12 18:01:32 +00002819void __kmp_get_schedule(int gtid, kmp_sched_t *kind, int *chunk) {
2820 kmp_info_t *thread;
2821 enum sched_type th_type;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002822
Jonathan Peyton30419822017-05-12 18:01:32 +00002823 KF_TRACE(10, ("__kmp_get_schedule: thread %d\n", gtid));
2824 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002825
Jonathan Peyton30419822017-05-12 18:01:32 +00002826 thread = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00002827
Jonathan Peyton30419822017-05-12 18:01:32 +00002828 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002829
Jonathan Peyton30419822017-05-12 18:01:32 +00002830 switch (th_type) {
2831 case kmp_sch_static:
2832 case kmp_sch_static_greedy:
2833 case kmp_sch_static_balanced:
2834 *kind = kmp_sched_static;
2835 *chunk = 0; // chunk was not set, try to show this fact via zero value
2836 return;
2837 case kmp_sch_static_chunked:
2838 *kind = kmp_sched_static;
2839 break;
2840 case kmp_sch_dynamic_chunked:
2841 *kind = kmp_sched_dynamic;
2842 break;
2843 case kmp_sch_guided_chunked:
2844 case kmp_sch_guided_iterative_chunked:
2845 case kmp_sch_guided_analytical_chunked:
2846 *kind = kmp_sched_guided;
2847 break;
2848 case kmp_sch_auto:
2849 *kind = kmp_sched_auto;
2850 break;
2851 case kmp_sch_trapezoidal:
2852 *kind = kmp_sched_trapezoidal;
2853 break;
Jonathan Peytona1234cf2016-10-07 18:01:35 +00002854#if KMP_STATIC_STEAL_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002855 case kmp_sch_static_steal:
2856 *kind = kmp_sched_static_steal;
2857 break;
Jonathan Peytona1234cf2016-10-07 18:01:35 +00002858#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00002859 default:
2860 KMP_FATAL(UnknownSchedulingType, th_type);
2861 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002862
Jonathan Peyton30419822017-05-12 18:01:32 +00002863 *chunk = thread->th.th_current_task->td_icvs.sched.chunk;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002864}
2865
Jonathan Peyton30419822017-05-12 18:01:32 +00002866int __kmp_get_ancestor_thread_num(int gtid, int level) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002867
Jonathan Peyton30419822017-05-12 18:01:32 +00002868 int ii, dd;
2869 kmp_team_t *team;
2870 kmp_info_t *thr;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002871
Jonathan Peyton30419822017-05-12 18:01:32 +00002872 KF_TRACE(10, ("__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level));
2873 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002874
Jonathan Peyton30419822017-05-12 18:01:32 +00002875 // validate level
2876 if (level == 0)
2877 return 0;
2878 if (level < 0)
2879 return -1;
2880 thr = __kmp_threads[gtid];
2881 team = thr->th.th_team;
2882 ii = team->t.t_level;
2883 if (level > ii)
2884 return -1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002885
2886#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002887 if (thr->th.th_teams_microtask) {
2888 // AC: we are in teams region where multiple nested teams have same level
2889 int tlevel = thr->th.th_teams_level; // the level of the teams construct
2890 if (level <=
2891 tlevel) { // otherwise usual algorithm works (will not touch the teams)
2892 KMP_DEBUG_ASSERT(ii >= tlevel);
2893 // AC: As we need to pass by the teams league, we need to artificially
2894 // increase ii
2895 if (ii == tlevel) {
2896 ii += 2; // three teams have same level
2897 } else {
2898 ii++; // two teams have same level
2899 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002900 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002901 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002902#endif
2903
Jonathan Peyton30419822017-05-12 18:01:32 +00002904 if (ii == level)
2905 return __kmp_tid_from_gtid(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002906
Jonathan Peyton30419822017-05-12 18:01:32 +00002907 dd = team->t.t_serialized;
2908 level++;
2909 while (ii > level) {
2910 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002911 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002912 if ((team->t.t_serialized) && (!dd)) {
2913 team = team->t.t_parent;
2914 continue;
2915 }
2916 if (ii > level) {
2917 team = team->t.t_parent;
2918 dd = team->t.t_serialized;
2919 ii--;
2920 }
2921 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002922
Jonathan Peyton30419822017-05-12 18:01:32 +00002923 return (dd > 1) ? (0) : (team->t.t_master_tid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002924}
2925
Jonathan Peyton30419822017-05-12 18:01:32 +00002926int __kmp_get_team_size(int gtid, int level) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002927
Jonathan Peyton30419822017-05-12 18:01:32 +00002928 int ii, dd;
2929 kmp_team_t *team;
2930 kmp_info_t *thr;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002931
Jonathan Peyton30419822017-05-12 18:01:32 +00002932 KF_TRACE(10, ("__kmp_get_team_size: thread %d %d\n", gtid, level));
2933 KMP_DEBUG_ASSERT(__kmp_init_serial);
Jim Cownie5e8470a2013-09-27 10:38:44 +00002934
Jonathan Peyton30419822017-05-12 18:01:32 +00002935 // validate level
2936 if (level == 0)
2937 return 1;
2938 if (level < 0)
2939 return -1;
2940 thr = __kmp_threads[gtid];
2941 team = thr->th.th_team;
2942 ii = team->t.t_level;
2943 if (level > ii)
2944 return -1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002945
2946#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00002947 if (thr->th.th_teams_microtask) {
2948 // AC: we are in teams region where multiple nested teams have same level
2949 int tlevel = thr->th.th_teams_level; // the level of the teams construct
2950 if (level <=
2951 tlevel) { // otherwise usual algorithm works (will not touch the teams)
2952 KMP_DEBUG_ASSERT(ii >= tlevel);
2953 // AC: As we need to pass by the teams league, we need to artificially
2954 // increase ii
2955 if (ii == tlevel) {
2956 ii += 2; // three teams have same level
2957 } else {
2958 ii++; // two teams have same level
2959 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002960 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002961 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002962#endif
2963
Jonathan Peyton30419822017-05-12 18:01:32 +00002964 while (ii > level) {
2965 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00002966 }
Jonathan Peyton30419822017-05-12 18:01:32 +00002967 if (team->t.t_serialized && (!dd)) {
2968 team = team->t.t_parent;
2969 continue;
2970 }
2971 if (ii > level) {
2972 team = team->t.t_parent;
2973 ii--;
2974 }
2975 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00002976
Jonathan Peyton30419822017-05-12 18:01:32 +00002977 return team->t.t_nproc;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002978}
2979
Jonathan Peyton30419822017-05-12 18:01:32 +00002980kmp_r_sched_t __kmp_get_schedule_global() {
2981 // This routine created because pairs (__kmp_sched, __kmp_chunk) and
2982 // (__kmp_static, __kmp_guided) may be changed by kmp_set_defaults
2983 // independently. So one can get the updated schedule here.
Jim Cownie5e8470a2013-09-27 10:38:44 +00002984
Jonathan Peyton30419822017-05-12 18:01:32 +00002985 kmp_r_sched_t r_sched;
Jim Cownie5e8470a2013-09-27 10:38:44 +00002986
Jonathan Peyton30419822017-05-12 18:01:32 +00002987 // create schedule from 4 globals: __kmp_sched, __kmp_chunk, __kmp_static,
2988 // __kmp_guided. __kmp_sched should keep original value, so that user can set
2989 // KMP_SCHEDULE multiple times, and thus have different run-time schedules in
2990 // different roots (even in OMP 2.5)
2991 if (__kmp_sched == kmp_sch_static) {
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00002992 // replace STATIC with more detailed schedule (balanced or greedy)
2993 r_sched.r_sched_type = __kmp_static;
Jonathan Peyton30419822017-05-12 18:01:32 +00002994 } else if (__kmp_sched == kmp_sch_guided_chunked) {
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00002995 // replace GUIDED with more detailed schedule (iterative or analytical)
2996 r_sched.r_sched_type = __kmp_guided;
2997 } else { // (STATIC_CHUNKED), or (DYNAMIC_CHUNKED), or other
2998 r_sched.r_sched_type = __kmp_sched;
Jonathan Peyton30419822017-05-12 18:01:32 +00002999 }
3000
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00003001 if (__kmp_chunk < KMP_DEFAULT_CHUNK) {
3002 // __kmp_chunk may be wrong here (if it was not ever set)
Jonathan Peyton30419822017-05-12 18:01:32 +00003003 r_sched.chunk = KMP_DEFAULT_CHUNK;
3004 } else {
3005 r_sched.chunk = __kmp_chunk;
3006 }
3007
3008 return r_sched;
3009}
3010
3011/* Allocate (realloc == FALSE) * or reallocate (realloc == TRUE)
3012 at least argc number of *t_argv entries for the requested team. */
3013static void __kmp_alloc_argv_entries(int argc, kmp_team_t *team, int realloc) {
3014
3015 KMP_DEBUG_ASSERT(team);
3016 if (!realloc || argc > team->t.t_max_argc) {
3017
3018 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: needed entries=%d, "
3019 "current entries=%d\n",
3020 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0));
3021 /* if previously allocated heap space for args, free them */
3022 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0])
3023 __kmp_free((void *)team->t.t_argv);
3024
3025 if (argc <= KMP_INLINE_ARGV_ENTRIES) {
3026 /* use unused space in the cache line for arguments */
3027 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES;
3028 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: inline allocate %d "
3029 "argv entries\n",
3030 team->t.t_id, team->t.t_max_argc));
3031 team->t.t_argv = &team->t.t_inline_argv[0];
3032 if (__kmp_storage_map) {
3033 __kmp_print_storage_map_gtid(
3034 -1, &team->t.t_inline_argv[0],
3035 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES],
3036 (sizeof(void *) * KMP_INLINE_ARGV_ENTRIES), "team_%d.t_inline_argv",
3037 team->t.t_id);
3038 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003039 } else {
Jonathan Peyton30419822017-05-12 18:01:32 +00003040 /* allocate space for arguments in the heap */
3041 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1))
3042 ? KMP_MIN_MALLOC_ARGV_ENTRIES
3043 : 2 * argc;
3044 KA_TRACE(100, ("__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
3045 "argv entries\n",
3046 team->t.t_id, team->t.t_max_argc));
3047 team->t.t_argv =
3048 (void **)__kmp_page_allocate(sizeof(void *) * team->t.t_max_argc);
3049 if (__kmp_storage_map) {
3050 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0],
3051 &team->t.t_argv[team->t.t_max_argc],
3052 sizeof(void *) * team->t.t_max_argc,
3053 "team_%d.t_argv", team->t.t_id);
3054 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003055 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003056 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003057}
3058
Jonathan Peyton30419822017-05-12 18:01:32 +00003059static void __kmp_allocate_team_arrays(kmp_team_t *team, int max_nth) {
3060 int i;
3061 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2;
3062 team->t.t_threads =
3063 (kmp_info_t **)__kmp_allocate(sizeof(kmp_info_t *) * max_nth);
3064 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate(
3065 sizeof(dispatch_shared_info_t) * num_disp_buff);
3066 team->t.t_dispatch =
3067 (kmp_disp_t *)__kmp_allocate(sizeof(kmp_disp_t) * max_nth);
3068 team->t.t_implicit_task_taskdata =
3069 (kmp_taskdata_t *)__kmp_allocate(sizeof(kmp_taskdata_t) * max_nth);
3070 team->t.t_max_nproc = max_nth;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003071
Jonathan Peyton30419822017-05-12 18:01:32 +00003072 /* setup dispatch buffers */
3073 for (i = 0; i < num_disp_buff; ++i) {
3074 team->t.t_disp_buffer[i].buffer_index = i;
Jonathan Peytondf6818b2016-06-14 17:57:47 +00003075#if OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003076 team->t.t_disp_buffer[i].doacross_buf_idx = i;
Jonathan Peyton71909c52016-03-02 22:42:06 +00003077#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003078 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003079}
3080
Jonathan Peyton30419822017-05-12 18:01:32 +00003081static void __kmp_free_team_arrays(kmp_team_t *team) {
3082 /* Note: this does not free the threads in t_threads (__kmp_free_threads) */
3083 int i;
3084 for (i = 0; i < team->t.t_max_nproc; ++i) {
3085 if (team->t.t_dispatch[i].th_disp_buffer != NULL) {
3086 __kmp_free(team->t.t_dispatch[i].th_disp_buffer);
3087 team->t.t_dispatch[i].th_disp_buffer = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003088 }
3089 }
Jonathan Peytonf6399362018-07-09 17:51:13 +00003090#if KMP_USE_HIER_SCHED
3091 __kmp_dispatch_free_hierarchies(team);
3092#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003093 __kmp_free(team->t.t_threads);
3094 __kmp_free(team->t.t_disp_buffer);
3095 __kmp_free(team->t.t_dispatch);
3096 __kmp_free(team->t.t_implicit_task_taskdata);
3097 team->t.t_threads = NULL;
3098 team->t.t_disp_buffer = NULL;
3099 team->t.t_dispatch = NULL;
3100 team->t.t_implicit_task_taskdata = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003101}
3102
Jonathan Peyton30419822017-05-12 18:01:32 +00003103static void __kmp_reallocate_team_arrays(kmp_team_t *team, int max_nth) {
3104 kmp_info_t **oldThreads = team->t.t_threads;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003105
Jonathan Peyton30419822017-05-12 18:01:32 +00003106 __kmp_free(team->t.t_disp_buffer);
3107 __kmp_free(team->t.t_dispatch);
3108 __kmp_free(team->t.t_implicit_task_taskdata);
3109 __kmp_allocate_team_arrays(team, max_nth);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003110
Jonathan Peyton30419822017-05-12 18:01:32 +00003111 KMP_MEMCPY(team->t.t_threads, oldThreads,
3112 team->t.t_nproc * sizeof(kmp_info_t *));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003113
Jonathan Peyton30419822017-05-12 18:01:32 +00003114 __kmp_free(oldThreads);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003115}
3116
Jonathan Peyton30419822017-05-12 18:01:32 +00003117static kmp_internal_control_t __kmp_get_global_icvs(void) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003118
Jonathan Peyton30419822017-05-12 18:01:32 +00003119 kmp_r_sched_t r_sched =
3120 __kmp_get_schedule_global(); // get current state of scheduling globals
Jim Cownie5e8470a2013-09-27 10:38:44 +00003121
3122#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003123 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003124#endif /* OMP_40_ENABLED */
3125
Jonathan Peyton30419822017-05-12 18:01:32 +00003126 kmp_internal_control_t g_icvs = {
3127 0, // int serial_nesting_level; //corresponds to value of th_team_serialized
3128 (kmp_int8)__kmp_dflt_nested, // int nested; //internal control
3129 // for nested parallelism (per thread)
3130 (kmp_int8)__kmp_global.g.g_dynamic, // internal control for dynamic
3131 // adjustment of threads (per thread)
3132 (kmp_int8)__kmp_env_blocktime, // int bt_set; //internal control for
3133 // whether blocktime is explicitly set
3134 __kmp_dflt_blocktime, // int blocktime; //internal control for blocktime
Jonathan Peytone1c7c132016-10-07 18:12:19 +00003135#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00003136 __kmp_bt_intervals, // int bt_intervals; //internal control for blocktime
3137// intervals
Jonathan Peytone1c7c132016-10-07 18:12:19 +00003138#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003139 __kmp_dflt_team_nth, // int nproc; //internal control for # of threads for
3140 // next parallel region (per thread)
3141 // (use a max ub on value if __kmp_parallel_initialize not called yet)
3142 __kmp_dflt_max_active_levels, // int max_active_levels; //internal control
3143 // for max_active_levels
3144 r_sched, // kmp_r_sched_t sched; //internal control for runtime schedule
3145// {sched,chunk} pair
Jim Cownie5e8470a2013-09-27 10:38:44 +00003146#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003147 __kmp_nested_proc_bind.bind_types[0],
3148 __kmp_default_device,
Jim Cownie5e8470a2013-09-27 10:38:44 +00003149#endif /* OMP_40_ENABLED */
Jonathan Peyton30419822017-05-12 18:01:32 +00003150 NULL // struct kmp_internal_control *next;
3151 };
Jim Cownie5e8470a2013-09-27 10:38:44 +00003152
Jonathan Peyton30419822017-05-12 18:01:32 +00003153 return g_icvs;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003154}
3155
Jonathan Peyton30419822017-05-12 18:01:32 +00003156static kmp_internal_control_t __kmp_get_x_global_icvs(const kmp_team_t *team) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003157
Jonathan Peyton30419822017-05-12 18:01:32 +00003158 kmp_internal_control_t gx_icvs;
3159 gx_icvs.serial_nesting_level =
3160 0; // probably =team->t.t_serial like in save_inter_controls
3161 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs);
3162 gx_icvs.next = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003163
Jonathan Peyton30419822017-05-12 18:01:32 +00003164 return gx_icvs;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003165}
3166
Jonathan Peyton30419822017-05-12 18:01:32 +00003167static void __kmp_initialize_root(kmp_root_t *root) {
3168 int f;
3169 kmp_team_t *root_team;
3170 kmp_team_t *hot_team;
3171 int hot_team_max_nth;
3172 kmp_r_sched_t r_sched =
3173 __kmp_get_schedule_global(); // get current state of scheduling globals
3174 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3175 KMP_DEBUG_ASSERT(root);
3176 KMP_ASSERT(!root->r.r_begin);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003177
Jonathan Peyton30419822017-05-12 18:01:32 +00003178 /* setup the root state structure */
3179 __kmp_init_lock(&root->r.r_begin_lock);
3180 root->r.r_begin = FALSE;
3181 root->r.r_active = FALSE;
3182 root->r.r_in_parallel = 0;
3183 root->r.r_blocktime = __kmp_dflt_blocktime;
3184 root->r.r_nested = __kmp_dflt_nested;
Jonathan Peytonf4392462017-07-27 20:58:41 +00003185 root->r.r_cg_nthreads = 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003186
Jonathan Peyton30419822017-05-12 18:01:32 +00003187 /* setup the root team for this task */
3188 /* allocate the root team structure */
3189 KF_TRACE(10, ("__kmp_initialize_root: before root_team\n"));
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003190
Jonathan Peyton30419822017-05-12 18:01:32 +00003191 root_team =
3192 __kmp_allocate_team(root,
3193 1, // new_nproc
3194 1, // max_nproc
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003195#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00003196 ompt_data_none, // root parallel id
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003197#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00003198#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003199 __kmp_nested_proc_bind.bind_types[0],
Jim Cownie5e8470a2013-09-27 10:38:44 +00003200#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003201 &r_icvs,
3202 0 // argc
3203 USE_NESTED_HOT_ARG(NULL) // master thread is unknown
3204 );
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00003205#if USE_DEBUGGER
Jonathan Peyton30419822017-05-12 18:01:32 +00003206 // Non-NULL value should be assigned to make the debugger display the root
3207 // team.
3208 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0));
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00003209#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00003210
Jonathan Peyton30419822017-05-12 18:01:32 +00003211 KF_TRACE(10, ("__kmp_initialize_root: after root_team = %p\n", root_team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003212
Jonathan Peyton30419822017-05-12 18:01:32 +00003213 root->r.r_root_team = root_team;
3214 root_team->t.t_control_stack_top = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003215
Jonathan Peyton30419822017-05-12 18:01:32 +00003216 /* initialize root team */
3217 root_team->t.t_threads[0] = NULL;
3218 root_team->t.t_nproc = 1;
3219 root_team->t.t_serialized = 1;
3220 // TODO???: root_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00003221 root_team->t.t_sched.sched = r_sched.sched;
Jonathan Peyton30419822017-05-12 18:01:32 +00003222 KA_TRACE(
3223 20,
3224 ("__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n",
3225 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003226
Jonathan Peyton30419822017-05-12 18:01:32 +00003227 /* setup the hot team for this task */
3228 /* allocate the hot team structure */
3229 KF_TRACE(10, ("__kmp_initialize_root: before hot_team\n"));
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003230
Jonathan Peyton30419822017-05-12 18:01:32 +00003231 hot_team =
3232 __kmp_allocate_team(root,
3233 1, // new_nproc
3234 __kmp_dflt_team_nth_ub * 2, // max_nproc
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003235#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00003236 ompt_data_none, // root parallel id
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003237#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00003238#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003239 __kmp_nested_proc_bind.bind_types[0],
Jim Cownie5e8470a2013-09-27 10:38:44 +00003240#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003241 &r_icvs,
3242 0 // argc
3243 USE_NESTED_HOT_ARG(NULL) // master thread is unknown
3244 );
3245 KF_TRACE(10, ("__kmp_initialize_root: after hot_team = %p\n", hot_team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003246
Jonathan Peyton30419822017-05-12 18:01:32 +00003247 root->r.r_hot_team = hot_team;
3248 root_team->t.t_control_stack_top = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003249
Jonathan Peyton30419822017-05-12 18:01:32 +00003250 /* first-time initialization */
3251 hot_team->t.t_parent = root_team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003252
Jonathan Peyton30419822017-05-12 18:01:32 +00003253 /* initialize hot team */
3254 hot_team_max_nth = hot_team->t.t_max_nproc;
3255 for (f = 0; f < hot_team_max_nth; ++f) {
3256 hot_team->t.t_threads[f] = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003257 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003258 hot_team->t.t_nproc = 1;
3259 // TODO???: hot_team->t.t_max_active_levels = __kmp_dflt_max_active_levels;
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00003260 hot_team->t.t_sched.sched = r_sched.sched;
Jonathan Peyton30419822017-05-12 18:01:32 +00003261 hot_team->t.t_size_changed = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003262}
3263
3264#ifdef KMP_DEBUG
3265
Jim Cownie5e8470a2013-09-27 10:38:44 +00003266typedef struct kmp_team_list_item {
Jonathan Peyton30419822017-05-12 18:01:32 +00003267 kmp_team_p const *entry;
3268 struct kmp_team_list_item *next;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003269} kmp_team_list_item_t;
Jonathan Peyton30419822017-05-12 18:01:32 +00003270typedef kmp_team_list_item_t *kmp_team_list_t;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003271
Jonathan Peyton30419822017-05-12 18:01:32 +00003272static void __kmp_print_structure_team_accum( // Add team to list of teams.
3273 kmp_team_list_t list, // List of teams.
3274 kmp_team_p const *team // Team to add.
3275 ) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003276
Jonathan Peyton30419822017-05-12 18:01:32 +00003277 // List must terminate with item where both entry and next are NULL.
3278 // Team is added to the list only once.
3279 // List is sorted in ascending order by team id.
3280 // Team id is *not* a key.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003281
Jonathan Peyton30419822017-05-12 18:01:32 +00003282 kmp_team_list_t l;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003283
Jonathan Peyton30419822017-05-12 18:01:32 +00003284 KMP_DEBUG_ASSERT(list != NULL);
3285 if (team == NULL) {
3286 return;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003287 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003288
Jonathan Peyton30419822017-05-12 18:01:32 +00003289 __kmp_print_structure_team_accum(list, team->t.t_parent);
3290 __kmp_print_structure_team_accum(list, team->t.t_next_pool);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003291
Jonathan Peyton30419822017-05-12 18:01:32 +00003292 // Search list for the team.
3293 l = list;
3294 while (l->next != NULL && l->entry != team) {
3295 l = l->next;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003296 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003297 if (l->next != NULL) {
3298 return; // Team has been added before, exit.
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003299 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003300
Jonathan Peyton30419822017-05-12 18:01:32 +00003301 // Team is not found. Search list again for insertion point.
3302 l = list;
3303 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) {
3304 l = l->next;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003305 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003306
Jonathan Peyton30419822017-05-12 18:01:32 +00003307 // Insert team.
3308 {
3309 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(
3310 sizeof(kmp_team_list_item_t));
3311 *item = *l;
3312 l->entry = team;
3313 l->next = item;
3314 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003315}
3316
Jonathan Peyton30419822017-05-12 18:01:32 +00003317static void __kmp_print_structure_team(char const *title, kmp_team_p const *team
Jim Cownie5e8470a2013-09-27 10:38:44 +00003318
Jonathan Peyton30419822017-05-12 18:01:32 +00003319 ) {
3320 __kmp_printf("%s", title);
3321 if (team != NULL) {
3322 __kmp_printf("%2x %p\n", team->t.t_id, team);
3323 } else {
3324 __kmp_printf(" - (nil)\n");
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003325 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003326}
3327
Jonathan Peyton30419822017-05-12 18:01:32 +00003328static void __kmp_print_structure_thread(char const *title,
3329 kmp_info_p const *thread) {
3330 __kmp_printf("%s", title);
3331 if (thread != NULL) {
3332 __kmp_printf("%2d %p\n", thread->th.th_info.ds.ds_gtid, thread);
3333 } else {
3334 __kmp_printf(" - (nil)\n");
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003335 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003336}
3337
Jonathan Peyton30419822017-05-12 18:01:32 +00003338void __kmp_print_structure(void) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003339
Jonathan Peyton30419822017-05-12 18:01:32 +00003340 kmp_team_list_t list;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003341
Jonathan Peyton30419822017-05-12 18:01:32 +00003342 // Initialize list of teams.
3343 list =
3344 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(sizeof(kmp_team_list_item_t));
3345 list->entry = NULL;
3346 list->next = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003347
Jonathan Peyton30419822017-05-12 18:01:32 +00003348 __kmp_printf("\n------------------------------\nGlobal Thread "
3349 "Table\n------------------------------\n");
3350 {
3351 int gtid;
3352 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3353 __kmp_printf("%2d", gtid);
3354 if (__kmp_threads != NULL) {
3355 __kmp_printf(" %p", __kmp_threads[gtid]);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003356 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003357 if (__kmp_root != NULL) {
3358 __kmp_printf(" %p", __kmp_root[gtid]);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003359 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003360 __kmp_printf("\n");
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003361 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003362 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003363
Jonathan Peyton30419822017-05-12 18:01:32 +00003364 // Print out __kmp_threads array.
3365 __kmp_printf("\n------------------------------\nThreads\n--------------------"
3366 "----------\n");
3367 if (__kmp_threads != NULL) {
3368 int gtid;
3369 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3370 kmp_info_t const *thread = __kmp_threads[gtid];
3371 if (thread != NULL) {
3372 __kmp_printf("GTID %2d %p:\n", gtid, thread);
3373 __kmp_printf(" Our Root: %p\n", thread->th.th_root);
3374 __kmp_print_structure_team(" Our Team: ", thread->th.th_team);
3375 __kmp_print_structure_team(" Serial Team: ",
3376 thread->th.th_serial_team);
3377 __kmp_printf(" Threads: %2d\n", thread->th.th_team_nproc);
3378 __kmp_print_structure_thread(" Master: ",
3379 thread->th.th_team_master);
3380 __kmp_printf(" Serialized?: %2d\n", thread->th.th_team_serialized);
3381 __kmp_printf(" Set NProc: %2d\n", thread->th.th_set_nproc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003382#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003383 __kmp_printf(" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003384#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003385 __kmp_print_structure_thread(" Next in pool: ",
3386 thread->th.th_next_pool);
3387 __kmp_printf("\n");
3388 __kmp_print_structure_team_accum(list, thread->th.th_team);
3389 __kmp_print_structure_team_accum(list, thread->th.th_serial_team);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003390 }
3391 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003392 } else {
3393 __kmp_printf("Threads array is not allocated.\n");
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003394 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003395
Jonathan Peyton30419822017-05-12 18:01:32 +00003396 // Print out __kmp_root array.
3397 __kmp_printf("\n------------------------------\nUbers\n----------------------"
3398 "--------\n");
3399 if (__kmp_root != NULL) {
3400 int gtid;
3401 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3402 kmp_root_t const *root = __kmp_root[gtid];
3403 if (root != NULL) {
3404 __kmp_printf("GTID %2d %p:\n", gtid, root);
3405 __kmp_print_structure_team(" Root Team: ", root->r.r_root_team);
3406 __kmp_print_structure_team(" Hot Team: ", root->r.r_hot_team);
3407 __kmp_print_structure_thread(" Uber Thread: ",
3408 root->r.r_uber_thread);
3409 __kmp_printf(" Active?: %2d\n", root->r.r_active);
3410 __kmp_printf(" Nested?: %2d\n", root->r.r_nested);
Jonathan Peyton61d44f12018-07-09 18:09:25 +00003411 __kmp_printf(" In Parallel: %2d\n",
3412 KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel));
Jonathan Peyton30419822017-05-12 18:01:32 +00003413 __kmp_printf("\n");
3414 __kmp_print_structure_team_accum(list, root->r.r_root_team);
3415 __kmp_print_structure_team_accum(list, root->r.r_hot_team);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003416 }
3417 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003418 } else {
3419 __kmp_printf("Ubers array is not allocated.\n");
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003420 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003421
Jonathan Peyton30419822017-05-12 18:01:32 +00003422 __kmp_printf("\n------------------------------\nTeams\n----------------------"
3423 "--------\n");
3424 while (list->next != NULL) {
3425 kmp_team_p const *team = list->entry;
3426 int i;
3427 __kmp_printf("Team %2x %p:\n", team->t.t_id, team);
3428 __kmp_print_structure_team(" Parent Team: ", team->t.t_parent);
3429 __kmp_printf(" Master TID: %2d\n", team->t.t_master_tid);
3430 __kmp_printf(" Max threads: %2d\n", team->t.t_max_nproc);
3431 __kmp_printf(" Levels of serial: %2d\n", team->t.t_serialized);
3432 __kmp_printf(" Number threads: %2d\n", team->t.t_nproc);
3433 for (i = 0; i < team->t.t_nproc; ++i) {
3434 __kmp_printf(" Thread %2d: ", i);
3435 __kmp_print_structure_thread("", team->t.t_threads[i]);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003436 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003437 __kmp_print_structure_team(" Next in pool: ", team->t.t_next_pool);
3438 __kmp_printf("\n");
3439 list = list->next;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003440 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003441
Jonathan Peyton30419822017-05-12 18:01:32 +00003442 // Print out __kmp_thread_pool and __kmp_team_pool.
3443 __kmp_printf("\n------------------------------\nPools\n----------------------"
3444 "--------\n");
3445 __kmp_print_structure_thread("Thread pool: ",
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00003446 CCAST(kmp_info_t *, __kmp_thread_pool));
Jonathan Peyton30419822017-05-12 18:01:32 +00003447 __kmp_print_structure_team("Team pool: ",
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00003448 CCAST(kmp_team_t *, __kmp_team_pool));
Jonathan Peyton30419822017-05-12 18:01:32 +00003449 __kmp_printf("\n");
Jim Cownie5e8470a2013-09-27 10:38:44 +00003450
Jonathan Peyton30419822017-05-12 18:01:32 +00003451 // Free team list.
3452 while (list != NULL) {
3453 kmp_team_list_item_t *item = list;
3454 list = list->next;
3455 KMP_INTERNAL_FREE(item);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003456 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003457}
3458
3459#endif
3460
Jim Cownie5e8470a2013-09-27 10:38:44 +00003461//---------------------------------------------------------------------------
3462// Stuff for per-thread fast random number generator
3463// Table of primes
Jim Cownie5e8470a2013-09-27 10:38:44 +00003464static const unsigned __kmp_primes[] = {
Jonathan Peyton30419822017-05-12 18:01:32 +00003465 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877,
3466 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231,
3467 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201,
3468 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3,
3469 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7,
3470 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9,
3471 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45,
3472 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7,
3473 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363,
3474 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3,
3475 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f};
Jim Cownie5e8470a2013-09-27 10:38:44 +00003476
3477//---------------------------------------------------------------------------
3478// __kmp_get_random: Get a random number using a linear congruential method.
Jonathan Peyton30419822017-05-12 18:01:32 +00003479unsigned short __kmp_get_random(kmp_info_t *thread) {
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003480 unsigned x = thread->th.th_x;
Jonathan Peyton30419822017-05-12 18:01:32 +00003481 unsigned short r = x >> 16;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003482
Jonathan Peyton30419822017-05-12 18:01:32 +00003483 thread->th.th_x = x * thread->th.th_a + 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003484
3485 KA_TRACE(30, ("__kmp_get_random: THREAD: %d, RETURN: %u\n",
Jonathan Peyton30419822017-05-12 18:01:32 +00003486 thread->th.th_info.ds.ds_tid, r));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003487
3488 return r;
3489}
3490//--------------------------------------------------------
3491// __kmp_init_random: Initialize a random number generator
Jonathan Peyton30419822017-05-12 18:01:32 +00003492void __kmp_init_random(kmp_info_t *thread) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00003493 unsigned seed = thread->th.th_info.ds.ds_tid;
3494
Jonathan Peyton30419822017-05-12 18:01:32 +00003495 thread->th.th_a =
3496 __kmp_primes[seed % (sizeof(__kmp_primes) / sizeof(__kmp_primes[0]))];
3497 thread->th.th_x = (seed + 1) * thread->th.th_a + 1;
3498 KA_TRACE(30,
3499 ("__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003500}
3501
Jim Cownie5e8470a2013-09-27 10:38:44 +00003502#if KMP_OS_WINDOWS
Jonathan Peyton30419822017-05-12 18:01:32 +00003503/* reclaim array entries for root threads that are already dead, returns number
3504 * reclaimed */
3505static int __kmp_reclaim_dead_roots(void) {
3506 int i, r = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003507
Jonathan Peyton30419822017-05-12 18:01:32 +00003508 for (i = 0; i < __kmp_threads_capacity; ++i) {
3509 if (KMP_UBER_GTID(i) &&
3510 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])) &&
3511 !__kmp_root[i]
3512 ->r.r_active) { // AC: reclaim only roots died in non-active state
3513 r += __kmp_unregister_root_other_thread(i);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003514 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003515 }
3516 return r;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003517}
3518#endif
3519
Jonathan Peyton30419822017-05-12 18:01:32 +00003520/* This function attempts to create free entries in __kmp_threads and
3521 __kmp_root, and returns the number of free entries generated.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003522
Jonathan Peyton30419822017-05-12 18:01:32 +00003523 For Windows* OS static library, the first mechanism used is to reclaim array
3524 entries for root threads that are already dead.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003525
Jonathan Peyton30419822017-05-12 18:01:32 +00003526 On all platforms, expansion is attempted on the arrays __kmp_threads_ and
3527 __kmp_root, with appropriate update to __kmp_threads_capacity. Array
3528 capacity is increased by doubling with clipping to __kmp_tp_capacity, if
3529 threadprivate cache array has been created. Synchronization with
3530 __kmpc_threadprivate_cached is done using __kmp_tp_cached_lock.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003531
Jonathan Peyton30419822017-05-12 18:01:32 +00003532 After any dead root reclamation, if the clipping value allows array expansion
Jonathan Peyton1800ece2018-01-10 18:27:01 +00003533 to result in the generation of a total of nNeed free slots, the function does
3534 that expansion. If not, nothing is done beyond the possible initial root
3535 thread reclamation.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003536
Jonathan Peyton30419822017-05-12 18:01:32 +00003537 If any argument is negative, the behavior is undefined. */
Jonathan Peyton1800ece2018-01-10 18:27:01 +00003538static int __kmp_expand_threads(int nNeed) {
Jonathan Peyton30419822017-05-12 18:01:32 +00003539 int added = 0;
Andrey Churbanov9e9333a2018-03-05 18:42:01 +00003540 int minimumRequiredCapacity;
3541 int newCapacity;
3542 kmp_info_t **newThreads;
3543 kmp_root_t **newRoot;
3544
3545// All calls to __kmp_expand_threads should be under __kmp_forkjoin_lock, so
3546// resizing __kmp_threads does not need additional protection if foreign
3547// threads are present
Jim Cownie5e8470a2013-09-27 10:38:44 +00003548
Jonathan Peyton8b3842f2018-10-05 17:59:39 +00003549#if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB
Jonathan Peyton30419822017-05-12 18:01:32 +00003550 /* only for Windows static library */
3551 /* reclaim array entries for root threads that are already dead */
3552 added = __kmp_reclaim_dead_roots();
Jim Cownie5e8470a2013-09-27 10:38:44 +00003553
Jonathan Peyton30419822017-05-12 18:01:32 +00003554 if (nNeed) {
3555 nNeed -= added;
3556 if (nNeed < 0)
3557 nNeed = 0;
3558 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003559#endif
Jonathan Peyton1800ece2018-01-10 18:27:01 +00003560 if (nNeed <= 0)
Jim Cownie5e8470a2013-09-27 10:38:44 +00003561 return added;
Jonathan Peyton30419822017-05-12 18:01:32 +00003562
Andrey Churbanov9e9333a2018-03-05 18:42:01 +00003563 // Note that __kmp_threads_capacity is not bounded by __kmp_max_nth. If
3564 // __kmp_max_nth is set to some value less than __kmp_sys_max_nth by the
3565 // user via KMP_DEVICE_THREAD_LIMIT, then __kmp_threads_capacity may become
3566 // > __kmp_max_nth in one of two ways:
3567 //
3568 // 1) The initialization thread (gtid = 0) exits. __kmp_threads[0]
3569 // may not be resused by another thread, so we may need to increase
3570 // __kmp_threads_capacity to __kmp_max_nth + 1.
3571 //
3572 // 2) New foreign root(s) are encountered. We always register new foreign
3573 // roots. This may cause a smaller # of threads to be allocated at
3574 // subsequent parallel regions, but the worker threads hang around (and
3575 // eventually go to sleep) and need slots in the __kmp_threads[] array.
3576 //
3577 // Anyway, that is the reason for moving the check to see if
3578 // __kmp_max_nth was exceeded into __kmp_reserve_threads()
3579 // instead of having it performed here. -BB
Jonathan Peyton30419822017-05-12 18:01:32 +00003580
Andrey Churbanov9e9333a2018-03-05 18:42:01 +00003581 KMP_DEBUG_ASSERT(__kmp_sys_max_nth >= __kmp_threads_capacity);
Jonathan Peyton30419822017-05-12 18:01:32 +00003582
Andrey Churbanov9e9333a2018-03-05 18:42:01 +00003583 /* compute expansion headroom to check if we can expand */
3584 if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) {
3585 /* possible expansion too small -- give up */
3586 return added;
Jonathan Peyton30419822017-05-12 18:01:32 +00003587 }
Andrey Churbanov9e9333a2018-03-05 18:42:01 +00003588 minimumRequiredCapacity = __kmp_threads_capacity + nNeed;
3589
3590 newCapacity = __kmp_threads_capacity;
3591 do {
3592 newCapacity = newCapacity <= (__kmp_sys_max_nth >> 1) ? (newCapacity << 1)
3593 : __kmp_sys_max_nth;
3594 } while (newCapacity < minimumRequiredCapacity);
3595 newThreads = (kmp_info_t **)__kmp_allocate(
3596 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * newCapacity + CACHE_LINE);
3597 newRoot =
3598 (kmp_root_t **)((char *)newThreads + sizeof(kmp_info_t *) * newCapacity);
3599 KMP_MEMCPY(newThreads, __kmp_threads,
3600 __kmp_threads_capacity * sizeof(kmp_info_t *));
3601 KMP_MEMCPY(newRoot, __kmp_root,
3602 __kmp_threads_capacity * sizeof(kmp_root_t *));
3603
3604 kmp_info_t **temp_threads = __kmp_threads;
3605 *(kmp_info_t * *volatile *)&__kmp_threads = newThreads;
3606 *(kmp_root_t * *volatile *)&__kmp_root = newRoot;
3607 __kmp_free(temp_threads);
3608 added += newCapacity - __kmp_threads_capacity;
3609 *(volatile int *)&__kmp_threads_capacity = newCapacity;
3610
3611 if (newCapacity > __kmp_tp_capacity) {
3612 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
3613 if (__kmp_tp_cached && newCapacity > __kmp_tp_capacity) {
3614 __kmp_threadprivate_resize_cache(newCapacity);
3615 } else { // increase __kmp_tp_capacity to correspond with kmp_threads size
3616 *(volatile int *)&__kmp_tp_capacity = newCapacity;
3617 }
3618 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
3619 }
3620
Jonathan Peyton30419822017-05-12 18:01:32 +00003621 return added;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003622}
3623
Jonathan Peyton30419822017-05-12 18:01:32 +00003624/* Register the current thread as a root thread and obtain our gtid. We must
3625 have the __kmp_initz_lock held at this point. Argument TRUE only if are the
3626 thread that calls from __kmp_do_serial_initialize() */
3627int __kmp_register_root(int initial_thread) {
3628 kmp_info_t *root_thread;
3629 kmp_root_t *root;
3630 int gtid;
3631 int capacity;
3632 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3633 KA_TRACE(20, ("__kmp_register_root: entered\n"));
3634 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00003635
Jonathan Peyton30419822017-05-12 18:01:32 +00003636 /* 2007-03-02:
3637 If initial thread did not invoke OpenMP RTL yet, and this thread is not an
3638 initial one, "__kmp_all_nth >= __kmp_threads_capacity" condition does not
3639 work as expected -- it may return false (that means there is at least one
3640 empty slot in __kmp_threads array), but it is possible the only free slot
3641 is #0, which is reserved for initial thread and so cannot be used for this
3642 one. Following code workarounds this bug.
Jim Cownie5e8470a2013-09-27 10:38:44 +00003643
Jonathan Peyton30419822017-05-12 18:01:32 +00003644 However, right solution seems to be not reserving slot #0 for initial
3645 thread because:
3646 (1) there is no magic in slot #0,
3647 (2) we cannot detect initial thread reliably (the first thread which does
3648 serial initialization may be not a real initial thread).
3649 */
3650 capacity = __kmp_threads_capacity;
3651 if (!initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) {
3652 --capacity;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003653 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003654
Jonathan Peyton30419822017-05-12 18:01:32 +00003655 /* see if there are too many threads */
Jonathan Peyton1800ece2018-01-10 18:27:01 +00003656 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) {
Jonathan Peyton30419822017-05-12 18:01:32 +00003657 if (__kmp_tp_cached) {
Jonathan Peyton6a393f72017-09-05 15:43:58 +00003658 __kmp_fatal(KMP_MSG(CantRegisterNewThread),
3659 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
3660 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
Jonathan Peyton30419822017-05-12 18:01:32 +00003661 } else {
Jonathan Peyton6a393f72017-09-05 15:43:58 +00003662 __kmp_fatal(KMP_MSG(CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads),
3663 __kmp_msg_null);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003664 }
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003665 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003666
3667 /* find an available thread slot */
3668 /* Don't reassign the zero slot since we need that to only be used by initial
3669 thread */
3670 for (gtid = (initial_thread ? 0 : 1); TCR_PTR(__kmp_threads[gtid]) != NULL;
3671 gtid++)
3672 ;
3673 KA_TRACE(1,
3674 ("__kmp_register_root: found slot in threads array: T#%d\n", gtid));
3675 KMP_ASSERT(gtid < __kmp_threads_capacity);
3676
3677 /* update global accounting */
3678 __kmp_all_nth++;
3679 TCW_4(__kmp_nth, __kmp_nth + 1);
3680
3681 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low
3682 // numbers of procs, and method #2 (keyed API call) for higher numbers.
3683 if (__kmp_adjust_gtid_mode) {
3684 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
3685 if (TCR_4(__kmp_gtid_mode) != 2) {
3686 TCW_4(__kmp_gtid_mode, 2);
3687 }
3688 } else {
3689 if (TCR_4(__kmp_gtid_mode) != 1) {
3690 TCW_4(__kmp_gtid_mode, 1);
3691 }
3692 }
3693 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003694
3695#ifdef KMP_ADJUST_BLOCKTIME
Jonathan Peyton30419822017-05-12 18:01:32 +00003696 /* Adjust blocktime to zero if necessary */
3697 /* Middle initialization might not have occurred yet */
3698 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
3699 if (__kmp_nth > __kmp_avail_proc) {
3700 __kmp_zero_bt = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003701 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003702 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003703#endif /* KMP_ADJUST_BLOCKTIME */
3704
Jonathan Peyton30419822017-05-12 18:01:32 +00003705 /* setup this new hierarchy */
3706 if (!(root = __kmp_root[gtid])) {
3707 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(sizeof(kmp_root_t));
3708 KMP_DEBUG_ASSERT(!root->r.r_root_team);
3709 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003710
Jonathan Peyton5375fe82016-11-14 21:13:44 +00003711#if KMP_STATS_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003712 // Initialize stats as soon as possible (right after gtid assignment).
3713 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid);
Jonathan Peytonf0682ac2018-07-30 17:41:08 +00003714 __kmp_stats_thread_ptr->startLife();
Jonathan Peyton30419822017-05-12 18:01:32 +00003715 KMP_SET_THREAD_STATE(SERIAL_REGION);
3716 KMP_INIT_PARTITIONED_TIMERS(OMP_serial);
Jonathan Peyton5375fe82016-11-14 21:13:44 +00003717#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003718 __kmp_initialize_root(root);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003719
Jonathan Peyton30419822017-05-12 18:01:32 +00003720 /* setup new root thread structure */
3721 if (root->r.r_uber_thread) {
3722 root_thread = root->r.r_uber_thread;
3723 } else {
3724 root_thread = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t));
3725 if (__kmp_storage_map) {
3726 __kmp_print_thread_storage_map(root_thread, gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003727 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003728 root_thread->th.th_info.ds.ds_gtid = gtid;
Joachim Protze82e94a52017-11-01 10:08:30 +00003729#if OMPT_SUPPORT
Jonathan Peyton3574f282018-10-04 14:57:04 +00003730 root_thread->th.ompt_thread_info.thread_data = ompt_data_none;
Joachim Protze82e94a52017-11-01 10:08:30 +00003731#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003732 root_thread->th.th_root = root;
3733 if (__kmp_env_consistency_check) {
3734 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid);
3735 }
3736#if USE_FAST_MEMORY
3737 __kmp_initialize_fast_memory(root_thread);
3738#endif /* USE_FAST_MEMORY */
Jim Cownie5e8470a2013-09-27 10:38:44 +00003739
Jonathan Peyton30419822017-05-12 18:01:32 +00003740#if KMP_USE_BGET
3741 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL);
3742 __kmp_initialize_bget(root_thread);
3743#endif
3744 __kmp_init_random(root_thread); // Initialize random number generator
3745 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003746
Jonathan Peyton30419822017-05-12 18:01:32 +00003747 /* setup the serial team held in reserve by the root thread */
3748 if (!root_thread->th.th_serial_team) {
3749 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3750 KF_TRACE(10, ("__kmp_register_root: before serial_team\n"));
3751 root_thread->th.th_serial_team =
3752 __kmp_allocate_team(root, 1, 1,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003753#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00003754 ompt_data_none, // root parallel id
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003755#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00003756#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003757 proc_bind_default,
Jim Cownie5e8470a2013-09-27 10:38:44 +00003758#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003759 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL));
3760 }
3761 KMP_ASSERT(root_thread->th.th_serial_team);
3762 KF_TRACE(10, ("__kmp_register_root: after serial_team = %p\n",
3763 root_thread->th.th_serial_team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003764
Jonathan Peyton30419822017-05-12 18:01:32 +00003765 /* drop root_thread into place */
3766 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003767
Jonathan Peyton30419822017-05-12 18:01:32 +00003768 root->r.r_root_team->t.t_threads[0] = root_thread;
3769 root->r.r_hot_team->t.t_threads[0] = root_thread;
3770 root_thread->th.th_serial_team->t.t_threads[0] = root_thread;
3771 // AC: the team created in reserve, not for execution (it is unused for now).
3772 root_thread->th.th_serial_team->t.t_serialized = 0;
3773 root->r.r_uber_thread = root_thread;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003774
Jonathan Peyton30419822017-05-12 18:01:32 +00003775 /* initialize the thread, get it ready to go */
3776 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid);
3777 TCW_4(__kmp_init_gtid, TRUE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003778
Jonathan Peyton30419822017-05-12 18:01:32 +00003779 /* prepare the master thread for get_gtid() */
3780 __kmp_gtid_set_specific(gtid);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003781
Jonathan Peyton7abf9d52016-05-26 18:19:10 +00003782#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +00003783 __kmp_itt_thread_name(gtid);
Jonathan Peyton7abf9d52016-05-26 18:19:10 +00003784#endif /* USE_ITT_BUILD */
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003785
Jonathan Peyton30419822017-05-12 18:01:32 +00003786#ifdef KMP_TDATA_GTID
3787 __kmp_gtid = gtid;
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00003788#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003789 __kmp_create_worker(gtid, root_thread, __kmp_stksize);
3790 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid);
3791
3792 KA_TRACE(20, ("__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
3793 "plain=%u\n",
3794 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team),
3795 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,
3796 KMP_INIT_BARRIER_STATE));
3797 { // Initialize barrier data.
3798 int b;
3799 for (b = 0; b < bs_last_barrier; ++b) {
3800 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE;
3801#if USE_DEBUGGER
3802 root_thread->th.th_bar[b].bb.b_worker_arrived = 0;
3803#endif
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00003804 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003805 }
3806 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived ==
3807 KMP_INIT_BARRIER_STATE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003808
Alp Toker763b9392014-02-28 09:42:41 +00003809#if KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00003810#if OMP_40_ENABLED
3811 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED;
3812 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED;
3813 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED;
3814 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED;
3815#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003816 if (TCR_4(__kmp_init_middle)) {
3817 __kmp_affinity_set_init_mask(gtid, TRUE);
3818 }
Alp Toker763b9392014-02-28 09:42:41 +00003819#endif /* KMP_AFFINITY_SUPPORTED */
Jonathan Peyton92ca6182018-09-07 18:25:49 +00003820#if OMP_50_ENABLED
3821 root_thread->th.th_def_allocator = __kmp_def_allocator;
3822#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00003823
Jonathan Peyton30419822017-05-12 18:01:32 +00003824 __kmp_root_counter++;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003825
Joachim Protze82e94a52017-11-01 10:08:30 +00003826#if OMPT_SUPPORT
3827 if (!initial_thread && ompt_enabled.enabled) {
3828
Joachim Protze489cdb72018-09-10 14:34:54 +00003829 kmp_info_t *root_thread = ompt_get_thread();
Joachim Protze82e94a52017-11-01 10:08:30 +00003830
3831 ompt_set_thread_state(root_thread, omp_state_overhead);
3832
3833 if (ompt_enabled.ompt_callback_thread_begin) {
3834 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)(
3835 ompt_thread_initial, __ompt_get_thread_data_internal());
3836 }
3837 ompt_data_t *task_data;
3838 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL);
3839 if (ompt_enabled.ompt_callback_task_create) {
3840 ompt_callbacks.ompt_callback(ompt_callback_task_create)(
3841 NULL, NULL, task_data, ompt_task_initial, 0, NULL);
3842 // initial task has nothing to return to
3843 }
3844
3845 ompt_set_thread_state(root_thread, omp_state_work_serial);
3846 }
3847#endif
3848
Jonathan Peyton30419822017-05-12 18:01:32 +00003849 KMP_MB();
3850 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003851
Jonathan Peyton30419822017-05-12 18:01:32 +00003852 return gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003853}
3854
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003855#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00003856static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr, int level,
3857 const int max_level) {
3858 int i, n, nth;
3859 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams;
3860 if (!hot_teams || !hot_teams[level].hot_team) {
3861 return 0;
3862 }
3863 KMP_DEBUG_ASSERT(level < max_level);
3864 kmp_team_t *team = hot_teams[level].hot_team;
3865 nth = hot_teams[level].hot_team_nth;
3866 n = nth - 1; // master is not freed
3867 if (level < max_level - 1) {
3868 for (i = 0; i < nth; ++i) {
3869 kmp_info_t *th = team->t.t_threads[i];
3870 n += __kmp_free_hot_teams(root, th, level + 1, max_level);
3871 if (i > 0 && th->th.th_hot_teams) {
3872 __kmp_free(th->th.th_hot_teams);
3873 th->th.th_hot_teams = NULL;
3874 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003875 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003876 }
3877 __kmp_free_team(root, team, NULL);
3878 return n;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003879}
3880#endif
3881
Jonathan Peyton30419822017-05-12 18:01:32 +00003882// Resets a root thread and clear its root and hot teams.
3883// Returns the number of __kmp_threads entries directly and indirectly freed.
3884static int __kmp_reset_root(int gtid, kmp_root_t *root) {
3885 kmp_team_t *root_team = root->r.r_root_team;
3886 kmp_team_t *hot_team = root->r.r_hot_team;
3887 int n = hot_team->t.t_nproc;
3888 int i;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003889
Jonathan Peyton30419822017-05-12 18:01:32 +00003890 KMP_DEBUG_ASSERT(!root->r.r_active);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003891
Jonathan Peyton30419822017-05-12 18:01:32 +00003892 root->r.r_root_team = NULL;
3893 root->r.r_hot_team = NULL;
3894 // __kmp_free_team() does not free hot teams, so we have to clear r_hot_team
3895 // before call to __kmp_free_team().
3896 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003897#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00003898 if (__kmp_hot_teams_max_level >
3899 0) { // need to free nested hot teams and their threads if any
3900 for (i = 0; i < hot_team->t.t_nproc; ++i) {
3901 kmp_info_t *th = hot_team->t.t_threads[i];
3902 if (__kmp_hot_teams_max_level > 1) {
3903 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level);
3904 }
3905 if (th->th.th_hot_teams) {
3906 __kmp_free(th->th.th_hot_teams);
3907 th->th.th_hot_teams = NULL;
3908 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003909 }
Jonathan Peyton30419822017-05-12 18:01:32 +00003910 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00003911#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003912 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL));
Jim Cownie5e8470a2013-09-27 10:38:44 +00003913
Jonathan Peyton30419822017-05-12 18:01:32 +00003914 // Before we can reap the thread, we need to make certain that all other
3915 // threads in the teams that had this root as ancestor have stopped trying to
3916 // steal tasks.
3917 if (__kmp_tasking_mode != tskm_immediate_exec) {
3918 __kmp_wait_to_unref_task_teams();
3919 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00003920
Jonathan Peyton30419822017-05-12 18:01:32 +00003921#if KMP_OS_WINDOWS
3922 /* Close Handle of root duplicated in __kmp_create_worker (tr #62919) */
3923 KA_TRACE(
3924 10, ("__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPEC
3925 "\n",
3926 (LPVOID) & (root->r.r_uber_thread->th),
3927 root->r.r_uber_thread->th.th_info.ds.ds_thread));
3928 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread);
3929#endif /* KMP_OS_WINDOWS */
Jim Cownie5e8470a2013-09-27 10:38:44 +00003930
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003931#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00003932 if (ompt_enabled.ompt_callback_thread_end) {
3933 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(
3934 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data));
Jonathan Peyton30419822017-05-12 18:01:32 +00003935 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00003936#endif
3937
Jonathan Peyton30419822017-05-12 18:01:32 +00003938 TCW_4(__kmp_nth,
3939 __kmp_nth - 1); // __kmp_reap_thread will decrement __kmp_all_nth.
Jonathan Peytonf4392462017-07-27 20:58:41 +00003940 root->r.r_cg_nthreads--;
3941
Jonathan Peyton30419822017-05-12 18:01:32 +00003942 __kmp_reap_thread(root->r.r_uber_thread, 1);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003943
Jonathan Peyton30419822017-05-12 18:01:32 +00003944 // We canot put root thread to __kmp_thread_pool, so we have to reap it istead
3945 // of freeing.
3946 root->r.r_uber_thread = NULL;
3947 /* mark root as no longer in use */
3948 root->r.r_begin = FALSE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003949
Jonathan Peyton30419822017-05-12 18:01:32 +00003950 return n;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003951}
3952
Jonathan Peyton30419822017-05-12 18:01:32 +00003953void __kmp_unregister_root_current_thread(int gtid) {
3954 KA_TRACE(1, ("__kmp_unregister_root_current_thread: enter T#%d\n", gtid));
3955 /* this lock should be ok, since unregister_root_current_thread is never
3956 called during an abort, only during a normal close. furthermore, if you
3957 have the forkjoin lock, you should never try to get the initz lock */
3958 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3959 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
3960 KC_TRACE(10, ("__kmp_unregister_root_current_thread: already finished, "
3961 "exiting T#%d\n",
3962 gtid));
3963 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
3964 return;
3965 }
3966 kmp_root_t *root = __kmp_root[gtid];
Jim Cownie77c2a632014-09-03 11:34:33 +00003967
Jonathan Peyton30419822017-05-12 18:01:32 +00003968 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
3969 KMP_ASSERT(KMP_UBER_GTID(gtid));
3970 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
3971 KMP_ASSERT(root->r.r_active == FALSE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003972
Jonathan Peyton30419822017-05-12 18:01:32 +00003973 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00003974
Jonathan Peytondf6818b2016-06-14 17:57:47 +00003975#if OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00003976 kmp_info_t *thread = __kmp_threads[gtid];
3977 kmp_team_t *team = thread->th.th_team;
3978 kmp_task_team_t *task_team = thread->th.th_task_team;
Andrey Churbanov535b6fa2015-05-07 17:41:51 +00003979
Jonathan Peyton30419822017-05-12 18:01:32 +00003980 // we need to wait for the proxy tasks before finishing the thread
3981 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks) {
Jonathan Peyton6d247f72015-09-10 21:33:50 +00003982#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00003983 // the runtime is shutting down so we won't report any events
Joachim Protze82e94a52017-11-01 10:08:30 +00003984 thread->th.ompt_thread_info.state = omp_state_undefined;
Jonathan Peyton6d247f72015-09-10 21:33:50 +00003985#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00003986 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL));
3987 }
Andrey Churbanov535b6fa2015-05-07 17:41:51 +00003988#endif
3989
Jonathan Peyton30419822017-05-12 18:01:32 +00003990 __kmp_reset_root(gtid, root);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003991
Jonathan Peyton30419822017-05-12 18:01:32 +00003992 /* free up this thread slot */
3993 __kmp_gtid_set_specific(KMP_GTID_DNE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00003994#ifdef KMP_TDATA_GTID
Jonathan Peyton30419822017-05-12 18:01:32 +00003995 __kmp_gtid = KMP_GTID_DNE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00003996#endif
3997
Jonathan Peyton30419822017-05-12 18:01:32 +00003998 KMP_MB();
3999 KC_TRACE(10,
4000 ("__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004001
Jonathan Peyton30419822017-05-12 18:01:32 +00004002 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004003}
4004
Jonathan Peyton2321d572015-06-08 19:25:25 +00004005#if KMP_OS_WINDOWS
Jim Cownie5e8470a2013-09-27 10:38:44 +00004006/* __kmp_forkjoin_lock must be already held
Jonathan Peyton30419822017-05-12 18:01:32 +00004007 Unregisters a root thread that is not the current thread. Returns the number
4008 of __kmp_threads entries freed as a result. */
4009static int __kmp_unregister_root_other_thread(int gtid) {
4010 kmp_root_t *root = __kmp_root[gtid];
4011 int r;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004012
Jonathan Peyton30419822017-05-12 18:01:32 +00004013 KA_TRACE(1, ("__kmp_unregister_root_other_thread: enter T#%d\n", gtid));
4014 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
4015 KMP_ASSERT(KMP_UBER_GTID(gtid));
4016 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
4017 KMP_ASSERT(root->r.r_active == FALSE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004018
Jonathan Peyton30419822017-05-12 18:01:32 +00004019 r = __kmp_reset_root(gtid, root);
4020 KC_TRACE(10,
4021 ("__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid));
4022 return r;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004023}
Jonathan Peyton2321d572015-06-08 19:25:25 +00004024#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00004025
Jim Cownie5e8470a2013-09-27 10:38:44 +00004026#if KMP_DEBUG
4027void __kmp_task_info() {
4028
Jonathan Peyton30419822017-05-12 18:01:32 +00004029 kmp_int32 gtid = __kmp_entry_gtid();
4030 kmp_int32 tid = __kmp_tid_from_gtid(gtid);
4031 kmp_info_t *this_thr = __kmp_threads[gtid];
4032 kmp_team_t *steam = this_thr->th.th_serial_team;
4033 kmp_team_t *team = this_thr->th.th_team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004034
Jonathan Peytonbaad3f62018-08-09 22:04:30 +00004035 __kmp_printf(
4036 "__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p steam=%p curtask=%p "
4037 "ptask=%p\n",
4038 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task,
4039 team->t.t_implicit_task_taskdata[tid].td_parent);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004040}
4041#endif // KMP_DEBUG
4042
Jonathan Peyton30419822017-05-12 18:01:32 +00004043/* TODO optimize with one big memclr, take out what isn't needed, split
4044 responsibility to workers as much as possible, and delay initialization of
4045 features as much as possible */
4046static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team,
4047 int tid, int gtid) {
4048 /* this_thr->th.th_info.ds.ds_gtid is setup in
4049 kmp_allocate_thread/create_worker.
4050 this_thr->th.th_serial_team is setup in __kmp_allocate_thread */
4051 kmp_info_t *master = team->t.t_threads[0];
4052 KMP_DEBUG_ASSERT(this_thr != NULL);
4053 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team);
4054 KMP_DEBUG_ASSERT(team);
4055 KMP_DEBUG_ASSERT(team->t.t_threads);
4056 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4057 KMP_DEBUG_ASSERT(master);
4058 KMP_DEBUG_ASSERT(master->th.th_root);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004059
Jonathan Peyton30419822017-05-12 18:01:32 +00004060 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00004061
Jonathan Peyton30419822017-05-12 18:01:32 +00004062 TCW_SYNC_PTR(this_thr->th.th_team, team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004063
Jonathan Peyton30419822017-05-12 18:01:32 +00004064 this_thr->th.th_info.ds.ds_tid = tid;
4065 this_thr->th.th_set_nproc = 0;
4066 if (__kmp_tasking_mode != tskm_immediate_exec)
4067 // When tasking is possible, threads are not safe to reap until they are
4068 // done tasking; this will be set when tasking code is exited in wait
4069 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
4070 else // no tasking --> always safe to reap
4071 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004072#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00004073 this_thr->th.th_set_proc_bind = proc_bind_default;
4074#if KMP_AFFINITY_SUPPORTED
4075 this_thr->th.th_new_place = this_thr->th.th_current_place;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004076#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004077#endif
4078 this_thr->th.th_root = master->th.th_root;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004079
Jonathan Peyton30419822017-05-12 18:01:32 +00004080 /* setup the thread's cache of the team structure */
4081 this_thr->th.th_team_nproc = team->t.t_nproc;
4082 this_thr->th.th_team_master = master;
4083 this_thr->th.th_team_serialized = team->t.t_serialized;
4084 TCW_PTR(this_thr->th.th_sleep_loc, NULL);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004085
Jonathan Peyton30419822017-05-12 18:01:32 +00004086 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004087
Jonathan Peyton30419822017-05-12 18:01:32 +00004088 KF_TRACE(10, ("__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n",
4089 tid, gtid, this_thr, this_thr->th.th_current_task));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004090
Jonathan Peyton30419822017-05-12 18:01:32 +00004091 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr,
4092 team, tid, TRUE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004093
Jonathan Peyton30419822017-05-12 18:01:32 +00004094 KF_TRACE(10, ("__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n",
4095 tid, gtid, this_thr, this_thr->th.th_current_task));
4096 // TODO: Initialize ICVs from parent; GEH - isn't that already done in
4097 // __kmp_initialize_team()?
Jim Cownie5e8470a2013-09-27 10:38:44 +00004098
Jonathan Peyton30419822017-05-12 18:01:32 +00004099 /* TODO no worksharing in speculative threads */
4100 this_thr->th.th_dispatch = &team->t.t_dispatch[tid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00004101
Jonathan Peyton30419822017-05-12 18:01:32 +00004102 this_thr->th.th_local.this_construct = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004103
Jonathan Peyton30419822017-05-12 18:01:32 +00004104 if (!this_thr->th.th_pri_common) {
4105 this_thr->th.th_pri_common =
4106 (struct common_table *)__kmp_allocate(sizeof(struct common_table));
4107 if (__kmp_storage_map) {
4108 __kmp_print_storage_map_gtid(
4109 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1,
4110 sizeof(struct common_table), "th_%d.th_pri_common\n", gtid);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00004111 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004112 this_thr->th.th_pri_head = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00004113 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00004114
Jonathan Peyton30419822017-05-12 18:01:32 +00004115 /* Initialize dynamic dispatch */
4116 {
4117 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch;
4118 // Use team max_nproc since this will never change for the team.
4119 size_t disp_size =
4120 sizeof(dispatch_private_info_t) *
4121 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers);
4122 KD_TRACE(10, ("__kmp_initialize_info: T#%d max_nproc: %d\n", gtid,
4123 team->t.t_max_nproc));
4124 KMP_ASSERT(dispatch);
4125 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4126 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004127
Jonathan Peyton30419822017-05-12 18:01:32 +00004128 dispatch->th_disp_index = 0;
Jonathan Peytondf6818b2016-06-14 17:57:47 +00004129#if OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00004130 dispatch->th_doacross_buf_idx = 0;
Jonathan Peyton71909c52016-03-02 22:42:06 +00004131#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004132 if (!dispatch->th_disp_buffer) {
4133 dispatch->th_disp_buffer =
4134 (dispatch_private_info_t *)__kmp_allocate(disp_size);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004135
Jonathan Peyton30419822017-05-12 18:01:32 +00004136 if (__kmp_storage_map) {
4137 __kmp_print_storage_map_gtid(
4138 gtid, &dispatch->th_disp_buffer[0],
4139 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1
4140 ? 1
4141 : __kmp_dispatch_num_buffers],
4142 disp_size, "th_%d.th_dispatch.th_disp_buffer "
4143 "(team_%d.t_dispatch[%d].th_disp_buffer)",
4144 gtid, team->t.t_id, gtid);
4145 }
4146 } else {
4147 memset(&dispatch->th_disp_buffer[0], '\0', disp_size);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004148 }
4149
Jonathan Peyton30419822017-05-12 18:01:32 +00004150 dispatch->th_dispatch_pr_current = 0;
4151 dispatch->th_dispatch_sh_current = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004152
Jonathan Peyton30419822017-05-12 18:01:32 +00004153 dispatch->th_deo_fcn = 0; /* ORDERED */
4154 dispatch->th_dxo_fcn = 0; /* END ORDERED */
4155 }
Andrey Churbanov6d224db2015-02-10 18:37:43 +00004156
Jonathan Peyton30419822017-05-12 18:01:32 +00004157 this_thr->th.th_next_pool = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004158
Jonathan Peyton30419822017-05-12 18:01:32 +00004159 if (!this_thr->th.th_task_state_memo_stack) {
4160 size_t i;
4161 this_thr->th.th_task_state_memo_stack =
4162 (kmp_uint8 *)__kmp_allocate(4 * sizeof(kmp_uint8));
4163 this_thr->th.th_task_state_top = 0;
4164 this_thr->th.th_task_state_stack_sz = 4;
4165 for (i = 0; i < this_thr->th.th_task_state_stack_sz;
4166 ++i) // zero init the stack
4167 this_thr->th.th_task_state_memo_stack[i] = 0;
4168 }
4169
4170 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
4171 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
4172
4173 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00004174}
4175
Jonathan Peyton30419822017-05-12 18:01:32 +00004176/* allocate a new thread for the requesting team. this is only called from
4177 within a forkjoin critical section. we will first try to get an available
4178 thread from the thread pool. if none is available, we will fork a new one
4179 assuming we are able to create a new one. this should be assured, as the
4180 caller should check on this first. */
4181kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team,
4182 int new_tid) {
4183 kmp_team_t *serial_team;
4184 kmp_info_t *new_thr;
4185 int new_gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004186
Jonathan Peyton30419822017-05-12 18:01:32 +00004187 KA_TRACE(20, ("__kmp_allocate_thread: T#%d\n", __kmp_get_gtid()));
4188 KMP_DEBUG_ASSERT(root && team);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004189#if !KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004190 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid()));
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004191#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004192 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00004193
Jonathan Peyton30419822017-05-12 18:01:32 +00004194 /* first, try to get one from the thread pool */
4195 if (__kmp_thread_pool) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00004196
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00004197 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool);
Jonathan Peyton30419822017-05-12 18:01:32 +00004198 __kmp_thread_pool = (volatile kmp_info_t *)new_thr->th.th_next_pool;
4199 if (new_thr == __kmp_thread_pool_insert_pt) {
4200 __kmp_thread_pool_insert_pt = NULL;
4201 }
4202 TCW_4(new_thr->th.th_in_pool, FALSE);
4203 // Don't touch th_active_in_pool or th_active.
4204 // The worker thread adjusts those flags as it sleeps/awakens.
4205 __kmp_thread_pool_nth--;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004206
Jonathan Peyton30419822017-05-12 18:01:32 +00004207 KA_TRACE(20, ("__kmp_allocate_thread: T#%d using thread T#%d\n",
4208 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid));
4209 KMP_ASSERT(!new_thr->th.th_team);
4210 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity);
4211 KMP_DEBUG_ASSERT(__kmp_thread_pool_nth >= 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004212
Jonathan Peyton30419822017-05-12 18:01:32 +00004213 /* setup the thread structure */
4214 __kmp_initialize_info(new_thr, team, new_tid,
4215 new_thr->th.th_info.ds.ds_gtid);
4216 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004217
Jonathan Peyton30419822017-05-12 18:01:32 +00004218 TCW_4(__kmp_nth, __kmp_nth + 1);
Jonathan Peytonf4392462017-07-27 20:58:41 +00004219 root->r.r_cg_nthreads++;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004220
Jonathan Peyton30419822017-05-12 18:01:32 +00004221 new_thr->th.th_task_state = 0;
4222 new_thr->th.th_task_state_top = 0;
4223 new_thr->th.th_task_state_stack_sz = 4;
Andrey Churbanov6d224db2015-02-10 18:37:43 +00004224
Jim Cownie5e8470a2013-09-27 10:38:44 +00004225#ifdef KMP_ADJUST_BLOCKTIME
Jonathan Peyton30419822017-05-12 18:01:32 +00004226 /* Adjust blocktime back to zero if necessary */
4227 /* Middle initialization might not have occurred yet */
4228 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4229 if (__kmp_nth > __kmp_avail_proc) {
4230 __kmp_zero_bt = TRUE;
4231 }
4232 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00004233#endif /* KMP_ADJUST_BLOCKTIME */
4234
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004235#if KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00004236 // If thread entered pool via __kmp_free_thread, wait_flag should !=
4237 // KMP_BARRIER_PARENT_FLAG.
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004238 int b;
Jonathan Peyton30419822017-05-12 18:01:32 +00004239 kmp_balign_t *balign = new_thr->th.th_bar;
4240 for (b = 0; b < bs_last_barrier; ++b)
4241 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004242#endif
4243
Jonathan Peyton30419822017-05-12 18:01:32 +00004244 KF_TRACE(10, ("__kmp_allocate_thread: T#%d using thread %p T#%d\n",
4245 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004246
Jim Cownie5e8470a2013-09-27 10:38:44 +00004247 KMP_MB();
4248 return new_thr;
Jonathan Peyton30419822017-05-12 18:01:32 +00004249 }
4250
4251 /* no, well fork a new one */
4252 KMP_ASSERT(__kmp_nth == __kmp_all_nth);
4253 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity);
4254
4255#if KMP_USE_MONITOR
4256 // If this is the first worker thread the RTL is creating, then also
4257 // launch the monitor thread. We try to do this as early as possible.
4258 if (!TCR_4(__kmp_init_monitor)) {
4259 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
4260 if (!TCR_4(__kmp_init_monitor)) {
4261 KF_TRACE(10, ("before __kmp_create_monitor\n"));
4262 TCW_4(__kmp_init_monitor, 1);
4263 __kmp_create_monitor(&__kmp_monitor);
4264 KF_TRACE(10, ("after __kmp_create_monitor\n"));
4265#if KMP_OS_WINDOWS
4266 // AC: wait until monitor has started. This is a fix for CQ232808.
4267 // The reason is that if the library is loaded/unloaded in a loop with
4268 // small (parallel) work in between, then there is high probability that
4269 // monitor thread started after the library shutdown. At shutdown it is
4270 // too late to cope with the problem, because when the master is in
4271 // DllMain (process detach) the monitor has no chances to start (it is
4272 // blocked), and master has no means to inform the monitor that the
4273 // library has gone, because all the memory which the monitor can access
4274 // is going to be released/reset.
4275 while (TCR_4(__kmp_init_monitor) < 2) {
4276 KMP_YIELD(TRUE);
4277 }
4278 KF_TRACE(10, ("after monitor thread has started\n"));
4279#endif
4280 }
4281 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
4282 }
4283#endif
4284
4285 KMP_MB();
4286 for (new_gtid = 1; TCR_PTR(__kmp_threads[new_gtid]) != NULL; ++new_gtid) {
4287 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity);
4288 }
4289
4290 /* allocate space for it. */
4291 new_thr = (kmp_info_t *)__kmp_allocate(sizeof(kmp_info_t));
4292
4293 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr);
4294
4295 if (__kmp_storage_map) {
4296 __kmp_print_thread_storage_map(new_thr, new_gtid);
4297 }
4298
4299 // add the reserve serialized team, initialized from the team's master thread
4300 {
4301 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team);
4302 KF_TRACE(10, ("__kmp_allocate_thread: before th_serial/serial_team\n"));
4303 new_thr->th.th_serial_team = serial_team =
4304 (kmp_team_t *)__kmp_allocate_team(root, 1, 1,
4305#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00004306 ompt_data_none, // root parallel id
Jonathan Peyton30419822017-05-12 18:01:32 +00004307#endif
4308#if OMP_40_ENABLED
4309 proc_bind_default,
4310#endif
4311 &r_icvs, 0 USE_NESTED_HOT_ARG(NULL));
4312 }
4313 KMP_ASSERT(serial_team);
4314 serial_team->t.t_serialized = 0; // AC: the team created in reserve, not for
4315 // execution (it is unused for now).
4316 serial_team->t.t_threads[0] = new_thr;
4317 KF_TRACE(10,
4318 ("__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n",
4319 new_thr));
4320
4321 /* setup the thread structures */
4322 __kmp_initialize_info(new_thr, team, new_tid, new_gtid);
4323
4324#if USE_FAST_MEMORY
4325 __kmp_initialize_fast_memory(new_thr);
4326#endif /* USE_FAST_MEMORY */
4327
4328#if KMP_USE_BGET
4329 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL);
4330 __kmp_initialize_bget(new_thr);
4331#endif
4332
4333 __kmp_init_random(new_thr); // Initialize random number generator
4334
4335 /* Initialize these only once when thread is grabbed for a team allocation */
4336 KA_TRACE(20,
4337 ("__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n",
4338 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
4339
4340 int b;
4341 kmp_balign_t *balign = new_thr->th.th_bar;
4342 for (b = 0; b < bs_last_barrier; ++b) {
4343 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE;
4344 balign[b].bb.team = NULL;
4345 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING;
4346 balign[b].bb.use_oncore_barrier = 0;
4347 }
4348
4349 new_thr->th.th_spin_here = FALSE;
4350 new_thr->th.th_next_waiting = 0;
Jonathan Peytona764af62018-07-19 19:17:00 +00004351#if KMP_OS_UNIX
4352 new_thr->th.th_blocking = false;
4353#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004354
4355#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
4356 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED;
4357 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED;
4358 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED;
4359 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED;
4360#endif
Jonathan Peyton92ca6182018-09-07 18:25:49 +00004361#if OMP_50_ENABLED
4362 new_thr->th.th_def_allocator = __kmp_def_allocator;
4363#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004364
4365 TCW_4(new_thr->th.th_in_pool, FALSE);
4366 new_thr->th.th_active_in_pool = FALSE;
4367 TCW_4(new_thr->th.th_active, TRUE);
4368
4369 /* adjust the global counters */
4370 __kmp_all_nth++;
4371 __kmp_nth++;
4372
Jonathan Peytonf4392462017-07-27 20:58:41 +00004373 root->r.r_cg_nthreads++;
4374
Jonathan Peyton30419822017-05-12 18:01:32 +00004375 // if __kmp_adjust_gtid_mode is set, then we use method #1 (sp search) for low
4376 // numbers of procs, and method #2 (keyed API call) for higher numbers.
4377 if (__kmp_adjust_gtid_mode) {
4378 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
4379 if (TCR_4(__kmp_gtid_mode) != 2) {
4380 TCW_4(__kmp_gtid_mode, 2);
4381 }
4382 } else {
4383 if (TCR_4(__kmp_gtid_mode) != 1) {
4384 TCW_4(__kmp_gtid_mode, 1);
4385 }
4386 }
4387 }
4388
4389#ifdef KMP_ADJUST_BLOCKTIME
4390 /* Adjust blocktime back to zero if necessary */
4391 /* Middle initialization might not have occurred yet */
4392 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4393 if (__kmp_nth > __kmp_avail_proc) {
4394 __kmp_zero_bt = TRUE;
4395 }
4396 }
4397#endif /* KMP_ADJUST_BLOCKTIME */
4398
4399 /* actually fork it and create the new worker thread */
4400 KF_TRACE(
4401 10, ("__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr));
4402 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize);
4403 KF_TRACE(10,
4404 ("__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr));
4405
4406 KA_TRACE(20, ("__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(),
4407 new_gtid));
4408 KMP_MB();
4409 return new_thr;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004410}
4411
Jonathan Peyton30419822017-05-12 18:01:32 +00004412/* Reinitialize team for reuse.
4413 The hot team code calls this case at every fork barrier, so EPCC barrier
4414 test are extremely sensitive to changes in it, esp. writes to the team
4415 struct, which cause a cache invalidation in all threads.
4416 IF YOU TOUCH THIS ROUTINE, RUN EPCC C SYNCBENCH ON A BIG-IRON MACHINE!!! */
4417static void __kmp_reinitialize_team(kmp_team_t *team,
4418 kmp_internal_control_t *new_icvs,
4419 ident_t *loc) {
4420 KF_TRACE(10, ("__kmp_reinitialize_team: enter this_thread=%p team=%p\n",
4421 team->t.t_threads[0], team));
4422 KMP_DEBUG_ASSERT(team && new_icvs);
4423 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);
4424 KMP_CHECK_UPDATE(team->t.t_ident, loc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004425
Jonathan Peyton30419822017-05-12 18:01:32 +00004426 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID());
Jonathan Peyton30419822017-05-12 18:01:32 +00004427 // Copy ICVs to the master thread's implicit taskdata
4428 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE);
4429 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);
Jim Cownie181b4bb2013-12-23 17:28:57 +00004430
Jonathan Peyton30419822017-05-12 18:01:32 +00004431 KF_TRACE(10, ("__kmp_reinitialize_team: exit this_thread=%p team=%p\n",
4432 team->t.t_threads[0], team));
Jim Cownie181b4bb2013-12-23 17:28:57 +00004433}
4434
Jonathan Peyton30419822017-05-12 18:01:32 +00004435/* Initialize the team data structure.
4436 This assumes the t_threads and t_max_nproc are already set.
4437 Also, we don't touch the arguments */
4438static void __kmp_initialize_team(kmp_team_t *team, int new_nproc,
4439 kmp_internal_control_t *new_icvs,
4440 ident_t *loc) {
4441 KF_TRACE(10, ("__kmp_initialize_team: enter: team=%p\n", team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004442
Jonathan Peyton30419822017-05-12 18:01:32 +00004443 /* verify */
4444 KMP_DEBUG_ASSERT(team);
4445 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc);
4446 KMP_DEBUG_ASSERT(team->t.t_threads);
4447 KMP_MB();
Jim Cownie181b4bb2013-12-23 17:28:57 +00004448
Jonathan Peyton30419822017-05-12 18:01:32 +00004449 team->t.t_master_tid = 0; /* not needed */
4450 /* team->t.t_master_bar; not needed */
4451 team->t.t_serialized = new_nproc > 1 ? 0 : 1;
4452 team->t.t_nproc = new_nproc;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004453
Jonathan Peyton30419822017-05-12 18:01:32 +00004454 /* team->t.t_parent = NULL; TODO not needed & would mess up hot team */
4455 team->t.t_next_pool = NULL;
4456 /* memset( team->t.t_threads, 0, sizeof(kmp_info_t*)*new_nproc ); would mess
4457 * up hot team */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004458
Jonathan Peyton30419822017-05-12 18:01:32 +00004459 TCW_SYNC_PTR(team->t.t_pkfn, NULL); /* not needed */
4460 team->t.t_invoke = NULL; /* not needed */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004461
Jonathan Peyton30419822017-05-12 18:01:32 +00004462 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00004463 team->t.t_sched.sched = new_icvs->sched.sched;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004464
4465#if KMP_ARCH_X86 || KMP_ARCH_X86_64
Jonathan Peyton30419822017-05-12 18:01:32 +00004466 team->t.t_fp_control_saved = FALSE; /* not needed */
4467 team->t.t_x87_fpu_control_word = 0; /* not needed */
4468 team->t.t_mxcsr = 0; /* not needed */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004469#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
4470
Jonathan Peyton30419822017-05-12 18:01:32 +00004471 team->t.t_construct = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004472
Jonathan Peyton30419822017-05-12 18:01:32 +00004473 team->t.t_ordered.dt.t_value = 0;
4474 team->t.t_master_active = FALSE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004475
Jonathan Peyton30419822017-05-12 18:01:32 +00004476 memset(&team->t.t_taskq, '\0', sizeof(kmp_taskq_t));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004477
4478#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00004479 team->t.t_copypriv_data = NULL; /* not necessary, but nice for debugging */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004480#endif
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00004481#if KMP_OS_WINDOWS
Jonathan Peyton30419822017-05-12 18:01:32 +00004482 team->t.t_copyin_counter = 0; /* for barrier-free copyin implementation */
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00004483#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00004484
Jonathan Peyton30419822017-05-12 18:01:32 +00004485 team->t.t_control_stack_top = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004486
Jonathan Peyton30419822017-05-12 18:01:32 +00004487 __kmp_reinitialize_team(team, new_icvs, loc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004488
Jonathan Peyton30419822017-05-12 18:01:32 +00004489 KMP_MB();
4490 KF_TRACE(10, ("__kmp_initialize_team: exit: team=%p\n", team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004491}
4492
Alp Toker98758b02014-03-02 04:12:06 +00004493#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
Jim Cownie5e8470a2013-09-27 10:38:44 +00004494/* Sets full mask for thread and returns old mask, no changes to structures. */
4495static void
Jonathan Peyton30419822017-05-12 18:01:32 +00004496__kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) {
4497 if (KMP_AFFINITY_CAPABLE()) {
4498 int status;
4499 if (old_mask != NULL) {
4500 status = __kmp_get_system_affinity(old_mask, TRUE);
4501 int error = errno;
4502 if (status != 0) {
Jonathan Peyton6a393f72017-09-05 15:43:58 +00004503 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError), KMP_ERR(error),
4504 __kmp_msg_null);
Jonathan Peyton30419822017-05-12 18:01:32 +00004505 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00004506 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004507 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE);
4508 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00004509}
4510#endif
4511
Alp Toker98758b02014-03-02 04:12:06 +00004512#if OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED
Jim Cownie5e8470a2013-09-27 10:38:44 +00004513
Jim Cownie5e8470a2013-09-27 10:38:44 +00004514// __kmp_partition_places() is the heart of the OpenMP 4.0 affinity mechanism.
4515// It calculats the worker + master thread's partition based upon the parent
Alp Toker8f2d3f02014-02-24 10:40:15 +00004516// thread's partition, and binds each worker to a thread in their partition.
Jim Cownie5e8470a2013-09-27 10:38:44 +00004517// The master thread's partition should already include its current binding.
Jonathan Peyton30419822017-05-12 18:01:32 +00004518static void __kmp_partition_places(kmp_team_t *team, int update_master_only) {
4519 // Copy the master thread's place partion to the team struct
4520 kmp_info_t *master_th = team->t.t_threads[0];
4521 KMP_DEBUG_ASSERT(master_th != NULL);
4522 kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
4523 int first_place = master_th->th.th_first_place;
4524 int last_place = master_th->th.th_last_place;
4525 int masters_place = master_th->th.th_current_place;
4526 team->t.t_first_place = first_place;
4527 team->t.t_last_place = last_place;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004528
Jonathan Peyton30419822017-05-12 18:01:32 +00004529 KA_TRACE(20, ("__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
4530 "bound to place %d partition = [%d,%d]\n",
4531 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]),
4532 team->t.t_id, masters_place, first_place, last_place));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004533
Jonathan Peyton30419822017-05-12 18:01:32 +00004534 switch (proc_bind) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00004535
Jonathan Peyton30419822017-05-12 18:01:32 +00004536 case proc_bind_default:
4537 // serial teams might have the proc_bind policy set to proc_bind_default. It
4538 // doesn't matter, as we don't rebind master thread for any proc_bind policy
4539 KMP_DEBUG_ASSERT(team->t.t_nproc == 1);
4540 break;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004541
Jonathan Peyton30419822017-05-12 18:01:32 +00004542 case proc_bind_master: {
4543 int f;
4544 int n_th = team->t.t_nproc;
4545 for (f = 1; f < n_th; f++) {
4546 kmp_info_t *th = team->t.t_threads[f];
4547 KMP_DEBUG_ASSERT(th != NULL);
4548 th->th.th_first_place = first_place;
4549 th->th.th_last_place = last_place;
4550 th->th.th_new_place = masters_place;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004551
Jonathan Peyton30419822017-05-12 18:01:32 +00004552 KA_TRACE(100, ("__kmp_partition_places: master: T#%d(%d:%d) place %d "
4553 "partition = [%d,%d]\n",
4554 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
4555 f, masters_place, first_place, last_place));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004556 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004557 } break;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004558
Jonathan Peyton30419822017-05-12 18:01:32 +00004559 case proc_bind_close: {
4560 int f;
4561 int n_th = team->t.t_nproc;
4562 int n_places;
4563 if (first_place <= last_place) {
4564 n_places = last_place - first_place + 1;
4565 } else {
4566 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4567 }
4568 if (n_th <= n_places) {
4569 int place = masters_place;
4570 for (f = 1; f < n_th; f++) {
4571 kmp_info_t *th = team->t.t_threads[f];
4572 KMP_DEBUG_ASSERT(th != NULL);
4573
4574 if (place == last_place) {
4575 place = first_place;
4576 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4577 place = 0;
4578 } else {
4579 place++;
4580 }
4581 th->th.th_first_place = first_place;
4582 th->th.th_last_place = last_place;
4583 th->th.th_new_place = place;
4584
4585 KA_TRACE(100, ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
4586 "partition = [%d,%d]\n",
4587 __kmp_gtid_from_thread(team->t.t_threads[f]),
4588 team->t.t_id, f, place, first_place, last_place));
4589 }
4590 } else {
4591 int S, rem, gap, s_count;
4592 S = n_th / n_places;
4593 s_count = 0;
4594 rem = n_th - (S * n_places);
4595 gap = rem > 0 ? n_places / rem : n_places;
4596 int place = masters_place;
4597 int gap_ct = gap;
4598 for (f = 0; f < n_th; f++) {
4599 kmp_info_t *th = team->t.t_threads[f];
4600 KMP_DEBUG_ASSERT(th != NULL);
4601
4602 th->th.th_first_place = first_place;
4603 th->th.th_last_place = last_place;
4604 th->th.th_new_place = place;
4605 s_count++;
4606
4607 if ((s_count == S) && rem && (gap_ct == gap)) {
4608 // do nothing, add an extra thread to place on next iteration
4609 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4610 // we added an extra thread to this place; move to next place
4611 if (place == last_place) {
4612 place = first_place;
4613 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4614 place = 0;
4615 } else {
4616 place++;
4617 }
4618 s_count = 0;
4619 gap_ct = 1;
4620 rem--;
4621 } else if (s_count == S) { // place full; don't add extra
4622 if (place == last_place) {
4623 place = first_place;
4624 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4625 place = 0;
4626 } else {
4627 place++;
4628 }
4629 gap_ct++;
4630 s_count = 0;
4631 }
4632
4633 KA_TRACE(100,
4634 ("__kmp_partition_places: close: T#%d(%d:%d) place %d "
4635 "partition = [%d,%d]\n",
4636 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f,
4637 th->th.th_new_place, first_place, last_place));
4638 }
4639 KMP_DEBUG_ASSERT(place == masters_place);
4640 }
4641 } break;
4642
4643 case proc_bind_spread: {
4644 int f;
4645 int n_th = team->t.t_nproc;
4646 int n_places;
4647 int thidx;
4648 if (first_place <= last_place) {
4649 n_places = last_place - first_place + 1;
4650 } else {
4651 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4652 }
4653 if (n_th <= n_places) {
Paul Osmialowskia0162792017-08-10 23:04:11 +00004654 int place = -1;
Jonathan Peyton30419822017-05-12 18:01:32 +00004655
Paul Osmialowskia0162792017-08-10 23:04:11 +00004656 if (n_places != static_cast<int>(__kmp_affinity_num_masks)) {
4657 int S = n_places / n_th;
4658 int s_count, rem, gap, gap_ct;
4659
4660 place = masters_place;
4661 rem = n_places - n_th * S;
4662 gap = rem ? n_th / rem : 1;
4663 gap_ct = gap;
4664 thidx = n_th;
4665 if (update_master_only == 1)
4666 thidx = 1;
4667 for (f = 0; f < thidx; f++) {
4668 kmp_info_t *th = team->t.t_threads[f];
4669 KMP_DEBUG_ASSERT(th != NULL);
4670
4671 th->th.th_first_place = place;
4672 th->th.th_new_place = place;
4673 s_count = 1;
4674 while (s_count < S) {
4675 if (place == last_place) {
4676 place = first_place;
4677 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4678 place = 0;
4679 } else {
4680 place++;
4681 }
4682 s_count++;
4683 }
4684 if (rem && (gap_ct == gap)) {
4685 if (place == last_place) {
4686 place = first_place;
4687 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4688 place = 0;
4689 } else {
4690 place++;
4691 }
4692 rem--;
4693 gap_ct = 0;
4694 }
4695 th->th.th_last_place = place;
4696 gap_ct++;
4697
Jonathan Peyton30419822017-05-12 18:01:32 +00004698 if (place == last_place) {
4699 place = first_place;
4700 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4701 place = 0;
4702 } else {
4703 place++;
4704 }
Paul Osmialowskia0162792017-08-10 23:04:11 +00004705
Jonathan Peyton94a114f2017-10-20 19:30:57 +00004706 KA_TRACE(100,
4707 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
4708 "partition = [%d,%d], __kmp_affinity_num_masks: %u\n",
4709 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
4710 f, th->th.th_new_place, th->th.th_first_place,
4711 th->th.th_last_place, __kmp_affinity_num_masks));
Jonathan Peyton30419822017-05-12 18:01:32 +00004712 }
Paul Osmialowskia0162792017-08-10 23:04:11 +00004713 } else {
4714 /* Having uniform space of available computation places I can create
4715 T partitions of round(P/T) size and put threads into the first
4716 place of each partition. */
4717 double current = static_cast<double>(masters_place);
4718 double spacing =
Jonathan Peyton94a114f2017-10-20 19:30:57 +00004719 (static_cast<double>(n_places + 1) / static_cast<double>(n_th));
Paul Osmialowskia0162792017-08-10 23:04:11 +00004720 int first, last;
4721 kmp_info_t *th;
4722
4723 thidx = n_th + 1;
4724 if (update_master_only == 1)
4725 thidx = 1;
4726 for (f = 0; f < thidx; f++) {
4727 first = static_cast<int>(current);
4728 last = static_cast<int>(current + spacing) - 1;
4729 KMP_DEBUG_ASSERT(last >= first);
4730 if (first >= n_places) {
4731 if (masters_place) {
4732 first -= n_places;
4733 last -= n_places;
4734 if (first == (masters_place + 1)) {
4735 KMP_DEBUG_ASSERT(f == n_th);
4736 first--;
4737 }
4738 if (last == masters_place) {
4739 KMP_DEBUG_ASSERT(f == (n_th - 1));
4740 last--;
4741 }
4742 } else {
4743 KMP_DEBUG_ASSERT(f == n_th);
4744 first = 0;
4745 last = 0;
4746 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004747 }
Paul Osmialowskia0162792017-08-10 23:04:11 +00004748 if (last >= n_places) {
4749 last = (n_places - 1);
4750 }
4751 place = first;
4752 current += spacing;
4753 if (f < n_th) {
4754 KMP_DEBUG_ASSERT(0 <= first);
4755 KMP_DEBUG_ASSERT(n_places > first);
4756 KMP_DEBUG_ASSERT(0 <= last);
4757 KMP_DEBUG_ASSERT(n_places > last);
4758 KMP_DEBUG_ASSERT(last_place >= first_place);
4759 th = team->t.t_threads[f];
4760 KMP_DEBUG_ASSERT(th);
4761 th->th.th_first_place = first;
4762 th->th.th_new_place = place;
4763 th->th.th_last_place = last;
Jonathan Peyton30419822017-05-12 18:01:32 +00004764
Jonathan Peyton94a114f2017-10-20 19:30:57 +00004765 KA_TRACE(100,
4766 ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
4767 "partition = [%d,%d], spacing = %.4f\n",
4768 __kmp_gtid_from_thread(team->t.t_threads[f]),
4769 team->t.t_id, f, th->th.th_new_place,
4770 th->th.th_first_place, th->th.th_last_place, spacing));
Paul Osmialowskia0162792017-08-10 23:04:11 +00004771 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004772 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004773 }
4774 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
4775 } else {
4776 int S, rem, gap, s_count;
4777 S = n_th / n_places;
4778 s_count = 0;
4779 rem = n_th - (S * n_places);
4780 gap = rem > 0 ? n_places / rem : n_places;
4781 int place = masters_place;
4782 int gap_ct = gap;
4783 thidx = n_th;
4784 if (update_master_only == 1)
4785 thidx = 1;
4786 for (f = 0; f < thidx; f++) {
4787 kmp_info_t *th = team->t.t_threads[f];
4788 KMP_DEBUG_ASSERT(th != NULL);
4789
4790 th->th.th_first_place = place;
4791 th->th.th_last_place = place;
4792 th->th.th_new_place = place;
4793 s_count++;
4794
4795 if ((s_count == S) && rem && (gap_ct == gap)) {
4796 // do nothing, add an extra thread to place on next iteration
4797 } else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4798 // we added an extra thread to this place; move on to next place
4799 if (place == last_place) {
4800 place = first_place;
4801 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4802 place = 0;
4803 } else {
4804 place++;
4805 }
4806 s_count = 0;
4807 gap_ct = 1;
4808 rem--;
4809 } else if (s_count == S) { // place is full; don't add extra thread
4810 if (place == last_place) {
4811 place = first_place;
4812 } else if (place == (int)(__kmp_affinity_num_masks - 1)) {
4813 place = 0;
4814 } else {
4815 place++;
4816 }
4817 gap_ct++;
4818 s_count = 0;
4819 }
4820
4821 KA_TRACE(100, ("__kmp_partition_places: spread: T#%d(%d:%d) place %d "
4822 "partition = [%d,%d]\n",
4823 __kmp_gtid_from_thread(team->t.t_threads[f]),
4824 team->t.t_id, f, th->th.th_new_place,
4825 th->th.th_first_place, th->th.th_last_place));
4826 }
4827 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
4828 }
4829 } break;
4830
4831 default:
4832 break;
4833 }
4834
4835 KA_TRACE(20, ("__kmp_partition_places: exit T#%d\n", team->t.t_id));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004836}
4837
Alp Toker98758b02014-03-02 04:12:06 +00004838#endif /* OMP_40_ENABLED && KMP_AFFINITY_SUPPORTED */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004839
Jonathan Peyton30419822017-05-12 18:01:32 +00004840/* allocate a new team data structure to use. take one off of the free pool if
4841 available */
Jim Cownie5e8470a2013-09-27 10:38:44 +00004842kmp_team_t *
Jonathan Peyton30419822017-05-12 18:01:32 +00004843__kmp_allocate_team(kmp_root_t *root, int new_nproc, int max_nproc,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00004844#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00004845 ompt_data_t ompt_parallel_data,
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00004846#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00004847#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00004848 kmp_proc_bind_t new_proc_bind,
Jim Cownie5e8470a2013-09-27 10:38:44 +00004849#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004850 kmp_internal_control_t *new_icvs,
4851 int argc USE_NESTED_HOT_ARG(kmp_info_t *master)) {
4852 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team);
4853 int f;
4854 kmp_team_t *team;
4855 int use_hot_team = !root->r.r_active;
4856 int level = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00004857
Jonathan Peyton30419822017-05-12 18:01:32 +00004858 KA_TRACE(20, ("__kmp_allocate_team: called\n"));
4859 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0);
4860 KMP_DEBUG_ASSERT(max_nproc >= new_nproc);
4861 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00004862
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004863#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004864 kmp_hot_team_ptr_t *hot_teams;
4865 if (master) {
4866 team = master->th.th_team;
4867 level = team->t.t_active_level;
4868 if (master->th.th_teams_microtask) { // in teams construct?
4869 if (master->th.th_teams_size.nteams > 1 &&
4870 ( // #teams > 1
4871 team->t.t_pkfn ==
4872 (microtask_t)__kmp_teams_master || // inner fork of the teams
4873 master->th.th_teams_level <
4874 team->t.t_level)) { // or nested parallel inside the teams
4875 ++level; // not increment if #teams==1, or for outer fork of the teams;
4876 // increment otherwise
4877 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004878 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004879 hot_teams = master->th.th_hot_teams;
4880 if (level < __kmp_hot_teams_max_level && hot_teams &&
4881 hot_teams[level]
4882 .hot_team) { // hot team has already been allocated for given level
4883 use_hot_team = 1;
4884 } else {
4885 use_hot_team = 0;
4886 }
4887 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004888#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00004889 // Optimization to use a "hot" team
4890 if (use_hot_team && new_nproc > 1) {
4891 KMP_DEBUG_ASSERT(new_nproc == max_nproc);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004892#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004893 team = hot_teams[level].hot_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004894#else
Jonathan Peyton30419822017-05-12 18:01:32 +00004895 team = root->r.r_hot_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004896#endif
4897#if KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00004898 if (__kmp_tasking_mode != tskm_immediate_exec) {
4899 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p "
4900 "task_team[1] = %p before reinit\n",
4901 team->t.t_task_team[0], team->t.t_task_team[1]));
4902 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00004903#endif
4904
Jonathan Peyton30419822017-05-12 18:01:32 +00004905 // Has the number of threads changed?
4906 /* Let's assume the most common case is that the number of threads is
4907 unchanged, and put that case first. */
4908 if (team->t.t_nproc == new_nproc) { // Check changes in number of threads
4909 KA_TRACE(20, ("__kmp_allocate_team: reusing hot team\n"));
4910 // This case can mean that omp_set_num_threads() was called and the hot
Jonathan Peyton642688b2017-06-01 16:46:36 +00004911 // team size was already reduced, so we check the special flag
Jonathan Peyton30419822017-05-12 18:01:32 +00004912 if (team->t.t_size_changed == -1) {
4913 team->t.t_size_changed = 1;
4914 } else {
4915 KMP_CHECK_UPDATE(team->t.t_size_changed, 0);
4916 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004917
Jonathan Peyton30419822017-05-12 18:01:32 +00004918 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
4919 kmp_r_sched_t new_sched = new_icvs->sched;
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00004920 // set master's schedule as new run-time schedule
4921 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004922
Jonathan Peyton30419822017-05-12 18:01:32 +00004923 __kmp_reinitialize_team(team, new_icvs,
4924 root->r.r_uber_thread->th.th_ident);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004925
Jonathan Peyton30419822017-05-12 18:01:32 +00004926 KF_TRACE(10, ("__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0,
4927 team->t.t_threads[0], team));
4928 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004929
4930#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00004931#if KMP_AFFINITY_SUPPORTED
4932 if ((team->t.t_size_changed == 0) &&
4933 (team->t.t_proc_bind == new_proc_bind)) {
4934 if (new_proc_bind == proc_bind_spread) {
4935 __kmp_partition_places(
4936 team, 1); // add flag to update only master for spread
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004937 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004938 KA_TRACE(200, ("__kmp_allocate_team: reusing hot team #%d bindings: "
4939 "proc_bind = %d, partition = [%d,%d]\n",
4940 team->t.t_id, new_proc_bind, team->t.t_first_place,
4941 team->t.t_last_place));
4942 } else {
4943 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
4944 __kmp_partition_places(team);
4945 }
4946#else
4947 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
4948#endif /* KMP_AFFINITY_SUPPORTED */
4949#endif /* OMP_40_ENABLED */
4950 } else if (team->t.t_nproc > new_nproc) {
4951 KA_TRACE(20,
4952 ("__kmp_allocate_team: decreasing hot team thread count to %d\n",
4953 new_nproc));
Jim Cownie5e8470a2013-09-27 10:38:44 +00004954
Jonathan Peyton30419822017-05-12 18:01:32 +00004955 team->t.t_size_changed = 1;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004956#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004957 if (__kmp_hot_teams_mode == 0) {
4958 // AC: saved number of threads should correspond to team's value in this
4959 // mode, can be bigger in mode 1, when hot team has threads in reserve
4960 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc);
4961 hot_teams[level].hot_team_nth = new_nproc;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004962#endif // KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004963 /* release the extra threads we don't need any more */
4964 for (f = new_nproc; f < team->t.t_nproc; f++) {
4965 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
4966 if (__kmp_tasking_mode != tskm_immediate_exec) {
4967 // When decreasing team size, threads no longer in the team should
4968 // unref task team.
4969 team->t.t_threads[f]->th.th_task_team = NULL;
4970 }
4971 __kmp_free_thread(team->t.t_threads[f]);
4972 team->t.t_threads[f] = NULL;
4973 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004974#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004975 } // (__kmp_hot_teams_mode == 0)
4976 else {
4977 // When keeping extra threads in team, switch threads to wait on own
4978 // b_go flag
4979 for (f = new_nproc; f < team->t.t_nproc; ++f) {
4980 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
4981 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar;
4982 for (int b = 0; b < bs_last_barrier; ++b) {
4983 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) {
4984 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
Andrey Churbanovd6e1d7e2016-08-11 13:04:00 +00004985 }
Jonathan Peyton30419822017-05-12 18:01:32 +00004986 KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0);
4987 }
4988 }
4989 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00004990#endif // KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00004991 team->t.t_nproc = new_nproc;
4992 // TODO???: team->t.t_max_active_levels = new_max_active_levels;
Jonathan Peytonba55a7b2017-11-29 22:47:52 +00004993 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched);
Jonathan Peyton30419822017-05-12 18:01:32 +00004994 __kmp_reinitialize_team(team, new_icvs,
4995 root->r.r_uber_thread->th.th_ident);
Jim Cownie5e8470a2013-09-27 10:38:44 +00004996
Jonathan Peyton30419822017-05-12 18:01:32 +00004997 /* update the remaining threads */
4998 for (f = 0; f < new_nproc; ++f) {
4999 team->t.t_threads[f]->th.th_team_nproc = new_nproc;
5000 }
5001 // restore the current task state of the master thread: should be the
5002 // implicit task
5003 KF_TRACE(10, ("__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0,
5004 team->t.t_threads[0], team));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005005
Jonathan Peyton30419822017-05-12 18:01:32 +00005006 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005007
5008#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00005009 for (f = 0; f < team->t.t_nproc; f++) {
5010 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
5011 team->t.t_threads[f]->th.th_team_nproc ==
5012 team->t.t_nproc);
5013 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005014#endif
5015
5016#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00005017 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5018#if KMP_AFFINITY_SUPPORTED
5019 __kmp_partition_places(team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005020#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00005021#endif
5022 } else { // team->t.t_nproc < new_nproc
Alp Toker98758b02014-03-02 04:12:06 +00005023#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00005024 kmp_affin_mask_t *old_mask;
5025 if (KMP_AFFINITY_CAPABLE()) {
5026 KMP_CPU_ALLOC(old_mask);
5027 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005028#endif
5029
Jonathan Peyton30419822017-05-12 18:01:32 +00005030 KA_TRACE(20,
5031 ("__kmp_allocate_team: increasing hot team thread count to %d\n",
5032 new_nproc));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005033
Jonathan Peyton30419822017-05-12 18:01:32 +00005034 team->t.t_size_changed = 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005035
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005036#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00005037 int avail_threads = hot_teams[level].hot_team_nth;
5038 if (new_nproc < avail_threads)
5039 avail_threads = new_nproc;
5040 kmp_info_t **other_threads = team->t.t_threads;
5041 for (f = team->t.t_nproc; f < avail_threads; ++f) {
5042 // Adjust barrier data of reserved threads (if any) of the team
5043 // Other data will be set in __kmp_initialize_info() below.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005044 int b;
Jonathan Peyton30419822017-05-12 18:01:32 +00005045 kmp_balign_t *balign = other_threads[f]->th.th_bar;
5046 for (b = 0; b < bs_last_barrier; ++b) {
5047 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5048 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00005049#if USE_DEBUGGER
Jonathan Peyton30419822017-05-12 18:01:32 +00005050 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
Jonathan Peyton8fbb49a2015-07-09 18:16:58 +00005051#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005052 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005053 }
5054 if (hot_teams[level].hot_team_nth >= new_nproc) {
5055 // we have all needed threads in reserve, no need to allocate any
5056 // this only possible in mode 1, cannot have reserved threads in mode 0
5057 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1);
5058 team->t.t_nproc = new_nproc; // just get reserved threads involved
5059 } else {
5060 // we may have some threads in reserve, but not enough
5061 team->t.t_nproc =
5062 hot_teams[level]
5063 .hot_team_nth; // get reserved threads involved if any
5064 hot_teams[level].hot_team_nth = new_nproc; // adjust hot team max size
5065#endif // KMP_NESTED_HOT_TEAMS
5066 if (team->t.t_max_nproc < new_nproc) {
5067 /* reallocate larger arrays */
5068 __kmp_reallocate_team_arrays(team, new_nproc);
5069 __kmp_reinitialize_team(team, new_icvs, NULL);
5070 }
5071
5072#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
5073 /* Temporarily set full mask for master thread before creation of
5074 workers. The reason is that workers inherit the affinity from master,
5075 so if a lot of workers are created on the single core quickly, they
5076 don't get a chance to set their own affinity for a long time. */
5077 __kmp_set_thread_affinity_mask_full_tmp(old_mask);
5078#endif
5079
5080 /* allocate new threads for the hot team */
5081 for (f = team->t.t_nproc; f < new_nproc; f++) {
5082 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f);
5083 KMP_DEBUG_ASSERT(new_worker);
5084 team->t.t_threads[f] = new_worker;
5085
5086 KA_TRACE(20,
5087 ("__kmp_allocate_team: team %d init T#%d arrived: "
5088 "join=%llu, plain=%llu\n",
5089 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f,
5090 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
5091 team->t.t_bar[bs_plain_barrier].b_arrived));
5092
5093 { // Initialize barrier data for new threads.
5094 int b;
5095 kmp_balign_t *balign = new_worker->th.th_bar;
5096 for (b = 0; b < bs_last_barrier; ++b) {
5097 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5098 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag !=
5099 KMP_BARRIER_PARENT_FLAG);
5100#if USE_DEBUGGER
5101 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5102#endif
5103 }
5104 }
5105 }
5106
5107#if KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED
5108 if (KMP_AFFINITY_CAPABLE()) {
5109 /* Restore initial master thread's affinity mask */
5110 __kmp_set_system_affinity(old_mask, TRUE);
5111 KMP_CPU_FREE(old_mask);
5112 }
5113#endif
5114#if KMP_NESTED_HOT_TEAMS
5115 } // end of check of t_nproc vs. new_nproc vs. hot_team_nth
5116#endif // KMP_NESTED_HOT_TEAMS
5117 /* make sure everyone is syncronized */
5118 int old_nproc = team->t.t_nproc; // save old value and use to update only
5119 // new threads below
5120 __kmp_initialize_team(team, new_nproc, new_icvs,
5121 root->r.r_uber_thread->th.th_ident);
5122
5123 /* reinitialize the threads */
5124 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc);
5125 for (f = 0; f < team->t.t_nproc; ++f)
5126 __kmp_initialize_info(team->t.t_threads[f], team, f,
5127 __kmp_gtid_from_tid(f, team));
5128 if (level) { // set th_task_state for new threads in nested hot team
5129 // __kmp_initialize_info() no longer zeroes th_task_state, so we should
5130 // only need to set the th_task_state for the new threads. th_task_state
5131 // for master thread will not be accurate until after this in
5132 // __kmp_fork_call(), so we look to the master's memo_stack to get the
5133 // correct value.
5134 for (f = old_nproc; f < team->t.t_nproc; ++f)
5135 team->t.t_threads[f]->th.th_task_state =
5136 team->t.t_threads[0]->th.th_task_state_memo_stack[level];
5137 } else { // set th_task_state for new threads in non-nested hot team
5138 int old_state =
5139 team->t.t_threads[0]->th.th_task_state; // copy master's state
5140 for (f = old_nproc; f < team->t.t_nproc; ++f)
5141 team->t.t_threads[f]->th.th_task_state = old_state;
5142 }
5143
5144#ifdef KMP_DEBUG
5145 for (f = 0; f < team->t.t_nproc; ++f) {
5146 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
5147 team->t.t_threads[f]->th.th_team_nproc ==
5148 team->t.t_nproc);
5149 }
5150#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005151
5152#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00005153 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5154#if KMP_AFFINITY_SUPPORTED
5155 __kmp_partition_places(team);
5156#endif
5157#endif
5158 } // Check changes in number of threads
5159
5160#if OMP_40_ENABLED
5161 kmp_info_t *master = team->t.t_threads[0];
5162 if (master->th.th_teams_microtask) {
5163 for (f = 1; f < new_nproc; ++f) {
5164 // propagate teams construct specific info to workers
5165 kmp_info_t *thr = team->t.t_threads[f];
5166 thr->th.th_teams_microtask = master->th.th_teams_microtask;
5167 thr->th.th_teams_level = master->th.th_teams_level;
5168 thr->th.th_teams_size = master->th.th_teams_size;
5169 }
5170 }
5171#endif /* OMP_40_ENABLED */
5172#if KMP_NESTED_HOT_TEAMS
5173 if (level) {
5174 // Sync barrier state for nested hot teams, not needed for outermost hot
5175 // team.
5176 for (f = 1; f < new_nproc; ++f) {
5177 kmp_info_t *thr = team->t.t_threads[f];
5178 int b;
5179 kmp_balign_t *balign = thr->th.th_bar;
5180 for (b = 0; b < bs_last_barrier; ++b) {
5181 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5182 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
5183#if USE_DEBUGGER
5184 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5185#endif
5186 }
5187 }
5188 }
5189#endif // KMP_NESTED_HOT_TEAMS
5190
5191 /* reallocate space for arguments if necessary */
5192 __kmp_alloc_argv_entries(argc, team, TRUE);
5193 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5194 // The hot team re-uses the previous task team,
5195 // if untouched during the previous release->gather phase.
5196
5197 KF_TRACE(10, (" hot_team = %p\n", team));
5198
5199#if KMP_DEBUG
5200 if (__kmp_tasking_mode != tskm_immediate_exec) {
5201 KA_TRACE(20, ("__kmp_allocate_team: hot team task_team[0] = %p "
5202 "task_team[1] = %p after reinit\n",
5203 team->t.t_task_team[0], team->t.t_task_team[1]));
5204 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005205#endif
5206
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005207#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00005208 __ompt_team_assign_id(team, ompt_parallel_data);
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005209#endif
5210
Jim Cownie5e8470a2013-09-27 10:38:44 +00005211 KMP_MB();
5212
Jim Cownie5e8470a2013-09-27 10:38:44 +00005213 return team;
Jonathan Peyton30419822017-05-12 18:01:32 +00005214 }
5215
5216 /* next, let's try to take one from the team pool */
5217 KMP_MB();
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00005218 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) {
Jonathan Peyton30419822017-05-12 18:01:32 +00005219 /* TODO: consider resizing undersized teams instead of reaping them, now
5220 that we have a resizing mechanism */
5221 if (team->t.t_max_nproc >= max_nproc) {
5222 /* take this team from the team pool */
5223 __kmp_team_pool = team->t.t_next_pool;
5224
5225 /* setup the team for fresh use */
5226 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5227
5228 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and "
5229 "task_team[1] %p to NULL\n",
5230 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5231 team->t.t_task_team[0] = NULL;
5232 team->t.t_task_team[1] = NULL;
5233
5234 /* reallocate space for arguments if necessary */
5235 __kmp_alloc_argv_entries(argc, team, TRUE);
5236 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5237
5238 KA_TRACE(
5239 20, ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5240 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5241 { // Initialize barrier data.
5242 int b;
5243 for (b = 0; b < bs_last_barrier; ++b) {
5244 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5245#if USE_DEBUGGER
5246 team->t.t_bar[b].b_master_arrived = 0;
5247 team->t.t_bar[b].b_team_arrived = 0;
5248#endif
5249 }
5250 }
5251
5252#if OMP_40_ENABLED
5253 team->t.t_proc_bind = new_proc_bind;
5254#endif
5255
5256 KA_TRACE(20, ("__kmp_allocate_team: using team from pool %d.\n",
5257 team->t.t_id));
5258
5259#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00005260 __ompt_team_assign_id(team, ompt_parallel_data);
Jonathan Peyton30419822017-05-12 18:01:32 +00005261#endif
5262
5263 KMP_MB();
5264
5265 return team;
5266 }
5267
Jonathan Peyton94a114f2017-10-20 19:30:57 +00005268 /* reap team if it is too small, then loop back and check the next one */
5269 // not sure if this is wise, but, will be redone during the hot-teams
5270 // rewrite.
5271 /* TODO: Use technique to find the right size hot-team, don't reap them */
Jonathan Peyton30419822017-05-12 18:01:32 +00005272 team = __kmp_reap_team(team);
5273 __kmp_team_pool = team;
5274 }
5275
5276 /* nothing available in the pool, no matter, make a new team! */
5277 KMP_MB();
5278 team = (kmp_team_t *)__kmp_allocate(sizeof(kmp_team_t));
5279
5280 /* and set it up */
5281 team->t.t_max_nproc = max_nproc;
5282 /* NOTE well, for some reason allocating one big buffer and dividing it up
5283 seems to really hurt performance a lot on the P4, so, let's not use this */
5284 __kmp_allocate_team_arrays(team, max_nproc);
5285
5286 KA_TRACE(20, ("__kmp_allocate_team: making a new team\n"));
5287 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5288
5289 KA_TRACE(20, ("__kmp_allocate_team: setting task_team[0] %p and task_team[1] "
5290 "%p to NULL\n",
5291 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5292 team->t.t_task_team[0] = NULL; // to be removed, as __kmp_allocate zeroes
5293 // memory, no need to duplicate
5294 team->t.t_task_team[1] = NULL; // to be removed, as __kmp_allocate zeroes
5295 // memory, no need to duplicate
5296
5297 if (__kmp_storage_map) {
5298 __kmp_print_team_storage_map("team", team, team->t.t_id, new_nproc);
5299 }
5300
5301 /* allocate space for arguments */
5302 __kmp_alloc_argv_entries(argc, team, FALSE);
5303 team->t.t_argc = argc;
5304
5305 KA_TRACE(20,
5306 ("__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5307 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5308 { // Initialize barrier data.
5309 int b;
5310 for (b = 0; b < bs_last_barrier; ++b) {
5311 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5312#if USE_DEBUGGER
5313 team->t.t_bar[b].b_master_arrived = 0;
5314 team->t.t_bar[b].b_team_arrived = 0;
5315#endif
5316 }
5317 }
5318
5319#if OMP_40_ENABLED
5320 team->t.t_proc_bind = new_proc_bind;
5321#endif
5322
5323#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00005324 __ompt_team_assign_id(team, ompt_parallel_data);
Jonathan Peyton30419822017-05-12 18:01:32 +00005325 team->t.ompt_serialized_team_info = NULL;
5326#endif
5327
5328 KMP_MB();
5329
5330 KA_TRACE(20, ("__kmp_allocate_team: done creating a new team %d.\n",
5331 team->t.t_id));
5332
5333 return team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005334}
5335
5336/* TODO implement hot-teams at all levels */
5337/* TODO implement lazy thread release on demand (disband request) */
5338
5339/* free the team. return it to the team pool. release all the threads
5340 * associated with it */
Jonathan Peyton30419822017-05-12 18:01:32 +00005341void __kmp_free_team(kmp_root_t *root,
5342 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) {
5343 int f;
5344 KA_TRACE(20, ("__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(),
5345 team->t.t_id));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005346
Jonathan Peyton30419822017-05-12 18:01:32 +00005347 /* verify state */
5348 KMP_DEBUG_ASSERT(root);
5349 KMP_DEBUG_ASSERT(team);
5350 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc);
5351 KMP_DEBUG_ASSERT(team->t.t_threads);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005352
Jonathan Peyton30419822017-05-12 18:01:32 +00005353 int use_hot_team = team == root->r.r_hot_team;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005354#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00005355 int level;
5356 kmp_hot_team_ptr_t *hot_teams;
5357 if (master) {
5358 level = team->t.t_active_level - 1;
5359 if (master->th.th_teams_microtask) { // in teams construct?
5360 if (master->th.th_teams_size.nteams > 1) {
5361 ++level; // level was not increased in teams construct for
5362 // team_of_masters
5363 }
5364 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
5365 master->th.th_teams_level == team->t.t_level) {
5366 ++level; // level was not increased in teams construct for
5367 // team_of_workers before the parallel
5368 } // team->t.t_level will be increased inside parallel
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005369 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005370 hot_teams = master->th.th_hot_teams;
5371 if (level < __kmp_hot_teams_max_level) {
5372 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team);
5373 use_hot_team = 1;
5374 }
5375 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005376#endif // KMP_NESTED_HOT_TEAMS
5377
Jonathan Peyton30419822017-05-12 18:01:32 +00005378 /* team is done working */
5379 TCW_SYNC_PTR(team->t.t_pkfn,
5380 NULL); // Important for Debugging Support Library.
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00005381#if KMP_OS_WINDOWS
Jonathan Peyton30419822017-05-12 18:01:32 +00005382 team->t.t_copyin_counter = 0; // init counter for possible reuse
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00005383#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00005384 // Do not reset pointer to parent team to NULL for hot teams.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005385
Jonathan Peyton30419822017-05-12 18:01:32 +00005386 /* if we are non-hot team, release our threads */
5387 if (!use_hot_team) {
5388 if (__kmp_tasking_mode != tskm_immediate_exec) {
5389 // Wait for threads to reach reapable state
5390 for (f = 1; f < team->t.t_nproc; ++f) {
5391 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5392 kmp_info_t *th = team->t.t_threads[f];
5393 volatile kmp_uint32 *state = &th->th.th_reap_state;
5394 while (*state != KMP_SAFE_TO_REAP) {
Andrey Churbanov581490e2017-02-06 18:53:32 +00005395#if KMP_OS_WINDOWS
Jonathan Peyton30419822017-05-12 18:01:32 +00005396 // On Windows a thread can be killed at any time, check this
5397 DWORD ecode;
5398 if (!__kmp_is_thread_alive(th, &ecode)) {
5399 *state = KMP_SAFE_TO_REAP; // reset the flag for dead thread
5400 break;
5401 }
Andrey Churbanov581490e2017-02-06 18:53:32 +00005402#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00005403 // first check if thread is sleeping
5404 kmp_flag_64 fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th);
5405 if (fl.is_sleeping())
5406 fl.resume(__kmp_gtid_from_thread(th));
5407 KMP_CPU_PAUSE();
5408 }
5409 }
Andrey Churbanov581490e2017-02-06 18:53:32 +00005410
Jonathan Peyton30419822017-05-12 18:01:32 +00005411 // Delete task teams
5412 int tt_idx;
5413 for (tt_idx = 0; tt_idx < 2; ++tt_idx) {
5414 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
5415 if (task_team != NULL) {
5416 for (f = 0; f < team->t.t_nproc;
5417 ++f) { // Have all threads unref task teams
5418 team->t.t_threads[f]->th.th_task_team = NULL;
5419 }
5420 KA_TRACE(
5421 20,
5422 ("__kmp_free_team: T#%d deactivating task_team %p on team %d\n",
5423 __kmp_get_gtid(), task_team, team->t.t_id));
Jonathan Peytonbaaccfa2015-11-16 22:48:41 +00005424#if KMP_NESTED_HOT_TEAMS
Jonathan Peyton30419822017-05-12 18:01:32 +00005425 __kmp_free_task_team(master, task_team);
Jonathan Peytonbaaccfa2015-11-16 22:48:41 +00005426#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00005427 team->t.t_task_team[tt_idx] = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005428 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005429 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005430 }
5431
Jonathan Peyton30419822017-05-12 18:01:32 +00005432 // Reset pointer to parent team only for non-hot teams.
5433 team->t.t_parent = NULL;
5434 team->t.t_level = 0;
5435 team->t.t_active_level = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005436
Jonathan Peyton30419822017-05-12 18:01:32 +00005437 /* free the worker threads */
5438 for (f = 1; f < team->t.t_nproc; ++f) {
5439 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5440 __kmp_free_thread(team->t.t_threads[f]);
5441 team->t.t_threads[f] = NULL;
5442 }
5443
5444 /* put the team back in the team pool */
5445 /* TODO limit size of team pool, call reap_team if pool too large */
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00005446 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool);
Jonathan Peyton30419822017-05-12 18:01:32 +00005447 __kmp_team_pool = (volatile kmp_team_t *)team;
5448 }
5449
5450 KMP_MB();
5451}
Jim Cownie5e8470a2013-09-27 10:38:44 +00005452
5453/* reap the team. destroy it, reclaim all its resources and free its memory */
Jonathan Peyton30419822017-05-12 18:01:32 +00005454kmp_team_t *__kmp_reap_team(kmp_team_t *team) {
5455 kmp_team_t *next_pool = team->t.t_next_pool;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005456
Jonathan Peyton30419822017-05-12 18:01:32 +00005457 KMP_DEBUG_ASSERT(team);
5458 KMP_DEBUG_ASSERT(team->t.t_dispatch);
5459 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
5460 KMP_DEBUG_ASSERT(team->t.t_threads);
5461 KMP_DEBUG_ASSERT(team->t.t_argv);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005462
Jonathan Peyton30419822017-05-12 18:01:32 +00005463 /* TODO clean the threads that are a part of this? */
Jim Cownie5e8470a2013-09-27 10:38:44 +00005464
Jonathan Peyton30419822017-05-12 18:01:32 +00005465 /* free stuff */
5466 __kmp_free_team_arrays(team);
5467 if (team->t.t_argv != &team->t.t_inline_argv[0])
5468 __kmp_free((void *)team->t.t_argv);
5469 __kmp_free(team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005470
Jonathan Peyton30419822017-05-12 18:01:32 +00005471 KMP_MB();
5472 return next_pool;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005473}
5474
Jim Cownie5e8470a2013-09-27 10:38:44 +00005475// Free the thread. Don't reap it, just place it on the pool of available
5476// threads.
5477//
5478// Changes for Quad issue 527845: We need a predictable OMP tid <-> gtid
5479// binding for the affinity mechanism to be useful.
5480//
5481// Now, we always keep the free list (__kmp_thread_pool) sorted by gtid.
5482// However, we want to avoid a potential performance problem by always
5483// scanning through the list to find the correct point at which to insert
5484// the thread (potential N**2 behavior). To do this we keep track of the
5485// last place a thread struct was inserted (__kmp_thread_pool_insert_pt).
5486// With single-level parallelism, threads will always be added to the tail
5487// of the list, kept track of by __kmp_thread_pool_insert_pt. With nested
5488// parallelism, all bets are off and we may need to scan through the entire
5489// free list.
5490//
5491// This change also has a potentially large performance benefit, for some
5492// applications. Previously, as threads were freed from the hot team, they
5493// would be placed back on the free list in inverse order. If the hot team
5494// grew back to it's original size, then the freed thread would be placed
5495// back on the hot team in reverse order. This could cause bad cache
5496// locality problems on programs where the size of the hot team regularly
5497// grew and shrunk.
5498//
5499// Now, for single-level parallelism, the OMP tid is alway == gtid.
Jonathan Peyton30419822017-05-12 18:01:32 +00005500void __kmp_free_thread(kmp_info_t *this_th) {
5501 int gtid;
5502 kmp_info_t **scan;
Jonathan Peytonf4392462017-07-27 20:58:41 +00005503 kmp_root_t *root = this_th->th.th_root;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005504
Jonathan Peyton30419822017-05-12 18:01:32 +00005505 KA_TRACE(20, ("__kmp_free_thread: T#%d putting T#%d back on free pool.\n",
5506 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005507
Jonathan Peyton30419822017-05-12 18:01:32 +00005508 KMP_DEBUG_ASSERT(this_th);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005509
Jonathan Peyton30419822017-05-12 18:01:32 +00005510 // When moving thread to pool, switch thread to wait on own b_go flag, and
5511 // uninitialized (NULL team).
5512 int b;
5513 kmp_balign_t *balign = this_th->th.th_bar;
5514 for (b = 0; b < bs_last_barrier; ++b) {
5515 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG)
5516 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
5517 balign[b].bb.team = NULL;
5518 balign[b].bb.leaf_kids = 0;
5519 }
5520 this_th->th.th_task_state = 0;
Andrey Churbanov3336aa02018-03-19 18:05:15 +00005521 this_th->th.th_reap_state = KMP_SAFE_TO_REAP;
Jonathan Peyton30419822017-05-12 18:01:32 +00005522
5523 /* put thread back on the free pool */
5524 TCW_PTR(this_th->th.th_team, NULL);
5525 TCW_PTR(this_th->th.th_root, NULL);
5526 TCW_PTR(this_th->th.th_dispatch, NULL); /* NOT NEEDED */
5527
Jonathan Peytonbff8ded2018-01-10 18:24:09 +00005528 /* If the implicit task assigned to this thread can be used by other threads
5529 * -> multiple threads can share the data and try to free the task at
5530 * __kmp_reap_thread at exit. This duplicate use of the task data can happen
5531 * with higher probability when hot team is disabled but can occurs even when
5532 * the hot team is enabled */
5533 __kmp_free_implicit_task(this_th);
5534 this_th->th.th_current_task = NULL;
5535
Jonathan Peyton30419822017-05-12 18:01:32 +00005536 // If the __kmp_thread_pool_insert_pt is already past the new insert
5537 // point, then we need to re-scan the entire list.
5538 gtid = this_th->th.th_info.ds.ds_gtid;
5539 if (__kmp_thread_pool_insert_pt != NULL) {
5540 KMP_DEBUG_ASSERT(__kmp_thread_pool != NULL);
5541 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) {
5542 __kmp_thread_pool_insert_pt = NULL;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005543 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005544 }
Jim Cownie4cc4bb42014-10-07 16:25:50 +00005545
Jonathan Peyton30419822017-05-12 18:01:32 +00005546 // Scan down the list to find the place to insert the thread.
5547 // scan is the address of a link in the list, possibly the address of
5548 // __kmp_thread_pool itself.
5549 //
5550 // In the absence of nested parallism, the for loop will have 0 iterations.
5551 if (__kmp_thread_pool_insert_pt != NULL) {
5552 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool);
5553 } else {
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00005554 scan = CCAST(kmp_info_t **, &__kmp_thread_pool);
Jonathan Peyton30419822017-05-12 18:01:32 +00005555 }
5556 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid);
5557 scan = &((*scan)->th.th_next_pool))
5558 ;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005559
Jonathan Peyton30419822017-05-12 18:01:32 +00005560 // Insert the new element on the list, and set __kmp_thread_pool_insert_pt
5561 // to its address.
5562 TCW_PTR(this_th->th.th_next_pool, *scan);
5563 __kmp_thread_pool_insert_pt = *scan = this_th;
5564 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) ||
5565 (this_th->th.th_info.ds.ds_gtid <
5566 this_th->th.th_next_pool->th.th_info.ds.ds_gtid));
5567 TCW_4(this_th->th.th_in_pool, TRUE);
5568 __kmp_thread_pool_nth++;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005569
Jonathan Peyton30419822017-05-12 18:01:32 +00005570 TCW_4(__kmp_nth, __kmp_nth - 1);
Jonathan Peytonf4392462017-07-27 20:58:41 +00005571 root->r.r_cg_nthreads--;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005572
5573#ifdef KMP_ADJUST_BLOCKTIME
Jonathan Peyton30419822017-05-12 18:01:32 +00005574 /* Adjust blocktime back to user setting or default if necessary */
5575 /* Middle initialization might never have occurred */
5576 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5577 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
5578 if (__kmp_nth <= __kmp_avail_proc) {
5579 __kmp_zero_bt = FALSE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005580 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005581 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005582#endif /* KMP_ADJUST_BLOCKTIME */
5583
Jonathan Peyton30419822017-05-12 18:01:32 +00005584 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00005585}
5586
Jim Cownie5e8470a2013-09-27 10:38:44 +00005587/* ------------------------------------------------------------------------ */
5588
Jonathan Peyton30419822017-05-12 18:01:32 +00005589void *__kmp_launch_thread(kmp_info_t *this_thr) {
5590 int gtid = this_thr->th.th_info.ds.ds_gtid;
5591 /* void *stack_data;*/
5592 kmp_team_t *(*volatile pteam);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005593
Jonathan Peyton30419822017-05-12 18:01:32 +00005594 KMP_MB();
5595 KA_TRACE(10, ("__kmp_launch_thread: T#%d start\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005596
Jonathan Peyton30419822017-05-12 18:01:32 +00005597 if (__kmp_env_consistency_check) {
5598 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid); // ATT: Memory leak?
5599 }
5600
5601#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00005602 ompt_data_t *thread_data;
5603 if (ompt_enabled.enabled) {
5604 thread_data = &(this_thr->th.ompt_thread_info.thread_data);
Jonathan Peyton3574f282018-10-04 14:57:04 +00005605 *thread_data = ompt_data_none;
Joachim Protze82e94a52017-11-01 10:08:30 +00005606
5607 this_thr->th.ompt_thread_info.state = omp_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00005608 this_thr->th.ompt_thread_info.wait_id = 0;
Joachim Protze82e94a52017-11-01 10:08:30 +00005609 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0);
5610 if (ompt_enabled.ompt_callback_thread_begin) {
5611 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)(
5612 ompt_thread_worker, thread_data);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005613 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005614 }
5615#endif
5616
Joachim Protze82e94a52017-11-01 10:08:30 +00005617#if OMPT_SUPPORT
5618 if (ompt_enabled.enabled) {
5619 this_thr->th.ompt_thread_info.state = omp_state_idle;
5620 }
5621#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00005622 /* This is the place where threads wait for work */
5623 while (!TCR_4(__kmp_global.g.g_done)) {
5624 KMP_DEBUG_ASSERT(this_thr == __kmp_threads[gtid]);
5625 KMP_MB();
5626
5627 /* wait for work to do */
5628 KA_TRACE(20, ("__kmp_launch_thread: T#%d waiting for work\n", gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00005629
Jonathan Peyton30419822017-05-12 18:01:32 +00005630 /* No tid yet since not part of a team */
5631 __kmp_fork_barrier(gtid, KMP_GTID_DNE);
5632
5633#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00005634 if (ompt_enabled.enabled) {
5635 this_thr->th.ompt_thread_info.state = omp_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00005636 }
5637#endif
5638
5639 pteam = (kmp_team_t * (*))(&this_thr->th.th_team);
5640
5641 /* have we been allocated? */
5642 if (TCR_SYNC_PTR(*pteam) && !TCR_4(__kmp_global.g.g_done)) {
Jonathan Peyton30419822017-05-12 18:01:32 +00005643 /* we were just woken up, so run our new task */
5644 if (TCR_SYNC_PTR((*pteam)->t.t_pkfn) != NULL) {
5645 int rc;
5646 KA_TRACE(20,
5647 ("__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n",
5648 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5649 (*pteam)->t.t_pkfn));
5650
5651 updateHWFPControl(*pteam);
5652
5653#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00005654 if (ompt_enabled.enabled) {
5655 this_thr->th.ompt_thread_info.state = omp_state_work_parallel;
Jonathan Peyton30419822017-05-12 18:01:32 +00005656 }
5657#endif
5658
Jonathan Peytonf0682ac2018-07-30 17:41:08 +00005659 rc = (*pteam)->t.t_invoke(gtid);
Jonathan Peyton30419822017-05-12 18:01:32 +00005660 KMP_ASSERT(rc);
5661
Jim Cownie5e8470a2013-09-27 10:38:44 +00005662 KMP_MB();
Jonathan Peyton30419822017-05-12 18:01:32 +00005663 KA_TRACE(20, ("__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n",
5664 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5665 (*pteam)->t.t_pkfn));
5666 }
Joachim Protze82e94a52017-11-01 10:08:30 +00005667#if OMPT_SUPPORT
5668 if (ompt_enabled.enabled) {
5669 /* no frame set while outside task */
Joachim Protzec255ca72017-11-05 14:11:10 +00005670 __ompt_get_task_info_object(0)->frame.exit_frame = NULL;
Joachim Protze82e94a52017-11-01 10:08:30 +00005671
5672 this_thr->th.ompt_thread_info.state = omp_state_overhead;
Jonathan Peyton30419822017-05-12 18:01:32 +00005673 }
5674#endif
Joachim Protze82e94a52017-11-01 10:08:30 +00005675 /* join barrier after parallel region */
5676 __kmp_join_barrier(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005677 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005678 }
5679 TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005680
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005681#if OMPT_SUPPORT
Joachim Protze82e94a52017-11-01 10:08:30 +00005682 if (ompt_enabled.ompt_callback_thread_end) {
5683 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(thread_data);
Jonathan Peyton30419822017-05-12 18:01:32 +00005684 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00005685#endif
5686
Jonathan Peyton30419822017-05-12 18:01:32 +00005687 this_thr->th.th_task_team = NULL;
5688 /* run the destructors for the threadprivate data for this thread */
5689 __kmp_common_destroy_gtid(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005690
Jonathan Peyton30419822017-05-12 18:01:32 +00005691 KA_TRACE(10, ("__kmp_launch_thread: T#%d done\n", gtid));
5692 KMP_MB();
5693 return this_thr;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005694}
5695
5696/* ------------------------------------------------------------------------ */
Jim Cownie5e8470a2013-09-27 10:38:44 +00005697
Jonathan Peyton30419822017-05-12 18:01:32 +00005698void __kmp_internal_end_dest(void *specific_gtid) {
5699#if KMP_COMPILER_ICC
5700#pragma warning(push)
5701#pragma warning(disable : 810) // conversion from "void *" to "int" may lose
5702// significant bits
5703#endif
5704 // Make sure no significant bits are lost
5705 int gtid = (kmp_intptr_t)specific_gtid - 1;
5706#if KMP_COMPILER_ICC
5707#pragma warning(pop)
5708#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005709
Jonathan Peyton30419822017-05-12 18:01:32 +00005710 KA_TRACE(30, ("__kmp_internal_end_dest: T#%d\n", gtid));
5711 /* NOTE: the gtid is stored as gitd+1 in the thread-local-storage
5712 * this is because 0 is reserved for the nothing-stored case */
Jim Cownie5e8470a2013-09-27 10:38:44 +00005713
Jonathan Peyton30419822017-05-12 18:01:32 +00005714 /* josh: One reason for setting the gtid specific data even when it is being
5715 destroyed by pthread is to allow gtid lookup through thread specific data
5716 (__kmp_gtid_get_specific). Some of the code, especially stat code,
5717 that gets executed in the call to __kmp_internal_end_thread, actually
5718 gets the gtid through the thread specific data. Setting it here seems
5719 rather inelegant and perhaps wrong, but allows __kmp_internal_end_thread
5720 to run smoothly.
5721 todo: get rid of this after we remove the dependence on
5722 __kmp_gtid_get_specific */
5723 if (gtid >= 0 && KMP_UBER_GTID(gtid))
5724 __kmp_gtid_set_specific(gtid);
5725#ifdef KMP_TDATA_GTID
5726 __kmp_gtid = gtid;
5727#endif
5728 __kmp_internal_end_thread(gtid);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005729}
5730
Jonathan Peyton99016992015-05-26 17:32:53 +00005731#if KMP_OS_UNIX && KMP_DYNAMIC_LIB
Jim Cownie5e8470a2013-09-27 10:38:44 +00005732
Jonathan Peyton30419822017-05-12 18:01:32 +00005733// 2009-09-08 (lev): It looks the destructor does not work. In simple test cases
5734// destructors work perfectly, but in real libomp.so I have no evidence it is
5735// ever called. However, -fini linker option in makefile.mk works fine.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005736
Jonathan Peyton30419822017-05-12 18:01:32 +00005737__attribute__((destructor)) void __kmp_internal_end_dtor(void) {
5738 __kmp_internal_end_atexit();
Jim Cownie5e8470a2013-09-27 10:38:44 +00005739}
5740
Jonathan Peyton30419822017-05-12 18:01:32 +00005741void __kmp_internal_end_fini(void) { __kmp_internal_end_atexit(); }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005742
5743#endif
5744
Jonathan Peyton30419822017-05-12 18:01:32 +00005745/* [Windows] josh: when the atexit handler is called, there may still be more
5746 than one thread alive */
5747void __kmp_internal_end_atexit(void) {
5748 KA_TRACE(30, ("__kmp_internal_end_atexit\n"));
5749 /* [Windows]
5750 josh: ideally, we want to completely shutdown the library in this atexit
5751 handler, but stat code that depends on thread specific data for gtid fails
5752 because that data becomes unavailable at some point during the shutdown, so
5753 we call __kmp_internal_end_thread instead. We should eventually remove the
5754 dependency on __kmp_get_specific_gtid in the stat code and use
5755 __kmp_internal_end_library to cleanly shutdown the library.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005756
Jonathan Peyton30419822017-05-12 18:01:32 +00005757 // TODO: Can some of this comment about GVS be removed?
5758 I suspect that the offending stat code is executed when the calling thread
5759 tries to clean up a dead root thread's data structures, resulting in GVS
5760 code trying to close the GVS structures for that thread, but since the stat
5761 code uses __kmp_get_specific_gtid to get the gtid with the assumption that
5762 the calling thread is cleaning up itself instead of another thread, it get
5763 confused. This happens because allowing a thread to unregister and cleanup
5764 another thread is a recent modification for addressing an issue.
5765 Based on the current design (20050722), a thread may end up
5766 trying to unregister another thread only if thread death does not trigger
5767 the calling of __kmp_internal_end_thread. For Linux* OS, there is the
5768 thread specific data destructor function to detect thread death. For
5769 Windows dynamic, there is DllMain(THREAD_DETACH). For Windows static, there
5770 is nothing. Thus, the workaround is applicable only for Windows static
5771 stat library. */
5772 __kmp_internal_end_library(-1);
5773#if KMP_OS_WINDOWS
5774 __kmp_close_console();
5775#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005776}
5777
Jonathan Peyton30419822017-05-12 18:01:32 +00005778static void __kmp_reap_thread(kmp_info_t *thread, int is_root) {
5779 // It is assumed __kmp_forkjoin_lock is acquired.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005780
Jonathan Peyton30419822017-05-12 18:01:32 +00005781 int gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005782
Jonathan Peyton30419822017-05-12 18:01:32 +00005783 KMP_DEBUG_ASSERT(thread != NULL);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005784
Jonathan Peyton30419822017-05-12 18:01:32 +00005785 gtid = thread->th.th_info.ds.ds_gtid;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005786
Jonathan Peyton30419822017-05-12 18:01:32 +00005787 if (!is_root) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00005788
Jonathan Peyton30419822017-05-12 18:01:32 +00005789 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
5790 /* Assume the threads are at the fork barrier here */
5791 KA_TRACE(
5792 20, ("__kmp_reap_thread: releasing T#%d from fork barrier for reap\n",
5793 gtid));
5794 /* Need release fence here to prevent seg faults for tree forkjoin barrier
5795 * (GEH) */
5796 ANNOTATE_HAPPENS_BEFORE(thread);
5797 kmp_flag_64 flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go, thread);
5798 __kmp_release_64(&flag);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005799 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005800
Jonathan Peyton30419822017-05-12 18:01:32 +00005801 // Terminate OS thread.
5802 __kmp_reap_worker(thread);
Jonathan Peyton7ca7ef02016-11-21 16:18:57 +00005803
Jonathan Peyton30419822017-05-12 18:01:32 +00005804 // The thread was killed asynchronously. If it was actively
5805 // spinning in the thread pool, decrement the global count.
5806 //
5807 // There is a small timing hole here - if the worker thread was just waking
5808 // up after sleeping in the pool, had reset it's th_active_in_pool flag but
5809 // not decremented the global counter __kmp_thread_pool_active_nth yet, then
5810 // the global counter might not get updated.
5811 //
5812 // Currently, this can only happen as the library is unloaded,
5813 // so there are no harmful side effects.
5814 if (thread->th.th_active_in_pool) {
5815 thread->th.th_active_in_pool = FALSE;
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00005816 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
5817 KMP_DEBUG_ASSERT(__kmp_thread_pool_active_nth >= 0);
Jonathan Peyton30419822017-05-12 18:01:32 +00005818 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005819
Jonathan Peyton30419822017-05-12 18:01:32 +00005820 // Decrement # of [worker] threads in the pool.
5821 KMP_DEBUG_ASSERT(__kmp_thread_pool_nth > 0);
5822 --__kmp_thread_pool_nth;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005823 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005824
Jonathan Peyton30419822017-05-12 18:01:32 +00005825 __kmp_free_implicit_task(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005826
Jonathan Peyton30419822017-05-12 18:01:32 +00005827// Free the fast memory for tasking
5828#if USE_FAST_MEMORY
5829 __kmp_free_fast_memory(thread);
5830#endif /* USE_FAST_MEMORY */
5831
5832 __kmp_suspend_uninitialize_thread(thread);
5833
5834 KMP_DEBUG_ASSERT(__kmp_threads[gtid] == thread);
5835 TCW_SYNC_PTR(__kmp_threads[gtid], NULL);
5836
5837 --__kmp_all_nth;
5838// __kmp_nth was decremented when thread is added to the pool.
Jim Cownie5e8470a2013-09-27 10:38:44 +00005839
5840#ifdef KMP_ADJUST_BLOCKTIME
Jonathan Peyton30419822017-05-12 18:01:32 +00005841 /* Adjust blocktime back to user setting or default if necessary */
5842 /* Middle initialization might never have occurred */
5843 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5844 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
5845 if (__kmp_nth <= __kmp_avail_proc) {
5846 __kmp_zero_bt = FALSE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005847 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005848 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005849#endif /* KMP_ADJUST_BLOCKTIME */
5850
Jonathan Peyton30419822017-05-12 18:01:32 +00005851 /* free the memory being used */
5852 if (__kmp_env_consistency_check) {
5853 if (thread->th.th_cons) {
5854 __kmp_free_cons_stack(thread->th.th_cons);
5855 thread->th.th_cons = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005856 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005857 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00005858
Jonathan Peyton30419822017-05-12 18:01:32 +00005859 if (thread->th.th_pri_common != NULL) {
5860 __kmp_free(thread->th.th_pri_common);
5861 thread->th.th_pri_common = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005862 }
Andrey Churbanov6d224db2015-02-10 18:37:43 +00005863
Jonathan Peyton30419822017-05-12 18:01:32 +00005864 if (thread->th.th_task_state_memo_stack != NULL) {
5865 __kmp_free(thread->th.th_task_state_memo_stack);
5866 thread->th.th_task_state_memo_stack = NULL;
5867 }
5868
5869#if KMP_USE_BGET
5870 if (thread->th.th_local.bget_data != NULL) {
5871 __kmp_finalize_bget(thread);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005872 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005873#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005874
Alp Toker98758b02014-03-02 04:12:06 +00005875#if KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00005876 if (thread->th.th_affin_mask != NULL) {
5877 KMP_CPU_FREE(thread->th.th_affin_mask);
5878 thread->th.th_affin_mask = NULL;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005879 }
Alp Toker98758b02014-03-02 04:12:06 +00005880#endif /* KMP_AFFINITY_SUPPORTED */
Jim Cownie5e8470a2013-09-27 10:38:44 +00005881
Jonathan Peytonf6399362018-07-09 17:51:13 +00005882#if KMP_USE_HIER_SCHED
5883 if (thread->th.th_hier_bar_data != NULL) {
5884 __kmp_free(thread->th.th_hier_bar_data);
5885 thread->th.th_hier_bar_data = NULL;
5886 }
5887#endif
5888
Jonathan Peyton30419822017-05-12 18:01:32 +00005889 __kmp_reap_team(thread->th.th_serial_team);
5890 thread->th.th_serial_team = NULL;
5891 __kmp_free(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00005892
Jonathan Peyton30419822017-05-12 18:01:32 +00005893 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00005894
5895} // __kmp_reap_thread
5896
Jonathan Peyton30419822017-05-12 18:01:32 +00005897static void __kmp_internal_end(void) {
5898 int i;
Jim Cownie5e8470a2013-09-27 10:38:44 +00005899
Jonathan Peyton30419822017-05-12 18:01:32 +00005900 /* First, unregister the library */
5901 __kmp_unregister_library();
Jim Cownie5e8470a2013-09-27 10:38:44 +00005902
Jonathan Peyton30419822017-05-12 18:01:32 +00005903#if KMP_OS_WINDOWS
5904 /* In Win static library, we can't tell when a root actually dies, so we
5905 reclaim the data structures for any root threads that have died but not
5906 unregistered themselves, in order to shut down cleanly.
5907 In Win dynamic library we also can't tell when a thread dies. */
5908 __kmp_reclaim_dead_roots(); // AC: moved here to always clean resources of
5909// dead roots
Jonathan Peytonb66d1aa2016-09-27 17:11:17 +00005910#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00005911
Jonathan Peyton30419822017-05-12 18:01:32 +00005912 for (i = 0; i < __kmp_threads_capacity; i++)
5913 if (__kmp_root[i])
5914 if (__kmp_root[i]->r.r_active)
5915 break;
5916 KMP_MB(); /* Flush all pending memory write invalidates. */
5917 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
5918
5919 if (i < __kmp_threads_capacity) {
5920#if KMP_USE_MONITOR
5921 // 2009-09-08 (lev): Other alive roots found. Why do we kill the monitor??
5922 KMP_MB(); /* Flush all pending memory write invalidates. */
5923
Jonathan Peyton94a114f2017-10-20 19:30:57 +00005924 // Need to check that monitor was initialized before reaping it. If we are
5925 // called form __kmp_atfork_child (which sets __kmp_init_parallel = 0), then
5926 // __kmp_monitor will appear to contain valid data, but it is only valid in
5927 // the parent process, not the child.
Jonathan Peyton30419822017-05-12 18:01:32 +00005928 // New behavior (201008): instead of keying off of the flag
5929 // __kmp_init_parallel, the monitor thread creation is keyed off
5930 // of the new flag __kmp_init_monitor.
5931 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
5932 if (TCR_4(__kmp_init_monitor)) {
5933 __kmp_reap_monitor(&__kmp_monitor);
5934 TCW_4(__kmp_init_monitor, 0);
5935 }
5936 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
5937 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n"));
5938#endif // KMP_USE_MONITOR
5939 } else {
5940/* TODO move this to cleanup code */
5941#ifdef KMP_DEBUG
5942 /* make sure that everything has properly ended */
5943 for (i = 0; i < __kmp_threads_capacity; i++) {
5944 if (__kmp_root[i]) {
5945 // KMP_ASSERT( ! KMP_UBER_GTID( i ) ); // AC:
5946 // there can be uber threads alive here
5947 KMP_ASSERT(!__kmp_root[i]->r.r_active); // TODO: can they be active?
5948 }
5949 }
5950#endif
5951
5952 KMP_MB();
5953
5954 // Reap the worker threads.
5955 // This is valid for now, but be careful if threads are reaped sooner.
5956 while (__kmp_thread_pool != NULL) { // Loop thru all the thread in the pool.
5957 // Get the next thread from the pool.
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00005958 kmp_info_t *thread = CCAST(kmp_info_t *, __kmp_thread_pool);
Jonathan Peyton30419822017-05-12 18:01:32 +00005959 __kmp_thread_pool = thread->th.th_next_pool;
5960 // Reap it.
5961 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP);
5962 thread->th.th_next_pool = NULL;
5963 thread->th.th_in_pool = FALSE;
5964 __kmp_reap_thread(thread, 0);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005965 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005966 __kmp_thread_pool_insert_pt = NULL;
5967
5968 // Reap teams.
5969 while (__kmp_team_pool != NULL) { // Loop thru all the teams in the pool.
5970 // Get the next team from the pool.
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00005971 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool);
Jonathan Peyton30419822017-05-12 18:01:32 +00005972 __kmp_team_pool = team->t.t_next_pool;
5973 // Reap it.
5974 team->t.t_next_pool = NULL;
5975 __kmp_reap_team(team);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00005976 }
Jonathan Peyton30419822017-05-12 18:01:32 +00005977
5978 __kmp_reap_task_teams();
5979
Jonathan Peytona764af62018-07-19 19:17:00 +00005980#if KMP_OS_UNIX
5981 // Threads that are not reaped should not access any resources since they
5982 // are going to be deallocated soon, so the shutdown sequence should wait
5983 // until all threads either exit the final spin-waiting loop or begin
5984 // sleeping after the given blocktime.
5985 for (i = 0; i < __kmp_threads_capacity; i++) {
5986 kmp_info_t *thr = __kmp_threads[i];
5987 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking))
5988 KMP_CPU_PAUSE();
5989 }
5990#endif
5991
Jonathan Peyton30419822017-05-12 18:01:32 +00005992 for (i = 0; i < __kmp_threads_capacity; ++i) {
5993 // TBD: Add some checking...
5994 // Something like KMP_DEBUG_ASSERT( __kmp_thread[ i ] == NULL );
5995 }
5996
5997 /* Make sure all threadprivate destructors get run by joining with all
5998 worker threads before resetting this flag */
5999 TCW_SYNC_4(__kmp_init_common, FALSE);
6000
6001 KA_TRACE(10, ("__kmp_internal_end: all workers reaped\n"));
6002 KMP_MB();
6003
6004#if KMP_USE_MONITOR
6005 // See note above: One of the possible fixes for CQ138434 / CQ140126
6006 //
6007 // FIXME: push both code fragments down and CSE them?
6008 // push them into __kmp_cleanup() ?
6009 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
6010 if (TCR_4(__kmp_init_monitor)) {
6011 __kmp_reap_monitor(&__kmp_monitor);
6012 TCW_4(__kmp_init_monitor, 0);
6013 }
6014 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
6015 KA_TRACE(10, ("__kmp_internal_end: monitor reaped\n"));
6016#endif
6017 } /* else !__kmp_global.t_active */
6018 TCW_4(__kmp_init_gtid, FALSE);
6019 KMP_MB(); /* Flush all pending memory write invalidates. */
6020
6021 __kmp_cleanup();
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006022#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00006023 ompt_fini();
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006024#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006025}
6026
Jonathan Peyton30419822017-05-12 18:01:32 +00006027void __kmp_internal_end_library(int gtid_req) {
6028 /* if we have already cleaned up, don't try again, it wouldn't be pretty */
6029 /* this shouldn't be a race condition because __kmp_internal_end() is the
6030 only place to clear __kmp_serial_init */
6031 /* we'll check this later too, after we get the lock */
6032 // 2009-09-06: We do not set g_abort without setting g_done. This check looks
6033 // redundaant, because the next check will work in any case.
6034 if (__kmp_global.g.g_abort) {
6035 KA_TRACE(11, ("__kmp_internal_end_library: abort, exiting\n"));
6036 /* TODO abort? */
6037 return;
6038 }
6039 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6040 KA_TRACE(10, ("__kmp_internal_end_library: already finished\n"));
6041 return;
6042 }
6043
6044 KMP_MB(); /* Flush all pending memory write invalidates. */
6045
6046 /* find out who we are and what we should do */
6047 {
6048 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6049 KA_TRACE(
6050 10, ("__kmp_internal_end_library: enter T#%d (%d)\n", gtid, gtid_req));
6051 if (gtid == KMP_GTID_SHUTDOWN) {
6052 KA_TRACE(10, ("__kmp_internal_end_library: !__kmp_init_runtime, system "
6053 "already shutdown\n"));
6054 return;
6055 } else if (gtid == KMP_GTID_MONITOR) {
6056 KA_TRACE(10, ("__kmp_internal_end_library: monitor thread, gtid not "
6057 "registered, or system shutdown\n"));
6058 return;
6059 } else if (gtid == KMP_GTID_DNE) {
6060 KA_TRACE(10, ("__kmp_internal_end_library: gtid not registered or system "
6061 "shutdown\n"));
6062 /* we don't know who we are, but we may still shutdown the library */
6063 } else if (KMP_UBER_GTID(gtid)) {
6064 /* unregister ourselves as an uber thread. gtid is no longer valid */
6065 if (__kmp_root[gtid]->r.r_active) {
6066 __kmp_global.g.g_abort = -1;
6067 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6068 KA_TRACE(10,
6069 ("__kmp_internal_end_library: root still active, abort T#%d\n",
6070 gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006071 return;
Jonathan Peyton30419822017-05-12 18:01:32 +00006072 } else {
6073 KA_TRACE(
6074 10,
6075 ("__kmp_internal_end_library: unregistering sibling T#%d\n", gtid));
6076 __kmp_unregister_root_current_thread(gtid);
6077 }
6078 } else {
6079/* worker threads may call this function through the atexit handler, if they
6080 * call exit() */
6081/* For now, skip the usual subsequent processing and just dump the debug buffer.
6082 TODO: do a thorough shutdown instead */
6083#ifdef DUMP_DEBUG_ON_EXIT
6084 if (__kmp_debug_buf)
6085 __kmp_dump_debug_buffer();
6086#endif
6087 return;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006088 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006089 }
6090 /* synchronize the termination process */
6091 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006092
Jonathan Peyton30419822017-05-12 18:01:32 +00006093 /* have we already finished */
6094 if (__kmp_global.g.g_abort) {
6095 KA_TRACE(10, ("__kmp_internal_end_library: abort, exiting\n"));
6096 /* TODO abort? */
6097 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6098 return;
6099 }
6100 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6101 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6102 return;
6103 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006104
Jonathan Peyton30419822017-05-12 18:01:32 +00006105 /* We need this lock to enforce mutex between this reading of
6106 __kmp_threads_capacity and the writing by __kmp_register_root.
6107 Alternatively, we can use a counter of roots that is atomically updated by
6108 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and
6109 __kmp_internal_end_*. */
6110 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006111
Jonathan Peyton30419822017-05-12 18:01:32 +00006112 /* now we can safely conduct the actual termination */
6113 __kmp_internal_end();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006114
Jonathan Peyton30419822017-05-12 18:01:32 +00006115 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6116 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006117
Jonathan Peyton30419822017-05-12 18:01:32 +00006118 KA_TRACE(10, ("__kmp_internal_end_library: exit\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006119
Jonathan Peyton30419822017-05-12 18:01:32 +00006120#ifdef DUMP_DEBUG_ON_EXIT
6121 if (__kmp_debug_buf)
6122 __kmp_dump_debug_buffer();
6123#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006124
Jonathan Peyton30419822017-05-12 18:01:32 +00006125#if KMP_OS_WINDOWS
6126 __kmp_close_console();
6127#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006128
Jonathan Peyton30419822017-05-12 18:01:32 +00006129 __kmp_fini_allocator();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006130
6131} // __kmp_internal_end_library
6132
Jonathan Peyton30419822017-05-12 18:01:32 +00006133void __kmp_internal_end_thread(int gtid_req) {
6134 int i;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006135
Jonathan Peyton30419822017-05-12 18:01:32 +00006136 /* if we have already cleaned up, don't try again, it wouldn't be pretty */
6137 /* this shouldn't be a race condition because __kmp_internal_end() is the
6138 * only place to clear __kmp_serial_init */
6139 /* we'll check this later too, after we get the lock */
6140 // 2009-09-06: We do not set g_abort without setting g_done. This check looks
6141 // redundant, because the next check will work in any case.
6142 if (__kmp_global.g.g_abort) {
6143 KA_TRACE(11, ("__kmp_internal_end_thread: abort, exiting\n"));
6144 /* TODO abort? */
6145 return;
6146 }
6147 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6148 KA_TRACE(10, ("__kmp_internal_end_thread: already finished\n"));
6149 return;
6150 }
6151
6152 KMP_MB(); /* Flush all pending memory write invalidates. */
6153
6154 /* find out who we are and what we should do */
6155 {
6156 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6157 KA_TRACE(10,
6158 ("__kmp_internal_end_thread: enter T#%d (%d)\n", gtid, gtid_req));
6159 if (gtid == KMP_GTID_SHUTDOWN) {
6160 KA_TRACE(10, ("__kmp_internal_end_thread: !__kmp_init_runtime, system "
6161 "already shutdown\n"));
6162 return;
6163 } else if (gtid == KMP_GTID_MONITOR) {
6164 KA_TRACE(10, ("__kmp_internal_end_thread: monitor thread, gtid not "
6165 "registered, or system shutdown\n"));
6166 return;
6167 } else if (gtid == KMP_GTID_DNE) {
6168 KA_TRACE(10, ("__kmp_internal_end_thread: gtid not registered or system "
6169 "shutdown\n"));
6170 return;
6171 /* we don't know who we are */
6172 } else if (KMP_UBER_GTID(gtid)) {
6173 /* unregister ourselves as an uber thread. gtid is no longer valid */
6174 if (__kmp_root[gtid]->r.r_active) {
6175 __kmp_global.g.g_abort = -1;
6176 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6177 KA_TRACE(10,
6178 ("__kmp_internal_end_thread: root still active, abort T#%d\n",
6179 gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006180 return;
Jonathan Peyton30419822017-05-12 18:01:32 +00006181 } else {
6182 KA_TRACE(10, ("__kmp_internal_end_thread: unregistering sibling T#%d\n",
6183 gtid));
6184 __kmp_unregister_root_current_thread(gtid);
6185 }
6186 } else {
6187 /* just a worker thread, let's leave */
6188 KA_TRACE(10, ("__kmp_internal_end_thread: worker thread T#%d\n", gtid));
6189
6190 if (gtid >= 0) {
6191 __kmp_threads[gtid]->th.th_task_team = NULL;
6192 }
6193
6194 KA_TRACE(10,
6195 ("__kmp_internal_end_thread: worker thread done, exiting T#%d\n",
6196 gtid));
6197 return;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006198 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006199 }
Jonathan Peyton8b3842f2018-10-05 17:59:39 +00006200#if KMP_DYNAMIC_LIB
Jonathan Peyton30419822017-05-12 18:01:32 +00006201 // AC: lets not shutdown the Linux* OS dynamic library at the exit of uber
6202 // thread, because we will better shutdown later in the library destructor.
6203 // The reason of this change is performance problem when non-openmp thread in
6204 // a loop forks and joins many openmp threads. We can save a lot of time
6205 // keeping worker threads alive until the program shutdown.
6206 // OM: Removed Linux* OS restriction to fix the crash on OS X* (DPD200239966)
6207 // and Windows(DPD200287443) that occurs when using critical sections from
6208 // foreign threads.
6209 KA_TRACE(10, ("__kmp_internal_end_thread: exiting T#%d\n", gtid_req));
6210 return;
6211#endif
6212 /* synchronize the termination process */
6213 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006214
Jonathan Peyton30419822017-05-12 18:01:32 +00006215 /* have we already finished */
6216 if (__kmp_global.g.g_abort) {
6217 KA_TRACE(10, ("__kmp_internal_end_thread: abort, exiting\n"));
6218 /* TODO abort? */
6219 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6220 return;
6221 }
6222 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6223 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6224 return;
6225 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006226
Jonathan Peyton30419822017-05-12 18:01:32 +00006227 /* We need this lock to enforce mutex between this reading of
6228 __kmp_threads_capacity and the writing by __kmp_register_root.
6229 Alternatively, we can use a counter of roots that is atomically updated by
6230 __kmp_get_global_thread_id_reg, __kmp_do_serial_initialize and
6231 __kmp_internal_end_*. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006232
Jonathan Peyton30419822017-05-12 18:01:32 +00006233 /* should we finish the run-time? are all siblings done? */
6234 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006235
Jonathan Peyton30419822017-05-12 18:01:32 +00006236 for (i = 0; i < __kmp_threads_capacity; ++i) {
6237 if (KMP_UBER_GTID(i)) {
6238 KA_TRACE(
6239 10,
6240 ("__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i));
6241 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6242 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6243 return;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006244 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006245 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006246
Jonathan Peyton30419822017-05-12 18:01:32 +00006247 /* now we can safely conduct the actual termination */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006248
Jonathan Peyton30419822017-05-12 18:01:32 +00006249 __kmp_internal_end();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006250
Jonathan Peyton30419822017-05-12 18:01:32 +00006251 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6252 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006253
Jonathan Peyton30419822017-05-12 18:01:32 +00006254 KA_TRACE(10, ("__kmp_internal_end_thread: exit T#%d\n", gtid_req));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006255
Jonathan Peyton30419822017-05-12 18:01:32 +00006256#ifdef DUMP_DEBUG_ON_EXIT
6257 if (__kmp_debug_buf)
6258 __kmp_dump_debug_buffer();
6259#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006260} // __kmp_internal_end_thread
6261
Jonathan Peyton30419822017-05-12 18:01:32 +00006262// -----------------------------------------------------------------------------
Jim Cownie5e8470a2013-09-27 10:38:44 +00006263// Library registration stuff.
6264
Jonathan Peyton30419822017-05-12 18:01:32 +00006265static long __kmp_registration_flag = 0;
6266// Random value used to indicate library initialization.
6267static char *__kmp_registration_str = NULL;
6268// Value to be saved in env var __KMP_REGISTERED_LIB_<pid>.
Jim Cownie5e8470a2013-09-27 10:38:44 +00006269
Jonathan Peyton30419822017-05-12 18:01:32 +00006270static inline char *__kmp_reg_status_name() {
6271 /* On RHEL 3u5 if linked statically, getpid() returns different values in
6272 each thread. If registration and unregistration go in different threads
6273 (omp_misc_other_root_exit.cpp test case), the name of registered_lib_env
6274 env var can not be found, because the name will contain different pid. */
6275 return __kmp_str_format("__KMP_REGISTERED_LIB_%d", (int)getpid());
Jim Cownie5e8470a2013-09-27 10:38:44 +00006276} // __kmp_reg_status_get
6277
Jonathan Peyton30419822017-05-12 18:01:32 +00006278void __kmp_register_library_startup(void) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006279
Jonathan Peyton30419822017-05-12 18:01:32 +00006280 char *name = __kmp_reg_status_name(); // Name of the environment variable.
6281 int done = 0;
6282 union {
6283 double dtime;
6284 long ltime;
6285 } time;
6286#if KMP_ARCH_X86 || KMP_ARCH_X86_64
6287 __kmp_initialize_system_tick();
6288#endif
6289 __kmp_read_system_time(&time.dtime);
6290 __kmp_registration_flag = 0xCAFE0000L | (time.ltime & 0x0000FFFFL);
6291 __kmp_registration_str =
6292 __kmp_str_format("%p-%lx-%s", &__kmp_registration_flag,
6293 __kmp_registration_flag, KMP_LIBRARY_FILE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006294
Jonathan Peyton30419822017-05-12 18:01:32 +00006295 KA_TRACE(50, ("__kmp_register_library_startup: %s=\"%s\"\n", name,
6296 __kmp_registration_str));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006297
Jonathan Peyton30419822017-05-12 18:01:32 +00006298 while (!done) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006299
Jonathan Peyton30419822017-05-12 18:01:32 +00006300 char *value = NULL; // Actual value of the environment variable.
Jim Cownie5e8470a2013-09-27 10:38:44 +00006301
Jonathan Peyton30419822017-05-12 18:01:32 +00006302 // Set environment variable, but do not overwrite if it is exist.
6303 __kmp_env_set(name, __kmp_registration_str, 0);
6304 // Check the variable is written.
6305 value = __kmp_env_get(name);
6306 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006307
Jonathan Peyton30419822017-05-12 18:01:32 +00006308 done = 1; // Ok, environment variable set successfully, exit the loop.
Jim Cownie5e8470a2013-09-27 10:38:44 +00006309
Jonathan Peyton30419822017-05-12 18:01:32 +00006310 } else {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006311
Jonathan Peyton30419822017-05-12 18:01:32 +00006312 // Oops. Write failed. Another copy of OpenMP RTL is in memory.
6313 // Check whether it alive or dead.
6314 int neighbor = 0; // 0 -- unknown status, 1 -- alive, 2 -- dead.
6315 char *tail = value;
6316 char *flag_addr_str = NULL;
6317 char *flag_val_str = NULL;
6318 char const *file_name = NULL;
6319 __kmp_str_split(tail, '-', &flag_addr_str, &tail);
6320 __kmp_str_split(tail, '-', &flag_val_str, &tail);
6321 file_name = tail;
6322 if (tail != NULL) {
6323 long *flag_addr = 0;
6324 long flag_val = 0;
Jonathan Peytonbaad3f62018-08-09 22:04:30 +00006325 KMP_SSCANF(flag_addr_str, "%p", RCAST(void**, &flag_addr));
Jonathan Peyton30419822017-05-12 18:01:32 +00006326 KMP_SSCANF(flag_val_str, "%lx", &flag_val);
6327 if (flag_addr != 0 && flag_val != 0 && strcmp(file_name, "") != 0) {
6328 // First, check whether environment-encoded address is mapped into
6329 // addr space.
6330 // If so, dereference it to see if it still has the right value.
6331 if (__kmp_is_address_mapped(flag_addr) && *flag_addr == flag_val) {
6332 neighbor = 1;
6333 } else {
6334 // If not, then we know the other copy of the library is no longer
6335 // running.
6336 neighbor = 2;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006337 }
6338 }
6339 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006340 switch (neighbor) {
6341 case 0: // Cannot parse environment variable -- neighbor status unknown.
6342 // Assume it is the incompatible format of future version of the
6343 // library. Assume the other library is alive.
6344 // WARN( ... ); // TODO: Issue a warning.
6345 file_name = "unknown library";
6346 // Attention! Falling to the next case. That's intentional.
6347 case 1: { // Neighbor is alive.
6348 // Check it is allowed.
6349 char *duplicate_ok = __kmp_env_get("KMP_DUPLICATE_LIB_OK");
6350 if (!__kmp_str_match_true(duplicate_ok)) {
6351 // That's not allowed. Issue fatal error.
Jonathan Peyton6a393f72017-09-05 15:43:58 +00006352 __kmp_fatal(KMP_MSG(DuplicateLibrary, KMP_LIBRARY_FILE, file_name),
6353 KMP_HNT(DuplicateLibrary), __kmp_msg_null);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006354 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006355 KMP_INTERNAL_FREE(duplicate_ok);
6356 __kmp_duplicate_library_ok = 1;
6357 done = 1; // Exit the loop.
6358 } break;
6359 case 2: { // Neighbor is dead.
6360 // Clear the variable and try to register library again.
6361 __kmp_env_unset(name);
6362 } break;
6363 default: { KMP_DEBUG_ASSERT(0); } break;
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006364 }
6365 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006366 KMP_INTERNAL_FREE((void *)value);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006367 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006368 KMP_INTERNAL_FREE((void *)name);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006369
6370} // func __kmp_register_library_startup
6371
Jonathan Peyton30419822017-05-12 18:01:32 +00006372void __kmp_unregister_library(void) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006373
Jonathan Peyton30419822017-05-12 18:01:32 +00006374 char *name = __kmp_reg_status_name();
6375 char *value = __kmp_env_get(name);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006376
Jonathan Peyton30419822017-05-12 18:01:32 +00006377 KMP_DEBUG_ASSERT(__kmp_registration_flag != 0);
6378 KMP_DEBUG_ASSERT(__kmp_registration_str != NULL);
6379 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
6380 // Ok, this is our variable. Delete it.
6381 __kmp_env_unset(name);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006382 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006383
Jonathan Peyton30419822017-05-12 18:01:32 +00006384 KMP_INTERNAL_FREE(__kmp_registration_str);
6385 KMP_INTERNAL_FREE(value);
6386 KMP_INTERNAL_FREE(name);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006387
Jonathan Peyton30419822017-05-12 18:01:32 +00006388 __kmp_registration_flag = 0;
6389 __kmp_registration_str = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006390
6391} // __kmp_unregister_library
6392
Jim Cownie5e8470a2013-09-27 10:38:44 +00006393// End of Library registration stuff.
Jonathan Peyton30419822017-05-12 18:01:32 +00006394// -----------------------------------------------------------------------------
Jim Cownie5e8470a2013-09-27 10:38:44 +00006395
Jonathan Peyton492e0a32017-06-13 17:17:26 +00006396#if KMP_MIC_SUPPORTED
Andrey Churbanov613edeb2015-02-20 18:14:43 +00006397
Jonathan Peyton30419822017-05-12 18:01:32 +00006398static void __kmp_check_mic_type() {
6399 kmp_cpuid_t cpuid_state = {0};
6400 kmp_cpuid_t *cs_p = &cpuid_state;
6401 __kmp_x86_cpuid(1, 0, cs_p);
6402 // We don't support mic1 at the moment
6403 if ((cs_p->eax & 0xff0) == 0xB10) {
6404 __kmp_mic_type = mic2;
6405 } else if ((cs_p->eax & 0xf0ff0) == 0x50670) {
6406 __kmp_mic_type = mic3;
6407 } else {
6408 __kmp_mic_type = non_mic;
6409 }
Andrey Churbanov613edeb2015-02-20 18:14:43 +00006410}
6411
Jonathan Peyton492e0a32017-06-13 17:17:26 +00006412#endif /* KMP_MIC_SUPPORTED */
Andrey Churbanov613edeb2015-02-20 18:14:43 +00006413
Jonathan Peyton30419822017-05-12 18:01:32 +00006414static void __kmp_do_serial_initialize(void) {
6415 int i, gtid;
6416 int size;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006417
Jonathan Peyton30419822017-05-12 18:01:32 +00006418 KA_TRACE(10, ("__kmp_do_serial_initialize: enter\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006419
Jonathan Peyton30419822017-05-12 18:01:32 +00006420 KMP_DEBUG_ASSERT(sizeof(kmp_int32) == 4);
6421 KMP_DEBUG_ASSERT(sizeof(kmp_uint32) == 4);
6422 KMP_DEBUG_ASSERT(sizeof(kmp_int64) == 8);
6423 KMP_DEBUG_ASSERT(sizeof(kmp_uint64) == 8);
6424 KMP_DEBUG_ASSERT(sizeof(kmp_intptr_t) == sizeof(void *));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006425
Jonathan Peyton82a13bf2015-09-21 18:01:02 +00006426#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00006427 ompt_pre_init();
Jonathan Peyton82a13bf2015-09-21 18:01:02 +00006428#endif
6429
Jonathan Peyton30419822017-05-12 18:01:32 +00006430 __kmp_validate_locks();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006431
Jonathan Peyton30419822017-05-12 18:01:32 +00006432 /* Initialize internal memory allocator */
6433 __kmp_init_allocator();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006434
Jonathan Peyton30419822017-05-12 18:01:32 +00006435 /* Register the library startup via an environment variable and check to see
6436 whether another copy of the library is already registered. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006437
Jonathan Peyton30419822017-05-12 18:01:32 +00006438 __kmp_register_library_startup();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006439
Jonathan Peyton30419822017-05-12 18:01:32 +00006440 /* TODO reinitialization of library */
6441 if (TCR_4(__kmp_global.g.g_done)) {
6442 KA_TRACE(10, ("__kmp_do_serial_initialize: reinitialization of library\n"));
6443 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006444
Jonathan Peyton30419822017-05-12 18:01:32 +00006445 __kmp_global.g.g_abort = 0;
6446 TCW_SYNC_4(__kmp_global.g.g_done, FALSE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006447
Jonathan Peyton30419822017-05-12 18:01:32 +00006448/* initialize the locks */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006449#if KMP_USE_ADAPTIVE_LOCKS
6450#if KMP_DEBUG_ADAPTIVE_LOCKS
Jonathan Peyton30419822017-05-12 18:01:32 +00006451 __kmp_init_speculative_stats();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006452#endif
6453#endif
Jonathan Peytonad579922015-12-17 16:19:05 +00006454#if KMP_STATS_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00006455 __kmp_stats_init();
Jonathan Peytonad579922015-12-17 16:19:05 +00006456#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006457 __kmp_init_lock(&__kmp_global_lock);
6458 __kmp_init_queuing_lock(&__kmp_dispatch_lock);
6459 __kmp_init_lock(&__kmp_debug_lock);
6460 __kmp_init_atomic_lock(&__kmp_atomic_lock);
6461 __kmp_init_atomic_lock(&__kmp_atomic_lock_1i);
6462 __kmp_init_atomic_lock(&__kmp_atomic_lock_2i);
6463 __kmp_init_atomic_lock(&__kmp_atomic_lock_4i);
6464 __kmp_init_atomic_lock(&__kmp_atomic_lock_4r);
6465 __kmp_init_atomic_lock(&__kmp_atomic_lock_8i);
6466 __kmp_init_atomic_lock(&__kmp_atomic_lock_8r);
6467 __kmp_init_atomic_lock(&__kmp_atomic_lock_8c);
6468 __kmp_init_atomic_lock(&__kmp_atomic_lock_10r);
6469 __kmp_init_atomic_lock(&__kmp_atomic_lock_16r);
6470 __kmp_init_atomic_lock(&__kmp_atomic_lock_16c);
6471 __kmp_init_atomic_lock(&__kmp_atomic_lock_20c);
6472 __kmp_init_atomic_lock(&__kmp_atomic_lock_32c);
6473 __kmp_init_bootstrap_lock(&__kmp_forkjoin_lock);
6474 __kmp_init_bootstrap_lock(&__kmp_exit_lock);
Jonathan Peytonb66d1aa2016-09-27 17:11:17 +00006475#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00006476 __kmp_init_bootstrap_lock(&__kmp_monitor_lock);
Jonathan Peytonb66d1aa2016-09-27 17:11:17 +00006477#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006478 __kmp_init_bootstrap_lock(&__kmp_tp_cached_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006479
Jonathan Peyton30419822017-05-12 18:01:32 +00006480 /* conduct initialization and initial setup of configuration */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006481
Jonathan Peyton30419822017-05-12 18:01:32 +00006482 __kmp_runtime_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006483
Jonathan Peyton492e0a32017-06-13 17:17:26 +00006484#if KMP_MIC_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00006485 __kmp_check_mic_type();
Andrey Churbanov613edeb2015-02-20 18:14:43 +00006486#endif
6487
Jonathan Peyton30419822017-05-12 18:01:32 +00006488// Some global variable initialization moved here from kmp_env_initialize()
Jim Cownie5e8470a2013-09-27 10:38:44 +00006489#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00006490 kmp_diag = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006491#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006492 __kmp_abort_delay = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006493
Jonathan Peyton30419822017-05-12 18:01:32 +00006494 // From __kmp_init_dflt_team_nth()
6495 /* assume the entire machine will be used */
6496 __kmp_dflt_team_nth_ub = __kmp_xproc;
6497 if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH) {
6498 __kmp_dflt_team_nth_ub = KMP_MIN_NTH;
6499 }
6500 if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) {
6501 __kmp_dflt_team_nth_ub = __kmp_sys_max_nth;
6502 }
6503 __kmp_max_nth = __kmp_sys_max_nth;
Jonathan Peytonf4392462017-07-27 20:58:41 +00006504 __kmp_cg_max_nth = __kmp_sys_max_nth;
Jonathan Peyton4f90c822017-08-02 20:04:45 +00006505 __kmp_teams_max_nth = __kmp_xproc; // set a "reasonable" default
6506 if (__kmp_teams_max_nth > __kmp_sys_max_nth) {
6507 __kmp_teams_max_nth = __kmp_sys_max_nth;
6508 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006509
Jonathan Peyton30419822017-05-12 18:01:32 +00006510 // Three vars below moved here from __kmp_env_initialize() "KMP_BLOCKTIME"
6511 // part
6512 __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME;
Jonathan Peytone1c7c132016-10-07 18:12:19 +00006513#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00006514 __kmp_monitor_wakeups =
6515 KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
6516 __kmp_bt_intervals =
6517 KMP_INTERVALS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
Jonathan Peytone1c7c132016-10-07 18:12:19 +00006518#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006519 // From "KMP_LIBRARY" part of __kmp_env_initialize()
6520 __kmp_library = library_throughput;
6521 // From KMP_SCHEDULE initialization
6522 __kmp_static = kmp_sch_static_balanced;
6523// AC: do not use analytical here, because it is non-monotonous
6524//__kmp_guided = kmp_sch_guided_iterative_chunked;
6525//__kmp_auto = kmp_sch_guided_analytical_chunked; // AC: it is the default, no
6526// need to repeat assignment
6527// Barrier initialization. Moved here from __kmp_env_initialize() Barrier branch
6528// bit control and barrier method control parts
Jim Cownie4cc4bb42014-10-07 16:25:50 +00006529#if KMP_FAST_REDUCTION_BARRIER
Jonathan Peyton30419822017-05-12 18:01:32 +00006530#define kmp_reduction_barrier_gather_bb ((int)1)
6531#define kmp_reduction_barrier_release_bb ((int)1)
6532#define kmp_reduction_barrier_gather_pat bp_hyper_bar
6533#define kmp_reduction_barrier_release_pat bp_hyper_bar
6534#endif // KMP_FAST_REDUCTION_BARRIER
6535 for (i = bs_plain_barrier; i < bs_last_barrier; i++) {
6536 __kmp_barrier_gather_branch_bits[i] = __kmp_barrier_gather_bb_dflt;
6537 __kmp_barrier_release_branch_bits[i] = __kmp_barrier_release_bb_dflt;
6538 __kmp_barrier_gather_pattern[i] = __kmp_barrier_gather_pat_dflt;
6539 __kmp_barrier_release_pattern[i] = __kmp_barrier_release_pat_dflt;
6540#if KMP_FAST_REDUCTION_BARRIER
6541 if (i == bs_reduction_barrier) { // tested and confirmed on ALTIX only (
6542 // lin_64 ): hyper,1
6543 __kmp_barrier_gather_branch_bits[i] = kmp_reduction_barrier_gather_bb;
6544 __kmp_barrier_release_branch_bits[i] = kmp_reduction_barrier_release_bb;
6545 __kmp_barrier_gather_pattern[i] = kmp_reduction_barrier_gather_pat;
6546 __kmp_barrier_release_pattern[i] = kmp_reduction_barrier_release_pat;
Andrey Churbanov613edeb2015-02-20 18:14:43 +00006547 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006548#endif // KMP_FAST_REDUCTION_BARRIER
6549 }
6550#if KMP_FAST_REDUCTION_BARRIER
6551#undef kmp_reduction_barrier_release_pat
6552#undef kmp_reduction_barrier_gather_pat
6553#undef kmp_reduction_barrier_release_bb
6554#undef kmp_reduction_barrier_gather_bb
6555#endif // KMP_FAST_REDUCTION_BARRIER
Jonathan Peyton492e0a32017-06-13 17:17:26 +00006556#if KMP_MIC_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00006557 if (__kmp_mic_type == mic2) { // KNC
6558 // AC: plane=3,2, forkjoin=2,1 are optimal for 240 threads on KNC
6559 __kmp_barrier_gather_branch_bits[bs_plain_barrier] = 3; // plain gather
6560 __kmp_barrier_release_branch_bits[bs_forkjoin_barrier] =
6561 1; // forkjoin release
6562 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6563 __kmp_barrier_release_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6564 }
6565#if KMP_FAST_REDUCTION_BARRIER
6566 if (__kmp_mic_type == mic2) { // KNC
6567 __kmp_barrier_gather_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6568 __kmp_barrier_release_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6569 }
Jonathan Peyton492e0a32017-06-13 17:17:26 +00006570#endif // KMP_FAST_REDUCTION_BARRIER
6571#endif // KMP_MIC_SUPPORTED
Jim Cownie5e8470a2013-09-27 10:38:44 +00006572
Jonathan Peyton30419822017-05-12 18:01:32 +00006573// From KMP_CHECKS initialization
Jim Cownie5e8470a2013-09-27 10:38:44 +00006574#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00006575 __kmp_env_checks = TRUE; /* development versions have the extra checks */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006576#else
Jonathan Peyton30419822017-05-12 18:01:32 +00006577 __kmp_env_checks = FALSE; /* port versions do not have the extra checks */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006578#endif
6579
Jonathan Peyton30419822017-05-12 18:01:32 +00006580 // From "KMP_FOREIGN_THREADS_THREADPRIVATE" initialization
6581 __kmp_foreign_tp = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006582
Jonathan Peyton30419822017-05-12 18:01:32 +00006583 __kmp_global.g.g_dynamic = FALSE;
6584 __kmp_global.g.g_dynamic_mode = dynamic_default;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006585
Jonathan Peyton30419822017-05-12 18:01:32 +00006586 __kmp_env_initialize(NULL);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00006587
Jonathan Peyton30419822017-05-12 18:01:32 +00006588// Print all messages in message catalog for testing purposes.
6589#ifdef KMP_DEBUG
6590 char const *val = __kmp_env_get("KMP_DUMP_CATALOG");
6591 if (__kmp_str_match_true(val)) {
6592 kmp_str_buf_t buffer;
6593 __kmp_str_buf_init(&buffer);
6594 __kmp_i18n_dump_catalog(&buffer);
6595 __kmp_printf("%s", buffer.str);
6596 __kmp_str_buf_free(&buffer);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006597 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006598 __kmp_env_free(&val);
6599#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006600
Jonathan Peyton30419822017-05-12 18:01:32 +00006601 __kmp_threads_capacity =
6602 __kmp_initial_threads_capacity(__kmp_dflt_team_nth_ub);
6603 // Moved here from __kmp_env_initialize() "KMP_ALL_THREADPRIVATE" part
6604 __kmp_tp_capacity = __kmp_default_tp_capacity(
6605 __kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006606
Jonathan Peyton30419822017-05-12 18:01:32 +00006607 // If the library is shut down properly, both pools must be NULL. Just in
6608 // case, set them to NULL -- some memory may leak, but subsequent code will
6609 // work even if pools are not freed.
6610 KMP_DEBUG_ASSERT(__kmp_thread_pool == NULL);
6611 KMP_DEBUG_ASSERT(__kmp_thread_pool_insert_pt == NULL);
6612 KMP_DEBUG_ASSERT(__kmp_team_pool == NULL);
6613 __kmp_thread_pool = NULL;
6614 __kmp_thread_pool_insert_pt = NULL;
6615 __kmp_team_pool = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006616
Jonathan Peyton30419822017-05-12 18:01:32 +00006617 /* Allocate all of the variable sized records */
6618 /* NOTE: __kmp_threads_capacity entries are allocated, but the arrays are
6619 * expandable */
6620 /* Since allocation is cache-aligned, just add extra padding at the end */
6621 size =
6622 (sizeof(kmp_info_t *) + sizeof(kmp_root_t *)) * __kmp_threads_capacity +
6623 CACHE_LINE;
6624 __kmp_threads = (kmp_info_t **)__kmp_allocate(size);
6625 __kmp_root = (kmp_root_t **)((char *)__kmp_threads +
6626 sizeof(kmp_info_t *) * __kmp_threads_capacity);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006627
Jonathan Peyton30419822017-05-12 18:01:32 +00006628 /* init thread counts */
6629 KMP_DEBUG_ASSERT(__kmp_all_nth ==
6630 0); // Asserts fail if the library is reinitializing and
6631 KMP_DEBUG_ASSERT(__kmp_nth == 0); // something was wrong in termination.
6632 __kmp_all_nth = 0;
6633 __kmp_nth = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006634
Jonathan Peyton30419822017-05-12 18:01:32 +00006635 /* setup the uber master thread and hierarchy */
6636 gtid = __kmp_register_root(TRUE);
6637 KA_TRACE(10, ("__kmp_do_serial_initialize T#%d\n", gtid));
6638 KMP_ASSERT(KMP_UBER_GTID(gtid));
6639 KMP_ASSERT(KMP_INITIAL_GTID(gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006640
Jonathan Peyton30419822017-05-12 18:01:32 +00006641 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006642
Jonathan Peyton30419822017-05-12 18:01:32 +00006643 __kmp_common_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006644
Jonathan Peyton30419822017-05-12 18:01:32 +00006645#if KMP_OS_UNIX
6646 /* invoke the child fork handler */
6647 __kmp_register_atfork();
6648#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006649
Jonathan Peyton8b3842f2018-10-05 17:59:39 +00006650#if !KMP_DYNAMIC_LIB
Jonathan Peyton30419822017-05-12 18:01:32 +00006651 {
6652 /* Invoke the exit handler when the program finishes, only for static
6653 library. For dynamic library, we already have _fini and DllMain. */
6654 int rc = atexit(__kmp_internal_end_atexit);
6655 if (rc != 0) {
Jonathan Peyton6a393f72017-09-05 15:43:58 +00006656 __kmp_fatal(KMP_MSG(FunctionError, "atexit()"), KMP_ERR(rc),
6657 __kmp_msg_null);
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00006658 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006659 }
6660#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006661
Jonathan Peyton30419822017-05-12 18:01:32 +00006662#if KMP_HANDLE_SIGNALS
6663#if KMP_OS_UNIX
6664 /* NOTE: make sure that this is called before the user installs their own
6665 signal handlers so that the user handlers are called first. this way they
6666 can return false, not call our handler, avoid terminating the library, and
6667 continue execution where they left off. */
6668 __kmp_install_signals(FALSE);
6669#endif /* KMP_OS_UNIX */
6670#if KMP_OS_WINDOWS
6671 __kmp_install_signals(TRUE);
6672#endif /* KMP_OS_WINDOWS */
6673#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006674
Jonathan Peyton30419822017-05-12 18:01:32 +00006675 /* we have finished the serial initialization */
6676 __kmp_init_counter++;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006677
Jonathan Peyton30419822017-05-12 18:01:32 +00006678 __kmp_init_serial = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006679
Jonathan Peyton30419822017-05-12 18:01:32 +00006680 if (__kmp_settings) {
6681 __kmp_env_print();
6682 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006683
6684#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00006685 if (__kmp_display_env || __kmp_display_env_verbose) {
6686 __kmp_env_print_2();
6687 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006688#endif // OMP_40_ENABLED
6689
Jonathan Peyton82a13bf2015-09-21 18:01:02 +00006690#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00006691 ompt_post_init();
Jonathan Peyton82a13bf2015-09-21 18:01:02 +00006692#endif
6693
Jonathan Peyton30419822017-05-12 18:01:32 +00006694 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006695
Jonathan Peyton30419822017-05-12 18:01:32 +00006696 KA_TRACE(10, ("__kmp_do_serial_initialize: exit\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006697}
6698
Jonathan Peyton30419822017-05-12 18:01:32 +00006699void __kmp_serial_initialize(void) {
6700 if (__kmp_init_serial) {
6701 return;
6702 }
6703 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6704 if (__kmp_init_serial) {
6705 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6706 return;
6707 }
6708 __kmp_do_serial_initialize();
6709 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6710}
6711
6712static void __kmp_do_middle_initialize(void) {
6713 int i, j;
6714 int prev_dflt_team_nth;
6715
6716 if (!__kmp_init_serial) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006717 __kmp_do_serial_initialize();
Jonathan Peyton30419822017-05-12 18:01:32 +00006718 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006719
Jonathan Peyton30419822017-05-12 18:01:32 +00006720 KA_TRACE(10, ("__kmp_middle_initialize: enter\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006721
Jonathan Peyton30419822017-05-12 18:01:32 +00006722 // Save the previous value for the __kmp_dflt_team_nth so that
6723 // we can avoid some reinitialization if it hasn't changed.
6724 prev_dflt_team_nth = __kmp_dflt_team_nth;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006725
Alp Toker98758b02014-03-02 04:12:06 +00006726#if KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00006727 // __kmp_affinity_initialize() will try to set __kmp_ncores to the
6728 // number of cores on the machine.
6729 __kmp_affinity_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006730
Jonathan Peyton30419822017-05-12 18:01:32 +00006731 // Run through the __kmp_threads array and set the affinity mask
6732 // for each root thread that is currently registered with the RTL.
6733 for (i = 0; i < __kmp_threads_capacity; i++) {
6734 if (TCR_PTR(__kmp_threads[i]) != NULL) {
6735 __kmp_affinity_set_init_mask(i, TRUE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006736 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006737 }
Alp Toker98758b02014-03-02 04:12:06 +00006738#endif /* KMP_AFFINITY_SUPPORTED */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006739
Jonathan Peyton30419822017-05-12 18:01:32 +00006740 KMP_ASSERT(__kmp_xproc > 0);
6741 if (__kmp_avail_proc == 0) {
6742 __kmp_avail_proc = __kmp_xproc;
6743 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006744
Jonathan Peyton30419822017-05-12 18:01:32 +00006745 // If there were empty places in num_threads list (OMP_NUM_THREADS=,,2,3),
6746 // correct them now
6747 j = 0;
6748 while ((j < __kmp_nested_nth.used) && !__kmp_nested_nth.nth[j]) {
6749 __kmp_nested_nth.nth[j] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub =
6750 __kmp_avail_proc;
6751 j++;
6752 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006753
Jonathan Peyton30419822017-05-12 18:01:32 +00006754 if (__kmp_dflt_team_nth == 0) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006755#ifdef KMP_DFLT_NTH_CORES
Jonathan Peyton30419822017-05-12 18:01:32 +00006756 // Default #threads = #cores
6757 __kmp_dflt_team_nth = __kmp_ncores;
6758 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
6759 "__kmp_ncores (%d)\n",
6760 __kmp_dflt_team_nth));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006761#else
Jonathan Peyton30419822017-05-12 18:01:32 +00006762 // Default #threads = #available OS procs
6763 __kmp_dflt_team_nth = __kmp_avail_proc;
6764 KA_TRACE(20, ("__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
6765 "__kmp_avail_proc(%d)\n",
6766 __kmp_dflt_team_nth));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006767#endif /* KMP_DFLT_NTH_CORES */
Jonathan Peyton30419822017-05-12 18:01:32 +00006768 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006769
Jonathan Peyton30419822017-05-12 18:01:32 +00006770 if (__kmp_dflt_team_nth < KMP_MIN_NTH) {
6771 __kmp_dflt_team_nth = KMP_MIN_NTH;
6772 }
6773 if (__kmp_dflt_team_nth > __kmp_sys_max_nth) {
6774 __kmp_dflt_team_nth = __kmp_sys_max_nth;
6775 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006776
Jonathan Peyton30419822017-05-12 18:01:32 +00006777 // There's no harm in continuing if the following check fails,
6778 // but it indicates an error in the previous logic.
6779 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006780
Jonathan Peyton30419822017-05-12 18:01:32 +00006781 if (__kmp_dflt_team_nth != prev_dflt_team_nth) {
6782 // Run through the __kmp_threads array and set the num threads icv for each
6783 // root thread that is currently registered with the RTL (which has not
6784 // already explicitly set its nthreads-var with a call to
6785 // omp_set_num_threads()).
6786 for (i = 0; i < __kmp_threads_capacity; i++) {
6787 kmp_info_t *thread = __kmp_threads[i];
6788 if (thread == NULL)
6789 continue;
6790 if (thread->th.th_current_task->td_icvs.nproc != 0)
6791 continue;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006792
Jonathan Peyton30419822017-05-12 18:01:32 +00006793 set__nproc(__kmp_threads[i], __kmp_dflt_team_nth);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006794 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006795 }
6796 KA_TRACE(
6797 20,
6798 ("__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n",
6799 __kmp_dflt_team_nth));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006800
6801#ifdef KMP_ADJUST_BLOCKTIME
Jonathan Peyton30419822017-05-12 18:01:32 +00006802 /* Adjust blocktime to zero if necessary now that __kmp_avail_proc is set */
6803 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
6804 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
6805 if (__kmp_nth > __kmp_avail_proc) {
6806 __kmp_zero_bt = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006807 }
Jonathan Peyton30419822017-05-12 18:01:32 +00006808 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006809#endif /* KMP_ADJUST_BLOCKTIME */
6810
Jonathan Peyton30419822017-05-12 18:01:32 +00006811 /* we have finished middle initialization */
6812 TCW_SYNC_4(__kmp_init_middle, TRUE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006813
Jonathan Peyton30419822017-05-12 18:01:32 +00006814 KA_TRACE(10, ("__kmp_do_middle_initialize: exit\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006815}
6816
Jonathan Peyton30419822017-05-12 18:01:32 +00006817void __kmp_middle_initialize(void) {
6818 if (__kmp_init_middle) {
6819 return;
6820 }
6821 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6822 if (__kmp_init_middle) {
6823 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6824 return;
6825 }
6826 __kmp_do_middle_initialize();
6827 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6828}
6829
6830void __kmp_parallel_initialize(void) {
6831 int gtid = __kmp_entry_gtid(); // this might be a new root
6832
6833 /* synchronize parallel initialization (for sibling) */
6834 if (TCR_4(__kmp_init_parallel))
6835 return;
6836 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6837 if (TCR_4(__kmp_init_parallel)) {
6838 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6839 return;
6840 }
6841
6842 /* TODO reinitialization after we have already shut down */
6843 if (TCR_4(__kmp_global.g.g_done)) {
6844 KA_TRACE(
6845 10,
6846 ("__kmp_parallel_initialize: attempt to init while shutting down\n"));
6847 __kmp_infinite_loop();
6848 }
6849
6850 /* jc: The lock __kmp_initz_lock is already held, so calling
6851 __kmp_serial_initialize would cause a deadlock. So we call
6852 __kmp_do_serial_initialize directly. */
6853 if (!__kmp_init_middle) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00006854 __kmp_do_middle_initialize();
Jonathan Peyton30419822017-05-12 18:01:32 +00006855 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006856
Jonathan Peyton30419822017-05-12 18:01:32 +00006857 /* begin initialization */
6858 KA_TRACE(10, ("__kmp_parallel_initialize: enter\n"));
6859 KMP_ASSERT(KMP_UBER_GTID(gtid));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006860
6861#if KMP_ARCH_X86 || KMP_ARCH_X86_64
Jonathan Peyton30419822017-05-12 18:01:32 +00006862 // Save the FP control regs.
6863 // Worker threads will set theirs to these values at thread startup.
6864 __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
6865 __kmp_store_mxcsr(&__kmp_init_mxcsr);
6866 __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006867#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
6868
6869#if KMP_OS_UNIX
Jonathan Peyton30419822017-05-12 18:01:32 +00006870#if KMP_HANDLE_SIGNALS
6871 /* must be after __kmp_serial_initialize */
6872 __kmp_install_signals(TRUE);
6873#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00006874#endif
6875
Jonathan Peyton30419822017-05-12 18:01:32 +00006876 __kmp_suspend_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006877
Jonathan Peyton749b4d52016-01-27 21:02:04 +00006878#if defined(USE_LOAD_BALANCE)
Jonathan Peyton30419822017-05-12 18:01:32 +00006879 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
6880 __kmp_global.g.g_dynamic_mode = dynamic_load_balance;
6881 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006882#else
Jonathan Peyton30419822017-05-12 18:01:32 +00006883 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
6884 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
6885 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006886#endif
6887
Jonathan Peyton30419822017-05-12 18:01:32 +00006888 if (__kmp_version) {
6889 __kmp_print_version_2();
6890 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006891
Jonathan Peyton30419822017-05-12 18:01:32 +00006892 /* we have finished parallel initialization */
6893 TCW_SYNC_4(__kmp_init_parallel, TRUE);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006894
Jonathan Peyton30419822017-05-12 18:01:32 +00006895 KMP_MB();
6896 KA_TRACE(10, ("__kmp_parallel_initialize: exit\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00006897
Jonathan Peyton30419822017-05-12 18:01:32 +00006898 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006899}
6900
Jim Cownie5e8470a2013-09-27 10:38:44 +00006901/* ------------------------------------------------------------------------ */
6902
Jonathan Peyton30419822017-05-12 18:01:32 +00006903void __kmp_run_before_invoked_task(int gtid, int tid, kmp_info_t *this_thr,
6904 kmp_team_t *team) {
6905 kmp_disp_t *dispatch;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006906
Jonathan Peyton30419822017-05-12 18:01:32 +00006907 KMP_MB();
Jim Cownie5e8470a2013-09-27 10:38:44 +00006908
Jonathan Peyton30419822017-05-12 18:01:32 +00006909 /* none of the threads have encountered any constructs, yet. */
6910 this_thr->th.th_local.this_construct = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006911#if KMP_CACHE_MANAGE
Jonathan Peyton30419822017-05-12 18:01:32 +00006912 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006913#endif /* KMP_CACHE_MANAGE */
Jonathan Peyton30419822017-05-12 18:01:32 +00006914 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch);
6915 KMP_DEBUG_ASSERT(dispatch);
6916 KMP_DEBUG_ASSERT(team->t.t_dispatch);
6917 // KMP_DEBUG_ASSERT( this_thr->th.th_dispatch == &team->t.t_dispatch[
6918 // this_thr->th.th_info.ds.ds_tid ] );
Jim Cownie5e8470a2013-09-27 10:38:44 +00006919
Jonathan Peyton30419822017-05-12 18:01:32 +00006920 dispatch->th_disp_index = 0; /* reset the dispatch buffer counter */
Jonathan Peytondf6818b2016-06-14 17:57:47 +00006921#if OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00006922 dispatch->th_doacross_buf_idx =
6923 0; /* reset the doacross dispatch buffer counter */
Jonathan Peyton71909c52016-03-02 22:42:06 +00006924#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006925 if (__kmp_env_consistency_check)
6926 __kmp_push_parallel(gtid, team->t.t_ident);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006927
Jonathan Peyton30419822017-05-12 18:01:32 +00006928 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00006929}
6930
Jonathan Peyton30419822017-05-12 18:01:32 +00006931void __kmp_run_after_invoked_task(int gtid, int tid, kmp_info_t *this_thr,
6932 kmp_team_t *team) {
6933 if (__kmp_env_consistency_check)
6934 __kmp_pop_parallel(gtid, team->t.t_ident);
Andrey Churbanovdf0d75e2016-10-27 11:43:07 +00006935
Jonathan Peyton30419822017-05-12 18:01:32 +00006936 __kmp_finish_implicit_task(this_thr);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006937}
6938
Jonathan Peyton30419822017-05-12 18:01:32 +00006939int __kmp_invoke_task_func(int gtid) {
6940 int rc;
6941 int tid = __kmp_tid_from_gtid(gtid);
6942 kmp_info_t *this_thr = __kmp_threads[gtid];
6943 kmp_team_t *team = this_thr->th.th_team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00006944
Jonathan Peyton30419822017-05-12 18:01:32 +00006945 __kmp_run_before_invoked_task(gtid, tid, this_thr, team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00006946#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +00006947 if (__itt_stack_caller_create_ptr) {
6948 __kmp_itt_stack_callee_enter(
6949 (__itt_caller)
6950 team->t.t_stack_id); // inform ittnotify about entering user's code
6951 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00006952#endif /* USE_ITT_BUILD */
Jim Cownie4cc4bb42014-10-07 16:25:50 +00006953#if INCLUDE_SSC_MARKS
Jonathan Peyton30419822017-05-12 18:01:32 +00006954 SSC_MARK_INVOKING();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00006955#endif
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006956
6957#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00006958 void *dummy;
6959 void **exit_runtime_p;
Joachim Protze82e94a52017-11-01 10:08:30 +00006960 ompt_data_t *my_task_data;
6961 ompt_data_t *my_parallel_data;
6962 int ompt_team_size;
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006963
Joachim Protze82e94a52017-11-01 10:08:30 +00006964 if (ompt_enabled.enabled) {
Joachim Protzec255ca72017-11-05 14:11:10 +00006965 exit_runtime_p = &(
6966 team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame.exit_frame);
Jonathan Peyton30419822017-05-12 18:01:32 +00006967 } else {
6968 exit_runtime_p = &dummy;
6969 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006970
Joachim Protze82e94a52017-11-01 10:08:30 +00006971 my_task_data =
6972 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data);
6973 my_parallel_data = &(team->t.ompt_team_info.parallel_data);
6974 if (ompt_enabled.ompt_callback_implicit_task) {
6975 ompt_team_size = team->t.t_nproc;
6976 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
6977 ompt_scope_begin, my_parallel_data, my_task_data, ompt_team_size,
6978 __kmp_tid_from_gtid(gtid));
Joachim Protze9be9cf22018-05-07 12:42:21 +00006979 OMPT_CUR_TASK_INFO(this_thr)->thread_num = __kmp_tid_from_gtid(gtid);
Jonathan Peyton30419822017-05-12 18:01:32 +00006980 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006981#endif
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006982
Jonathan Peyton30419822017-05-12 18:01:32 +00006983 {
6984 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
6985 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
6986 rc =
6987 __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid,
6988 tid, (int)team->t.t_argc, (void **)team->t.t_argv
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006989#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00006990 ,
6991 exit_runtime_p
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006992#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006993 );
Jonas Hahnfeld8a270642016-09-14 13:59:19 +00006994#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00006995 *exit_runtime_p = NULL;
Jonas Hahnfeld8a270642016-09-14 13:59:19 +00006996#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00006997 }
Andrey Churbanovd7d088f2015-04-29 16:42:24 +00006998
Jim Cownie5e8470a2013-09-27 10:38:44 +00006999#if USE_ITT_BUILD
Jonathan Peyton30419822017-05-12 18:01:32 +00007000 if (__itt_stack_caller_create_ptr) {
7001 __kmp_itt_stack_callee_leave(
7002 (__itt_caller)
7003 team->t.t_stack_id); // inform ittnotify about leaving user's code
7004 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007005#endif /* USE_ITT_BUILD */
Jonathan Peyton30419822017-05-12 18:01:32 +00007006 __kmp_run_after_invoked_task(gtid, tid, this_thr, team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007007
Jonathan Peyton30419822017-05-12 18:01:32 +00007008 return rc;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007009}
7010
7011#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00007012void __kmp_teams_master(int gtid) {
7013 // This routine is called by all master threads in teams construct
7014 kmp_info_t *thr = __kmp_threads[gtid];
7015 kmp_team_t *team = thr->th.th_team;
7016 ident_t *loc = team->t.t_ident;
7017 thr->th.th_set_nproc = thr->th.th_teams_size.nth;
7018 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask);
7019 KMP_DEBUG_ASSERT(thr->th.th_set_nproc);
7020 KA_TRACE(20, ("__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid,
7021 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask));
7022// Launch league of teams now, but not let workers execute
7023// (they hang on fork barrier until next parallel)
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007024#if INCLUDE_SSC_MARKS
Jonathan Peyton30419822017-05-12 18:01:32 +00007025 SSC_MARK_FORKING();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007026#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00007027 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc,
Jonathan Peyton30419822017-05-12 18:01:32 +00007028 (microtask_t)thr->th.th_teams_microtask, // "wrapped" task
7029 VOLATILE_CAST(launch_t) __kmp_invoke_task_func, NULL);
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007030#if INCLUDE_SSC_MARKS
Jonathan Peyton30419822017-05-12 18:01:32 +00007031 SSC_MARK_JOINING();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007032#endif
Jonathan Peyton61118492016-05-20 19:03:38 +00007033
Jonathan Peyton30419822017-05-12 18:01:32 +00007034 // AC: last parameter "1" eliminates join barrier which won't work because
7035 // worker threads are in a fork barrier waiting for more parallel regions
7036 __kmp_join_call(loc, gtid
Jonathan Peytonf89fbbb2015-08-31 18:15:00 +00007037#if OMPT_SUPPORT
Jonathan Peyton30419822017-05-12 18:01:32 +00007038 ,
7039 fork_context_intel
Jonathan Peytonf89fbbb2015-08-31 18:15:00 +00007040#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00007041 ,
7042 1);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007043}
7044
Jonathan Peyton30419822017-05-12 18:01:32 +00007045int __kmp_invoke_teams_master(int gtid) {
7046 kmp_info_t *this_thr = __kmp_threads[gtid];
7047 kmp_team_t *team = this_thr->th.th_team;
7048#if KMP_DEBUG
7049 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized)
7050 KMP_DEBUG_ASSERT((void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn ==
7051 (void *)__kmp_teams_master);
7052#endif
7053 __kmp_run_before_invoked_task(gtid, 0, this_thr, team);
7054 __kmp_teams_master(gtid);
7055 __kmp_run_after_invoked_task(gtid, 0, this_thr, team);
7056 return 1;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007057}
7058#endif /* OMP_40_ENABLED */
7059
7060/* this sets the requested number of threads for the next parallel region
Jonathan Peyton30419822017-05-12 18:01:32 +00007061 encountered by this team. since this should be enclosed in the forkjoin
7062 critical section it should avoid race conditions with assymmetrical nested
7063 parallelism */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007064
Jonathan Peyton30419822017-05-12 18:01:32 +00007065void __kmp_push_num_threads(ident_t *id, int gtid, int num_threads) {
7066 kmp_info_t *thr = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00007067
Jonathan Peyton30419822017-05-12 18:01:32 +00007068 if (num_threads > 0)
7069 thr->th.th_set_nproc = num_threads;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007070}
7071
7072#if OMP_40_ENABLED
7073
7074/* this sets the requested number of teams for the teams region and/or
Jonathan Peyton30419822017-05-12 18:01:32 +00007075 the number of threads for the next parallel region encountered */
7076void __kmp_push_num_teams(ident_t *id, int gtid, int num_teams,
7077 int num_threads) {
7078 kmp_info_t *thr = __kmp_threads[gtid];
7079 KMP_DEBUG_ASSERT(num_teams >= 0);
7080 KMP_DEBUG_ASSERT(num_threads >= 0);
Jonathan Peyton1be692e2015-11-30 20:14:05 +00007081
Jonathan Peyton30419822017-05-12 18:01:32 +00007082 if (num_teams == 0)
7083 num_teams = 1; // default number of teams is 1.
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007084 if (num_teams > __kmp_teams_max_nth) { // if too many teams requested?
Jonathan Peyton30419822017-05-12 18:01:32 +00007085 if (!__kmp_reserve_warn) {
7086 __kmp_reserve_warn = 1;
7087 __kmp_msg(kmp_ms_warning,
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007088 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth),
Jonathan Peyton30419822017-05-12 18:01:32 +00007089 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007090 }
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007091 num_teams = __kmp_teams_max_nth;
Jonathan Peyton30419822017-05-12 18:01:32 +00007092 }
7093 // Set number of teams (number of threads in the outer "parallel" of the
7094 // teams)
7095 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007096
Jonathan Peyton30419822017-05-12 18:01:32 +00007097 // Remember the number of threads for inner parallel regions
7098 if (num_threads == 0) {
7099 if (!TCR_4(__kmp_init_middle))
7100 __kmp_middle_initialize(); // get __kmp_avail_proc calculated
7101 num_threads = __kmp_avail_proc / num_teams;
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007102 if (num_teams * num_threads > __kmp_teams_max_nth) {
Jonathan Peyton30419822017-05-12 18:01:32 +00007103 // adjust num_threads w/o warning as it is not user setting
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007104 num_threads = __kmp_teams_max_nth / num_teams;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007105 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007106 } else {
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007107 if (num_teams * num_threads > __kmp_teams_max_nth) {
7108 int new_threads = __kmp_teams_max_nth / num_teams;
Jonathan Peyton30419822017-05-12 18:01:32 +00007109 if (!__kmp_reserve_warn) { // user asked for too many threads
Jonathan Peyton4f90c822017-08-02 20:04:45 +00007110 __kmp_reserve_warn = 1; // that conflicts with KMP_TEAMS_THREAD_LIMIT
Jonathan Peyton30419822017-05-12 18:01:32 +00007111 __kmp_msg(kmp_ms_warning,
7112 KMP_MSG(CantFormThrTeam, num_threads, new_threads),
7113 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
7114 }
7115 num_threads = new_threads;
7116 }
7117 }
7118 thr->th.th_teams_size.nth = num_threads;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007119}
7120
Jim Cownie5e8470a2013-09-27 10:38:44 +00007121// Set the proc_bind var to use in the following parallel region.
Jonathan Peyton30419822017-05-12 18:01:32 +00007122void __kmp_push_proc_bind(ident_t *id, int gtid, kmp_proc_bind_t proc_bind) {
7123 kmp_info_t *thr = __kmp_threads[gtid];
7124 thr->th.th_set_proc_bind = proc_bind;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007125}
7126
7127#endif /* OMP_40_ENABLED */
7128
7129/* Launch the worker threads into the microtask. */
7130
Jonathan Peyton30419822017-05-12 18:01:32 +00007131void __kmp_internal_fork(ident_t *id, int gtid, kmp_team_t *team) {
7132 kmp_info_t *this_thr = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00007133
7134#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00007135 int f;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007136#endif /* KMP_DEBUG */
7137
Jonathan Peyton30419822017-05-12 18:01:32 +00007138 KMP_DEBUG_ASSERT(team);
7139 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7140 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7141 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007142
Jonathan Peyton30419822017-05-12 18:01:32 +00007143 team->t.t_construct = 0; /* no single directives seen yet */
7144 team->t.t_ordered.dt.t_value =
7145 0; /* thread 0 enters the ordered section first */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007146
Jonathan Peyton30419822017-05-12 18:01:32 +00007147 /* Reset the identifiers on the dispatch buffer */
7148 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
7149 if (team->t.t_max_nproc > 1) {
7150 int i;
7151 for (i = 0; i < __kmp_dispatch_num_buffers; ++i) {
7152 team->t.t_disp_buffer[i].buffer_index = i;
Jonathan Peytondf6818b2016-06-14 17:57:47 +00007153#if OMP_45_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00007154 team->t.t_disp_buffer[i].doacross_buf_idx = i;
Jonathan Peyton71909c52016-03-02 22:42:06 +00007155#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00007156 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007157 } else {
7158 team->t.t_disp_buffer[0].buffer_index = 0;
7159#if OMP_45_ENABLED
7160 team->t.t_disp_buffer[0].doacross_buf_idx = 0;
7161#endif
7162 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007163
Jonathan Peyton30419822017-05-12 18:01:32 +00007164 KMP_MB(); /* Flush all pending memory write invalidates. */
7165 KMP_ASSERT(this_thr->th.th_team == team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007166
7167#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00007168 for (f = 0; f < team->t.t_nproc; f++) {
7169 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
7170 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc);
7171 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007172#endif /* KMP_DEBUG */
7173
Jonathan Peyton30419822017-05-12 18:01:32 +00007174 /* release the worker threads so they may begin working */
7175 __kmp_fork_barrier(gtid, 0);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007176}
7177
Jonathan Peyton30419822017-05-12 18:01:32 +00007178void __kmp_internal_join(ident_t *id, int gtid, kmp_team_t *team) {
7179 kmp_info_t *this_thr = __kmp_threads[gtid];
Jim Cownie5e8470a2013-09-27 10:38:44 +00007180
Jonathan Peyton30419822017-05-12 18:01:32 +00007181 KMP_DEBUG_ASSERT(team);
7182 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7183 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7184 KMP_MB(); /* Flush all pending memory write invalidates. */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007185
Jonathan Peyton30419822017-05-12 18:01:32 +00007186/* Join barrier after fork */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007187
7188#ifdef KMP_DEBUG
Jonathan Peyton30419822017-05-12 18:01:32 +00007189 if (__kmp_threads[gtid] &&
7190 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) {
7191 __kmp_printf("GTID: %d, __kmp_threads[%d]=%p\n", gtid, gtid,
7192 __kmp_threads[gtid]);
7193 __kmp_printf("__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, "
7194 "team->t.t_nproc=%d\n",
7195 gtid, __kmp_threads[gtid]->th.th_team_nproc, team,
7196 team->t.t_nproc);
7197 __kmp_print_structure();
7198 }
7199 KMP_DEBUG_ASSERT(__kmp_threads[gtid] &&
7200 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007201#endif /* KMP_DEBUG */
7202
Jonathan Peyton30419822017-05-12 18:01:32 +00007203 __kmp_join_barrier(gtid); /* wait for everyone */
Joachim Protze82e94a52017-11-01 10:08:30 +00007204#if OMPT_SUPPORT
Jonas Hahnfeld82768d02018-02-23 16:46:25 +00007205 if (ompt_enabled.enabled &&
7206 this_thr->th.ompt_thread_info.state == omp_state_wait_barrier_implicit) {
7207 int ds_tid = this_thr->th.th_info.ds.ds_tid;
7208 ompt_data_t *task_data = OMPT_CUR_TASK_DATA(this_thr);
Joachim Protze82e94a52017-11-01 10:08:30 +00007209 this_thr->th.ompt_thread_info.state = omp_state_overhead;
7210#if OMPT_OPTIONAL
7211 void *codeptr = NULL;
7212 if (KMP_MASTER_TID(ds_tid) &&
7213 (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait) ||
7214 ompt_callbacks.ompt_callback(ompt_callback_sync_region)))
7215 codeptr = OMPT_CUR_TEAM_INFO(this_thr)->master_return_address;
7216
7217 if (ompt_enabled.ompt_callback_sync_region_wait) {
7218 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
Jonas Hahnfeld82768d02018-02-23 16:46:25 +00007219 ompt_sync_region_barrier, ompt_scope_end, NULL, task_data, codeptr);
Joachim Protze82e94a52017-11-01 10:08:30 +00007220 }
7221 if (ompt_enabled.ompt_callback_sync_region) {
7222 ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
Jonas Hahnfeld82768d02018-02-23 16:46:25 +00007223 ompt_sync_region_barrier, ompt_scope_end, NULL, task_data, codeptr);
Joachim Protze82e94a52017-11-01 10:08:30 +00007224 }
7225#endif
7226 if (!KMP_MASTER_TID(ds_tid) && ompt_enabled.ompt_callback_implicit_task) {
7227 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
Jonas Hahnfeld82768d02018-02-23 16:46:25 +00007228 ompt_scope_end, NULL, task_data, 0, ds_tid);
Joachim Protze82e94a52017-11-01 10:08:30 +00007229 }
Joachim Protze82e94a52017-11-01 10:08:30 +00007230 }
7231#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00007232
Jonathan Peyton30419822017-05-12 18:01:32 +00007233 KMP_MB(); /* Flush all pending memory write invalidates. */
7234 KMP_ASSERT(this_thr->th.th_team == team);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007235}
7236
Jim Cownie5e8470a2013-09-27 10:38:44 +00007237/* ------------------------------------------------------------------------ */
7238
7239#ifdef USE_LOAD_BALANCE
7240
Jim Cownie5e8470a2013-09-27 10:38:44 +00007241// Return the worker threads actively spinning in the hot team, if we
7242// are at the outermost level of parallelism. Otherwise, return 0.
Jonathan Peyton30419822017-05-12 18:01:32 +00007243static int __kmp_active_hot_team_nproc(kmp_root_t *root) {
7244 int i;
7245 int retval;
7246 kmp_team_t *hot_team;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007247
Jonathan Peyton30419822017-05-12 18:01:32 +00007248 if (root->r.r_active) {
7249 return 0;
7250 }
7251 hot_team = root->r.r_hot_team;
7252 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
7253 return hot_team->t.t_nproc - 1; // Don't count master thread
7254 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007255
Jonathan Peyton30419822017-05-12 18:01:32 +00007256 // Skip the master thread - it is accounted for elsewhere.
7257 retval = 0;
7258 for (i = 1; i < hot_team->t.t_nproc; i++) {
7259 if (hot_team->t.t_threads[i]->th.th_active) {
7260 retval++;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007261 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007262 }
7263 return retval;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007264}
7265
Jim Cownie5e8470a2013-09-27 10:38:44 +00007266// Perform an automatic adjustment to the number of
7267// threads used by the next parallel region.
Jonathan Peyton30419822017-05-12 18:01:32 +00007268static int __kmp_load_balance_nproc(kmp_root_t *root, int set_nproc) {
7269 int retval;
7270 int pool_active;
7271 int hot_team_active;
7272 int team_curr_active;
7273 int system_active;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007274
Jonathan Peyton30419822017-05-12 18:01:32 +00007275 KB_TRACE(20, ("__kmp_load_balance_nproc: called root:%p set_nproc:%d\n", root,
7276 set_nproc));
7277 KMP_DEBUG_ASSERT(root);
7278 KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0]
7279 ->th.th_current_task->td_icvs.dynamic == TRUE);
7280 KMP_DEBUG_ASSERT(set_nproc > 1);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007281
Jonathan Peyton30419822017-05-12 18:01:32 +00007282 if (set_nproc == 1) {
7283 KB_TRACE(20, ("__kmp_load_balance_nproc: serial execution.\n"));
7284 return 1;
7285 }
7286
7287 // Threads that are active in the thread pool, active in the hot team for this
7288 // particular root (if we are at the outer par level), and the currently
7289 // executing thread (to become the master) are available to add to the new
7290 // team, but are currently contributing to the system load, and must be
7291 // accounted for.
Jonathan Peyton37e2ef52018-07-09 17:36:22 +00007292 pool_active = __kmp_thread_pool_active_nth;
Jonathan Peyton30419822017-05-12 18:01:32 +00007293 hot_team_active = __kmp_active_hot_team_nproc(root);
7294 team_curr_active = pool_active + hot_team_active + 1;
7295
7296 // Check the system load.
7297 system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active);
7298 KB_TRACE(30, ("__kmp_load_balance_nproc: system active = %d pool active = %d "
7299 "hot team active = %d\n",
7300 system_active, pool_active, hot_team_active));
7301
7302 if (system_active < 0) {
7303 // There was an error reading the necessary info from /proc, so use the
7304 // thread limit algorithm instead. Once we set __kmp_global.g.g_dynamic_mode
7305 // = dynamic_thread_limit, we shouldn't wind up getting back here.
7306 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
7307 KMP_WARNING(CantLoadBalUsing, "KMP_DYNAMIC_MODE=thread limit");
7308
7309 // Make this call behave like the thread limit algorithm.
7310 retval = __kmp_avail_proc - __kmp_nth +
7311 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
7312 if (retval > set_nproc) {
7313 retval = set_nproc;
7314 }
7315 if (retval < KMP_MIN_NTH) {
7316 retval = KMP_MIN_NTH;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007317 }
7318
Jonathan Peyton30419822017-05-12 18:01:32 +00007319 KB_TRACE(20, ("__kmp_load_balance_nproc: thread limit exit. retval:%d\n",
7320 retval));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007321 return retval;
Jonathan Peyton30419822017-05-12 18:01:32 +00007322 }
7323
7324 // There is a slight delay in the load balance algorithm in detecting new
7325 // running procs. The real system load at this instant should be at least as
7326 // large as the #active omp thread that are available to add to the team.
7327 if (system_active < team_curr_active) {
7328 system_active = team_curr_active;
7329 }
7330 retval = __kmp_avail_proc - system_active + team_curr_active;
7331 if (retval > set_nproc) {
7332 retval = set_nproc;
7333 }
7334 if (retval < KMP_MIN_NTH) {
7335 retval = KMP_MIN_NTH;
7336 }
7337
7338 KB_TRACE(20, ("__kmp_load_balance_nproc: exit. retval:%d\n", retval));
7339 return retval;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007340} // __kmp_load_balance_nproc()
7341
7342#endif /* USE_LOAD_BALANCE */
7343
Jim Cownie5e8470a2013-09-27 10:38:44 +00007344/* ------------------------------------------------------------------------ */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007345
7346/* NOTE: this is called with the __kmp_init_lock held */
Jonathan Peyton30419822017-05-12 18:01:32 +00007347void __kmp_cleanup(void) {
7348 int f;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007349
Jonathan Peyton30419822017-05-12 18:01:32 +00007350 KA_TRACE(10, ("__kmp_cleanup: enter\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007351
Jonathan Peyton30419822017-05-12 18:01:32 +00007352 if (TCR_4(__kmp_init_parallel)) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00007353#if KMP_HANDLE_SIGNALS
Jonathan Peyton30419822017-05-12 18:01:32 +00007354 __kmp_remove_signals();
Jim Cownie5e8470a2013-09-27 10:38:44 +00007355#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00007356 TCW_4(__kmp_init_parallel, FALSE);
7357 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007358
Jonathan Peyton30419822017-05-12 18:01:32 +00007359 if (TCR_4(__kmp_init_middle)) {
Alp Toker763b9392014-02-28 09:42:41 +00007360#if KMP_AFFINITY_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00007361 __kmp_affinity_uninitialize();
Alp Toker763b9392014-02-28 09:42:41 +00007362#endif /* KMP_AFFINITY_SUPPORTED */
Jonathan Peyton30419822017-05-12 18:01:32 +00007363 __kmp_cleanup_hierarchy();
7364 TCW_4(__kmp_init_middle, FALSE);
7365 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007366
Jonathan Peyton30419822017-05-12 18:01:32 +00007367 KA_TRACE(10, ("__kmp_cleanup: go serial cleanup\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007368
Jonathan Peyton30419822017-05-12 18:01:32 +00007369 if (__kmp_init_serial) {
7370 __kmp_runtime_destroy();
7371 __kmp_init_serial = FALSE;
7372 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007373
Andrey Churbanov9e9333a2018-03-05 18:42:01 +00007374 __kmp_cleanup_threadprivate_caches();
7375
Jonathan Peyton30419822017-05-12 18:01:32 +00007376 for (f = 0; f < __kmp_threads_capacity; f++) {
7377 if (__kmp_root[f] != NULL) {
7378 __kmp_free(__kmp_root[f]);
7379 __kmp_root[f] = NULL;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007380 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007381 }
7382 __kmp_free(__kmp_threads);
7383 // __kmp_threads and __kmp_root were allocated at once, as single block, so
7384 // there is no need in freeing __kmp_root.
7385 __kmp_threads = NULL;
7386 __kmp_root = NULL;
7387 __kmp_threads_capacity = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007388
Andrey Churbanov5c56fb52015-02-20 18:05:17 +00007389#if KMP_USE_DYNAMIC_LOCK
Jonathan Peyton30419822017-05-12 18:01:32 +00007390 __kmp_cleanup_indirect_user_locks();
Andrey Churbanov5c56fb52015-02-20 18:05:17 +00007391#else
Jonathan Peyton30419822017-05-12 18:01:32 +00007392 __kmp_cleanup_user_locks();
Andrey Churbanov5c56fb52015-02-20 18:05:17 +00007393#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00007394
Jonathan Peyton30419822017-05-12 18:01:32 +00007395#if KMP_AFFINITY_SUPPORTED
Andrey Churbanovc47afcd2017-07-03 11:24:08 +00007396 KMP_INTERNAL_FREE(CCAST(char *, __kmp_cpuinfo_file));
Jonathan Peyton30419822017-05-12 18:01:32 +00007397 __kmp_cpuinfo_file = NULL;
7398#endif /* KMP_AFFINITY_SUPPORTED */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007399
Jonathan Peyton30419822017-05-12 18:01:32 +00007400#if KMP_USE_ADAPTIVE_LOCKS
7401#if KMP_DEBUG_ADAPTIVE_LOCKS
7402 __kmp_print_speculative_stats();
7403#endif
7404#endif
7405 KMP_INTERNAL_FREE(__kmp_nested_nth.nth);
7406 __kmp_nested_nth.nth = NULL;
7407 __kmp_nested_nth.size = 0;
7408 __kmp_nested_nth.used = 0;
7409 KMP_INTERNAL_FREE(__kmp_nested_proc_bind.bind_types);
7410 __kmp_nested_proc_bind.bind_types = NULL;
7411 __kmp_nested_proc_bind.size = 0;
7412 __kmp_nested_proc_bind.used = 0;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007413
Jonathan Peyton30419822017-05-12 18:01:32 +00007414 __kmp_i18n_catclose();
Jim Cownie5e8470a2013-09-27 10:38:44 +00007415
Jonathan Peytonf6399362018-07-09 17:51:13 +00007416#if KMP_USE_HIER_SCHED
7417 __kmp_hier_scheds.deallocate();
7418#endif
7419
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007420#if KMP_STATS_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00007421 __kmp_stats_fini();
Jim Cownie4cc4bb42014-10-07 16:25:50 +00007422#endif
7423
Jonathan Peyton30419822017-05-12 18:01:32 +00007424 KA_TRACE(10, ("__kmp_cleanup: exit\n"));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007425}
7426
7427/* ------------------------------------------------------------------------ */
Jonathan Peyton30419822017-05-12 18:01:32 +00007428
7429int __kmp_ignore_mppbeg(void) {
7430 char *env;
7431
7432 if ((env = getenv("KMP_IGNORE_MPPBEG")) != NULL) {
7433 if (__kmp_str_match_false(env))
7434 return FALSE;
7435 }
7436 // By default __kmpc_begin() is no-op.
7437 return TRUE;
7438}
7439
7440int __kmp_ignore_mppend(void) {
7441 char *env;
7442
7443 if ((env = getenv("KMP_IGNORE_MPPEND")) != NULL) {
7444 if (__kmp_str_match_false(env))
7445 return FALSE;
7446 }
7447 // By default __kmpc_end() is no-op.
7448 return TRUE;
7449}
7450
7451void __kmp_internal_begin(void) {
7452 int gtid;
7453 kmp_root_t *root;
7454
7455 /* this is a very important step as it will register new sibling threads
7456 and assign these new uber threads a new gtid */
7457 gtid = __kmp_entry_gtid();
7458 root = __kmp_threads[gtid]->th.th_root;
7459 KMP_ASSERT(KMP_UBER_GTID(gtid));
7460
7461 if (root->r.r_begin)
7462 return;
7463 __kmp_acquire_lock(&root->r.r_begin_lock, gtid);
7464 if (root->r.r_begin) {
7465 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7466 return;
7467 }
7468
7469 root->r.r_begin = TRUE;
7470
7471 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7472}
7473
Jim Cownie5e8470a2013-09-27 10:38:44 +00007474/* ------------------------------------------------------------------------ */
7475
Jonathan Peyton30419822017-05-12 18:01:32 +00007476void __kmp_user_set_library(enum library_type arg) {
7477 int gtid;
7478 kmp_root_t *root;
7479 kmp_info_t *thread;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007480
Jonathan Peyton30419822017-05-12 18:01:32 +00007481 /* first, make sure we are initialized so we can get our gtid */
7482
7483 gtid = __kmp_entry_gtid();
7484 thread = __kmp_threads[gtid];
7485
7486 root = thread->th.th_root;
7487
7488 KA_TRACE(20, ("__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg,
7489 library_serial));
7490 if (root->r.r_in_parallel) { /* Must be called in serial section of top-level
7491 thread */
7492 KMP_WARNING(SetLibraryIncorrectCall);
7493 return;
7494 }
7495
7496 switch (arg) {
7497 case library_serial:
7498 thread->th.th_set_nproc = 0;
7499 set__nproc(thread, 1);
7500 break;
7501 case library_turnaround:
7502 thread->th.th_set_nproc = 0;
7503 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
7504 : __kmp_dflt_team_nth_ub);
7505 break;
7506 case library_throughput:
7507 thread->th.th_set_nproc = 0;
7508 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
7509 : __kmp_dflt_team_nth_ub);
7510 break;
7511 default:
7512 KMP_FATAL(UnknownLibraryType, arg);
7513 }
7514
7515 __kmp_aux_set_library(arg);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007516}
7517
Jonathan Peyton30419822017-05-12 18:01:32 +00007518void __kmp_aux_set_stacksize(size_t arg) {
7519 if (!__kmp_init_serial)
7520 __kmp_serial_initialize();
Jim Cownie5e8470a2013-09-27 10:38:44 +00007521
7522#if KMP_OS_DARWIN
Jonathan Peyton30419822017-05-12 18:01:32 +00007523 if (arg & (0x1000 - 1)) {
7524 arg &= ~(0x1000 - 1);
7525 if (arg + 0x1000) /* check for overflow if we round up */
7526 arg += 0x1000;
7527 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007528#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00007529 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007530
Jonathan Peyton30419822017-05-12 18:01:32 +00007531 /* only change the default stacksize before the first parallel region */
7532 if (!TCR_4(__kmp_init_parallel)) {
7533 size_t value = arg; /* argument is in bytes */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007534
Jonathan Peyton30419822017-05-12 18:01:32 +00007535 if (value < __kmp_sys_min_stksize)
7536 value = __kmp_sys_min_stksize;
7537 else if (value > KMP_MAX_STKSIZE)
7538 value = KMP_MAX_STKSIZE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007539
Jonathan Peyton30419822017-05-12 18:01:32 +00007540 __kmp_stksize = value;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007541
Jonathan Peyton30419822017-05-12 18:01:32 +00007542 __kmp_env_stksize = TRUE; /* was KMP_STACKSIZE specified? */
7543 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007544
Jonathan Peyton30419822017-05-12 18:01:32 +00007545 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007546}
7547
7548/* set the behaviour of the runtime library */
7549/* TODO this can cause some odd behaviour with sibling parallelism... */
Jonathan Peyton30419822017-05-12 18:01:32 +00007550void __kmp_aux_set_library(enum library_type arg) {
7551 __kmp_library = arg;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007552
Jonathan Peyton30419822017-05-12 18:01:32 +00007553 switch (__kmp_library) {
7554 case library_serial: {
7555 KMP_INFORM(LibraryIsSerial);
7556 (void)__kmp_change_library(TRUE);
7557 } break;
7558 case library_turnaround:
7559 (void)__kmp_change_library(TRUE);
7560 break;
7561 case library_throughput:
7562 (void)__kmp_change_library(FALSE);
7563 break;
7564 default:
7565 KMP_FATAL(UnknownLibraryType, arg);
7566 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007567}
7568
7569/* ------------------------------------------------------------------------ */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007570
Jonathan Peyton30419822017-05-12 18:01:32 +00007571void __kmp_aux_set_blocktime(int arg, kmp_info_t *thread, int tid) {
7572 int blocktime = arg; /* argument is in milliseconds */
Jonathan Peytone1c7c132016-10-07 18:12:19 +00007573#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00007574 int bt_intervals;
Jonathan Peytone1c7c132016-10-07 18:12:19 +00007575#endif
Jonathan Peyton30419822017-05-12 18:01:32 +00007576 int bt_set;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007577
Jonathan Peyton30419822017-05-12 18:01:32 +00007578 __kmp_save_internal_controls(thread);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007579
Jonathan Peyton30419822017-05-12 18:01:32 +00007580 /* Normalize and set blocktime for the teams */
7581 if (blocktime < KMP_MIN_BLOCKTIME)
7582 blocktime = KMP_MIN_BLOCKTIME;
7583 else if (blocktime > KMP_MAX_BLOCKTIME)
7584 blocktime = KMP_MAX_BLOCKTIME;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007585
Jonathan Peyton30419822017-05-12 18:01:32 +00007586 set__blocktime_team(thread->th.th_team, tid, blocktime);
7587 set__blocktime_team(thread->th.th_serial_team, 0, blocktime);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007588
Jonathan Peytone1c7c132016-10-07 18:12:19 +00007589#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00007590 /* Calculate and set blocktime intervals for the teams */
7591 bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007592
Jonathan Peyton30419822017-05-12 18:01:32 +00007593 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals);
7594 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals);
Jonathan Peytone1c7c132016-10-07 18:12:19 +00007595#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00007596
Jonathan Peyton30419822017-05-12 18:01:32 +00007597 /* Set whether blocktime has been set to "TRUE" */
7598 bt_set = TRUE;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007599
Jonathan Peyton30419822017-05-12 18:01:32 +00007600 set__bt_set_team(thread->th.th_team, tid, bt_set);
7601 set__bt_set_team(thread->th.th_serial_team, 0, bt_set);
Jonathan Peytone1c7c132016-10-07 18:12:19 +00007602#if KMP_USE_MONITOR
Jonathan Peyton30419822017-05-12 18:01:32 +00007603 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
7604 "bt_intervals=%d, monitor_updates=%d\n",
7605 __kmp_gtid_from_tid(tid, thread->th.th_team),
7606 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals,
7607 __kmp_monitor_wakeups));
Samuel Antao33515192016-10-20 13:20:17 +00007608#else
Jonathan Peyton30419822017-05-12 18:01:32 +00007609 KF_TRACE(10, ("kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n",
7610 __kmp_gtid_from_tid(tid, thread->th.th_team),
7611 thread->th.th_team->t.t_id, tid, blocktime));
Jonathan Peytone1c7c132016-10-07 18:12:19 +00007612#endif
Jim Cownie5e8470a2013-09-27 10:38:44 +00007613}
7614
Jonathan Peyton30419822017-05-12 18:01:32 +00007615void __kmp_aux_set_defaults(char const *str, int len) {
7616 if (!__kmp_init_serial) {
7617 __kmp_serial_initialize();
Jonathan Peytonbd3a7632017-09-27 20:36:27 +00007618 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007619 __kmp_env_initialize(str);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007620
Jonathan Peyton30419822017-05-12 18:01:32 +00007621 if (__kmp_settings
Jim Cownie5e8470a2013-09-27 10:38:44 +00007622#if OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00007623 || __kmp_display_env || __kmp_display_env_verbose
Jim Cownie5e8470a2013-09-27 10:38:44 +00007624#endif // OMP_40_ENABLED
Jonathan Peyton30419822017-05-12 18:01:32 +00007625 ) {
7626 __kmp_env_print();
7627 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007628} // __kmp_aux_set_defaults
7629
7630/* ------------------------------------------------------------------------ */
Jonathan Peyton30419822017-05-12 18:01:32 +00007631/* internal fast reduction routines */
Jim Cownie5e8470a2013-09-27 10:38:44 +00007632
Jim Cownie5e8470a2013-09-27 10:38:44 +00007633PACKED_REDUCTION_METHOD_T
Jonathan Peyton30419822017-05-12 18:01:32 +00007634__kmp_determine_reduction_method(
7635 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size,
7636 void *reduce_data, void (*reduce_func)(void *lhs_data, void *rhs_data),
7637 kmp_critical_name *lck) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00007638
Jonathan Peyton30419822017-05-12 18:01:32 +00007639 // Default reduction method: critical construct ( lck != NULL, like in current
7640 // PAROPT )
7641 // If ( reduce_data!=NULL && reduce_func!=NULL ): the tree-reduction method
7642 // can be selected by RTL
7643 // If loc->flags contains KMP_IDENT_ATOMIC_REDUCE, the atomic reduce method
7644 // can be selected by RTL
7645 // Finally, it's up to OpenMP RTL to make a decision on which method to select
7646 // among generated by PAROPT.
Jim Cownie5e8470a2013-09-27 10:38:44 +00007647
Jonathan Peyton30419822017-05-12 18:01:32 +00007648 PACKED_REDUCTION_METHOD_T retval;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007649
Jonathan Peyton30419822017-05-12 18:01:32 +00007650 int team_size;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007651
Jonathan Peyton30419822017-05-12 18:01:32 +00007652 KMP_DEBUG_ASSERT(loc); // it would be nice to test ( loc != 0 )
7653 KMP_DEBUG_ASSERT(lck); // it would be nice to test ( lck != 0 )
Jim Cownie5e8470a2013-09-27 10:38:44 +00007654
Jonathan Peyton30419822017-05-12 18:01:32 +00007655#define FAST_REDUCTION_ATOMIC_METHOD_GENERATED \
7656 ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE)) == (KMP_IDENT_ATOMIC_REDUCE))
7657#define FAST_REDUCTION_TREE_METHOD_GENERATED ((reduce_data) && (reduce_func))
Jim Cownie5e8470a2013-09-27 10:38:44 +00007658
Jonathan Peyton30419822017-05-12 18:01:32 +00007659 retval = critical_reduce_block;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007660
Jonathan Peyton30419822017-05-12 18:01:32 +00007661 // another choice of getting a team size (with 1 dynamic deference) is slower
7662 team_size = __kmp_get_team_num_threads(global_tid);
7663 if (team_size == 1) {
Jim Cownie5e8470a2013-09-27 10:38:44 +00007664
Jonathan Peyton30419822017-05-12 18:01:32 +00007665 retval = empty_reduce_block;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007666
Jonathan Peyton30419822017-05-12 18:01:32 +00007667 } else {
Jim Cownie5e8470a2013-09-27 10:38:44 +00007668
Jonathan Peyton30419822017-05-12 18:01:32 +00007669 int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007670
Jonathan Peyton30419822017-05-12 18:01:32 +00007671#if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS64
Jim Cownie5e8470a2013-09-27 10:38:44 +00007672
Kamil Rytarowskia56ac942018-12-09 16:40:33 +00007673#if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
Kamil Rytarowski7e1ea992018-12-09 16:46:48 +00007674 KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD
Jim Cownie5e8470a2013-09-27 10:38:44 +00007675
Jonathan Peyton30419822017-05-12 18:01:32 +00007676 int teamsize_cutoff = 4;
Jonathan Peyton91b78702015-06-08 19:39:07 +00007677
Jonathan Peyton492e0a32017-06-13 17:17:26 +00007678#if KMP_MIC_SUPPORTED
Jonathan Peyton30419822017-05-12 18:01:32 +00007679 if (__kmp_mic_type != non_mic) {
7680 teamsize_cutoff = 8;
7681 }
Andrey Churbanov613edeb2015-02-20 18:14:43 +00007682#endif
Jonathan Peytonbaad3f62018-08-09 22:04:30 +00007683 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
Jonathan Peyton30419822017-05-12 18:01:32 +00007684 if (tree_available) {
7685 if (team_size <= teamsize_cutoff) {
7686 if (atomic_available) {
7687 retval = atomic_reduce_block;
Jim Cownie5e8470a2013-09-27 10:38:44 +00007688 }
Jonathan Peyton30419822017-05-12 18:01:32 +00007689 } else {
7690 retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
7691 }
7692 } else if (atomic_available) {
7693 retval = atomic_reduce_block;
7694 }
7695#else
7696#error "Unknown or unsupported OS"
Jonathan Peyton17e53b92018-12-10 18:26:50 +00007697#endif // KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||
7698 // KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD
Jim Cownie5e8470a2013-09-27 10:38:44 +00007699
Jonathan Peyton30419822017-05-12 18:01:32 +00007700#elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS
7701
Andrey Churbanov855d0982018-11-07 12:27:38 +00007702#if KMP_OS_LINUX || KMP_OS_WINDOWS || KMP_OS_HURD
Jonathan Peyton30419822017-05-12 18:01:32 +00007703
7704 // basic tuning
7705
7706 if (atomic_available) {
7707 if (num_vars <= 2) { // && ( team_size <= 8 ) due to false-sharing ???
7708 retval = atomic_reduce_block;
7709 }
7710 } // otherwise: use critical section
7711
7712#elif KMP_OS_DARWIN
7713
Jonathan Peytonbaad3f62018-08-09 22:04:30 +00007714 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
Jonathan Peyton30419822017-05-12 18:01:32 +00007715 if (atomic_available && (num_vars <= 3)) {
7716 retval = atomic_reduce_block;
7717 } else if (tree_available) {
7718 if ((reduce_size > (9 * sizeof(kmp_real64))) &&
7719 (reduce_size < (2000 * sizeof(kmp_real64)))) {
7720 retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER;
7721 }
7722 } // otherwise: use critical section
7723
7724#else
7725#error "Unknown or unsupported OS"
7726#endif
7727
7728#else
7729#error "Unknown or unsupported architecture"
7730#endif
7731 }
7732
7733 // KMP_FORCE_REDUCTION
7734
7735 // If the team is serialized (team_size == 1), ignore the forced reduction
7736 // method and stay with the unsynchronized method (empty_reduce_block)
7737 if (__kmp_force_reduction_method != reduction_method_not_defined &&
7738 team_size != 1) {
7739
7740 PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block;
7741
7742 int atomic_available, tree_available;
7743
7744 switch ((forced_retval = __kmp_force_reduction_method)) {
7745 case critical_reduce_block:
7746 KMP_ASSERT(lck); // lck should be != 0
7747 break;
7748
7749 case atomic_reduce_block:
7750 atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
7751 if (!atomic_available) {
7752 KMP_WARNING(RedMethodNotSupported, "atomic");
7753 forced_retval = critical_reduce_block;
7754 }
7755 break;
7756
7757 case tree_reduce_block:
7758 tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
7759 if (!tree_available) {
7760 KMP_WARNING(RedMethodNotSupported, "tree");
7761 forced_retval = critical_reduce_block;
7762 } else {
7763#if KMP_FAST_REDUCTION_BARRIER
7764 forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
7765#endif
7766 }
7767 break;
7768
7769 default:
7770 KMP_ASSERT(0); // "unsupported method specified"
Jim Cownie5e8470a2013-09-27 10:38:44 +00007771 }
7772
Jonathan Peyton30419822017-05-12 18:01:32 +00007773 retval = forced_retval;
7774 }
Jim Cownie5e8470a2013-09-27 10:38:44 +00007775
Jonathan Peyton30419822017-05-12 18:01:32 +00007776 KA_TRACE(10, ("reduction method selected=%08x\n", retval));
Jim Cownie5e8470a2013-09-27 10:38:44 +00007777
Jonathan Peyton30419822017-05-12 18:01:32 +00007778#undef FAST_REDUCTION_TREE_METHOD_GENERATED
7779#undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED
7780
7781 return (retval);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007782}
7783
7784// this function is for testing set/get/determine reduce method
Jonathan Peyton30419822017-05-12 18:01:32 +00007785kmp_int32 __kmp_get_reduce_method(void) {
7786 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8);
Jim Cownie5e8470a2013-09-27 10:38:44 +00007787}