blob: fc666fc5976e9662353cba7f32c02fd86d32317d [file] [log] [blame]
Rich Felkerf68a3462013-09-16 10:54:31 -04001#define _GNU_SOURCE
Rich Felker0b44a032011-02-12 00:22:29 -05002#include "pthread_impl.h"
Rich Felkerdba68bf2011-07-30 08:02:14 -04003#include "stdio_impl.h"
Szabolcs Nagyb20760c2013-09-15 02:00:32 +00004#include "libc.h"
Rich Felkerefd4d872012-11-08 17:04:20 -05005#include <sys/mman.h>
Rich Felker0b44a032011-02-12 00:22:29 -05006
Rich Felkerb2486a82011-04-06 20:27:07 -04007static void dummy_0()
8{
9}
Rich Felkerdcd60372012-10-05 11:51:50 -040010weak_alias(dummy_0, __acquire_ptc);
11weak_alias(dummy_0, __release_ptc);
Rich Felkera6054e32011-04-19 23:09:14 -040012weak_alias(dummy_0, __pthread_tsd_run_dtors);
Rich Felkerfd80cfa2011-04-03 02:33:50 -040013
Rich Felker0c05bd32012-09-06 23:34:10 -040014_Noreturn void pthread_exit(void *result)
Rich Felker1a9a2ff2011-02-13 19:58:30 -050015{
Rich Felker1ebde9c2011-04-17 17:06:05 -040016 pthread_t self = pthread_self();
Rich Felkerd0ba0982013-04-26 16:16:04 -040017 sigset_t set;
Rich Felker1a9a2ff2011-02-13 19:58:30 -050018
Rich Felkerafc35d52012-02-09 02:33:08 -050019 self->result = result;
20
21 while (self->cancelbuf) {
22 void (*f)(void *) = self->cancelbuf->__f;
23 void *x = self->cancelbuf->__x;
24 self->cancelbuf = self->cancelbuf->__next;
25 f(x);
Rich Felker1ebde9c2011-04-17 17:06:05 -040026 }
Rich Felker1a9a2ff2011-02-13 19:58:30 -050027
Rich Felkera6054e32011-04-19 23:09:14 -040028 __pthread_tsd_run_dtors();
Rich Felker1a9a2ff2011-02-13 19:58:30 -050029
Rich Felkerbbbe87e2012-07-12 11:23:43 -040030 __lock(self->exitlock);
Rich Felkerf58c8a02011-06-14 01:25:17 -040031
Rich Felker5fcebcd2011-03-10 18:31:37 -050032 /* Mark this thread dead before decrementing count */
Rich Felkerbbbe87e2012-07-12 11:23:43 -040033 __lock(self->killlock);
Rich Felker5fcebcd2011-03-10 18:31:37 -050034 self->dead = 1;
Rich Felker19eb13b2011-02-19 11:04:36 -050035
Rich Felker6e531f92013-04-26 16:04:30 -040036 /* Block all signals before decrementing the live thread count.
37 * This is important to ensure that dynamically allocated TLS
38 * is not under-allocated/over-committed, and possibly for other
39 * reasons as well. */
Rich Felker2c074b02013-04-26 19:48:01 -040040 __block_all_sigs(&set);
Rich Felker23f21c32013-04-26 15:47:44 -040041
Rich Felkerd674f852013-04-26 17:51:22 -040042 /* Wait to unlock the kill lock, which governs functions like
43 * pthread_kill which target a thread id, until signals have
44 * been blocked. This precludes observation of the thread id
45 * as a live thread (with application code running in it) after
46 * the thread was reported dead by ESRCH being returned. */
47 __unlock(self->killlock);
48
Rich Felkerd0ba0982013-04-26 16:16:04 -040049 /* It's impossible to determine whether this is "the last thread"
50 * until performing the atomic decrement, since multiple threads
51 * could exit at the same time. For the last thread, revert the
52 * decrement and unblock signals to give the atexit handlers and
53 * stdio cleanup code a consistent state. */
54 if (a_fetch_add(&libc.threads_minus_1, -1)==0) {
55 libc.threads_minus_1 = 0;
Rich Felker2c074b02013-04-26 19:48:01 -040056 __restore_sigs(&set);
Rich Felkerd0ba0982013-04-26 16:16:04 -040057 exit(0);
58 }
Rich Felkerfb11b6b2011-02-19 10:38:57 -050059
Rich Felker5fcebcd2011-03-10 18:31:37 -050060 if (self->detached && self->map_base) {
Rich Felker6e531f92013-04-26 16:04:30 -040061 /* Detached threads must avoid the kernel clear_child_tid
62 * feature, since the virtual address will have been
63 * unmapped and possibly already reused by a new mapping
64 * at the time the kernel would perform the write. In
65 * the case of threads that started out detached, the
66 * initial clone flags are correct, but if the thread was
67 * detached later (== 2), we need to clear it here. */
68 if (self->detached == 2) __syscall(SYS_set_tid_address, 0);
69
70 /* The following call unmaps the thread's stack mapping
71 * and then exits without touching the stack. */
Rich Felker1a9a2ff2011-02-13 19:58:30 -050072 __unmapself(self->map_base, self->map_size);
Rich Felker5fcebcd2011-03-10 18:31:37 -050073 }
Rich Felker1a9a2ff2011-02-13 19:58:30 -050074
Rich Felker0c05bd32012-09-06 23:34:10 -040075 for (;;) __syscall(SYS_exit, 0);
Rich Felker1a9a2ff2011-02-13 19:58:30 -050076}
Rich Felker0b44a032011-02-12 00:22:29 -050077
Rich Felkercfd892f2012-05-23 14:13:54 -040078void __do_cleanup_push(struct __ptcb *cb)
Rich Felker5f37fc12011-08-03 19:57:46 -040079{
80 struct pthread *self = pthread_self();
81 cb->__next = self->cancelbuf;
82 self->cancelbuf = cb;
83}
84
Rich Felkercfd892f2012-05-23 14:13:54 -040085void __do_cleanup_pop(struct __ptcb *cb)
Rich Felker5f37fc12011-08-03 19:57:46 -040086{
Rich Felkerafc35d52012-02-09 02:33:08 -050087 __pthread_self()->cancelbuf = cb->__next;
Rich Felker5f37fc12011-08-03 19:57:46 -040088}
89
Rich Felker3f72cda2011-09-18 10:14:37 -040090static int start(void *p)
Rich Felker0b44a032011-02-12 00:22:29 -050091{
Rich Felker3f72cda2011-09-18 10:14:37 -040092 pthread_t self = p;
Rich Felker1e21e782012-11-11 15:38:04 -050093 if (self->startlock[0]) {
94 __wait(self->startlock, 0, 1, 1);
95 if (self->startlock[0]) {
96 self->detached = 2;
97 pthread_exit(0);
98 }
Rich Felker2c074b02013-04-26 19:48:01 -040099 __restore_sigs(self->sigmask);
Rich Felker1e21e782012-11-11 15:38:04 -0500100 }
Rich Felker99b8a252011-05-07 23:23:58 -0400101 if (self->unblock_cancel)
Rich Felker2f437042012-08-09 22:52:13 -0400102 __syscall(SYS_rt_sigprocmask, SIG_UNBLOCK,
Rich Felkerccc7b4c2013-03-26 23:07:31 -0400103 SIGPT_SET, 0, _NSIG/8);
Rich Felker0b44a032011-02-12 00:22:29 -0500104 pthread_exit(self->start(self->start_arg));
Rich Felker3f72cda2011-09-18 10:14:37 -0400105 return 0;
Rich Felker0b44a032011-02-12 00:22:29 -0500106}
107
Rich Felker0b44a032011-02-12 00:22:29 -0500108#define ROUND(x) (((x)+PAGE_SIZE-1)&-PAGE_SIZE)
109
110/* pthread_key_create.c overrides this */
111static const size_t dummy = 0;
112weak_alias(dummy, __pthread_tsd_size);
113
Rich Felkerdba68bf2011-07-30 08:02:14 -0400114static FILE *const dummy_file = 0;
115weak_alias(dummy_file, __stdin_used);
116weak_alias(dummy_file, __stdout_used);
117weak_alias(dummy_file, __stderr_used);
118
119static void init_file_lock(FILE *f)
120{
121 if (f && f->lock<0) f->lock = 0;
122}
123
Rich Felkerdcd60372012-10-05 11:51:50 -0400124void *__copy_tls(unsigned char *);
Rich Felker8431d792012-10-04 16:35:46 -0400125
Rich Felkerd5142642013-02-01 22:10:40 -0500126int pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict attrp, void *(*entry)(void *), void *restrict arg)
Rich Felker0b44a032011-02-12 00:22:29 -0500127{
Rich Felker0b44a032011-02-12 00:22:29 -0500128 int ret;
Rich Felkerd5142642013-02-01 22:10:40 -0500129 size_t size, guard;
Rich Felker0b44a032011-02-12 00:22:29 -0500130 struct pthread *self = pthread_self(), *new;
Rich Felker14a835b2013-03-31 23:25:55 -0400131 unsigned char *map = 0, *stack = 0, *tsd = 0, *stack_limit;
Rich Felkerf68a3462013-09-16 10:54:31 -0400132 unsigned flags = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND
133 | CLONE_PARENT | CLONE_THREAD | CLONE_SYSVSEM | CLONE_SETTLS
134 | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | CLONE_DETACHED;
Rich Felker1e21e782012-11-11 15:38:04 -0500135 int do_sched = 0;
Rich Felkerd5142642013-02-01 22:10:40 -0500136 pthread_attr_t attr = {0};
Rich Felker0b44a032011-02-12 00:22:29 -0500137
Rich Felker7fd39952011-04-03 16:15:15 -0400138 if (!self) return ENOSYS;
Rich Felker9080cc12011-04-17 16:53:54 -0400139 if (!libc.threaded) {
Rich Felkerdba68bf2011-07-30 08:02:14 -0400140 for (FILE *f=libc.ofl_head; f; f=f->next)
141 init_file_lock(f);
142 init_file_lock(__stdin_used);
143 init_file_lock(__stdout_used);
144 init_file_lock(__stderr_used);
Rich Felker9080cc12011-04-17 16:53:54 -0400145 libc.threaded = 1;
146 }
Rich Felkerd5142642013-02-01 22:10:40 -0500147 if (attrp) attr = *attrp;
Rich Felker0b44a032011-02-12 00:22:29 -0500148
Rich Felkerdcd60372012-10-05 11:51:50 -0400149 __acquire_ptc();
150
Rich Felkerd5142642013-02-01 22:10:40 -0500151 if (attr._a_stackaddr) {
152 size_t need = libc.tls_size + __pthread_tsd_size;
153 size = attr._a_stacksize + DEFAULT_STACK_SIZE;
154 stack = (void *)(attr._a_stackaddr & -16);
Rich Felkerced64992013-04-06 01:15:08 -0400155 stack_limit = (void *)(attr._a_stackaddr - size);
Rich Felkerd5142642013-02-01 22:10:40 -0500156 /* Use application-provided stack for TLS only when
157 * it does not take more than ~12% or 2k of the
158 * application's stack space. */
159 if (need < size/8 && need < 2048) {
160 tsd = stack - __pthread_tsd_size;
161 stack = tsd - libc.tls_size;
162 } else {
163 size = ROUND(need);
164 guard = 0;
Rich Felker819006a2012-06-09 19:53:29 -0400165 }
Rich Felkerd5142642013-02-01 22:10:40 -0500166 } else {
167 guard = ROUND(DEFAULT_GUARD_SIZE + attr._a_guardsize);
168 size = guard + ROUND(DEFAULT_STACK_SIZE + attr._a_stacksize
169 + libc.tls_size + __pthread_tsd_size);
170 }
171
172 if (!tsd) {
Rich Felker8431d792012-10-04 16:35:46 -0400173 if (guard) {
174 map = mmap(0, size, PROT_NONE, MAP_PRIVATE|MAP_ANON, -1, 0);
Rich Felker72768ea2013-02-01 22:25:19 -0500175 if (map == MAP_FAILED) goto fail;
Rich Felker8431d792012-10-04 16:35:46 -0400176 if (mprotect(map+guard, size-guard, PROT_READ|PROT_WRITE)) {
177 munmap(map, size);
Rich Felker72768ea2013-02-01 22:25:19 -0500178 goto fail;
Rich Felker8431d792012-10-04 16:35:46 -0400179 }
180 } else {
181 map = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0);
Rich Felker72768ea2013-02-01 22:25:19 -0500182 if (map == MAP_FAILED) goto fail;
Rich Felker8431d792012-10-04 16:35:46 -0400183 }
Rich Felker819006a2012-06-09 19:53:29 -0400184 tsd = map + size - __pthread_tsd_size;
Rich Felker14a835b2013-03-31 23:25:55 -0400185 if (!stack) {
186 stack = tsd - libc.tls_size;
187 stack_limit = map + guard;
188 }
Rich Felker11e4b922011-05-07 23:39:48 -0400189 }
Rich Felkerd5142642013-02-01 22:10:40 -0500190
191 new = __copy_tls(tsd - libc.tls_size);
Rich Felker0b44a032011-02-12 00:22:29 -0500192 new->map_base = map;
193 new->map_size = size;
Rich Felker14a835b2013-03-31 23:25:55 -0400194 new->stack = stack;
195 new->stack_size = stack - stack_limit;
Rich Felker0b44a032011-02-12 00:22:29 -0500196 new->pid = self->pid;
197 new->errno_ptr = &new->errno_val;
198 new->start = entry;
199 new->start_arg = arg;
200 new->self = new;
201 new->tsd = (void *)tsd;
Rich Felkerd5142642013-02-01 22:10:40 -0500202 if (attr._a_detach) {
Rich Felker92f83962012-07-11 23:36:46 -0400203 new->detached = 1;
Rich Felkerf68a3462013-09-16 10:54:31 -0400204 flags -= CLONE_CHILD_CLEARTID;
Rich Felker92f83962012-07-11 23:36:46 -0400205 }
Rich Felkerd5142642013-02-01 22:10:40 -0500206 if (attr._a_sched) {
Rich Felker1e21e782012-11-11 15:38:04 -0500207 do_sched = new->startlock[0] = 1;
Rich Felker2c074b02013-04-26 19:48:01 -0400208 __block_app_sigs(new->sigmask);
Rich Felker1e21e782012-11-11 15:38:04 -0500209 }
Rich Felkerbf619d82011-03-29 12:58:22 -0400210 new->unblock_cancel = self->cancel;
Rich Felker0a96a372012-10-07 21:43:46 -0400211 new->canary = self->canary;
Rich Felker0b44a032011-02-12 00:22:29 -0500212
Rich Felker0b44a032011-02-12 00:22:29 -0500213 a_inc(&libc.threads_minus_1);
Rich Felker9ec42832012-10-15 18:51:53 -0400214 ret = __clone(start, stack, flags, new, &new->tid, TP_ADJ(new), &new->tid);
Rich Felker0b44a032011-02-12 00:22:29 -0500215
Rich Felkerdcd60372012-10-05 11:51:50 -0400216 __release_ptc();
Rich Felker0b44a032011-02-12 00:22:29 -0500217
Rich Felker1e21e782012-11-11 15:38:04 -0500218 if (do_sched) {
Rich Felker2c074b02013-04-26 19:48:01 -0400219 __restore_sigs(new->sigmask);
Rich Felker1e21e782012-11-11 15:38:04 -0500220 }
221
Rich Felker0b44a032011-02-12 00:22:29 -0500222 if (ret < 0) {
223 a_dec(&libc.threads_minus_1);
Rich Felker077549e2013-02-01 22:23:24 -0500224 if (map) munmap(map, size);
Rich Felker59666802011-02-15 02:20:21 -0500225 return EAGAIN;
Rich Felker0b44a032011-02-12 00:22:29 -0500226 }
Rich Felker1e21e782012-11-11 15:38:04 -0500227
228 if (do_sched) {
229 ret = __syscall(SYS_sched_setscheduler, new->tid,
Rich Felkerd5142642013-02-01 22:10:40 -0500230 attr._a_policy, &attr._a_prio);
Rich Felker1e21e782012-11-11 15:38:04 -0500231 a_store(new->startlock, ret<0 ? 2 : 0);
232 __wake(new->startlock, 1, 1);
233 if (ret < 0) return -ret;
234 }
235
Rich Felker0b44a032011-02-12 00:22:29 -0500236 *res = new;
237 return 0;
Rich Felker72768ea2013-02-01 22:25:19 -0500238fail:
239 __release_ptc();
240 return EAGAIN;
Rich Felker0b44a032011-02-12 00:22:29 -0500241}