Rich Felker | 22e5bbd | 2019-04-10 17:10:36 -0400 | [diff] [blame] | 1 | #define SYSCALL_NO_TLS 1 |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 2 | #include <elf.h> |
| 3 | #include <limits.h> |
Rich Felker | efd4d87 | 2012-11-08 17:04:20 -0500 | [diff] [blame] | 4 | #include <sys/mman.h> |
| 5 | #include <string.h> |
Rich Felker | bd67959 | 2015-03-06 13:27:08 -0500 | [diff] [blame] | 6 | #include <stddef.h> |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 7 | #include "pthread_impl.h" |
| 8 | #include "libc.h" |
| 9 | #include "atomic.h" |
Rich Felker | 98221c3 | 2014-03-23 23:19:30 -0400 | [diff] [blame] | 10 | #include "syscall.h" |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 11 | |
Rich Felker | 8f11e61 | 2019-02-15 22:29:01 -0500 | [diff] [blame] | 12 | volatile int __thread_list_lock; |
| 13 | |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 14 | int __init_tp(void *p) |
| 15 | { |
| 16 | pthread_t td = p; |
| 17 | td->self = td; |
Rich Felker | 64e3228 | 2014-06-10 03:36:56 -0400 | [diff] [blame] | 18 | int r = __set_thread_area(TP_ADJ(p)); |
| 19 | if (r < 0) return -1; |
| 20 | if (!r) libc.can_do_threads = 1; |
Rich Felker | cdba6b2 | 2018-05-05 21:33:58 -0400 | [diff] [blame] | 21 | td->detach_state = DT_JOINABLE; |
Rich Felker | 8f11e61 | 2019-02-15 22:29:01 -0500 | [diff] [blame] | 22 | td->tid = __syscall(SYS_set_tid_address, &__thread_list_lock); |
Rich Felker | 0bc0309 | 2014-07-02 19:33:19 -0400 | [diff] [blame] | 23 | td->locale = &libc.global_locale; |
Rich Felker | 4e98cce | 2015-04-10 00:54:48 -0400 | [diff] [blame] | 24 | td->robust_list.head = &td->robust_list.head; |
Rich Felker | 22e5bbd | 2019-04-10 17:10:36 -0400 | [diff] [blame] | 25 | td->sysinfo = __sysinfo; |
Rich Felker | 8f11e61 | 2019-02-15 22:29:01 -0500 | [diff] [blame] | 26 | td->next = td->prev = td; |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 27 | return 0; |
| 28 | } |
| 29 | |
Rich Felker | bd67959 | 2015-03-06 13:27:08 -0500 | [diff] [blame] | 30 | static struct builtin_tls { |
| 31 | char c; |
| 32 | struct pthread pt; |
| 33 | void *space[16]; |
| 34 | } builtin_tls[1]; |
| 35 | #define MIN_TLS_ALIGN offsetof(struct builtin_tls, pt) |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 36 | |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 37 | static struct tls_module main_tls; |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 38 | |
Rich Felker | dcd6037 | 2012-10-05 11:51:50 -0400 | [diff] [blame] | 39 | void *__copy_tls(unsigned char *mem) |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 40 | { |
Rich Felker | 6a2eaa3 | 2012-10-06 16:51:03 -0400 | [diff] [blame] | 41 | pthread_t td; |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 42 | struct tls_module *p; |
| 43 | size_t i; |
Rich Felker | b6d701a | 2018-10-12 00:30:34 -0400 | [diff] [blame] | 44 | uintptr_t *dtv; |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 45 | |
Rich Felker | 9ec4283 | 2012-10-15 18:51:53 -0400 | [diff] [blame] | 46 | #ifdef TLS_ABOVE_TP |
Rich Felker | b6d701a | 2018-10-12 00:30:34 -0400 | [diff] [blame] | 47 | dtv = (uintptr_t*)(mem + libc.tls_size) - (libc.tls_cnt + 1); |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 48 | |
| 49 | mem += -((uintptr_t)mem + sizeof(struct pthread)) & (libc.tls_align-1); |
Rich Felker | 9ec4283 | 2012-10-15 18:51:53 -0400 | [diff] [blame] | 50 | td = (pthread_t)mem; |
| 51 | mem += sizeof(struct pthread); |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 52 | |
| 53 | for (i=1, p=libc.tls_head; p; i++, p=p->next) { |
Rich Felker | b6d701a | 2018-10-12 00:30:34 -0400 | [diff] [blame] | 54 | dtv[i] = (uintptr_t)(mem + p->offset) + DTP_OFFSET; |
| 55 | memcpy(mem + p->offset, p->image, p->len); |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 56 | } |
Rich Felker | 9ec4283 | 2012-10-15 18:51:53 -0400 | [diff] [blame] | 57 | #else |
Rich Felker | b6d701a | 2018-10-12 00:30:34 -0400 | [diff] [blame] | 58 | dtv = (uintptr_t *)mem; |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 59 | |
Rich Felker | e172c7b | 2012-12-25 21:51:11 -0500 | [diff] [blame] | 60 | mem += libc.tls_size - sizeof(struct pthread); |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 61 | mem -= (uintptr_t)mem & (libc.tls_align-1); |
Rich Felker | 6a2eaa3 | 2012-10-06 16:51:03 -0400 | [diff] [blame] | 62 | td = (pthread_t)mem; |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 63 | |
| 64 | for (i=1, p=libc.tls_head; p; i++, p=p->next) { |
Rich Felker | b6d701a | 2018-10-12 00:30:34 -0400 | [diff] [blame] | 65 | dtv[i] = (uintptr_t)(mem - p->offset) + DTP_OFFSET; |
| 66 | memcpy(mem - p->offset, p->image, p->len); |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 67 | } |
Rich Felker | 9ec4283 | 2012-10-15 18:51:53 -0400 | [diff] [blame] | 68 | #endif |
Rich Felker | b6d701a | 2018-10-12 00:30:34 -0400 | [diff] [blame] | 69 | dtv[0] = libc.tls_cnt; |
Szabolcs Nagy | 204a69d | 2015-03-11 12:48:12 +0000 | [diff] [blame] | 70 | td->dtv = td->dtv_copy = dtv; |
Rich Felker | 6a2eaa3 | 2012-10-06 16:51:03 -0400 | [diff] [blame] | 71 | return td; |
| 72 | } |
| 73 | |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 74 | #if ULONG_MAX == 0xffffffff |
| 75 | typedef Elf32_Phdr Phdr; |
| 76 | #else |
| 77 | typedef Elf64_Phdr Phdr; |
| 78 | #endif |
| 79 | |
Rich Felker | 9b95fd0 | 2018-09-05 12:43:34 -0400 | [diff] [blame] | 80 | extern weak hidden const size_t _DYNAMIC[]; |
Rich Felker | 5bf7eba | 2016-12-20 14:19:32 -0500 | [diff] [blame] | 81 | |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 82 | static void static_init_tls(size_t *aux) |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 83 | { |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 84 | unsigned char *p; |
Rich Felker | 3a5aa8e | 2012-11-01 22:58:17 -0400 | [diff] [blame] | 85 | size_t n; |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 86 | Phdr *phdr, *tls_phdr=0; |
| 87 | size_t base = 0; |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 88 | void *mem; |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 89 | |
Rich Felker | 0a96a37 | 2012-10-07 21:43:46 -0400 | [diff] [blame] | 90 | for (p=(void *)aux[AT_PHDR],n=aux[AT_PHNUM]; n; n--,p+=aux[AT_PHENT]) { |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 91 | phdr = (void *)p; |
| 92 | if (phdr->p_type == PT_PHDR) |
| 93 | base = aux[AT_PHDR] - phdr->p_vaddr; |
Rich Felker | 5bf7eba | 2016-12-20 14:19:32 -0500 | [diff] [blame] | 94 | if (phdr->p_type == PT_DYNAMIC && _DYNAMIC) |
| 95 | base = (size_t)_DYNAMIC - phdr->p_vaddr; |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 96 | if (phdr->p_type == PT_TLS) |
| 97 | tls_phdr = phdr; |
Rich Felker | 7b3348a | 2018-09-18 23:54:18 -0400 | [diff] [blame] | 98 | if (phdr->p_type == PT_GNU_STACK && |
| 99 | phdr->p_memsz > __default_stacksize) |
| 100 | __default_stacksize = |
| 101 | phdr->p_memsz < DEFAULT_STACK_MAX ? |
| 102 | phdr->p_memsz : DEFAULT_STACK_MAX; |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 103 | } |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 104 | |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 105 | if (tls_phdr) { |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 106 | main_tls.image = (void *)(base + tls_phdr->p_vaddr); |
| 107 | main_tls.len = tls_phdr->p_filesz; |
| 108 | main_tls.size = tls_phdr->p_memsz; |
| 109 | main_tls.align = tls_phdr->p_align; |
| 110 | libc.tls_cnt = 1; |
| 111 | libc.tls_head = &main_tls; |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 112 | } |
Rich Felker | 6a2eaa3 | 2012-10-06 16:51:03 -0400 | [diff] [blame] | 113 | |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 114 | main_tls.size += (-main_tls.size - (uintptr_t)main_tls.image) |
| 115 | & (main_tls.align-1); |
Szabolcs Nagy | 610c5a8 | 2018-06-02 01:52:01 +0200 | [diff] [blame] | 116 | #ifdef TLS_ABOVE_TP |
| 117 | main_tls.offset = GAP_ABOVE_TP; |
Szabolcs Nagy | a60b9e0 | 2019-05-13 18:47:11 +0000 | [diff] [blame] | 118 | main_tls.offset += (-GAP_ABOVE_TP + (uintptr_t)main_tls.image) |
| 119 | & (main_tls.align-1); |
Szabolcs Nagy | 610c5a8 | 2018-06-02 01:52:01 +0200 | [diff] [blame] | 120 | #else |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 121 | main_tls.offset = main_tls.size; |
| 122 | #endif |
Szabolcs Nagy | 610c5a8 | 2018-06-02 01:52:01 +0200 | [diff] [blame] | 123 | if (main_tls.align < MIN_TLS_ALIGN) main_tls.align = MIN_TLS_ALIGN; |
Rich Felker | 6a2eaa3 | 2012-10-06 16:51:03 -0400 | [diff] [blame] | 124 | |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 125 | libc.tls_align = main_tls.align; |
| 126 | libc.tls_size = 2*sizeof(void *) + sizeof(struct pthread) |
Szabolcs Nagy | 610c5a8 | 2018-06-02 01:52:01 +0200 | [diff] [blame] | 127 | #ifdef TLS_ABOVE_TP |
| 128 | + main_tls.offset |
| 129 | #endif |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 130 | + main_tls.size + main_tls.align |
Rich Felker | abead1b | 2015-04-23 18:51:02 -0400 | [diff] [blame] | 131 | + MIN_TLS_ALIGN-1 & -MIN_TLS_ALIGN; |
Rich Felker | 6a2eaa3 | 2012-10-06 16:51:03 -0400 | [diff] [blame] | 132 | |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 133 | if (libc.tls_size > sizeof builtin_tls) { |
Szabolcs Nagy | d86af2a | 2014-08-13 17:07:44 +0200 | [diff] [blame] | 134 | #ifndef SYS_mmap2 |
| 135 | #define SYS_mmap2 SYS_mmap |
Rich Felker | 98221c3 | 2014-03-23 23:19:30 -0400 | [diff] [blame] | 136 | #endif |
Szabolcs Nagy | d86af2a | 2014-08-13 17:07:44 +0200 | [diff] [blame] | 137 | mem = (void *)__syscall( |
| 138 | SYS_mmap2, |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 139 | 0, libc.tls_size, PROT_READ|PROT_WRITE, |
| 140 | MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); |
| 141 | /* -4095...-1 cast to void * will crash on dereference anyway, |
| 142 | * so don't bloat the init code checking for error codes and |
| 143 | * explicitly calling a_crash(). */ |
| 144 | } else { |
| 145 | mem = builtin_tls; |
| 146 | } |
Rich Felker | 98221c3 | 2014-03-23 23:19:30 -0400 | [diff] [blame] | 147 | |
Rich Felker | 19a1fe6 | 2015-04-13 19:24:51 -0400 | [diff] [blame] | 148 | /* Failure to initialize thread pointer is always fatal. */ |
| 149 | if (__init_tp(__copy_tls(mem)) < 0) |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 150 | a_crash(); |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 151 | } |
Rich Felker | d56460c | 2015-11-12 15:50:26 -0500 | [diff] [blame] | 152 | |
| 153 | weak_alias(static_init_tls, __init_tls); |