Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 1 | #include <elf.h> |
| 2 | #include <limits.h> |
Rich Felker | efd4d87 | 2012-11-08 17:04:20 -0500 | [diff] [blame] | 3 | #include <sys/mman.h> |
| 4 | #include <string.h> |
Rich Felker | bd67959 | 2015-03-06 13:27:08 -0500 | [diff] [blame] | 5 | #include <stddef.h> |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 6 | #include "pthread_impl.h" |
| 7 | #include "libc.h" |
| 8 | #include "atomic.h" |
Rich Felker | 98221c3 | 2014-03-23 23:19:30 -0400 | [diff] [blame] | 9 | #include "syscall.h" |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 10 | |
Rich Felker | 5f51d52 | 2015-04-23 17:04:31 -0400 | [diff] [blame] | 11 | #ifndef SHARED |
| 12 | static |
| 13 | #endif |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 14 | int __init_tp(void *p) |
| 15 | { |
| 16 | pthread_t td = p; |
| 17 | td->self = td; |
Rich Felker | 64e3228 | 2014-06-10 03:36:56 -0400 | [diff] [blame] | 18 | int r = __set_thread_area(TP_ADJ(p)); |
| 19 | if (r < 0) return -1; |
| 20 | if (!r) libc.can_do_threads = 1; |
Rich Felker | 83dc6eb | 2014-07-05 23:29:55 -0400 | [diff] [blame] | 21 | td->tid = __syscall(SYS_set_tid_address, &td->tid); |
Rich Felker | 0bc0309 | 2014-07-02 19:33:19 -0400 | [diff] [blame] | 22 | td->locale = &libc.global_locale; |
Rich Felker | 4e98cce | 2015-04-10 00:54:48 -0400 | [diff] [blame] | 23 | td->robust_list.head = &td->robust_list.head; |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 24 | return 0; |
| 25 | } |
| 26 | |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 27 | #ifndef SHARED |
| 28 | |
Rich Felker | bd67959 | 2015-03-06 13:27:08 -0500 | [diff] [blame] | 29 | static struct builtin_tls { |
| 30 | char c; |
| 31 | struct pthread pt; |
| 32 | void *space[16]; |
| 33 | } builtin_tls[1]; |
| 34 | #define MIN_TLS_ALIGN offsetof(struct builtin_tls, pt) |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 35 | |
Rich Felker | 7c6c290 | 2013-08-03 16:27:30 -0400 | [diff] [blame] | 36 | struct tls_image { |
| 37 | void *image; |
| 38 | size_t len, size, align; |
Rich Felker | c267fb8 | 2015-04-22 02:36:54 -0400 | [diff] [blame] | 39 | } __static_tls; |
Rich Felker | 7c6c290 | 2013-08-03 16:27:30 -0400 | [diff] [blame] | 40 | |
| 41 | #define T __static_tls |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 42 | |
Rich Felker | dcd6037 | 2012-10-05 11:51:50 -0400 | [diff] [blame] | 43 | void *__copy_tls(unsigned char *mem) |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 44 | { |
Rich Felker | 6a2eaa3 | 2012-10-06 16:51:03 -0400 | [diff] [blame] | 45 | pthread_t td; |
Rich Felker | 7c6c290 | 2013-08-03 16:27:30 -0400 | [diff] [blame] | 46 | if (!T.image) return mem; |
Rich Felker | 6a2eaa3 | 2012-10-06 16:51:03 -0400 | [diff] [blame] | 47 | void **dtv = (void *)mem; |
| 48 | dtv[0] = (void *)1; |
Rich Felker | 9ec4283 | 2012-10-15 18:51:53 -0400 | [diff] [blame] | 49 | #ifdef TLS_ABOVE_TP |
| 50 | mem += sizeof(void *) * 2; |
Rich Felker | 7c6c290 | 2013-08-03 16:27:30 -0400 | [diff] [blame] | 51 | mem += -((uintptr_t)mem + sizeof(struct pthread)) & (T.align-1); |
Rich Felker | 9ec4283 | 2012-10-15 18:51:53 -0400 | [diff] [blame] | 52 | td = (pthread_t)mem; |
| 53 | mem += sizeof(struct pthread); |
| 54 | #else |
Rich Felker | e172c7b | 2012-12-25 21:51:11 -0500 | [diff] [blame] | 55 | mem += libc.tls_size - sizeof(struct pthread); |
Rich Felker | 7c6c290 | 2013-08-03 16:27:30 -0400 | [diff] [blame] | 56 | mem -= (uintptr_t)mem & (T.align-1); |
Rich Felker | 6a2eaa3 | 2012-10-06 16:51:03 -0400 | [diff] [blame] | 57 | td = (pthread_t)mem; |
Rich Felker | 7c6c290 | 2013-08-03 16:27:30 -0400 | [diff] [blame] | 58 | mem -= T.size; |
Rich Felker | 9ec4283 | 2012-10-15 18:51:53 -0400 | [diff] [blame] | 59 | #endif |
Szabolcs Nagy | 204a69d | 2015-03-11 12:48:12 +0000 | [diff] [blame] | 60 | td->dtv = td->dtv_copy = dtv; |
Rich Felker | 6a2eaa3 | 2012-10-06 16:51:03 -0400 | [diff] [blame] | 61 | dtv[1] = mem; |
Rich Felker | 7c6c290 | 2013-08-03 16:27:30 -0400 | [diff] [blame] | 62 | memcpy(mem, T.image, T.len); |
Rich Felker | 6a2eaa3 | 2012-10-06 16:51:03 -0400 | [diff] [blame] | 63 | return td; |
| 64 | } |
| 65 | |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 66 | #if ULONG_MAX == 0xffffffff |
| 67 | typedef Elf32_Phdr Phdr; |
| 68 | #else |
| 69 | typedef Elf64_Phdr Phdr; |
| 70 | #endif |
| 71 | |
Rich Felker | 0a96a37 | 2012-10-07 21:43:46 -0400 | [diff] [blame] | 72 | void __init_tls(size_t *aux) |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 73 | { |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 74 | unsigned char *p; |
Rich Felker | 3a5aa8e | 2012-11-01 22:58:17 -0400 | [diff] [blame] | 75 | size_t n; |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 76 | Phdr *phdr, *tls_phdr=0; |
| 77 | size_t base = 0; |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 78 | void *mem; |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 79 | |
Rich Felker | 0a96a37 | 2012-10-07 21:43:46 -0400 | [diff] [blame] | 80 | for (p=(void *)aux[AT_PHDR],n=aux[AT_PHNUM]; n; n--,p+=aux[AT_PHENT]) { |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 81 | phdr = (void *)p; |
| 82 | if (phdr->p_type == PT_PHDR) |
| 83 | base = aux[AT_PHDR] - phdr->p_vaddr; |
| 84 | if (phdr->p_type == PT_TLS) |
| 85 | tls_phdr = phdr; |
| 86 | } |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 87 | |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 88 | if (tls_phdr) { |
| 89 | T.image = (void *)(base + tls_phdr->p_vaddr); |
| 90 | T.len = tls_phdr->p_filesz; |
| 91 | T.size = tls_phdr->p_memsz; |
| 92 | T.align = tls_phdr->p_align; |
| 93 | } |
Rich Felker | 6a2eaa3 | 2012-10-06 16:51:03 -0400 | [diff] [blame] | 94 | |
Rich Felker | 7c6c290 | 2013-08-03 16:27:30 -0400 | [diff] [blame] | 95 | T.size += (-T.size - (uintptr_t)T.image) & (T.align-1); |
Rich Felker | bd67959 | 2015-03-06 13:27:08 -0500 | [diff] [blame] | 96 | if (T.align < MIN_TLS_ALIGN) T.align = MIN_TLS_ALIGN; |
Rich Felker | 6a2eaa3 | 2012-10-06 16:51:03 -0400 | [diff] [blame] | 97 | |
Rich Felker | abead1b | 2015-04-23 18:51:02 -0400 | [diff] [blame] | 98 | libc.tls_size = 2*sizeof(void *)+T.size+T.align+sizeof(struct pthread) |
| 99 | + MIN_TLS_ALIGN-1 & -MIN_TLS_ALIGN; |
Rich Felker | 6a2eaa3 | 2012-10-06 16:51:03 -0400 | [diff] [blame] | 100 | |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 101 | if (libc.tls_size > sizeof builtin_tls) { |
Szabolcs Nagy | d86af2a | 2014-08-13 17:07:44 +0200 | [diff] [blame] | 102 | #ifndef SYS_mmap2 |
| 103 | #define SYS_mmap2 SYS_mmap |
Rich Felker | 98221c3 | 2014-03-23 23:19:30 -0400 | [diff] [blame] | 104 | #endif |
Szabolcs Nagy | d86af2a | 2014-08-13 17:07:44 +0200 | [diff] [blame] | 105 | mem = (void *)__syscall( |
| 106 | SYS_mmap2, |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 107 | 0, libc.tls_size, PROT_READ|PROT_WRITE, |
| 108 | MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); |
| 109 | /* -4095...-1 cast to void * will crash on dereference anyway, |
| 110 | * so don't bloat the init code checking for error codes and |
| 111 | * explicitly calling a_crash(). */ |
| 112 | } else { |
| 113 | mem = builtin_tls; |
| 114 | } |
Rich Felker | 98221c3 | 2014-03-23 23:19:30 -0400 | [diff] [blame] | 115 | |
Rich Felker | 19a1fe6 | 2015-04-13 19:24:51 -0400 | [diff] [blame] | 116 | /* Failure to initialize thread pointer is always fatal. */ |
| 117 | if (__init_tp(__copy_tls(mem)) < 0) |
Rich Felker | dab441a | 2014-03-24 16:57:11 -0400 | [diff] [blame] | 118 | a_crash(); |
Rich Felker | 8431d79 | 2012-10-04 16:35:46 -0400 | [diff] [blame] | 119 | } |
| 120 | #else |
| 121 | void __init_tls(size_t *auxv) { } |
| 122 | #endif |