Pirama Arumuga Nainar | 799172d | 2016-03-03 15:50:30 -0800 | [diff] [blame^] | 1 | /* ===---------- emutls.c - Implements __emutls_get_address ---------------=== |
| 2 | * |
| 3 | * The LLVM Compiler Infrastructure |
| 4 | * |
| 5 | * This file is dual licensed under the MIT and the University of Illinois Open |
| 6 | * Source Licenses. See LICENSE.TXT for details. |
| 7 | * |
| 8 | * ===----------------------------------------------------------------------=== |
| 9 | */ |
| 10 | #include <pthread.h> |
| 11 | #include <stdint.h> |
| 12 | #include <stdlib.h> |
| 13 | #include <string.h> |
| 14 | |
| 15 | #include "int_lib.h" |
| 16 | #include "int_util.h" |
| 17 | |
| 18 | /* Default is not to use posix_memalign, so systems like Android |
| 19 | * can use thread local data without heavier POSIX memory allocators. |
| 20 | */ |
| 21 | #ifndef EMUTLS_USE_POSIX_MEMALIGN |
| 22 | #define EMUTLS_USE_POSIX_MEMALIGN 0 |
| 23 | #endif |
| 24 | |
| 25 | /* For every TLS variable xyz, |
| 26 | * there is one __emutls_control variable named __emutls_v.xyz. |
| 27 | * If xyz has non-zero initial value, __emutls_v.xyz's "value" |
| 28 | * will point to __emutls_t.xyz, which has the initial value. |
| 29 | */ |
| 30 | typedef struct __emutls_control { |
| 31 | size_t size; /* size of the object in bytes */ |
| 32 | size_t align; /* alignment of the object in bytes */ |
| 33 | union { |
| 34 | uintptr_t index; /* data[index-1] is the object address */ |
| 35 | void* address; /* object address, when in single thread env */ |
| 36 | } object; |
| 37 | void* value; /* null or non-zero initial value for the object */ |
| 38 | } __emutls_control; |
| 39 | |
| 40 | static __inline void *emutls_memalign_alloc(size_t align, size_t size) { |
| 41 | void *base; |
| 42 | #if EMUTLS_USE_POSIX_MEMALIGN |
| 43 | if (posix_memalign(&base, align, size) != 0) |
| 44 | abort(); |
| 45 | #else |
| 46 | #define EXTRA_ALIGN_PTR_BYTES (align - 1 + sizeof(void*)) |
| 47 | char* object; |
| 48 | if ((object = malloc(EXTRA_ALIGN_PTR_BYTES + size)) == NULL) |
| 49 | abort(); |
| 50 | base = (void*)(((uintptr_t)(object + EXTRA_ALIGN_PTR_BYTES)) |
| 51 | & ~(uintptr_t)(align - 1)); |
| 52 | |
| 53 | ((void**)base)[-1] = object; |
| 54 | #endif |
| 55 | return base; |
| 56 | } |
| 57 | |
| 58 | static __inline void emutls_memalign_free(void *base) { |
| 59 | #if EMUTLS_USE_POSIX_MEMALIGN |
| 60 | free(base); |
| 61 | #else |
| 62 | /* The mallocated address is in ((void**)base)[-1] */ |
| 63 | free(((void**)base)[-1]); |
| 64 | #endif |
| 65 | } |
| 66 | |
| 67 | /* Emulated TLS objects are always allocated at run-time. */ |
| 68 | static __inline void *emutls_allocate_object(__emutls_control *control) { |
| 69 | /* Use standard C types, check with gcc's emutls.o. */ |
| 70 | typedef unsigned int gcc_word __attribute__((mode(word))); |
| 71 | typedef unsigned int gcc_pointer __attribute__((mode(pointer))); |
| 72 | COMPILE_TIME_ASSERT(sizeof(size_t) == sizeof(gcc_word)); |
| 73 | COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(gcc_pointer)); |
| 74 | COMPILE_TIME_ASSERT(sizeof(uintptr_t) == sizeof(void*)); |
| 75 | |
| 76 | size_t size = control->size; |
| 77 | size_t align = control->align; |
| 78 | if (align < sizeof(void*)) |
| 79 | align = sizeof(void*); |
| 80 | /* Make sure that align is power of 2. */ |
| 81 | if ((align & (align - 1)) != 0) |
| 82 | abort(); |
| 83 | |
| 84 | void* base = emutls_memalign_alloc(align, size); |
| 85 | if (control->value) |
| 86 | memcpy(base, control->value, size); |
| 87 | else |
| 88 | memset(base, 0, size); |
| 89 | return base; |
| 90 | } |
| 91 | |
| 92 | static pthread_mutex_t emutls_mutex = PTHREAD_MUTEX_INITIALIZER; |
| 93 | |
| 94 | static size_t emutls_num_object = 0; /* number of allocated TLS objects */ |
| 95 | |
| 96 | typedef struct emutls_address_array { |
| 97 | uintptr_t size; /* number of elements in the 'data' array */ |
| 98 | void* data[]; |
| 99 | } emutls_address_array; |
| 100 | |
| 101 | static pthread_key_t emutls_pthread_key; |
| 102 | |
| 103 | static void emutls_key_destructor(void* ptr) { |
| 104 | emutls_address_array* array = (emutls_address_array*)ptr; |
| 105 | uintptr_t i; |
| 106 | for (i = 0; i < array->size; ++i) { |
| 107 | if (array->data[i]) |
| 108 | emutls_memalign_free(array->data[i]); |
| 109 | } |
| 110 | free(ptr); |
| 111 | } |
| 112 | |
| 113 | static void emutls_init(void) { |
| 114 | if (pthread_key_create(&emutls_pthread_key, emutls_key_destructor) != 0) |
| 115 | abort(); |
| 116 | } |
| 117 | |
| 118 | /* Returns control->object.index; set index if not allocated yet. */ |
| 119 | static __inline uintptr_t emutls_get_index(__emutls_control *control) { |
| 120 | uintptr_t index = __atomic_load_n(&control->object.index, __ATOMIC_ACQUIRE); |
| 121 | if (!index) { |
| 122 | static pthread_once_t once = PTHREAD_ONCE_INIT; |
| 123 | pthread_once(&once, emutls_init); |
| 124 | pthread_mutex_lock(&emutls_mutex); |
| 125 | index = control->object.index; |
| 126 | if (!index) { |
| 127 | index = ++emutls_num_object; |
| 128 | __atomic_store_n(&control->object.index, index, __ATOMIC_RELEASE); |
| 129 | } |
| 130 | pthread_mutex_unlock(&emutls_mutex); |
| 131 | } |
| 132 | return index; |
| 133 | } |
| 134 | |
| 135 | /* Updates newly allocated thread local emutls_address_array. */ |
| 136 | static __inline void emutls_check_array_set_size(emutls_address_array *array, |
| 137 | uintptr_t size) { |
| 138 | if (array == NULL) |
| 139 | abort(); |
| 140 | array->size = size; |
| 141 | pthread_setspecific(emutls_pthread_key, (void*)array); |
| 142 | } |
| 143 | |
| 144 | /* Returns the new 'data' array size, number of elements, |
| 145 | * which must be no smaller than the given index. |
| 146 | */ |
| 147 | static __inline uintptr_t emutls_new_data_array_size(uintptr_t index) { |
| 148 | /* Need to allocate emutls_address_array with one extra slot |
| 149 | * to store the data array size. |
| 150 | * Round up the emutls_address_array size to multiple of 16. |
| 151 | */ |
| 152 | return ((index + 1 + 15) & ~((uintptr_t)15)) - 1; |
| 153 | } |
| 154 | |
| 155 | /* Returns the thread local emutls_address_array. |
| 156 | * Extends its size if necessary to hold address at index. |
| 157 | */ |
| 158 | static __inline emutls_address_array * |
| 159 | emutls_get_address_array(uintptr_t index) { |
| 160 | emutls_address_array* array = pthread_getspecific(emutls_pthread_key); |
| 161 | if (array == NULL) { |
| 162 | uintptr_t new_size = emutls_new_data_array_size(index); |
| 163 | array = calloc(new_size + 1, sizeof(void*)); |
| 164 | emutls_check_array_set_size(array, new_size); |
| 165 | } else if (index > array->size) { |
| 166 | uintptr_t orig_size = array->size; |
| 167 | uintptr_t new_size = emutls_new_data_array_size(index); |
| 168 | array = realloc(array, (new_size + 1) * sizeof(void*)); |
| 169 | if (array) |
| 170 | memset(array->data + orig_size, 0, |
| 171 | (new_size - orig_size) * sizeof(void*)); |
| 172 | emutls_check_array_set_size(array, new_size); |
| 173 | } |
| 174 | return array; |
| 175 | } |
| 176 | |
| 177 | void* __emutls_get_address(__emutls_control* control) { |
| 178 | uintptr_t index = emutls_get_index(control); |
| 179 | emutls_address_array* array = emutls_get_address_array(index); |
| 180 | if (array->data[index - 1] == NULL) |
| 181 | array->data[index - 1] = emutls_allocate_object(control); |
| 182 | return array->data[index - 1]; |
| 183 | } |