| /* Copyright (c) 2008-2010, Google Inc. |
| * All rights reserved. |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions are |
| * met: |
| * |
| * * Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * * Neither the name of Google Inc. nor the names of its |
| * contributors may be used to endorse or promote products derived from |
| * this software without specific prior written permission. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| */ |
| |
| // This file is part of ThreadSanitizer, a dynamic data race detector. |
| // Author: Konstantin Serebryany. |
| // Author: Timur Iskhodzhanov. |
| // |
| // See ts_util.h for mode details. |
| |
| #include "common_util.h" |
| #include "thread_sanitizer.h" |
| #include "ts_stats.h" |
| #include "ts_lock.h" |
| #include <stdarg.h> |
| |
| FLAGS *G_flags = NULL; |
| |
| #if defined(_MSC_VER) |
| |
| #pragma comment(lib, "winmm.lib") |
| |
| # ifdef TS_PIN |
| # include "pin.H" |
| # endif |
| namespace WINDOWS |
| { |
| // This is the way of including winows.h recommended by PIN docs. |
| #include<Windows.h> |
| } |
| int getpid() { return WINDOWS::GetCurrentProcessId(); } |
| #endif |
| |
| #if defined(TS_VALGRIND) |
| size_t TimeInMilliSeconds() { |
| return VG_(read_millisecond_timer)(); |
| } |
| #else |
| // TODO(kcc): implement this. |
| size_t TimeInMilliSeconds() { |
| #ifdef __GNUC__ |
| return time(0) * 1000; |
| #else |
| return WINDOWS::timeGetTime(); |
| #endif |
| } |
| #endif |
| |
| Stats *G_stats; |
| |
| #ifndef TS_LLVM |
| bool GetNameAndOffsetOfGlobalObject(uintptr_t addr, |
| string *name, uintptr_t *offset) { |
| # ifdef TS_VALGRIND |
| const int kBufLen = 1023; |
| char buff[kBufLen+1]; |
| PtrdiffT off; |
| if (VG_(get_datasym_and_offset)(addr, reinterpret_cast<Char*>(buff), |
| kBufLen, &off)) { |
| *name = buff; |
| *offset = off; |
| return true; |
| } |
| return false; |
| # else |
| return false; |
| # endif // TS_VALGRIND |
| } |
| #endif // TS_LLVM |
| |
| |
| #ifndef TS_VALGRIND |
| void GetThreadStack(int tid, uintptr_t *min_addr, uintptr_t *max_addr) { |
| *min_addr = 0xfffa; |
| *max_addr = 0xfffb; |
| } |
| #endif |
| |
| static int n_errs_found; |
| |
| void SetNumberOfFoundErrors(int n_errs) { |
| n_errs_found = n_errs; |
| } |
| |
| int GetNumberOfFoundErrors() { |
| return n_errs_found; |
| } |
| |
| |
| #if !defined(TS_VALGRIND) && !defined(TS_LLVM) |
| FILE *G_out = stderr; |
| #endif |
| |
| #ifdef TS_LLVM |
| FILE *G_out; |
| #endif |
| |
| static string RemoveUnsupportedFormat(const char *str) { |
| #ifdef _MSC_VER |
| // replace "%'" with "%" |
| string res; |
| size_t n = strlen(str); |
| if (n == 0) { |
| return ""; |
| } |
| res.reserve(n); |
| res.push_back(str[0]); |
| for (size_t i = 1; i < n; i++) { |
| if (str[i] == '\'' && *res.rbegin() == '%') continue; |
| res.push_back(str[i]); |
| } |
| return res; |
| #else |
| return str; |
| #endif |
| } |
| |
| void Printf(const char *format, ...) { |
| #ifdef TS_VALGRIND |
| va_list args; |
| va_start(args, format); |
| VG_(vprintf)(format, args); |
| va_end(args); |
| #else |
| va_list args; |
| va_start(args, format); |
| vfprintf(G_out, RemoveUnsupportedFormat(format).c_str(), args); |
| fflush(G_out); |
| va_end(args); |
| #endif |
| } |
| |
| // Like Print(), but prepend each line with ==XXXXX==, |
| // where XXXXX is the pid. |
| void Report(const char *format, ...) { |
| int buff_size = 1024*16; |
| char *buff = new char[buff_size]; |
| CHECK(buff); |
| DCHECK(G_flags); |
| |
| va_list args; |
| |
| while (1) { |
| va_start(args, format); |
| int ret = vsnprintf(buff, buff_size, |
| RemoveUnsupportedFormat(format).c_str(), args); |
| va_end(args); |
| if (ret < buff_size) break; |
| delete [] buff; |
| buff_size *= 2; |
| buff = new char[buff_size]; |
| CHECK(buff); |
| // Printf("Resized buff: %d\n", buff_size); |
| } |
| |
| char pid_buff[100]; |
| snprintf(pid_buff, sizeof(pid_buff), "==%d== ", getpid()); |
| |
| string res; |
| #ifndef TS_LLVM |
| int len = strlen(buff); |
| #else |
| int len = __real_strlen(buff); |
| #endif |
| bool last_was_new_line = true; |
| for (int i = 0; i < len; i++) { |
| if (G_flags->show_pid && last_was_new_line) |
| res += pid_buff; |
| last_was_new_line = (buff[i] == '\n'); |
| res += buff[i]; |
| } |
| |
| delete [] buff; |
| |
| Printf("%s", res.c_str()); |
| } |
| |
| long my_strtol(const char *str, char **end, int base) { |
| #ifdef TS_VALGRIND |
| if (base == 16 || (base == 0 && str && str[0] == '0' && str[1] == 'x')) { |
| return VG_(strtoll16)((Char*)str, (Char**)end); |
| } |
| return VG_(strtoll10)((Char*)str, (Char**)end); |
| #else |
| return strtoll(str, end, base); |
| #endif |
| } |
| |
| // Not thread-safe. Need to make it thread-local if we allow |
| // malloc to be called concurrently. |
| MallocCostCenterStack g_malloc_stack; |
| |
| size_t GetVmSizeInMb() { |
| #ifdef VGO_linux |
| const char *path ="/proc/self/statm"; // see 'man proc' |
| uintptr_t counter = G_stats->read_proc_self_stats++; |
| if (counter >= 1024 && ((counter & (counter - 1)) == 0)) |
| Report("INFO: reading %s for %ld'th time\n", path, counter); |
| int fd = OpenFileReadOnly(path, false); |
| if (fd < 0) return 0; |
| char buff[128]; |
| int n_read = read(fd, buff, sizeof(buff) - 1); |
| buff[n_read] = 0; |
| close(fd); |
| char *end; |
| size_t vm_size_in_pages = my_strtol(buff, &end, 10); |
| return vm_size_in_pages >> 8; |
| #else |
| return 0; |
| #endif |
| } |
| |
| static string StripTemplatesFromFunctionName(const string &fname) { |
| // Returns "" in case of error. |
| |
| string ret; |
| size_t read_pointer = 0, braces_depth = 0; |
| |
| while (read_pointer < fname.size()) { |
| size_t next_brace = fname.find_first_of("<>", read_pointer); |
| if (next_brace == fname.npos) { |
| if (braces_depth > 0) { |
| // This can happen on Visual Studio if we reach the ~2000 char limit. |
| CHECK(fname.size() > 256); |
| return ""; |
| } |
| ret += (fname.c_str() + read_pointer); |
| break; |
| } |
| |
| if (braces_depth == 0) { |
| ret.append(fname, read_pointer, next_brace - read_pointer); |
| } |
| |
| if (next_brace > 0) { |
| // We could have found one of the following operators. |
| const char *OP[] = {">>=", "<<=", |
| ">>", "<<", |
| ">=", "<=", |
| "->", "->*", |
| "<", ">"}; |
| |
| bool operator_name = false; |
| for (size_t i = 0; i < TS_ARRAY_SIZE(OP); i++) { |
| size_t op_offset = ((string)OP[i]).find(fname[next_brace]); |
| if (op_offset == string::npos) |
| continue; |
| if (next_brace >= 8 + op_offset && // 8 == strlen("operator"); |
| "operator" == fname.substr(next_brace - (8 + op_offset), 8) && |
| OP[i] == fname.substr(next_brace - op_offset, strlen(OP[i]))) { |
| operator_name = true; |
| ret += OP[i] + op_offset; |
| next_brace += strlen(OP[i] + op_offset); |
| read_pointer = next_brace; |
| break; |
| } |
| } |
| |
| if (operator_name) |
| continue; |
| } |
| |
| if (fname[next_brace] == '<') { |
| braces_depth++; |
| read_pointer = next_brace + 1; |
| } else if (fname[next_brace] == '>') { |
| if (braces_depth == 0) { |
| // Going to `braces_depth == -1` IS possible at least for this function on Windows: |
| // "std::operator<<char,std::char_traits<char>,std::allocator<char> >". |
| // Oh, well... Return an empty string and let the caller decide. |
| return ""; |
| } |
| braces_depth--; |
| read_pointer = next_brace + 1; |
| } else |
| CHECK(0); |
| } |
| if (braces_depth != 0) { |
| CHECK(fname.size() > 256); |
| return ""; |
| } |
| return ret; |
| } |
| |
| static string StripParametersFromFunctionName(const string &demangled) { |
| // Returns "" in case of error. |
| |
| string fname = demangled; |
| |
| // Strip stuff like "(***)" and "(anonymous namespace)" -> they are tricky. |
| size_t found = fname.npos; |
| while ((found = fname.find(", ")) != fname.npos) |
| fname.erase(found+1, 1); |
| while ((found = fname.find("(**")) != fname.npos) |
| fname.erase(found+2, 1); |
| while ((found = fname.find("(*)")) != fname.npos) |
| fname.erase(found, 3); |
| while ((found = fname.find("const()")) != fname.npos) |
| fname.erase(found+5, 2); |
| while ((found = fname.find("const volatile")) != fname.npos && |
| found > 1 && found + 14 == fname.size()) |
| fname.erase(found-1); |
| while ((found = fname.find("(anonymous namespace)")) != fname.npos) |
| fname.erase(found, 21); |
| |
| if (fname.find_first_of("(") == fname.npos) |
| return fname; |
| DCHECK(count(fname.begin(), fname.end(), '(') == |
| count(fname.begin(), fname.end(), ')')); |
| |
| string ret; |
| bool returns_fun_ptr = false; |
| size_t braces_depth = 0, read_pointer = 0; |
| |
| size_t first_parenthesis = fname.find("("); |
| if (first_parenthesis != fname.npos) { |
| DCHECK(fname.find_first_of(")") != fname.npos); |
| DCHECK(fname.find_first_of(")") > first_parenthesis); |
| DCHECK(fname[first_parenthesis] == '('); |
| if (first_parenthesis + 2 < fname.size() && |
| fname[first_parenthesis - 1] == ' ' && |
| fname[first_parenthesis + 1] == '*' && |
| fname[first_parenthesis + 2] != ' ') { |
| // Return value type is a function pointer |
| read_pointer = first_parenthesis + 2; |
| while (fname[read_pointer] == '*' || fname[read_pointer] == '&') |
| read_pointer++; |
| braces_depth = 1; |
| returns_fun_ptr = true; |
| } |
| } |
| |
| while (read_pointer < fname.size()) { |
| size_t next_brace = fname.find_first_of("()", read_pointer); |
| if (next_brace == fname.npos) { |
| if (braces_depth != 0) { |
| // Overflow? |
| return ""; |
| } |
| size_t _const = fname.find(" const", read_pointer); |
| if (_const == fname.npos) { |
| ret += (fname.c_str() + read_pointer); |
| } else { |
| CHECK(_const + 6 == fname.size()); |
| ret.append(fname, read_pointer, _const - read_pointer); |
| } |
| break; |
| } |
| |
| if (braces_depth == (returns_fun_ptr ? 1 : 0)) { |
| ret.append(fname, read_pointer, next_brace - read_pointer); |
| returns_fun_ptr = false; |
| } |
| |
| if (fname[next_brace] == '(') { |
| if (next_brace >= 8 && fname[next_brace+1] == ')' && |
| "operator" == fname.substr(next_brace - 8, 8)) { |
| ret += "()"; |
| read_pointer = next_brace + 2; |
| } else { |
| braces_depth++; |
| read_pointer = next_brace + 1; |
| } |
| } else if (fname[next_brace] == ')') { |
| CHECK(braces_depth > 0); |
| braces_depth--; |
| read_pointer = next_brace + 1; |
| } else |
| CHECK(0); |
| } |
| if (braces_depth != 0) |
| return ""; |
| |
| // Special case: on Linux, Valgrind prepends the return type for template |
| // functions. And on Windows we may see `scalar deleting destructor'. |
| // And we may see "operaror new" etc. |
| // And some STL code inserts const& between the return type and the function |
| // name. |
| // Oh, well... |
| size_t space_or_tick; |
| while (ret != "") { |
| space_or_tick = ret.find_first_of("` "); |
| if (space_or_tick != ret.npos && ret[space_or_tick] == ' ' && |
| ret.substr(0, space_or_tick).find("operator") == string::npos) { |
| ret = ret.substr(space_or_tick + 1); |
| } else if (space_or_tick != ret.npos && space_or_tick + 1 == ret.size()) { |
| ret = ret.substr(0, space_or_tick); |
| } else { |
| break; |
| } |
| } |
| return ret; |
| } |
| |
| string NormalizeFunctionName(const string &demangled) { |
| if (demangled[1] == '[' && strchr("+-=", demangled[0]) != NULL) { |
| // Objective-C function |
| return demangled; |
| } |
| |
| if (demangled.find_first_of("<>()") == demangled.npos) { |
| // C function or a well-formatted function name. |
| return demangled; |
| } |
| |
| if (demangled == "(below main)" || demangled == "(no symbols)") |
| return demangled; |
| |
| const char* const MALFORMED = "(malformed frame)"; |
| |
| string fname = StripTemplatesFromFunctionName(demangled); |
| if (fname.size() == 0) { |
| if (DEBUG_MODE) |
| Printf("PANIC: `%s`\n", demangled.c_str()); |
| return MALFORMED; |
| } |
| |
| fname = StripParametersFromFunctionName(fname); |
| if (fname.size() == 0) { |
| CHECK(demangled.size() >= 256); |
| if (DEBUG_MODE) |
| Printf("PANIC: `%s`\n", demangled.c_str()); |
| return MALFORMED; |
| } |
| |
| return fname; |
| } |
| |
| void OpenFileWriteStringAndClose(const string &file_name, const string &str) { |
| #ifdef TS_VALGRIND |
| SysRes sres = VG_(open)((const Char*)file_name.c_str(), |
| VKI_O_WRONLY|VKI_O_CREAT|VKI_O_TRUNC, |
| VKI_S_IRUSR|VKI_S_IWUSR); |
| if (sr_isError(sres)) { |
| Report("WARNING: can not open file %s\n", file_name.c_str()); |
| exit(1); |
| } |
| int fd = sr_Res(sres); |
| write(fd, str.c_str(), str.size()); |
| close(fd); |
| #else |
| CHECK(0); |
| #endif |
| } |
| |
| //--------- Sockets ------------------ {{{1 |
| #if defined(TS_PIN) && defined(__GNUC__) |
| #include <sys/types.h> |
| #include <sys/socket.h> |
| #include <netinet/in.h> |
| #include <netdb.h> |
| FILE *OpenSocketForWriting(const string &host_and_port) { |
| size_t col = host_and_port.find(":"); |
| if (col == string::npos) return NULL; |
| string host = host_and_port.substr(0, col); |
| string port_str = host_and_port.substr(col + 1); |
| int sockfd; |
| struct sockaddr_in serv_addr; |
| struct hostent *server; |
| sockfd = socket(AF_INET, SOCK_STREAM, 0); |
| if (sockfd < 0) return NULL; |
| server = gethostbyname(host.c_str()); |
| if (server == 0) return NULL; |
| memset(&serv_addr, 0, sizeof(serv_addr)); |
| serv_addr.sin_family = AF_INET; |
| memcpy((char *)&serv_addr.sin_addr.s_addr, |
| (char *)server->h_addr, |
| server->h_length); |
| serv_addr.sin_port = htons(atoi(port_str.c_str())); |
| if (connect(sockfd, (struct sockaddr*)&serv_addr, sizeof(serv_addr)) < 0) |
| return NULL; |
| return fdopen(sockfd, "w"); |
| } |
| #else |
| FILE *OpenSocketForWriting(const string &host_and_port) { |
| return NULL; // unimplemented. |
| } |
| #endif |
| //--------- TSLock ------------------ {{{1 |
| #ifdef _MSC_VER |
| //# define TS_LOCK_PIPE |
| # define TS_LOCK_PIN |
| #else |
| # define TS_LOCK_FUTEX |
| #endif |
| |
| #if defined(TS_LOCK_PIPE) && defined(TS_PIN) |
| #ifdef __GNUC__ |
| #include <unistd.h> |
| // Lock based on pipe's send/receive. The idea (but not the code) |
| // is shamelessly stolen from valgrind's /coregrind/m_scheduler/sema.c |
| struct TSLock::Rep { |
| bool held; |
| char pipe_char; |
| int pipe_fd[2]; |
| |
| void Write() { |
| char buf[2]; |
| buf[0] = pipe_char; |
| buf[1] = 0; |
| int res = write(pipe_fd[1], buf, 1); |
| CHECK(res == 1); |
| } |
| bool Read() { |
| char buf[2]; |
| buf[0] = 0; |
| buf[1] = 0; |
| int res = read(pipe_fd[0], buf, 1); |
| if (res != 1) |
| return false; |
| //Printf("rep::Read: %c\n", buf[0]); |
| |
| pipe_char++; |
| if (pipe_char == 'Z' + 1) pipe_char = 'A'; |
| return true; |
| } |
| void Open() { |
| CHECK(0 == pipe(pipe_fd)); |
| CHECK(pipe_fd[0] != pipe_fd[1]); |
| pipe_char = 'A'; |
| } |
| void Close() { |
| close(pipe_fd[0]); |
| close(pipe_fd[1]); |
| } |
| }; |
| #elif defined(_MSC_VER) |
| struct TSLock::Rep { |
| bool held; |
| char pipe_char; |
| WINDOWS::HANDLE pipe_fd[2]; |
| void Write() { |
| char buf[2]; |
| buf[0] = pipe_char; |
| buf[1] = 0; |
| WINDOWS::DWORD n_written = 0; |
| int res = WINDOWS::WriteFile(pipe_fd[1], buf, 1, &n_written, NULL); |
| CHECK(res != 0 && n_written == 1); |
| } |
| bool Read() { |
| char buf[2]; |
| buf[0] = 0; |
| buf[1] = 0; |
| WINDOWS::DWORD n_read = 0; |
| int res = WINDOWS::ReadFile(pipe_fd[0], buf, 1, &n_read, NULL); |
| if (res == 0 && n_read == 0) |
| return false; |
| //Printf("rep::Read: %c\n", buf[0]); |
| |
| pipe_char++; |
| if (pipe_char == 'Z' + 1) pipe_char = 'A'; |
| return true; |
| } |
| void Open() { |
| CHECK(WINDOWS::CreatePipe(&pipe_fd[0], &pipe_fd[1], NULL, 0)); |
| CHECK(pipe_fd[0] != pipe_fd[1]); |
| pipe_char = 'A'; |
| } |
| void Close() { |
| WINDOWS::CloseHandle(pipe_fd[0]); |
| WINDOWS::CloseHandle(pipe_fd[1]); |
| } |
| }; |
| #endif |
| |
| TSLock::TSLock() { |
| rep_ = new Rep; |
| rep_->held = false; |
| rep_->Open(); |
| rep_->Write(); |
| } |
| TSLock::~TSLock() { |
| rep_->Close(); |
| } |
| void TSLock::Lock() { |
| while(rep_->Read() == false) |
| ; |
| rep_->held = true; |
| } |
| void TSLock::Unlock() { |
| rep_->held = false; |
| rep_->Write(); |
| } |
| void TSLock::AssertHeld() { |
| DCHECK(rep_->held); |
| } |
| #endif // __GNUC__ & TS_LOCK_PIPE |
| |
| #if defined(TS_LOCK_PIN) && defined(TS_PIN) |
| #include "pin.H" |
| struct TSLock::Rep { |
| PIN_LOCK lock; |
| bool held; |
| }; |
| |
| TSLock::TSLock() { |
| rep_ = new Rep(); |
| rep_->held = false; |
| InitLock(&rep_->lock); |
| } |
| TSLock::~TSLock() { |
| delete rep_; |
| } |
| void TSLock::Lock() { |
| GetLock(&rep_->lock, __LINE__); |
| rep_->held = true; |
| } |
| void TSLock::Unlock() { |
| rep_->held = false; |
| ReleaseLock(&rep_->lock); |
| } |
| void TSLock::AssertHeld() { |
| DCHECK(rep_->held); |
| } |
| #endif // TS_LOCK_PIN |
| |
| #if defined(TS_WRAP_PTHREAD_LOCK) |
| #include "tsan_rtl_wrap.h" |
| |
| struct TSLock::Rep { |
| pthread_mutex_t lock; |
| bool held; |
| }; |
| TSLock::TSLock() { |
| rep_ = new Rep(); |
| rep_->held = false; |
| __real_pthread_mutex_init(&rep_->lock, NULL); |
| } |
| TSLock::~TSLock() { |
| __real_pthread_mutex_destroy(&rep_->lock); |
| delete rep_; |
| } |
| void TSLock::Lock() { |
| __real_pthread_mutex_lock(&rep_->lock); |
| rep_->held = true; |
| } |
| void TSLock::Unlock() { |
| rep_->held = false; |
| __real_pthread_mutex_unlock(&rep_->lock); |
| } |
| void TSLock::AssertHeld() { |
| DCHECK(rep_->held); |
| } |
| #endif // TS_LLVM |
| |
| #if defined(TS_LOCK_FUTEX) && defined(__GNUC__) && \ |
| (defined (TS_PIN) || defined (TS_LLVM)) |
| #include <linux/futex.h> |
| #include <sys/time.h> |
| #include <syscall.h> |
| |
| // Simple futex-based lock. |
| // The idea is taken from "Futexes Are Tricky" by Ulrich Drepper |
| |
| TSLock::TSLock() { |
| rep_ = 0; |
| ANNOTATE_BENIGN_RACE(&rep_, "Benign race on TSLock::rep_"); |
| ANNOTATE_RWLOCK_CREATE(this); |
| } |
| TSLock::~TSLock() { |
| ANNOTATE_RWLOCK_DESTROY(this); |
| DCHECK(rep_ == 0); |
| } |
| void TSLock::Lock() { |
| int *p = (int*)&rep_; |
| const int kSpinCount = 100; |
| DCHECK(kSpinCount > 0); |
| int c; |
| for (int i = 0; i < kSpinCount; i++) { |
| c = __sync_val_compare_and_swap(p, 0, 1); |
| if (c == 0) break; |
| } |
| if (c == 0) { |
| // The mutex was unlocked. Now it's ours. Done. |
| ANNOTATE_RWLOCK_ACQUIRED(this, /*is_w*/true); |
| return; |
| } |
| DCHECK(c == 1 || c == 2); |
| // We are going to block on this lock. Make sure others know that. |
| if (c != 2) { |
| c = __sync_lock_test_and_set(p, 2); |
| } |
| // Block. |
| int n_waits = 0; |
| while (c != 0) { |
| syscall(SYS_futex, p, FUTEX_WAIT, 2, 0, 0, 0); |
| n_waits++; |
| c = __sync_lock_test_and_set(p, 2); |
| } |
| ANNOTATE_RWLOCK_ACQUIRED(this, /*is_w*/true); |
| G_stats->futex_wait += n_waits; |
| } |
| void TSLock::Unlock() { |
| ANNOTATE_RWLOCK_RELEASED(this, /*is_w*/true); |
| int *p = (int*)&rep_; |
| DCHECK(*p == 1 || *p == 2); |
| int c = __sync_sub_and_fetch(p, 1); |
| DCHECK(c == 0 || c == 1); |
| if (c == 1) { |
| *p = 0; |
| syscall(SYS_futex, p, FUTEX_WAKE, 1, 0, 0, 0); |
| } |
| } |
| void TSLock::AssertHeld() { |
| DCHECK(rep_); |
| } |
| #endif |
| |
| // Same as above to compile Go's rtl |
| // No annotations in this version: it should be simple as possible. |
| #if defined(TS_LOCK_FUTEX) && defined(__GNUC__) && \ |
| (defined (TS_GO)) |
| #include <linux/futex.h> // TODO(mpimenov): portability? |
| #include <sys/time.h> |
| #include <syscall.h> |
| |
| // Simple futex-based lock. |
| // The idea is taken from "Futexes Are Tricky" by Ulrich Drepper |
| |
| TSLock::TSLock() { |
| rep_ = 0; |
| } |
| TSLock::~TSLock() { |
| DCHECK(rep_ == 0); |
| } |
| void TSLock::Lock() { |
| int *p = (int*)&rep_; |
| const int kSpinCount = 100; |
| DCHECK(kSpinCount > 0); |
| int c; |
| for (int i = 0; i < kSpinCount; i++) { |
| c = __sync_val_compare_and_swap(p, 0, 1); |
| if (c == 0) break; |
| } |
| if (c == 0) { |
| // The mutex was unlocked. Now it's ours. Done. |
| return; |
| } |
| DCHECK(c == 1 || c == 2); |
| // We are going to block on this lock. Make sure others know that. |
| if (c != 2) { |
| c = __sync_lock_test_and_set(p, 2); |
| } |
| // Block. |
| int n_waits = 0; |
| while (c != 0) { |
| syscall(SYS_futex, p, FUTEX_WAIT, 2, 0, 0, 0); |
| n_waits++; |
| c = __sync_lock_test_and_set(p, 2); |
| } |
| G_stats->futex_wait += n_waits; |
| } |
| void TSLock::Unlock() { |
| int *p = (int*)&rep_; |
| DCHECK(*p == 1 || *p == 2); |
| int c = __sync_sub_and_fetch(p, 1); |
| DCHECK(c == 0 || c == 1); |
| if (c == 1) { |
| *p = 0; |
| syscall(SYS_futex, p, FUTEX_WAKE, 1, 0, 0, 0); |
| } |
| } |
| void TSLock::AssertHeld() { |
| DCHECK(rep_); |
| } |
| #endif // (TS_LOCK_FUTEX) (__GNUC__) && (TS_GO) |
| |
| //--------------- Atomics ----------------- {{{1 |
| #if defined (_MSC_VER) && TS_SERIALIZED == 0 |
| uintptr_t AtomicExchange(uintptr_t *ptr, uintptr_t new_value) { |
| return _InterlockedExchange((volatile WINDOWS::LONG*)ptr, new_value); |
| } |
| |
| void ReleaseStore(uintptr_t *ptr, uintptr_t value) { |
| *(volatile uintptr_t*)ptr = value; |
| // TODO(kcc): anything to add here? |
| } |
| |
| int32_t NoBarrier_AtomicIncrement(int32_t* ptr) { |
| return _InterlockedIncrement((volatile WINDOWS::LONG *)ptr); |
| } |
| |
| int32_t NoBarrier_AtomicDecrement(int32_t* ptr) { |
| return _InterlockedDecrement((volatile WINDOWS::LONG *)ptr); |
| } |
| #endif // _MSC_VER && TS_SERIALIZED |
| //--------------- YIELD ----------------- {{{1 |
| #if defined (_MSC_VER) |
| #include <intrin.h> |
| void YIELD() { |
| WINDOWS::Sleep(0); |
| } |
| void PROCESSOR_YIELD() { |
| _mm_pause(); |
| } |
| #elif defined(TS_VALGRIND) |
| void YIELD() { |
| } |
| void PROCESSOR_YIELD() { |
| } |
| #elif defined(__GNUC__) |
| void YIELD() { |
| sched_yield(); |
| } |
| void PROCESSOR_YIELD() { |
| __asm__ __volatile__ ("pause"); |
| } |
| #else |
| #error "Unknown config" |
| #endif |
| |
| // end. {{{1 |
| // vim:shiftwidth=2:softtabstop=2:expandtab:tw=80 |