Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 1 | #!/bin/bash |
Craig Tiller | 6169d5f | 2016-03-31 07:46:18 -0700 | [diff] [blame] | 2 | # Copyright 2015, Google Inc. |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 3 | # All rights reserved. |
| 4 | # |
| 5 | # Redistribution and use in source and binary forms, with or without |
| 6 | # modification, are permitted provided that the following conditions are |
| 7 | # met: |
| 8 | # |
| 9 | # * Redistributions of source code must retain the above copyright |
| 10 | # notice, this list of conditions and the following disclaimer. |
| 11 | # * Redistributions in binary form must reproduce the above |
| 12 | # copyright notice, this list of conditions and the following disclaimer |
| 13 | # in the documentation and/or other materials provided with the |
| 14 | # distribution. |
| 15 | # * Neither the name of Google Inc. nor the names of its |
| 16 | # contributors may be used to endorse or promote products derived from |
| 17 | # this software without specific prior written permission. |
| 18 | # |
| 19 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 20 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 21 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 22 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 23 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 24 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 25 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 26 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 27 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 28 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 29 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 30 | |
| 31 | # Initializes a fresh GCE VM to become a jenkins linux performance worker. |
| 32 | # You shouldn't run this script on your own, |
| 33 | # use create_linux_performance_worker.sh instead. |
| 34 | |
| 35 | set -ex |
| 36 | |
| 37 | sudo apt-get update |
| 38 | |
Jan Tattermusch | bb1a453 | 2016-03-30 18:04:01 -0700 | [diff] [blame] | 39 | # Install Java 8 JDK (to build gRPC Java) |
| 40 | sudo apt-get install -y openjdk-8-jdk |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 41 | sudo apt-get install -y unzip lsof |
| 42 | |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 43 | sudo apt-get install -y \ |
| 44 | autoconf \ |
| 45 | autotools-dev \ |
| 46 | build-essential \ |
| 47 | bzip2 \ |
| 48 | ccache \ |
| 49 | curl \ |
| 50 | gcc \ |
| 51 | gcc-multilib \ |
| 52 | git \ |
| 53 | gyp \ |
| 54 | lcov \ |
| 55 | libc6 \ |
| 56 | libc6-dbg \ |
| 57 | libc6-dev \ |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 58 | libcurl4-openssl-dev \ |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 59 | libgtest-dev \ |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 60 | libreadline-dev \ |
| 61 | libssl-dev \ |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 62 | libtool \ |
| 63 | make \ |
| 64 | strace \ |
| 65 | pypy \ |
| 66 | python-dev \ |
| 67 | python-pip \ |
| 68 | python-setuptools \ |
| 69 | python-yaml \ |
Ken Payson | 1efb601 | 2016-06-08 13:06:44 -0700 | [diff] [blame] | 70 | python3-dev \ |
| 71 | python3-pip \ |
| 72 | python3-setuptools \ |
| 73 | python3-yaml \ |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 74 | telnet \ |
| 75 | unzip \ |
| 76 | wget \ |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 77 | zip \ |
| 78 | zlib1g-dev |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 79 | |
| 80 | # perftools |
| 81 | sudo apt-get install -y google-perftools libgoogle-perftools-dev |
| 82 | |
Jan Tattermusch | 1128019 | 2016-05-06 13:28:46 -0700 | [diff] [blame] | 83 | # netperf |
| 84 | sudo apt-get install -y netperf |
| 85 | |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 86 | # C++ dependencies |
| 87 | sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang |
| 88 | |
| 89 | # Python dependencies |
| 90 | sudo pip install tabulate |
Jan Tattermusch | 962c387 | 2016-04-21 10:09:06 -0700 | [diff] [blame] | 91 | sudo pip install google-api-python-client |
Jan Tattermusch | ab0a727 | 2016-10-15 21:12:41 +0200 | [diff] [blame] | 92 | sudo pip install virtualenv |
| 93 | |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 94 | # Building gRPC Python depends on python3.4 being installed, but python3.4 |
| 95 | # is not available on Ubuntu 16.10, so install from source |
| 96 | curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz |
| 97 | tar xzvf Python-3.4.6.tgz |
| 98 | cd Python-3.4.6 |
| 99 | ./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" |
| 100 | sudo make altinstall |
| 101 | cd .. |
| 102 | rm Python-3.4.6.tgz |
Jan Tattermusch | 962c387 | 2016-04-21 10:09:06 -0700 | [diff] [blame] | 103 | |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 104 | curl -O https://bootstrap.pypa.io/get-pip.py |
| 105 | sudo pypy get-pip.py |
| 106 | sudo pypy -m pip install tabulate |
Jan Tattermusch | 962c387 | 2016-04-21 10:09:06 -0700 | [diff] [blame] | 107 | sudo pip install google-api-python-client |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 108 | |
Jan Tattermusch | b19b62c | 2016-03-31 17:18:54 -0700 | [diff] [blame] | 109 | # Node dependencies (nvm has to be installed under user jenkins) |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 110 | touch .profile |
| 111 | curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash |
Jan Tattermusch | 15d2979 | 2016-05-09 23:27:05 -0700 | [diff] [blame] | 112 | source ~/.nvm/nvm.sh |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 113 | nvm install 0.12 && npm config set cache /tmp/npm-cache |
murgatroid99 | 957fe8d | 2016-05-02 15:03:55 -0700 | [diff] [blame] | 114 | nvm install 4 && npm config set cache /tmp/npm-cache |
| 115 | nvm install 5 && npm config set cache /tmp/npm-cache |
| 116 | nvm alias default 4 |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 117 | |
Jan Tattermusch | 8e258d3 | 2016-10-14 14:04:44 +0200 | [diff] [blame] | 118 | # C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives) |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 119 | sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF |
| 120 | echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list |
| 121 | sudo apt-get update |
| 122 | sudo apt-get install -y mono-devel nuget |
| 123 | |
Jan Tattermusch | 8e258d3 | 2016-10-14 14:04:44 +0200 | [diff] [blame] | 124 | # C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 125 | sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' |
Jan Tattermusch | 8e258d3 | 2016-10-14 14:04:44 +0200 | [diff] [blame] | 126 | sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 |
| 127 | sudo apt-get update |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 128 | sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 |
Jan Tattermusch | 17ce30e | 2017-04-05 18:10:50 +0200 | [diff] [blame] | 129 | sudo apt-get install -y dotnet-dev-1.0.1 |
Alexander Polcyn | e495928 | 2016-08-15 12:20:00 -0700 | [diff] [blame] | 130 | |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 131 | # Ruby dependencies |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 132 | git clone https://github.com/rbenv/rbenv.git ~/.rbenv |
| 133 | export PATH="$HOME/.rbenv/bin:$PATH" |
| 134 | eval "$(rbenv init -)" |
| 135 | |
| 136 | git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build |
| 137 | export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH" |
| 138 | |
| 139 | rbenv install 2.4.0 |
| 140 | rbenv global 2.4.0 |
| 141 | ruby -v |
Jan Tattermusch | bb1a453 | 2016-03-30 18:04:01 -0700 | [diff] [blame] | 142 | |
Jan Tattermusch | 962c387 | 2016-04-21 10:09:06 -0700 | [diff] [blame] | 143 | # Install bundler (prerequisite for gRPC Ruby) |
Jan Tattermusch | 962c387 | 2016-04-21 10:09:06 -0700 | [diff] [blame] | 144 | gem install bundler |
| 145 | |
Jan Tattermusch | bb1a453 | 2016-03-30 18:04:01 -0700 | [diff] [blame] | 146 | # Java dependencies - nothing as we already have Java JDK 8 |
Jan Tattermusch | 03b44ab | 2016-05-10 09:20:59 -0700 | [diff] [blame] | 147 | |
| 148 | # Go dependencies |
Alexander Polcyn | c4fb2e7 | 2016-10-07 11:00:45 -0700 | [diff] [blame] | 149 | # Currently, the golang package available via apt-get doesn't have the latest go. |
Alexander Polcyn | 46b9be7 | 2016-10-12 14:33:13 -0700 | [diff] [blame] | 150 | # Significant performance improvements with grpc-go have been observed after |
| 151 | # upgrading from go 1.5 to a later version, so a later go version is preferred. |
Alexander Polcyn | c4fb2e7 | 2016-10-07 11:00:45 -0700 | [diff] [blame] | 152 | # Following go install instructions from https://golang.org/doc/install |
Alexander Polcyn | 08ad565 | 2017-02-27 13:45:15 -0800 | [diff] [blame] | 153 | GO_VERSION=1.8 |
Alexander Polcyn | c4fb2e7 | 2016-10-07 11:00:45 -0700 | [diff] [blame] | 154 | OS=linux |
| 155 | ARCH=amd64 |
| 156 | curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz |
| 157 | sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz |
| 158 | # Put go on the PATH, keep the usual installation dir |
| 159 | sudo ln -s /usr/local/go/bin/go /usr/bin/go |
| 160 | rm go$GO_VERSION.$OS-$ARCH.tar.gz |
Alexander Polcyn | 9f08d11 | 2016-10-24 12:25:02 -0700 | [diff] [blame] | 161 | |
| 162 | # Install perf, to profile benchmarks. (need to get the right linux-tools-<> for kernel version) |
| 163 | sudo apt-get install -y linux-tools-common linux-tools-generic linux-tools-`uname -r` |
| 164 | # see http://unix.stackexchange.com/questions/14227/do-i-need-root-admin-permissions-to-run-userspace-perf-tool-perf-events-ar |
| 165 | echo 0 | sudo tee /proc/sys/kernel/perf_event_paranoid |
| 166 | # see http://stackoverflow.com/questions/21284906/perf-couldnt-record-kernel-reference-relocation-symbol |
| 167 | echo 0 | sudo tee /proc/sys/kernel/kptr_restrict |
| 168 | |
| 169 | # qps workers under perf appear to need a lot of mmap pages under certain scenarios and perf args in |
| 170 | # order to not lose perf events or time out |
| 171 | echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb |
| 172 | |
| 173 | # Fetch scripts to generate flame graphs from perf data collected |
| 174 | # on benchmarks |
| 175 | git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph |
| 176 | |
Alexander Polcyn | d88168b | 2017-03-28 14:13:37 -0700 | [diff] [blame] | 177 | # Install scipy and numpy for benchmarking scripts |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 178 | sudo apt-get install -y python-scipy python-numpy |
Matt Kwong | 654c367 | 2017-04-11 14:31:39 -0700 | [diff] [blame] | 179 | |
| 180 | # Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ |
| 181 | # This needs to happen as the last step to prevent Jenkins master from connecting |
| 182 | # to a machine that hasn't been properly setup yet. |
| 183 | cat jenkins_master.pub | sudo tee --append ~jenkins/.ssh/authorized_keys |
| 184 | |
| 185 | # Restart for VM to pick up kernel update |
| 186 | echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' |
| 187 | sleep 10 |
| 188 | sudo reboot |