Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 1 | #!/bin/bash |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 2 | # Copyright 2015 gRPC authors. |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 3 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 7 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 9 | # |
Jan Tattermusch | 7897ae9 | 2017-06-07 22:57:36 +0200 | [diff] [blame] | 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 15 | |
| 16 | # Initializes a fresh GCE VM to become a jenkins linux performance worker. |
| 17 | # You shouldn't run this script on your own, |
| 18 | # use create_linux_performance_worker.sh instead. |
| 19 | |
| 20 | set -ex |
| 21 | |
| 22 | sudo apt-get update |
| 23 | |
Jan Tattermusch | bb1a453 | 2016-03-30 18:04:01 -0700 | [diff] [blame] | 24 | # Install Java 8 JDK (to build gRPC Java) |
| 25 | sudo apt-get install -y openjdk-8-jdk |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 26 | sudo apt-get install -y unzip lsof |
| 27 | |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 28 | sudo apt-get install -y \ |
| 29 | autoconf \ |
| 30 | autotools-dev \ |
| 31 | build-essential \ |
| 32 | bzip2 \ |
| 33 | ccache \ |
| 34 | curl \ |
| 35 | gcc \ |
| 36 | gcc-multilib \ |
| 37 | git \ |
| 38 | gyp \ |
| 39 | lcov \ |
| 40 | libc6 \ |
| 41 | libc6-dbg \ |
| 42 | libc6-dev \ |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 43 | libcurl4-openssl-dev \ |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 44 | libgtest-dev \ |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 45 | libreadline-dev \ |
| 46 | libssl-dev \ |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 47 | libtool \ |
| 48 | make \ |
| 49 | strace \ |
| 50 | pypy \ |
| 51 | python-dev \ |
| 52 | python-pip \ |
| 53 | python-setuptools \ |
| 54 | python-yaml \ |
Ken Payson | 1efb601 | 2016-06-08 13:06:44 -0700 | [diff] [blame] | 55 | python3-dev \ |
| 56 | python3-pip \ |
| 57 | python3-setuptools \ |
| 58 | python3-yaml \ |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 59 | telnet \ |
| 60 | unzip \ |
| 61 | wget \ |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 62 | zip \ |
| 63 | zlib1g-dev |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 64 | |
| 65 | # perftools |
| 66 | sudo apt-get install -y google-perftools libgoogle-perftools-dev |
| 67 | |
Jan Tattermusch | 1128019 | 2016-05-06 13:28:46 -0700 | [diff] [blame] | 68 | # netperf |
| 69 | sudo apt-get install -y netperf |
| 70 | |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 71 | # C++ dependencies |
| 72 | sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang |
| 73 | |
| 74 | # Python dependencies |
Nathaniel Manista | 8bba3bf | 2017-06-26 16:33:25 +0000 | [diff] [blame] | 75 | sudo pip install --upgrade pip==9.0.1 |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 76 | sudo pip install tabulate |
Jan Tattermusch | 962c387 | 2016-04-21 10:09:06 -0700 | [diff] [blame] | 77 | sudo pip install google-api-python-client |
Jan Tattermusch | ab0a727 | 2016-10-15 21:12:41 +0200 | [diff] [blame] | 78 | sudo pip install virtualenv |
| 79 | |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 80 | # Building gRPC Python depends on python3.4 being installed, but python3.4 |
| 81 | # is not available on Ubuntu 16.10, so install from source |
| 82 | curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz |
| 83 | tar xzvf Python-3.4.6.tgz |
| 84 | cd Python-3.4.6 |
| 85 | ./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" |
| 86 | sudo make altinstall |
| 87 | cd .. |
| 88 | rm Python-3.4.6.tgz |
Jan Tattermusch | 962c387 | 2016-04-21 10:09:06 -0700 | [diff] [blame] | 89 | |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 90 | curl -O https://bootstrap.pypa.io/get-pip.py |
| 91 | sudo pypy get-pip.py |
| 92 | sudo pypy -m pip install tabulate |
Jan Tattermusch | 962c387 | 2016-04-21 10:09:06 -0700 | [diff] [blame] | 93 | sudo pip install google-api-python-client |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 94 | |
Jan Tattermusch | b19b62c | 2016-03-31 17:18:54 -0700 | [diff] [blame] | 95 | # Node dependencies (nvm has to be installed under user jenkins) |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 96 | touch .profile |
| 97 | curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash |
Jan Tattermusch | 15d2979 | 2016-05-09 23:27:05 -0700 | [diff] [blame] | 98 | source ~/.nvm/nvm.sh |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 99 | nvm install 0.12 && npm config set cache /tmp/npm-cache |
murgatroid99 | 957fe8d | 2016-05-02 15:03:55 -0700 | [diff] [blame] | 100 | nvm install 4 && npm config set cache /tmp/npm-cache |
| 101 | nvm install 5 && npm config set cache /tmp/npm-cache |
| 102 | nvm alias default 4 |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 103 | |
Jan Tattermusch | 8e258d3 | 2016-10-14 14:04:44 +0200 | [diff] [blame] | 104 | # C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives) |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 105 | sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF |
| 106 | echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list |
| 107 | sudo apt-get update |
| 108 | sudo apt-get install -y mono-devel nuget |
| 109 | |
Jan Tattermusch | 8e258d3 | 2016-10-14 14:04:44 +0200 | [diff] [blame] | 110 | # C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 111 | sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' |
Jan Tattermusch | 8e258d3 | 2016-10-14 14:04:44 +0200 | [diff] [blame] | 112 | sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 |
| 113 | sudo apt-get update |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 114 | sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 |
Jan Tattermusch | 17ce30e | 2017-04-05 18:10:50 +0200 | [diff] [blame] | 115 | sudo apt-get install -y dotnet-dev-1.0.1 |
Alexander Polcyn | e495928 | 2016-08-15 12:20:00 -0700 | [diff] [blame] | 116 | |
Jan Tattermusch | a30fc5d | 2016-03-25 14:00:23 -0700 | [diff] [blame] | 117 | # Ruby dependencies |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 118 | git clone https://github.com/rbenv/rbenv.git ~/.rbenv |
| 119 | export PATH="$HOME/.rbenv/bin:$PATH" |
| 120 | eval "$(rbenv init -)" |
| 121 | |
| 122 | git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build |
| 123 | export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH" |
| 124 | |
| 125 | rbenv install 2.4.0 |
| 126 | rbenv global 2.4.0 |
| 127 | ruby -v |
Jan Tattermusch | bb1a453 | 2016-03-30 18:04:01 -0700 | [diff] [blame] | 128 | |
Jan Tattermusch | 962c387 | 2016-04-21 10:09:06 -0700 | [diff] [blame] | 129 | # Install bundler (prerequisite for gRPC Ruby) |
Jan Tattermusch | 962c387 | 2016-04-21 10:09:06 -0700 | [diff] [blame] | 130 | gem install bundler |
| 131 | |
Zhouyihai Ding | 4917b08 | 2017-09-21 12:52:50 -0700 | [diff] [blame] | 132 | # PHP dependencies |
| 133 | sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev |
| 134 | curl -sS https://getcomposer.org/installer | php |
| 135 | sudo mv composer.phar /usr/local/bin/composer |
| 136 | |
Jan Tattermusch | bb1a453 | 2016-03-30 18:04:01 -0700 | [diff] [blame] | 137 | # Java dependencies - nothing as we already have Java JDK 8 |
Jan Tattermusch | 03b44ab | 2016-05-10 09:20:59 -0700 | [diff] [blame] | 138 | |
| 139 | # Go dependencies |
Alexander Polcyn | c4fb2e7 | 2016-10-07 11:00:45 -0700 | [diff] [blame] | 140 | # Currently, the golang package available via apt-get doesn't have the latest go. |
Alexander Polcyn | 46b9be7 | 2016-10-12 14:33:13 -0700 | [diff] [blame] | 141 | # Significant performance improvements with grpc-go have been observed after |
| 142 | # upgrading from go 1.5 to a later version, so a later go version is preferred. |
Alexander Polcyn | c4fb2e7 | 2016-10-07 11:00:45 -0700 | [diff] [blame] | 143 | # Following go install instructions from https://golang.org/doc/install |
Alexander Polcyn | 08ad565 | 2017-02-27 13:45:15 -0800 | [diff] [blame] | 144 | GO_VERSION=1.8 |
Alexander Polcyn | c4fb2e7 | 2016-10-07 11:00:45 -0700 | [diff] [blame] | 145 | OS=linux |
| 146 | ARCH=amd64 |
| 147 | curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz |
| 148 | sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz |
| 149 | # Put go on the PATH, keep the usual installation dir |
| 150 | sudo ln -s /usr/local/go/bin/go /usr/bin/go |
| 151 | rm go$GO_VERSION.$OS-$ARCH.tar.gz |
Alexander Polcyn | 9f08d11 | 2016-10-24 12:25:02 -0700 | [diff] [blame] | 152 | |
| 153 | # Install perf, to profile benchmarks. (need to get the right linux-tools-<> for kernel version) |
| 154 | sudo apt-get install -y linux-tools-common linux-tools-generic linux-tools-`uname -r` |
| 155 | # see http://unix.stackexchange.com/questions/14227/do-i-need-root-admin-permissions-to-run-userspace-perf-tool-perf-events-ar |
| 156 | echo 0 | sudo tee /proc/sys/kernel/perf_event_paranoid |
| 157 | # see http://stackoverflow.com/questions/21284906/perf-couldnt-record-kernel-reference-relocation-symbol |
| 158 | echo 0 | sudo tee /proc/sys/kernel/kptr_restrict |
| 159 | |
| 160 | # qps workers under perf appear to need a lot of mmap pages under certain scenarios and perf args in |
| 161 | # order to not lose perf events or time out |
| 162 | echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb |
| 163 | |
| 164 | # Fetch scripts to generate flame graphs from perf data collected |
| 165 | # on benchmarks |
| 166 | git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph |
| 167 | |
Alexander Polcyn | d88168b | 2017-03-28 14:13:37 -0700 | [diff] [blame] | 168 | # Install scipy and numpy for benchmarking scripts |
Matt Kwong | 6ae9175 | 2017-04-26 19:03:18 -0700 | [diff] [blame] | 169 | sudo apt-get install -y python-scipy python-numpy |
Matt Kwong | 654c367 | 2017-04-11 14:31:39 -0700 | [diff] [blame] | 170 | |
| 171 | # Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ |
| 172 | # This needs to happen as the last step to prevent Jenkins master from connecting |
| 173 | # to a machine that hasn't been properly setup yet. |
| 174 | cat jenkins_master.pub | sudo tee --append ~jenkins/.ssh/authorized_keys |
| 175 | |
| 176 | # Restart for VM to pick up kernel update |
| 177 | echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' |
| 178 | sleep 10 |
| 179 | sudo reboot |