Matt Kwong | b95aa40 | 2017-10-13 13:25:23 -0700 | [diff] [blame] | 1 | #!/bin/bash |
| 2 | # Copyright 2017 gRPC authors. |
| 3 | # |
| 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
| 7 | # |
| 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | # |
| 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
| 15 | |
| 16 | # Initializes a fresh GCE VM to become a Kokoro Linux performance worker. |
| 17 | # You shouldn't run this script on your own, |
| 18 | # use create_linux_kokoro_performance_worker.sh instead. |
| 19 | |
| 20 | set -ex |
| 21 | |
| 22 | sudo apt-get update |
| 23 | |
| 24 | # Install Java 8 JDK (to build gRPC Java) |
| 25 | sudo apt-get install -y openjdk-8-jdk |
| 26 | sudo apt-get install -y unzip lsof |
| 27 | |
| 28 | sudo apt-get install -y \ |
| 29 | autoconf \ |
| 30 | autotools-dev \ |
| 31 | build-essential \ |
| 32 | bzip2 \ |
| 33 | ccache \ |
| 34 | curl \ |
| 35 | gcc \ |
| 36 | gcc-multilib \ |
| 37 | git \ |
| 38 | gyp \ |
| 39 | lcov \ |
| 40 | libc6 \ |
| 41 | libc6-dbg \ |
| 42 | libc6-dev \ |
| 43 | libcurl4-openssl-dev \ |
| 44 | libgtest-dev \ |
| 45 | libreadline-dev \ |
| 46 | libssl-dev \ |
| 47 | libtool \ |
| 48 | make \ |
| 49 | strace \ |
| 50 | pypy \ |
| 51 | python-dev \ |
| 52 | python-pip \ |
| 53 | python-setuptools \ |
| 54 | python-yaml \ |
| 55 | python3-dev \ |
| 56 | python3-pip \ |
| 57 | python3-setuptools \ |
| 58 | python3-yaml \ |
| 59 | telnet \ |
| 60 | unzip \ |
| 61 | wget \ |
| 62 | zip \ |
| 63 | zlib1g-dev |
| 64 | |
| 65 | # perftools |
| 66 | sudo apt-get install -y google-perftools libgoogle-perftools-dev |
| 67 | |
| 68 | # netperf |
| 69 | sudo apt-get install -y netperf |
| 70 | |
| 71 | # C++ dependencies |
| 72 | sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang |
| 73 | |
| 74 | # Python dependencies |
| 75 | sudo pip install --upgrade pip==9.0.1 |
| 76 | sudo pip install tabulate |
| 77 | sudo pip install google-api-python-client |
| 78 | sudo pip install virtualenv |
| 79 | |
| 80 | # Building gRPC Python depends on python3.4 being installed, but python3.4 |
| 81 | # is not available on Ubuntu 16.10, so install from source |
| 82 | curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz |
| 83 | tar xzvf Python-3.4.6.tgz |
| 84 | cd Python-3.4.6 |
| 85 | ./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" |
| 86 | sudo make altinstall |
| 87 | cd .. |
| 88 | rm Python-3.4.6.tgz |
| 89 | |
| 90 | curl -O https://bootstrap.pypa.io/get-pip.py |
| 91 | sudo pypy get-pip.py |
| 92 | sudo pypy -m pip install tabulate |
| 93 | sudo pip install google-api-python-client |
| 94 | |
| 95 | # Node dependencies (nvm has to be installed under user kbuilder) |
| 96 | touch .profile |
| 97 | curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.25.4/install.sh | bash |
| 98 | source ~/.nvm/nvm.sh |
| 99 | nvm install 0.12 && npm config set cache /tmp/npm-cache |
| 100 | nvm install 4 && npm config set cache /tmp/npm-cache |
| 101 | nvm install 5 && npm config set cache /tmp/npm-cache |
| 102 | nvm alias default 4 |
| 103 | |
| 104 | # C# mono dependencies (http://www.mono-project.com/docs/getting-started/install/linux/#debian-ubuntu-and-derivatives) |
| 105 | sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF |
| 106 | echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list |
| 107 | sudo apt-get update |
| 108 | sudo apt-get install -y mono-devel nuget |
| 109 | |
| 110 | # C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) |
| 111 | sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' |
| 112 | sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 |
| 113 | sudo apt-get update |
| 114 | sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 |
| 115 | sudo apt-get install -y dotnet-dev-1.0.1 |
| 116 | |
| 117 | # Ruby dependencies |
| 118 | gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 |
| 119 | curl -sSL https://get.rvm.io | bash -s stable --ruby |
| 120 | source ~/.rvm/scripts/rvm |
| 121 | |
| 122 | git clone https://github.com/rbenv/rbenv.git ~/.rbenv |
| 123 | export PATH="$HOME/.rbenv/bin:$PATH" |
| 124 | eval "$(rbenv init -)" |
| 125 | |
| 126 | git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build |
| 127 | export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH" |
| 128 | |
| 129 | rbenv install 2.4.0 |
| 130 | rbenv global 2.4.0 |
| 131 | ruby -v |
| 132 | |
| 133 | # Install bundler (prerequisite for gRPC Ruby) |
| 134 | gem install bundler |
| 135 | |
| 136 | # PHP dependencies |
| 137 | sudo apt-get install -y php php-dev phpunit php-pear unzip zlib1g-dev |
| 138 | curl -sS https://getcomposer.org/installer | php |
| 139 | sudo mv composer.phar /usr/local/bin/composer |
| 140 | |
| 141 | # Java dependencies - nothing as we already have Java JDK 8 |
| 142 | |
| 143 | # Go dependencies |
| 144 | # Currently, the golang package available via apt-get doesn't have the latest go. |
| 145 | # Significant performance improvements with grpc-go have been observed after |
| 146 | # upgrading from go 1.5 to a later version, so a later go version is preferred. |
| 147 | # Following go install instructions from https://golang.org/doc/install |
| 148 | GO_VERSION=1.8 |
| 149 | OS=linux |
| 150 | ARCH=amd64 |
| 151 | curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.${OS}-${ARCH}.tar.gz |
| 152 | sudo tar -C /usr/local -xzf go$GO_VERSION.$OS-$ARCH.tar.gz |
| 153 | # Put go on the PATH, keep the usual installation dir |
| 154 | sudo ln -s /usr/local/go/bin/go /usr/bin/go |
| 155 | rm go$GO_VERSION.$OS-$ARCH.tar.gz |
| 156 | |
| 157 | # Install perf, to profile benchmarks. (need to get the right linux-tools-<> for kernel version) |
| 158 | sudo apt-get install -y linux-tools-common linux-tools-generic linux-tools-`uname -r` |
| 159 | # see http://unix.stackexchange.com/questions/14227/do-i-need-root-admin-permissions-to-run-userspace-perf-tool-perf-events-ar |
| 160 | echo 0 | sudo tee /proc/sys/kernel/perf_event_paranoid |
| 161 | # see http://stackoverflow.com/questions/21284906/perf-couldnt-record-kernel-reference-relocation-symbol |
| 162 | echo 0 | sudo tee /proc/sys/kernel/kptr_restrict |
| 163 | |
| 164 | # qps workers under perf appear to need a lot of mmap pages under certain scenarios and perf args in |
| 165 | # order to not lose perf events or time out |
| 166 | echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb |
| 167 | |
| 168 | # Fetch scripts to generate flame graphs from perf data collected |
| 169 | # on benchmarks |
| 170 | git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph |
| 171 | |
| 172 | # Install scipy and numpy for benchmarking scripts |
| 173 | sudo apt-get install -y python-scipy python-numpy |
| 174 | |
| 175 | # Add pubkey of Kokoro driver VM to allow SSH |
| 176 | cat kokoro_performance.pub | sudo tee --append ~kbuilder/.ssh/authorized_keys |
| 177 | |
| 178 | # Restart for VM to pick up kernel update |
| 179 | echo 'Successfully initialized the linux worker, going for reboot in 10 seconds' |
| 180 | sleep 10 |
| 181 | sudo reboot |