Nathaniel Manista | cbf21da | 2016-02-02 22:17:44 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python2.7 |
Jan Tattermusch | be538a1 | 2016-01-28 14:58:15 -0800 | [diff] [blame] | 2 | # Copyright 2016, Google Inc. |
| 3 | # All rights reserved. |
| 4 | # |
| 5 | # Redistribution and use in source and binary forms, with or without |
| 6 | # modification, are permitted provided that the following conditions are |
| 7 | # met: |
| 8 | # |
| 9 | # * Redistributions of source code must retain the above copyright |
| 10 | # notice, this list of conditions and the following disclaimer. |
| 11 | # * Redistributions in binary form must reproduce the above |
| 12 | # copyright notice, this list of conditions and the following disclaimer |
| 13 | # in the documentation and/or other materials provided with the |
| 14 | # distribution. |
| 15 | # * Neither the name of Google Inc. nor the names of its |
| 16 | # contributors may be used to endorse or promote products derived from |
| 17 | # this software without specific prior written permission. |
| 18 | # |
| 19 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 20 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 21 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 22 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 23 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 24 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 25 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 26 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 27 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 28 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 29 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 30 | |
| 31 | """Definition of targets to build artifacts.""" |
| 32 | |
| 33 | import jobset |
| 34 | |
| 35 | |
| 36 | def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, |
| 37 | flake_retries=0, timeout_retries=0): |
| 38 | """Creates jobspec for a task running under docker.""" |
| 39 | environ = environ.copy() |
| 40 | environ['RUN_COMMAND'] = shell_command |
| 41 | |
| 42 | docker_args=[] |
| 43 | for k,v in environ.iteritems(): |
| 44 | docker_args += ['-e', '%s=%s' % (k, v)] |
| 45 | docker_env = {'DOCKERFILE_DIR': dockerfile_dir, |
| 46 | 'DOCKER_RUN_SCRIPT': 'tools/jenkins/docker_run.sh', |
| 47 | 'OUTPUT_DIR': 'artifacts'} |
| 48 | jobspec = jobset.JobSpec( |
| 49 | cmdline=['tools/jenkins/build_and_run_docker.sh'] + docker_args, |
| 50 | environ=docker_env, |
| 51 | shortname='build_artifact.%s' % (name), |
| 52 | timeout_seconds=30*60, |
| 53 | flake_retries=flake_retries, |
| 54 | timeout_retries=timeout_retries) |
| 55 | return jobspec |
| 56 | |
| 57 | |
| 58 | def create_jobspec(name, cmdline, environ=None, shell=False, |
| 59 | flake_retries=0, timeout_retries=0): |
| 60 | """Creates jobspec.""" |
| 61 | jobspec = jobset.JobSpec( |
| 62 | cmdline=cmdline, |
| 63 | environ=environ, |
| 64 | shortname='build_artifact.%s' % (name), |
Jan Tattermusch | 5b762b6 | 2016-02-09 08:18:02 -0800 | [diff] [blame] | 65 | timeout_seconds=30*60, |
Jan Tattermusch | be538a1 | 2016-01-28 14:58:15 -0800 | [diff] [blame] | 66 | flake_retries=flake_retries, |
| 67 | timeout_retries=timeout_retries, |
| 68 | shell=shell) |
| 69 | return jobspec |
| 70 | |
| 71 | |
| 72 | def macos_arch_env(arch): |
| 73 | """Returns environ specifying -arch arguments for make.""" |
| 74 | if arch == 'x86': |
| 75 | arch_arg = '-arch i386' |
| 76 | elif arch == 'x64': |
| 77 | arch_arg = '-arch x86_64' |
| 78 | else: |
| 79 | raise Exception('Unsupported arch') |
| 80 | return {'CFLAGS': arch_arg, 'LDFLAGS': arch_arg} |
| 81 | |
Jan Tattermusch | e046f71 | 2016-02-18 16:06:10 -0800 | [diff] [blame] | 82 | _MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7' |
| 83 | |
| 84 | _ARCH_FLAG_MAP = { |
| 85 | 'x86': '-m32', |
| 86 | 'x64': '-m64' |
| 87 | } |
Jan Tattermusch | be538a1 | 2016-01-28 14:58:15 -0800 | [diff] [blame] | 88 | |
Jan Tattermusch | 8640f92 | 2016-02-01 18:58:46 -0800 | [diff] [blame] | 89 | class PythonArtifact: |
| 90 | """Builds Python artifacts.""" |
| 91 | |
| 92 | def __init__(self, platform, arch): |
| 93 | self.name = 'python_%s_%s' % (platform, arch) |
| 94 | self.platform = platform |
| 95 | self.arch = arch |
| 96 | self.labels = ['artifact', 'python', platform, arch] |
| 97 | |
| 98 | def pre_build_jobspecs(self): |
| 99 | return [] |
| 100 | |
| 101 | def build_jobspec(self): |
| 102 | if self.platform == 'windows': |
| 103 | raise Exception('Not supported yet.') |
| 104 | else: |
Jan Tattermusch | e066717 | 2016-02-03 15:57:57 -0800 | [diff] [blame] | 105 | environ = {} |
Jan Tattermusch | 8640f92 | 2016-02-01 18:58:46 -0800 | [diff] [blame] | 106 | if self.platform == 'linux': |
Jan Tattermusch | 7cf8bf4 | 2016-02-03 13:41:07 -0800 | [diff] [blame] | 107 | if self.arch == 'x86': |
| 108 | environ['SETARCH_CMD'] = 'linux32' |
Jan Tattermusch | 8640f92 | 2016-02-01 18:58:46 -0800 | [diff] [blame] | 109 | return create_docker_jobspec(self.name, |
| 110 | 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, |
Jan Tattermusch | 7cf8bf4 | 2016-02-03 13:41:07 -0800 | [diff] [blame] | 111 | 'tools/run_tests/build_artifact_python.sh', |
| 112 | environ=environ) |
Jan Tattermusch | 8640f92 | 2016-02-01 18:58:46 -0800 | [diff] [blame] | 113 | else: |
Jan Tattermusch | e066717 | 2016-02-03 15:57:57 -0800 | [diff] [blame] | 114 | environ['SKIP_PIP_INSTALL'] = 'TRUE' |
Jan Tattermusch | 8640f92 | 2016-02-01 18:58:46 -0800 | [diff] [blame] | 115 | return create_jobspec(self.name, |
Jan Tattermusch | e066717 | 2016-02-03 15:57:57 -0800 | [diff] [blame] | 116 | ['tools/run_tests/build_artifact_python.sh'], |
| 117 | environ=environ) |
Jan Tattermusch | 8640f92 | 2016-02-01 18:58:46 -0800 | [diff] [blame] | 118 | |
| 119 | def __str__(self): |
| 120 | return self.name |
| 121 | |
| 122 | |
Jan Tattermusch | 4437213 | 2016-02-01 16:20:03 -0800 | [diff] [blame] | 123 | class RubyArtifact: |
| 124 | """Builds ruby native gem.""" |
| 125 | |
| 126 | def __init__(self, platform, arch): |
| 127 | self.name = 'ruby_native_gem_%s_%s' % (platform, arch) |
| 128 | self.platform = platform |
| 129 | self.arch = arch |
| 130 | self.labels = ['artifact', 'ruby', platform, arch] |
| 131 | |
| 132 | def pre_build_jobspecs(self): |
| 133 | return [] |
| 134 | |
| 135 | def build_jobspec(self): |
| 136 | if self.platform == 'windows': |
| 137 | raise Exception("Not supported yet") |
| 138 | else: |
| 139 | if self.platform == 'linux': |
| 140 | environ = {} |
| 141 | if self.arch == 'x86': |
Jan Tattermusch | 7cf8bf4 | 2016-02-03 13:41:07 -0800 | [diff] [blame] | 142 | environ['SETARCH_CMD'] = 'linux32' |
Jan Tattermusch | 4437213 | 2016-02-01 16:20:03 -0800 | [diff] [blame] | 143 | return create_docker_jobspec(self.name, |
| 144 | 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, |
| 145 | 'tools/run_tests/build_artifact_ruby.sh', |
| 146 | environ=environ) |
| 147 | else: |
| 148 | return create_jobspec(self.name, |
| 149 | ['tools/run_tests/build_artifact_ruby.sh']) |
| 150 | |
| 151 | |
Jan Tattermusch | be538a1 | 2016-01-28 14:58:15 -0800 | [diff] [blame] | 152 | class CSharpExtArtifact: |
| 153 | """Builds C# native extension library""" |
| 154 | |
| 155 | def __init__(self, platform, arch): |
| 156 | self.name = 'csharp_ext_%s_%s' % (platform, arch) |
| 157 | self.platform = platform |
| 158 | self.arch = arch |
| 159 | self.labels = ['artifact', 'csharp', platform, arch] |
| 160 | |
| 161 | def pre_build_jobspecs(self): |
| 162 | if self.platform == 'windows': |
| 163 | return [create_jobspec('prebuild_%s' % self.name, |
| 164 | ['tools\\run_tests\\pre_build_c.bat'], |
| 165 | shell=True, |
| 166 | flake_retries=5, |
| 167 | timeout_retries=2)] |
| 168 | else: |
| 169 | return [] |
| 170 | |
| 171 | def build_jobspec(self): |
| 172 | if self.platform == 'windows': |
| 173 | msbuild_platform = 'Win32' if self.arch == 'x86' else self.arch |
| 174 | return create_jobspec(self.name, |
| 175 | ['tools\\run_tests\\build_artifact_csharp.bat', |
| 176 | 'vsprojects\\grpc_csharp_ext.sln', |
| 177 | '/p:Configuration=Release', |
| 178 | '/p:PlatformToolset=v120', |
| 179 | '/p:Platform=%s' % msbuild_platform], |
| 180 | shell=True) |
| 181 | else: |
| 182 | environ = {'CONFIG': 'opt', |
| 183 | 'EMBED_OPENSSL': 'true', |
Craig Tiller | 71ea4a1 | 2016-02-04 15:06:41 -0800 | [diff] [blame] | 184 | 'EMBED_ZLIB': 'true', |
| 185 | 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE'} |
Jan Tattermusch | be538a1 | 2016-01-28 14:58:15 -0800 | [diff] [blame] | 186 | if self.platform == 'linux': |
| 187 | return create_docker_jobspec(self.name, |
| 188 | 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch, |
| 189 | 'tools/run_tests/build_artifact_csharp.sh') |
| 190 | else: |
| 191 | environ.update(macos_arch_env(self.arch)) |
| 192 | return create_jobspec(self.name, |
| 193 | ['tools/run_tests/build_artifact_csharp.sh'], |
| 194 | environ=environ) |
| 195 | |
| 196 | def __str__(self): |
| 197 | return self.name |
| 198 | |
Jan Tattermusch | e046f71 | 2016-02-18 16:06:10 -0800 | [diff] [blame] | 199 | |
murgatroid99 | 673f65b | 2016-02-01 11:19:07 -0800 | [diff] [blame] | 200 | node_gyp_arch_map = { |
| 201 | 'x86': 'ia32', |
| 202 | 'x64': 'x64' |
| 203 | } |
| 204 | |
| 205 | class NodeExtArtifact: |
| 206 | """Builds Node native extension""" |
| 207 | |
| 208 | def __init__(self, platform, arch): |
| 209 | self.name = 'node_ext_{0}_{1}'.format(platform, arch) |
| 210 | self.platform = platform |
| 211 | self.arch = arch |
| 212 | self.gyp_arch = node_gyp_arch_map[arch] |
| 213 | self.labels = ['artifact', 'node', platform, arch] |
| 214 | |
| 215 | def pre_build_jobspecs(self): |
| 216 | return [] |
| 217 | |
| 218 | def build_jobspec(self): |
| 219 | if self.platform == 'windows': |
| 220 | return create_jobspec(self.name, |
| 221 | ['tools\\run_tests\\build_artifact_node.bat', |
| 222 | self.gyp_arch], |
| 223 | shell=True) |
| 224 | else: |
| 225 | if self.platform == 'linux': |
| 226 | return create_docker_jobspec( |
| 227 | self.name, |
| 228 | 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), |
| 229 | 'tools/run_tests/build_artifact_node.sh {}'.format(self.gyp_arch)) |
| 230 | else: |
| 231 | return create_jobspec(self.name, |
| 232 | ['tools/run_tests/build_artifact_node.sh', |
| 233 | self.gyp_arch]) |
| 234 | |
Jan Tattermusch | be538a1 | 2016-01-28 14:58:15 -0800 | [diff] [blame] | 235 | |
Jan Tattermusch | e046f71 | 2016-02-18 16:06:10 -0800 | [diff] [blame] | 236 | class ProtocArtifact: |
| 237 | """Builds protoc and protoc-plugin artifacts""" |
| 238 | |
| 239 | def __init__(self, platform, arch): |
| 240 | self.name = 'protoc_%s_%s' % (platform, arch) |
| 241 | self.platform = platform |
| 242 | self.arch = arch |
| 243 | self.labels = ['artifact', 'protoc', platform, arch] |
| 244 | |
| 245 | def pre_build_jobspecs(self): |
| 246 | return [] |
| 247 | |
| 248 | def build_jobspec(self): |
| 249 | if self.platform != 'windows': |
| 250 | cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch] |
Jan Tattermusch | e7b7d86 | 2016-02-19 09:49:35 -0800 | [diff] [blame] | 251 | ldflags = '%s' % _ARCH_FLAG_MAP[self.arch] |
| 252 | if self.platform != 'macos': |
Jan Tattermusch | 07591a5 | 2016-02-19 10:28:24 -0800 | [diff] [blame] | 253 | ldflags += ' -static-libgcc -static-libstdc++ -s' |
Jan Tattermusch | e046f71 | 2016-02-18 16:06:10 -0800 | [diff] [blame] | 254 | environ={'CONFIG': 'opt', |
| 255 | 'CXXFLAGS': cxxflags, |
| 256 | 'LDFLAGS': ldflags, |
| 257 | 'PROTOBUF_LDFLAGS_EXTRA': ldflags} |
| 258 | if self.platform == 'linux': |
| 259 | return create_docker_jobspec(self.name, |
| 260 | 'tools/dockerfile/grpc_artifact_protoc', |
| 261 | 'tools/run_tests/build_artifact_protoc.sh', |
| 262 | environ=environ) |
| 263 | else: |
Jan Tattermusch | e7b7d86 | 2016-02-19 09:49:35 -0800 | [diff] [blame] | 264 | environ['CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG |
Jan Tattermusch | e046f71 | 2016-02-18 16:06:10 -0800 | [diff] [blame] | 265 | return create_jobspec(self.name, |
| 266 | ['tools/run_tests/build_artifact_protoc.sh'], |
| 267 | environ=environ) |
| 268 | else: |
| 269 | raise Exception('Not yet supported') |
| 270 | |
| 271 | def __str__(self): |
| 272 | return self.name |
| 273 | |
| 274 | |
Jan Tattermusch | be538a1 | 2016-01-28 14:58:15 -0800 | [diff] [blame] | 275 | def targets(): |
| 276 | """Gets list of supported targets""" |
murgatroid99 | 41a9e83 | 2016-02-03 09:47:35 -0800 | [diff] [blame] | 277 | return ([Cls(platform, arch) |
| 278 | for Cls in (CSharpExtArtifact, NodeExtArtifact) |
| 279 | for platform in ('linux', 'macos', 'windows') |
| 280 | for arch in ('x86', 'x64')] + |
| 281 | [PythonArtifact('linux', 'x86'), |
| 282 | PythonArtifact('linux', 'x64'), |
Jan Tattermusch | e066717 | 2016-02-03 15:57:57 -0800 | [diff] [blame] | 283 | PythonArtifact('macos', 'x64'), |
murgatroid99 | 41a9e83 | 2016-02-03 09:47:35 -0800 | [diff] [blame] | 284 | RubyArtifact('linux', 'x86'), |
murgatroid99 | 33c808b | 2016-02-03 15:27:41 -0800 | [diff] [blame] | 285 | RubyArtifact('linux', 'x64'), |
Jan Tattermusch | e046f71 | 2016-02-18 16:06:10 -0800 | [diff] [blame] | 286 | RubyArtifact('macos', 'x64'), |
| 287 | ProtocArtifact('linux', 'x86'), |
| 288 | ProtocArtifact('linux', 'x64'), |
| 289 | ProtocArtifact('macos', 'x86'), |
| 290 | ProtocArtifact('macos', 'x64')]) |