blob: cf056ec9293491888b7376ac3d76c062e1a47740 [file] [log] [blame]
Nathaniel Manistacbf21da2016-02-02 22:17:44 +00001#!/usr/bin/env python2.7
Jan Tattermuschbe538a12016-01-28 14:58:15 -08002# Copyright 2016, Google Inc.
3# All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met:
8#
9# * Redistributions of source code must retain the above copyright
10# notice, this list of conditions and the following disclaimer.
11# * Redistributions in binary form must reproduce the above
12# copyright notice, this list of conditions and the following disclaimer
13# in the documentation and/or other materials provided with the
14# distribution.
15# * Neither the name of Google Inc. nor the names of its
16# contributors may be used to endorse or promote products derived from
17# this software without specific prior written permission.
18#
19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31"""Definition of targets to build artifacts."""
32
33import jobset
34
35
36def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={},
37 flake_retries=0, timeout_retries=0):
38 """Creates jobspec for a task running under docker."""
39 environ = environ.copy()
40 environ['RUN_COMMAND'] = shell_command
41
42 docker_args=[]
43 for k,v in environ.iteritems():
44 docker_args += ['-e', '%s=%s' % (k, v)]
45 docker_env = {'DOCKERFILE_DIR': dockerfile_dir,
46 'DOCKER_RUN_SCRIPT': 'tools/jenkins/docker_run.sh',
47 'OUTPUT_DIR': 'artifacts'}
48 jobspec = jobset.JobSpec(
49 cmdline=['tools/jenkins/build_and_run_docker.sh'] + docker_args,
50 environ=docker_env,
51 shortname='build_artifact.%s' % (name),
52 timeout_seconds=30*60,
53 flake_retries=flake_retries,
54 timeout_retries=timeout_retries)
55 return jobspec
56
57
58def create_jobspec(name, cmdline, environ=None, shell=False,
59 flake_retries=0, timeout_retries=0):
60 """Creates jobspec."""
61 jobspec = jobset.JobSpec(
62 cmdline=cmdline,
63 environ=environ,
64 shortname='build_artifact.%s' % (name),
Jan Tattermusch5b762b62016-02-09 08:18:02 -080065 timeout_seconds=30*60,
Jan Tattermuschbe538a12016-01-28 14:58:15 -080066 flake_retries=flake_retries,
67 timeout_retries=timeout_retries,
68 shell=shell)
69 return jobspec
70
71
72def macos_arch_env(arch):
73 """Returns environ specifying -arch arguments for make."""
74 if arch == 'x86':
75 arch_arg = '-arch i386'
76 elif arch == 'x64':
77 arch_arg = '-arch x86_64'
78 else:
79 raise Exception('Unsupported arch')
80 return {'CFLAGS': arch_arg, 'LDFLAGS': arch_arg}
81
Jan Tattermusche046f712016-02-18 16:06:10 -080082_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7'
83
84_ARCH_FLAG_MAP = {
85 'x86': '-m32',
86 'x64': '-m64'
87}
Jan Tattermuschbe538a12016-01-28 14:58:15 -080088
Jan Tattermusch8640f922016-02-01 18:58:46 -080089class PythonArtifact:
90 """Builds Python artifacts."""
91
92 def __init__(self, platform, arch):
93 self.name = 'python_%s_%s' % (platform, arch)
94 self.platform = platform
95 self.arch = arch
96 self.labels = ['artifact', 'python', platform, arch]
97
98 def pre_build_jobspecs(self):
99 return []
100
101 def build_jobspec(self):
102 if self.platform == 'windows':
103 raise Exception('Not supported yet.')
104 else:
Jan Tattermusche0667172016-02-03 15:57:57 -0800105 environ = {}
Jan Tattermusch8640f922016-02-01 18:58:46 -0800106 if self.platform == 'linux':
Jan Tattermusch7cf8bf42016-02-03 13:41:07 -0800107 if self.arch == 'x86':
108 environ['SETARCH_CMD'] = 'linux32'
Jan Tattermusch8640f922016-02-01 18:58:46 -0800109 return create_docker_jobspec(self.name,
110 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch,
Jan Tattermusch7cf8bf42016-02-03 13:41:07 -0800111 'tools/run_tests/build_artifact_python.sh',
112 environ=environ)
Jan Tattermusch8640f922016-02-01 18:58:46 -0800113 else:
Jan Tattermusche0667172016-02-03 15:57:57 -0800114 environ['SKIP_PIP_INSTALL'] = 'TRUE'
Jan Tattermusch8640f922016-02-01 18:58:46 -0800115 return create_jobspec(self.name,
Jan Tattermusche0667172016-02-03 15:57:57 -0800116 ['tools/run_tests/build_artifact_python.sh'],
117 environ=environ)
Jan Tattermusch8640f922016-02-01 18:58:46 -0800118
119 def __str__(self):
120 return self.name
121
122
Jan Tattermusch44372132016-02-01 16:20:03 -0800123class RubyArtifact:
124 """Builds ruby native gem."""
125
126 def __init__(self, platform, arch):
127 self.name = 'ruby_native_gem_%s_%s' % (platform, arch)
128 self.platform = platform
129 self.arch = arch
130 self.labels = ['artifact', 'ruby', platform, arch]
131
132 def pre_build_jobspecs(self):
133 return []
134
135 def build_jobspec(self):
136 if self.platform == 'windows':
137 raise Exception("Not supported yet")
138 else:
139 if self.platform == 'linux':
140 environ = {}
141 if self.arch == 'x86':
Jan Tattermusch7cf8bf42016-02-03 13:41:07 -0800142 environ['SETARCH_CMD'] = 'linux32'
Jan Tattermusch44372132016-02-01 16:20:03 -0800143 return create_docker_jobspec(self.name,
144 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch,
145 'tools/run_tests/build_artifact_ruby.sh',
146 environ=environ)
147 else:
148 return create_jobspec(self.name,
149 ['tools/run_tests/build_artifact_ruby.sh'])
150
151
Jan Tattermuschbe538a12016-01-28 14:58:15 -0800152class CSharpExtArtifact:
153 """Builds C# native extension library"""
154
155 def __init__(self, platform, arch):
156 self.name = 'csharp_ext_%s_%s' % (platform, arch)
157 self.platform = platform
158 self.arch = arch
159 self.labels = ['artifact', 'csharp', platform, arch]
160
161 def pre_build_jobspecs(self):
162 if self.platform == 'windows':
163 return [create_jobspec('prebuild_%s' % self.name,
164 ['tools\\run_tests\\pre_build_c.bat'],
165 shell=True,
166 flake_retries=5,
167 timeout_retries=2)]
168 else:
169 return []
170
171 def build_jobspec(self):
172 if self.platform == 'windows':
173 msbuild_platform = 'Win32' if self.arch == 'x86' else self.arch
174 return create_jobspec(self.name,
175 ['tools\\run_tests\\build_artifact_csharp.bat',
176 'vsprojects\\grpc_csharp_ext.sln',
177 '/p:Configuration=Release',
178 '/p:PlatformToolset=v120',
179 '/p:Platform=%s' % msbuild_platform],
180 shell=True)
181 else:
182 environ = {'CONFIG': 'opt',
183 'EMBED_OPENSSL': 'true',
Craig Tiller71ea4a12016-02-04 15:06:41 -0800184 'EMBED_ZLIB': 'true',
185 'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE'}
Jan Tattermuschbe538a12016-01-28 14:58:15 -0800186 if self.platform == 'linux':
187 return create_docker_jobspec(self.name,
188 'tools/dockerfile/grpc_artifact_linux_%s' % self.arch,
189 'tools/run_tests/build_artifact_csharp.sh')
190 else:
191 environ.update(macos_arch_env(self.arch))
192 return create_jobspec(self.name,
193 ['tools/run_tests/build_artifact_csharp.sh'],
194 environ=environ)
195
196 def __str__(self):
197 return self.name
198
Jan Tattermusche046f712016-02-18 16:06:10 -0800199
murgatroid99673f65b2016-02-01 11:19:07 -0800200node_gyp_arch_map = {
201 'x86': 'ia32',
202 'x64': 'x64'
203}
204
205class NodeExtArtifact:
206 """Builds Node native extension"""
207
208 def __init__(self, platform, arch):
209 self.name = 'node_ext_{0}_{1}'.format(platform, arch)
210 self.platform = platform
211 self.arch = arch
212 self.gyp_arch = node_gyp_arch_map[arch]
213 self.labels = ['artifact', 'node', platform, arch]
214
215 def pre_build_jobspecs(self):
216 return []
217
218 def build_jobspec(self):
219 if self.platform == 'windows':
220 return create_jobspec(self.name,
221 ['tools\\run_tests\\build_artifact_node.bat',
222 self.gyp_arch],
223 shell=True)
224 else:
225 if self.platform == 'linux':
226 return create_docker_jobspec(
227 self.name,
228 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch),
229 'tools/run_tests/build_artifact_node.sh {}'.format(self.gyp_arch))
230 else:
231 return create_jobspec(self.name,
232 ['tools/run_tests/build_artifact_node.sh',
233 self.gyp_arch])
234
Jan Tattermuschbe538a12016-01-28 14:58:15 -0800235
Jan Tattermusche046f712016-02-18 16:06:10 -0800236class ProtocArtifact:
237 """Builds protoc and protoc-plugin artifacts"""
238
239 def __init__(self, platform, arch):
240 self.name = 'protoc_%s_%s' % (platform, arch)
241 self.platform = platform
242 self.arch = arch
243 self.labels = ['artifact', 'protoc', platform, arch]
244
245 def pre_build_jobspecs(self):
246 return []
247
248 def build_jobspec(self):
249 if self.platform != 'windows':
250 cxxflags = '-DNDEBUG %s' % _ARCH_FLAG_MAP[self.arch]
Jan Tattermusche7b7d862016-02-19 09:49:35 -0800251 ldflags = '%s' % _ARCH_FLAG_MAP[self.arch]
252 if self.platform != 'macos':
Jan Tattermusch07591a52016-02-19 10:28:24 -0800253 ldflags += ' -static-libgcc -static-libstdc++ -s'
Jan Tattermusche046f712016-02-18 16:06:10 -0800254 environ={'CONFIG': 'opt',
255 'CXXFLAGS': cxxflags,
256 'LDFLAGS': ldflags,
257 'PROTOBUF_LDFLAGS_EXTRA': ldflags}
258 if self.platform == 'linux':
259 return create_docker_jobspec(self.name,
260 'tools/dockerfile/grpc_artifact_protoc',
261 'tools/run_tests/build_artifact_protoc.sh',
262 environ=environ)
263 else:
Jan Tattermusche7b7d862016-02-19 09:49:35 -0800264 environ['CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG
Jan Tattermusche046f712016-02-18 16:06:10 -0800265 return create_jobspec(self.name,
266 ['tools/run_tests/build_artifact_protoc.sh'],
267 environ=environ)
268 else:
269 raise Exception('Not yet supported')
270
271 def __str__(self):
272 return self.name
273
274
Jan Tattermuschbe538a12016-01-28 14:58:15 -0800275def targets():
276 """Gets list of supported targets"""
murgatroid9941a9e832016-02-03 09:47:35 -0800277 return ([Cls(platform, arch)
278 for Cls in (CSharpExtArtifact, NodeExtArtifact)
279 for platform in ('linux', 'macos', 'windows')
280 for arch in ('x86', 'x64')] +
281 [PythonArtifact('linux', 'x86'),
282 PythonArtifact('linux', 'x64'),
Jan Tattermusche0667172016-02-03 15:57:57 -0800283 PythonArtifact('macos', 'x64'),
murgatroid9941a9e832016-02-03 09:47:35 -0800284 RubyArtifact('linux', 'x86'),
murgatroid9933c808b2016-02-03 15:27:41 -0800285 RubyArtifact('linux', 'x64'),
Jan Tattermusche046f712016-02-18 16:06:10 -0800286 RubyArtifact('macos', 'x64'),
287 ProtocArtifact('linux', 'x86'),
288 ProtocArtifact('linux', 'x64'),
289 ProtocArtifact('macos', 'x86'),
290 ProtocArtifact('macos', 'x64')])