Add Debian10 GCC Docker Build
Bug: skia:9632
Change-Id: I779fa02cd7227b4d68cc8b0d1c1781473ad0741d
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/254499
Commit-Queue: Ben Wagner aka dogben <benjaminwagner@google.com>
Reviewed-by: Kevin Lubick <kjlubick@google.com>
Reviewed-by: Herb Derby <herb@google.com>
diff --git a/gn/BUILD.gn b/gn/BUILD.gn
index 1324283..49c0c95 100644
--- a/gn/BUILD.gn
+++ b/gn/BUILD.gn
@@ -350,6 +350,17 @@
cflags_cc += [
"-Wnon-virtual-dtor",
"-Wno-noexcept-type",
+
+ # TODO(dogben): The following are new between GCC 6 (stretch) and GCC 8 (buster); they should
+ # all be investigated.
+ "-Wno-array-bounds",
+ "-Wno-class-memaccess",
+ "-Wno-stringop-overflow",
+ "-Wno-restrict",
+ "-Wno-stringop-truncation",
+ "-Wno-sizeof-pointer-memaccess",
+ "-Wno-parentheses",
+ "-Wno-format-truncation",
]
}
diff --git a/infra/bots/gen_tasks_logic/gen_tasks_logic.go b/infra/bots/gen_tasks_logic/gen_tasks_logic.go
index d3c8f39..9aeb237 100644
--- a/infra/bots/gen_tasks_logic/gen_tasks_logic.go
+++ b/infra/bots/gen_tasks_logic/gen_tasks_logic.go
@@ -588,6 +588,9 @@
d := map[string]string{
"pool": b.cfg.Pool,
}
+ if strings.Contains(parts["extra_config"], "Docker") && parts["role"] == "Build" {
+ return b.dockerGceDimensions()
+ }
if os, ok := parts["os"]; ok {
d["os"], ok = map[string]string{
"Android": "Android",
@@ -978,7 +981,7 @@
// usesDocker adds attributes to tasks which use docker.
func usesDocker(t *specs.TaskSpec, name string) {
- if strings.Contains(name, "EMCC") || strings.Contains(name, "SKQP") || strings.Contains(name, "LottieWeb") || strings.Contains(name, "CMake") {
+ if strings.Contains(name, "EMCC") || strings.Contains(name, "SKQP") || strings.Contains(name, "LottieWeb") || strings.Contains(name, "CMake") || strings.Contains(name, "Docker") {
t.Caches = append(t.Caches, CACHES_DOCKER...)
}
}
diff --git a/infra/bots/jobs.json b/infra/bots/jobs.json
index ec7cfa5..bd204d8 100644
--- a/infra/bots/jobs.json
+++ b/infra/bots/jobs.json
@@ -82,6 +82,7 @@
"Build-Debian9-GCC-x86_64-Release-NoGPU",
"Build-Debian9-GCC-x86_64-Release-SK_CPU_LIMIT_SSE41",
"Build-Debian9-GCC-x86_64-Release-Shared",
+ "Build-Debian10-GCC-x86_64-Release-Docker",
"Build-Mac-Clang-arm-Debug-iOS",
"Build-Mac-Clang-arm-Release-iOS",
"Build-Mac-Clang-arm64-Debug-Android",
diff --git a/infra/bots/recipe_modules/build/api.py b/infra/bots/recipe_modules/build/api.py
index 0edf5c0..c4964d9 100644
--- a/infra/bots/recipe_modules/build/api.py
+++ b/infra/bots/recipe_modules/build/api.py
@@ -14,6 +14,7 @@
from . import chromecast
from . import cmake
from . import default
+from . import docker
from . import flutter
from . import pathkit
from . import skqp
@@ -47,6 +48,9 @@
elif 'CMake' in b:
self.compile_fn = cmake.compile_fn
self.copy_fn = cmake.copy_build_products
+ elif 'Docker' in b:
+ self.compile_fn = docker.compile_fn
+ self.copy_fn = docker.copy_build_products
else:
self.compile_fn = default.compile_fn
self.copy_fn = default.copy_build_products
diff --git a/infra/bots/recipe_modules/build/docker.py b/infra/bots/recipe_modules/build/docker.py
new file mode 100644
index 0000000..c00beff
--- /dev/null
+++ b/infra/bots/recipe_modules/build/docker.py
@@ -0,0 +1,67 @@
+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+from . import util
+
+
+def compile_fn(api, checkout_root, out_dir):
+ compiler = api.vars.builder_cfg.get('compiler', '')
+ configuration = api.vars.builder_cfg.get('configuration', '')
+ extra_tokens = api.vars.extra_tokens
+ extra_tokens.remove('Docker')
+ os = api.vars.builder_cfg.get('os', '')
+ target_arch = api.vars.builder_cfg.get('target_arch', '')
+
+ args = {
+ 'target_cpu': '"%s"' % target_arch,
+ 'werror': 'true'
+ }
+ if configuration != 'Debug':
+ args['is_debug'] = 'false'
+
+ image = None
+ if os == 'Debian10':
+ if compiler == 'GCC':
+ if not extra_tokens:
+ image = (
+ 'gcr.io/skia-public/gcc-debian10@sha256:'
+ '89a72df1e2fdea6f774a3fa4199bb9aaa4a0526a3ac1f233e604d689b694f95c')
+ args['cc'] = '"gcc"'
+ args['cxx'] = '"g++"'
+ if not image:
+ raise Exception('Not implemented: ' + api.vars.builder_name)
+
+ # We always perform an incremental compile, since out dir is cached across
+ # compile tasks. However, we need to force a recompile when the toolchain
+ # changes. The simplest way to do that is using a C define that changes
+ # anytime the image changes.
+ args['extra_cflags'] = '["-DDUMMY_docker_image=%s"]' % image
+
+ # We want to make sure the directories exist and were created by chrome-bot.
+ # (Note that the docker --mount option, unlike the --volume option, does not
+ # create this dir as root if it doesn't exist.)
+ api.file.ensure_directory('mkdirs out_dir', out_dir, mode=0777)
+
+ # Format the GN args for this build.
+ gn_args = ' '.join('%s=%s' % (k, v) for (k, v) in sorted(args.iteritems()))
+
+ # Run the compile script inside the docker container. It expects two mounts:
+ # the start_dir at /SRC and the output directory at /OUT.
+ src_mnt = 'type=bind,source=%s,target=/SRC' % checkout_root
+ out_mnt = 'type=bind,source=%s,target=/OUT' % out_dir
+ inner_script_path = ('/SRC/recipe_bundle/skia/infra/bots/recipe_modules'
+ '/build/resources/docker-compile.sh')
+ cmd = ['docker', 'run', '--rm', '--mount', src_mnt, '--mount', out_mnt, image,
+ inner_script_path, gn_args]
+ # Override DOCKER_CONFIG set by Kitchen.
+ env = {'DOCKER_CONFIG': '/home/chrome-bot/.docker'}
+ with api.env(env):
+ api.run(
+ api.step,
+ 'Run build script in Docker',
+ cmd=cmd)
+
+def copy_build_products(api, src, dst):
+ util.copy_listed_files(api, src, dst, util.DEFAULT_BUILD_PRODUCTS)
diff --git a/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian10-GCC-x86_64-Release-Docker.json b/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian10-GCC-x86_64-Release-Docker.json
new file mode 100644
index 0000000..c588882
--- /dev/null
+++ b/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian10-GCC-x86_64-Release-Docker.json
@@ -0,0 +1,78 @@
+[
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/cache/work/skia/out/Build-Debian10-GCC-x86_64-Release-Docker/Release"
+ ],
+ "infra_step": true,
+ "name": "mkdirs out_dir"
+ },
+ {
+ "cmd": [
+ "docker",
+ "run",
+ "--rm",
+ "--mount",
+ "type=bind,source=[START_DIR]/cache/work,target=/SRC",
+ "--mount",
+ "type=bind,source=[START_DIR]/cache/work/skia/out/Build-Debian10-GCC-x86_64-Release-Docker/Release,target=/OUT",
+ "gcr.io/skia-public/gcc-debian10@sha256:89a72df1e2fdea6f774a3fa4199bb9aaa4a0526a3ac1f233e604d689b694f95c",
+ "/SRC/recipe_bundle/skia/infra/bots/recipe_modules/build/resources/docker-compile.sh",
+ "cc=\"gcc\" cxx=\"g++\" extra_cflags=[\"-DDUMMY_docker_image=gcr.io/skia-public/gcc-debian10@sha256:89a72df1e2fdea6f774a3fa4199bb9aaa4a0526a3ac1f233e604d689b694f95c\"] is_debug=false target_cpu=\"x86_64\" werror=true"
+ ],
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "DOCKER_CONFIG": "/home/chrome-bot/.docker",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
+ "name": "Run build script in Docker"
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products = ['dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'hello-opencl', 'hello-opencl.exe', 'nanobench', 'nanobench.exe', 'skpbench', 'skpbench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skottie_tool', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n",
+ "[START_DIR]/cache/work/skia/out/Build-Debian10-GCC-x86_64-Release-Docker/Release",
+ "[START_DIR]/[SWARM_OUT_DIR]/out/Release"
+ ],
+ "infra_step": true,
+ "name": "copy build products",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@import errno@@@",
+ "@@@STEP_LOG_LINE@python.inline@import glob@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@import shutil@@@",
+ "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@",
+ "@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@",
+ "@@@STEP_LOG_LINE@python.inline@build_products = ['dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'hello-opencl', 'hello-opencl.exe', 'nanobench', 'nanobench.exe', 'skpbench', 'skpbench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skottie_tool', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@try:@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@",
+ "@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@",
+ "@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@",
+ "@@@STEP_LOG_LINE@python.inline@ raise@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@for pattern in build_products:@@@",
+ "@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@",
+ "@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@",
+ "@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@",
+ "@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@",
+ "@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@",
+ "@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/infra/bots/recipe_modules/build/examples/full.expected/unknown-docker-image.json b/infra/bots/recipe_modules/build/examples/full.expected/unknown-docker-image.json
new file mode 100644
index 0000000..8c2bdfd
--- /dev/null
+++ b/infra/bots/recipe_modules/build/examples/full.expected/unknown-docker-image.json
@@ -0,0 +1,35 @@
+[
+ {
+ "cmd": [],
+ "name": "RECIPE CRASH (Uncaught exception)",
+ "~followup_annotations": [
+ "@@@STEP_EXCEPTION@@@",
+ "The recipe has crashed at point 'Uncaught exception'!",
+ "",
+ "Traceback (most recent call last):",
+ " File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", in run_steps",
+ " raw_result = recipe_obj.run_steps(api, engine)",
+ " File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", in run_steps",
+ " properties_def, api=api)",
+ " File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/property_invoker.py\", in invoke_with_properties",
+ " arg_names, **additional_args)",
+ " File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/property_invoker.py\", in _invoke_with_properties",
+ " return callable_obj(*props, **additional_args)",
+ " File \"RECIPE_REPO[skia]/infra/bots/recipe_modules/build/examples/full.py\", line 22, in RunSteps",
+ " api.build(checkout_root=checkout_root, out_dir=out_dir)",
+ " File \"RECIPE_REPO[recipe_engine]/recipe_engine/recipe_api.py\", in _inner",
+ " return func(*a, **kw)",
+ " File \"RECIPE_REPO[skia]/infra/bots/recipe_modules/build/api.py\", line 61, in __call__",
+ " self.compile_fn(self.m, checkout_root, out_dir)",
+ " File \"RECIPE_REPO[skia]/infra/bots/recipe_modules/build/docker.py\", line 34, in compile_fn",
+ " raise Exception('Not implemented: ' + api.vars.builder_name)",
+ "Exception: Not implemented: Build-Unix-GCC-x86_64-Release-Docker"
+ ]
+ },
+ {
+ "failure": {
+ "humanReason": "Uncaught Exception: Exception('Not implemented: Build-Unix-GCC-x86_64-Release-Docker',)"
+ },
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/infra/bots/recipe_modules/build/examples/full.py b/infra/bots/recipe_modules/build/examples/full.py
index 32f922f..8bed742 100644
--- a/infra/bots/recipe_modules/build/examples/full.py
+++ b/infra/bots/recipe_modules/build/examples/full.py
@@ -58,6 +58,7 @@
'Build-Debian9-GCC-x86_64-Release-ANGLE',
'Build-Debian9-GCC-x86_64-Release-NoGPU',
'Build-Debian9-GCC-x86_64-Release-Shared',
+ 'Build-Debian10-GCC-x86_64-Release-Docker',
'Build-Mac-Clang-arm-Debug-iOS',
'Build-Mac-Clang-arm64-Debug-Android_Vulkan',
'Build-Mac-Clang-arm64-Debug-iOS',
@@ -92,3 +93,9 @@
if 'Win' in buildername and not 'LenovoYogaC630' in buildername:
test += api.platform('win', 64)
yield test
+
+ yield (
+ api.test('unknown-docker-image') +
+ api.properties(**defaultProps('Build-Unix-GCC-x86_64-Release-Docker')) +
+ api.expect_exception('Exception')
+ )
diff --git a/infra/bots/recipe_modules/build/resources/docker-compile.sh b/infra/bots/recipe_modules/build/resources/docker-compile.sh
new file mode 100755
index 0000000..57c9a66
--- /dev/null
+++ b/infra/bots/recipe_modules/build/resources/docker-compile.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Copyright 2019 Google LLC
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This assumes it is being run inside a docker container with the following
+# mounts:
+# /SRC: Swarming start dir
+# /OUT: output directory for gn and ninja
+
+set -e
+set -x
+
+export PATH="/SRC/recipe_bundle/depot_tools:${PATH}"
+
+cd /SRC/skia
+./bin/fetch-gn
+./bin/gn gen /OUT "--args=$1"
+ninja -C /OUT
diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json
index 9868b0b..35a70fb 100755
--- a/infra/bots/tasks.json
+++ b/infra/bots/tasks.json
@@ -1,5 +1,10 @@
{
"jobs": {
+ "Build-Debian10-GCC-x86_64-Release-Docker": {
+ "tasks": [
+ "Build-Debian10-GCC-x86_64-Release-Docker"
+ ]
+ },
"Build-Debian9-Clang-TAP-Presubmit-G3_Framework": {
"tasks": [
"Build-Debian9-Clang-TAP-Presubmit-G3_Framework"
@@ -2996,6 +3001,75 @@
}
},
"tasks": {
+ "Build-Debian10-GCC-x86_64-Release-Docker": {
+ "caches": [
+ {
+ "name": "vpython",
+ "path": "cache/vpython"
+ },
+ {
+ "name": "docker",
+ "path": "cache/docker"
+ }
+ ],
+ "cipd_packages": [
+ {
+ "name": "infra/tools/luci/kitchen/${platform}",
+ "path": ".",
+ "version": "git_revision:d8f38ca9494b5af249942631f9cee45927f6b4bc"
+ },
+ {
+ "name": "infra/tools/luci-auth/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "git_revision:2c805f1c716f6c5ad2126b27ec88b8585a09481e"
+ },
+ {
+ "name": "infra/tools/luci/vpython/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "git_revision:f96db4b66034c859090be3c47eb38227277f228b"
+ }
+ ],
+ "command": [
+ "cipd_bin_packages/vpython${EXECUTABLE_SUFFIX}",
+ "skia/infra/bots/run_recipe.py",
+ "${ISOLATED_OUTDIR}",
+ "compile",
+ "{\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"buildername\":\"Build-Debian10-GCC-x86_64-Release-Docker\",\"swarm_out_dir\":\"build\"}",
+ "skia"
+ ],
+ "dependencies": [
+ "Housekeeper-PerCommit-BundleRecipes"
+ ],
+ "dimensions": [
+ "cpu:x86-64-Haswell_GCE",
+ "gpu:none",
+ "machine_type:n1-standard-16",
+ "os:Debian-9.8",
+ "pool:Skia",
+ "docker_installed:true"
+ ],
+ "env_prefixes": {
+ "PATH": [
+ "cipd_bin_packages",
+ "cipd_bin_packages/bin"
+ ],
+ "VPYTHON_VIRTUALENV_ROOT": [
+ "cache/vpython"
+ ]
+ },
+ "execution_timeout_ns": 3600000000000,
+ "extra_tags": {
+ "log_location": "logdog://logs.chromium.org/skia/${SWARMING_TASK_ID}/+/annotations"
+ },
+ "idempotent": true,
+ "io_timeout_ns": 3600000000000,
+ "isolate": "compile.isolate",
+ "max_attempts": 2,
+ "outputs": [
+ "build"
+ ],
+ "service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
+ },
"Build-Debian9-Clang-TAP-Presubmit-G3_Framework": {
"caches": [
{
diff --git a/infra/gcc/Debian10/Dockerfile b/infra/gcc/Debian10/Dockerfile
new file mode 100644
index 0000000..2800ca5
--- /dev/null
+++ b/infra/gcc/Debian10/Dockerfile
@@ -0,0 +1,9 @@
+FROM debian:10-slim
+
+RUN apt-get update && apt-get upgrade -y && apt-get install -y \
+ build-essential \
+ ca-certificates \
+ libfontconfig-dev \
+ libglu-dev \
+ python \
+ && rm -rf /var/lib/apt/lists/*
diff --git a/infra/gcc/Makefile b/infra/gcc/Makefile
new file mode 100644
index 0000000..1fabeb6
--- /dev/null
+++ b/infra/gcc/Makefile
@@ -0,0 +1,5 @@
+
+publish_Debian10:
+ docker build -t gcc-debian10 ./Debian10/
+ docker tag gcc-debian10 gcr.io/skia-public/gcc-debian10
+ docker push gcr.io/skia-public/gcc-debian10