Use docker recipe module for Docker Build
Bug: skia:9632
Change-Id: I85e0314404105edf6044775d8ff2433104e2b59e
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/254969
Commit-Queue: Ben Wagner aka dogben <benjaminwagner@google.com>
Reviewed-by: Kevin Lubick <kjlubick@google.com>
diff --git a/infra/bots/recipe_modules/build/__init__.py b/infra/bots/recipe_modules/build/__init__.py
index a7325b4..8a8ba88 100644
--- a/infra/bots/recipe_modules/build/__init__.py
+++ b/infra/bots/recipe_modules/build/__init__.py
@@ -4,6 +4,7 @@
DEPS = [
'depot_tools/gclient',
+ 'docker',
'env',
'infra',
'recipe_engine/context',
diff --git a/infra/bots/recipe_modules/build/docker.py b/infra/bots/recipe_modules/build/docker.py
index c00beff..9638411 100644
--- a/infra/bots/recipe_modules/build/docker.py
+++ b/infra/bots/recipe_modules/build/docker.py
@@ -39,29 +39,12 @@
# anytime the image changes.
args['extra_cflags'] = '["-DDUMMY_docker_image=%s"]' % image
- # We want to make sure the directories exist and were created by chrome-bot.
- # (Note that the docker --mount option, unlike the --volume option, does not
- # create this dir as root if it doesn't exist.)
- api.file.ensure_directory('mkdirs out_dir', out_dir, mode=0777)
-
# Format the GN args for this build.
gn_args = ' '.join('%s=%s' % (k, v) for (k, v) in sorted(args.iteritems()))
-
- # Run the compile script inside the docker container. It expects two mounts:
- # the start_dir at /SRC and the output directory at /OUT.
- src_mnt = 'type=bind,source=%s,target=/SRC' % checkout_root
- out_mnt = 'type=bind,source=%s,target=/OUT' % out_dir
- inner_script_path = ('/SRC/recipe_bundle/skia/infra/bots/recipe_modules'
- '/build/resources/docker-compile.sh')
- cmd = ['docker', 'run', '--rm', '--mount', src_mnt, '--mount', out_mnt, image,
- inner_script_path, gn_args]
- # Override DOCKER_CONFIG set by Kitchen.
- env = {'DOCKER_CONFIG': '/home/chrome-bot/.docker'}
- with api.env(env):
- api.run(
- api.step,
- 'Run build script in Docker',
- cmd=cmd)
+ api.docker.run('Run build script in Docker', image, checkout_root, out_dir,
+ 'recipe_bundle/skia/infra/bots/recipe_modules'
+ '/build/resources/docker-compile.sh',
+ args=[gn_args])
def copy_build_products(api, src, dst):
util.copy_listed_files(api, src, dst, util.DEFAULT_BUILD_PRODUCTS)
diff --git a/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian10-GCC-x86_64-Release-Docker.json b/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian10-GCC-x86_64-Release-Docker.json
index c588882..1243946 100644
--- a/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian10-GCC-x86_64-Release-Docker.json
+++ b/infra/bots/recipe_modules/build/examples/full.expected/Build-Debian10-GCC-x86_64-Release-Docker.json
@@ -1,5 +1,9 @@
[
{
+ "cmd": [],
+ "name": "Docker setup"
+ },
+ {
"cmd": [
"vpython",
"-u",
@@ -12,12 +16,52 @@
"[START_DIR]/cache/work/skia/out/Build-Debian10-GCC-x86_64-Release-Docker/Release"
],
"infra_step": true,
- "name": "mkdirs out_dir"
+ "name": "Docker setup.mkdirs out_dir",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "chmod",
+ "777",
+ "[START_DIR]/cache/work/skia/out/Build-Debian10-GCC-x86_64-Release-Docker/Release"
+ ],
+ "infra_step": true,
+ "name": "Docker setup.chmod 777 [START_DIR]/cache/work/skia/out/Build-Debian10-GCC-x86_64-Release-Docker/Release",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "chmod",
+ "755",
+ "[START_DIR]/cache/work"
+ ],
+ "infra_step": true,
+ "name": "Docker setup.chmod 755 [START_DIR]/cache/work",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "chmod",
+ "0755",
+ "recipe_bundle/skia/infra/bots/recipe_modules/build/resources/docker-compile.sh"
+ ],
+ "infra_step": true,
+ "name": "Docker setup.chmod 0755 recipe_bundle/skia/infra/bots/recipe_modules/build/resources/docker-compile.sh",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
},
{
"cmd": [
"docker",
"run",
+ "--shm-size=2gb",
"--rm",
"--mount",
"type=bind,source=[START_DIR]/cache/work,target=/SRC",
@@ -28,9 +72,7 @@
"cc=\"gcc\" cxx=\"g++\" extra_cflags=[\"-DDUMMY_docker_image=gcr.io/skia-public/gcc-debian10@sha256:89a72df1e2fdea6f774a3fa4199bb9aaa4a0526a3ac1f233e604d689b694f95c\"] is_debug=false target_cpu=\"x86_64\" werror=true"
],
"env": {
- "CHROME_HEADLESS": "1",
- "DOCKER_CONFIG": "/home/chrome-bot/.docker",
- "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ "DOCKER_CONFIG": "/home/chrome-bot/.docker"
},
"name": "Run build script in Docker"
},
diff --git a/infra/bots/recipe_modules/docker/api.py b/infra/bots/recipe_modules/docker/api.py
index 4412288..944989e 100644
--- a/infra/bots/recipe_modules/docker/api.py
+++ b/infra/bots/recipe_modules/docker/api.py
@@ -24,8 +24,9 @@
# Setup. Docker runs as a different user, so we need to give it access to
# read, write, and execute certain files.
with self.m.step.nest('Docker setup'):
- # Make sure out_dir exists, otherwise Docker will create it with root
- # permissions.
+ # Make sure out_dir exists, otherwise mounting will fail.
+ # (Note that the docker --mount option, unlike the --volume option, does
+ # not create this dir as root if it doesn't exist.)
self.m.file.ensure_directory('mkdirs out_dir', out_dir, mode=0777)
# ensure_directory won't change the permissions if the dir already exists,
# so we need to do that explicitly.
@@ -55,8 +56,8 @@
# Run.
cmd = [
'docker', 'run', '--shm-size=2gb', '--rm',
- '--volume', '%s:%s' % (src_dir, MOUNT_SRC),
- '--volume', '%s:%s' % (out_dir, MOUNT_OUT),
+ '--mount', 'type=bind,source=%s,target=%s' % (src_dir, MOUNT_SRC),
+ '--mount', 'type=bind,source=%s,target=%s' % (out_dir, MOUNT_OUT),
]
if docker_args:
cmd.extend(docker_args)
diff --git a/infra/bots/recipe_modules/docker/examples/full.expected/test.json b/infra/bots/recipe_modules/docker/examples/full.expected/test.json
index 4b301e7..9a01816 100644
--- a/infra/bots/recipe_modules/docker/examples/full.expected/test.json
+++ b/infra/bots/recipe_modules/docker/examples/full.expected/test.json
@@ -123,10 +123,10 @@
"run",
"--shm-size=2gb",
"--rm",
- "--volume",
- "/host-src:/SRC",
- "--volume",
- "/host-out:/OUT",
+ "--mount",
+ "type=bind,source=/host-src,target=/SRC",
+ "--mount",
+ "type=bind,source=/host-out,target=/OUT",
"--cpus",
"2",
"my.docker.image",
diff --git a/infra/bots/recipes/perf_canvaskit.expected/Perf-Debian9-EMCC-GCE-CPU-AVX2-wasm-Release-All-CanvasKit.json b/infra/bots/recipes/perf_canvaskit.expected/Perf-Debian9-EMCC-GCE-CPU-AVX2-wasm-Release-All-CanvasKit.json
index 6e94c63..eb11ac6 100644
--- a/infra/bots/recipes/perf_canvaskit.expected/Perf-Debian9-EMCC-GCE-CPU-AVX2-wasm-Release-All-CanvasKit.json
+++ b/infra/bots/recipes/perf_canvaskit.expected/Perf-Debian9-EMCC-GCE-CPU-AVX2-wasm-Release-All-CanvasKit.json
@@ -196,10 +196,10 @@
"run",
"--shm-size=2gb",
"--rm",
- "--volume",
- "[START_DIR]:/SRC",
- "--volume",
- "[START_DIR]/[SWARM_OUT_DIR]:/OUT",
+ "--mount",
+ "type=bind,source=[START_DIR],target=/SRC",
+ "--mount",
+ "type=bind,source=[START_DIR]/[SWARM_OUT_DIR],target=/OUT",
"gcr.io/skia-public/perf-karma-chrome-tests:77.0.3865.120_v1",
"/SRC/skia/infra/canvaskit/perf_canvaskit.sh",
"--builder",
diff --git a/infra/bots/recipes/perf_canvaskit.expected/pathkit_trybot.json b/infra/bots/recipes/perf_canvaskit.expected/pathkit_trybot.json
index a07760f..d84f0f3 100644
--- a/infra/bots/recipes/perf_canvaskit.expected/pathkit_trybot.json
+++ b/infra/bots/recipes/perf_canvaskit.expected/pathkit_trybot.json
@@ -196,10 +196,10 @@
"run",
"--shm-size=2gb",
"--rm",
- "--volume",
- "[START_DIR]:/SRC",
- "--volume",
- "[START_DIR]/[SWARM_OUT_DIR]:/OUT",
+ "--mount",
+ "type=bind,source=[START_DIR],target=/SRC",
+ "--mount",
+ "type=bind,source=[START_DIR]/[SWARM_OUT_DIR],target=/OUT",
"gcr.io/skia-public/perf-karma-chrome-tests:77.0.3865.120_v1",
"/SRC/skia/infra/canvaskit/perf_canvaskit.sh",
"--builder",
diff --git a/infra/bots/recipes/perf_pathkit.expected/Perf-Debian9-EMCC-GCE-CPU-AVX2-asmjs-Release-All-PathKit.json b/infra/bots/recipes/perf_pathkit.expected/Perf-Debian9-EMCC-GCE-CPU-AVX2-asmjs-Release-All-PathKit.json
index d16399e..e7f23d2 100644
--- a/infra/bots/recipes/perf_pathkit.expected/Perf-Debian9-EMCC-GCE-CPU-AVX2-asmjs-Release-All-PathKit.json
+++ b/infra/bots/recipes/perf_pathkit.expected/Perf-Debian9-EMCC-GCE-CPU-AVX2-asmjs-Release-All-PathKit.json
@@ -211,10 +211,10 @@
"run",
"--shm-size=2gb",
"--rm",
- "--volume",
- "[START_DIR]:/SRC",
- "--volume",
- "[START_DIR]/[SWARM_OUT_DIR]:/OUT",
+ "--mount",
+ "type=bind,source=[START_DIR],target=/SRC",
+ "--mount",
+ "type=bind,source=[START_DIR]/[SWARM_OUT_DIR],target=/OUT",
"--env",
"ASM_JS=1",
"gcr.io/skia-public/perf-karma-chrome-tests:77.0.3865.120_v1",
diff --git a/infra/bots/recipes/perf_pathkit.expected/Perf-Debian9-EMCC-GCE-CPU-AVX2-wasm-Release-All-PathKit.json b/infra/bots/recipes/perf_pathkit.expected/Perf-Debian9-EMCC-GCE-CPU-AVX2-wasm-Release-All-PathKit.json
index 61e883f..19b2de0 100644
--- a/infra/bots/recipes/perf_pathkit.expected/Perf-Debian9-EMCC-GCE-CPU-AVX2-wasm-Release-All-PathKit.json
+++ b/infra/bots/recipes/perf_pathkit.expected/Perf-Debian9-EMCC-GCE-CPU-AVX2-wasm-Release-All-PathKit.json
@@ -211,10 +211,10 @@
"run",
"--shm-size=2gb",
"--rm",
- "--volume",
- "[START_DIR]:/SRC",
- "--volume",
- "[START_DIR]/[SWARM_OUT_DIR]:/OUT",
+ "--mount",
+ "type=bind,source=[START_DIR],target=/SRC",
+ "--mount",
+ "type=bind,source=[START_DIR]/[SWARM_OUT_DIR],target=/OUT",
"gcr.io/skia-public/perf-karma-chrome-tests:77.0.3865.120_v1",
"/SRC/skia/infra/pathkit/perf_pathkit.sh",
"--builder",
diff --git a/infra/bots/recipes/perf_pathkit.expected/pathkit_trybot.json b/infra/bots/recipes/perf_pathkit.expected/pathkit_trybot.json
index fd925bd..8aefe66 100644
--- a/infra/bots/recipes/perf_pathkit.expected/pathkit_trybot.json
+++ b/infra/bots/recipes/perf_pathkit.expected/pathkit_trybot.json
@@ -211,10 +211,10 @@
"run",
"--shm-size=2gb",
"--rm",
- "--volume",
- "[START_DIR]:/SRC",
- "--volume",
- "[START_DIR]/[SWARM_OUT_DIR]:/OUT",
+ "--mount",
+ "type=bind,source=[START_DIR],target=/SRC",
+ "--mount",
+ "type=bind,source=[START_DIR]/[SWARM_OUT_DIR],target=/OUT",
"gcr.io/skia-public/perf-karma-chrome-tests:77.0.3865.120_v1",
"/SRC/skia/infra/pathkit/perf_pathkit.sh",
"--builder",
diff --git a/infra/bots/recipes/test_canvaskit.expected/Test-Debian9-EMCC-GCE-GPU-WEBGL1-wasm-Debug-All-CanvasKit.json b/infra/bots/recipes/test_canvaskit.expected/Test-Debian9-EMCC-GCE-GPU-WEBGL1-wasm-Debug-All-CanvasKit.json
index 4e4f0ae..890a1bf 100644
--- a/infra/bots/recipes/test_canvaskit.expected/Test-Debian9-EMCC-GCE-GPU-WEBGL1-wasm-Debug-All-CanvasKit.json
+++ b/infra/bots/recipes/test_canvaskit.expected/Test-Debian9-EMCC-GCE-GPU-WEBGL1-wasm-Debug-All-CanvasKit.json
@@ -211,10 +211,10 @@
"run",
"--shm-size=2gb",
"--rm",
- "--volume",
- "[START_DIR]:/SRC",
- "--volume",
- "[START_DIR]/[SWARM_OUT_DIR]:/OUT",
+ "--mount",
+ "type=bind,source=[START_DIR],target=/SRC",
+ "--mount",
+ "type=bind,source=[START_DIR]/[SWARM_OUT_DIR],target=/OUT",
"gcr.io/skia-public/gold-karma-chrome-tests:77.0.3865.120_v2",
"/SRC/skia/infra/canvaskit/test_canvaskit.sh",
"--builder",
diff --git a/infra/bots/recipes/test_canvaskit.expected/canvaskit_trybot.json b/infra/bots/recipes/test_canvaskit.expected/canvaskit_trybot.json
index 6de3a05..74f6d16 100644
--- a/infra/bots/recipes/test_canvaskit.expected/canvaskit_trybot.json
+++ b/infra/bots/recipes/test_canvaskit.expected/canvaskit_trybot.json
@@ -211,10 +211,10 @@
"run",
"--shm-size=2gb",
"--rm",
- "--volume",
- "[START_DIR]:/SRC",
- "--volume",
- "[START_DIR]/[SWARM_OUT_DIR]:/OUT",
+ "--mount",
+ "type=bind,source=[START_DIR],target=/SRC",
+ "--mount",
+ "type=bind,source=[START_DIR]/[SWARM_OUT_DIR],target=/OUT",
"gcr.io/skia-public/gold-karma-chrome-tests:77.0.3865.120_v2",
"/SRC/skia/infra/canvaskit/test_canvaskit.sh",
"--builder",
diff --git a/infra/bots/recipes/test_pathkit.expected/Test-Debian9-EMCC-GCE-CPU-AVX2-asmjs-Debug-All-PathKit.json b/infra/bots/recipes/test_pathkit.expected/Test-Debian9-EMCC-GCE-CPU-AVX2-asmjs-Debug-All-PathKit.json
index 3caf5a5..9c1cd23 100644
--- a/infra/bots/recipes/test_pathkit.expected/Test-Debian9-EMCC-GCE-CPU-AVX2-asmjs-Debug-All-PathKit.json
+++ b/infra/bots/recipes/test_pathkit.expected/Test-Debian9-EMCC-GCE-CPU-AVX2-asmjs-Debug-All-PathKit.json
@@ -149,10 +149,10 @@
"run",
"--shm-size=2gb",
"--rm",
- "--volume",
- "[START_DIR]:/SRC",
- "--volume",
- "[START_DIR]/[SWARM_OUT_DIR]:/OUT",
+ "--mount",
+ "type=bind,source=[START_DIR],target=/SRC",
+ "--mount",
+ "type=bind,source=[START_DIR]/[SWARM_OUT_DIR],target=/OUT",
"--env",
"ASM_JS=1",
"gcr.io/skia-public/gold-karma-chrome-tests:77.0.3865.120_v2",
diff --git a/infra/bots/recipes/test_pathkit.expected/Test-Debian9-EMCC-GCE-CPU-AVX2-asmjs-Release-All-PathKit.json b/infra/bots/recipes/test_pathkit.expected/Test-Debian9-EMCC-GCE-CPU-AVX2-asmjs-Release-All-PathKit.json
index 50244c6..4ca0d47 100644
--- a/infra/bots/recipes/test_pathkit.expected/Test-Debian9-EMCC-GCE-CPU-AVX2-asmjs-Release-All-PathKit.json
+++ b/infra/bots/recipes/test_pathkit.expected/Test-Debian9-EMCC-GCE-CPU-AVX2-asmjs-Release-All-PathKit.json
@@ -196,10 +196,10 @@
"run",
"--shm-size=2gb",
"--rm",
- "--volume",
- "[START_DIR]:/SRC",
- "--volume",
- "[START_DIR]/[SWARM_OUT_DIR]:/OUT",
+ "--mount",
+ "type=bind,source=[START_DIR],target=/SRC",
+ "--mount",
+ "type=bind,source=[START_DIR]/[SWARM_OUT_DIR],target=/OUT",
"--env",
"ASM_JS=1",
"gcr.io/skia-public/gold-karma-chrome-tests:77.0.3865.120_v2",
diff --git a/infra/bots/recipes/test_pathkit.expected/Test-Debian9-EMCC-GCE-CPU-AVX2-wasm-Debug-All-PathKit.json b/infra/bots/recipes/test_pathkit.expected/Test-Debian9-EMCC-GCE-CPU-AVX2-wasm-Debug-All-PathKit.json
index 0092c6e..b9e7806 100644
--- a/infra/bots/recipes/test_pathkit.expected/Test-Debian9-EMCC-GCE-CPU-AVX2-wasm-Debug-All-PathKit.json
+++ b/infra/bots/recipes/test_pathkit.expected/Test-Debian9-EMCC-GCE-CPU-AVX2-wasm-Debug-All-PathKit.json
@@ -196,10 +196,10 @@
"run",
"--shm-size=2gb",
"--rm",
- "--volume",
- "[START_DIR]:/SRC",
- "--volume",
- "[START_DIR]/[SWARM_OUT_DIR]:/OUT",
+ "--mount",
+ "type=bind,source=[START_DIR],target=/SRC",
+ "--mount",
+ "type=bind,source=[START_DIR]/[SWARM_OUT_DIR],target=/OUT",
"gcr.io/skia-public/gold-karma-chrome-tests:77.0.3865.120_v2",
"/SRC/skia/infra/pathkit/test_pathkit.sh",
"--builder",
diff --git a/infra/bots/recipes/test_pathkit.expected/pathkit_trybot.json b/infra/bots/recipes/test_pathkit.expected/pathkit_trybot.json
index b0b10b3..9de671e 100644
--- a/infra/bots/recipes/test_pathkit.expected/pathkit_trybot.json
+++ b/infra/bots/recipes/test_pathkit.expected/pathkit_trybot.json
@@ -196,10 +196,10 @@
"run",
"--shm-size=2gb",
"--rm",
- "--volume",
- "[START_DIR]:/SRC",
- "--volume",
- "[START_DIR]/[SWARM_OUT_DIR]:/OUT",
+ "--mount",
+ "type=bind,source=[START_DIR],target=/SRC",
+ "--mount",
+ "type=bind,source=[START_DIR]/[SWARM_OUT_DIR],target=/OUT",
"gcr.io/skia-public/gold-karma-chrome-tests:77.0.3865.120_v2",
"/SRC/skia/infra/pathkit/test_pathkit.sh",
"--builder",