Update win_toolchain, and refactor how it's built
The old win_toolchain script required a Chromium checkout, and
extracted portions of the win_toolchain from that to build the
Skia asset. Instead, use the depot_tools script that assembles
a toolchain from a locally installed MSVC.
The create script doesn't do that, but relies on the user to
run that script first. Automating everything would be a nice
follow-up.
With the new strategy, the toolchain directory is simpler, and
no longer contains the depot_tools kruft or extra directories.
Adjust the bot scripts accordingly. (Renaming the directory to
win_toolchain from 't' would be a nice touch, too).
Finally, I built the new toolchain with the updated process,
and included the ARM64 compiler and libraries, so we can set
up a bot to build Windows ARM64.
Docs-Preview: https://skia.org/?cl=176968
Bug: skia:8569
Change-Id: I4bdf3cfb29d50f4464853445d0226241e70c33b4
Reviewed-on: https://skia-review.googlesource.com/c/176968
Reviewed-by: Mike Klein <mtklein@google.com>
Reviewed-by: Eric Boren <borenet@google.com>
Commit-Queue: Brian Osman <brianosman@google.com>
diff --git a/infra/bots/assets/win_toolchain/VERSION b/infra/bots/assets/win_toolchain/VERSION
index 301160a..f11c82a 100644
--- a/infra/bots/assets/win_toolchain/VERSION
+++ b/infra/bots/assets/win_toolchain/VERSION
@@ -1 +1 @@
-8
\ No newline at end of file
+9
\ No newline at end of file
diff --git a/infra/bots/assets/win_toolchain/create.py b/infra/bots/assets/win_toolchain/create.py
index e96cde9..429ee80 100755
--- a/infra/bots/assets/win_toolchain/create.py
+++ b/infra/bots/assets/win_toolchain/create.py
@@ -6,12 +6,21 @@
# found in the LICENSE file.
-"""Download an updated VS toolchain"""
+"""
+Create an updated VS toolchain
+Before you can run this script, you need a collated VC toolchain + Windows SDK.
+To generate that, run depot_tools/win_toolchain/package_from_installed.py
+That script pulls all of the compiler and SDK bits from your locally installed
+version of Visual Studio. The comments in that script include instructions on
+which components need to be installed (C++, ARM64, etc...)
+
+That script produces a .zip file with a SHA filename. Unzip that file, then
+pass the unzipped directory as the src_dir to this script.
+"""
import argparse
import common
-import json
import os
import shlex
import shutil
@@ -19,8 +28,6 @@
import sys
import utils
-import win_toolchain_utils
-
# By default the toolchain includes a bunch of unnecessary stuff with long path
# names. Trim out directories with these names.
@@ -33,9 +40,6 @@
'AccChecker',
]
-REPO_CHROME = 'https://chromium.googlesource.com/chromium/src.git'
-
-
def filter_toolchain_files(dirname, files):
"""Callback for shutil.copytree. Return lists of files to skip."""
split = dirname.split(os.path.sep)
@@ -45,83 +49,18 @@
return files
return []
-
-def get_toolchain_dir(toolchain_dir_output):
- """Find the toolchain directory."""
- prefix = 'vs_path = '
- for line in toolchain_dir_output.splitlines():
- if line.startswith(prefix):
- return line[len(prefix):].strip('"')
- raise Exception('Unable to find toolchain dir in output:\n%s' % (
- toolchain_dir_output))
-
-
-def gen_toolchain(chrome_path, msvs_version, target_dir):
- """Update the VS toolchain and copy it to the target_dir."""
- with utils.chdir(os.path.join(chrome_path, 'src')):
- subprocess.check_call([utils.GCLIENT, 'sync'])
- depot_tools = subprocess.check_output([
- 'python', os.path.join('build', 'find_depot_tools.py')]).rstrip()
- with utils.git_branch():
- vs_toolchain_py = os.path.join('build', 'vs_toolchain.py')
- env = os.environ.copy()
- env['GYP_MSVS_VERSION'] = msvs_version
- subprocess.check_call(['python', vs_toolchain_py, 'update'], env=env)
- output = subprocess.check_output(['python', vs_toolchain_py,
- 'get_toolchain_dir'], env=env).rstrip()
- src_dir = get_toolchain_dir(output)
- # Mock out absolute paths in win_toolchain.json.
- win_toolchain_utils.abstract(os.path.join('build', 'win_toolchain.json'),
- os.path.dirname(depot_tools))
-
- # Copy the toolchain files to the target_dir.
- build = os.path.join(os.getcwd(), 'build')
- dst_build = os.path.join(target_dir, 'src', 'build')
- os.makedirs(dst_build)
- for f in ('find_depot_tools.py', 'vs_toolchain.py', 'win_toolchain.json'):
- shutil.copyfile(os.path.join(build, f), os.path.join(dst_build, f))
-
- shutil.copytree(os.path.join(os.getcwd(), 'tools', 'gyp', 'pylib'),
- os.path.join(target_dir, 'src', 'tools', 'gyp', 'pylib'))
-
- dst_depot_tools = os.path.join(target_dir, 'depot_tools')
- os.makedirs(dst_depot_tools)
- for f in ('gclient.py', 'breakpad.py'):
- shutil.copyfile(os.path.join(depot_tools, f),
- os.path.join(dst_depot_tools, f))
- toolchain_dst = os.path.join(
- target_dir, 'depot_tools', os.path.relpath(src_dir, depot_tools))
- shutil.copytree(src_dir, toolchain_dst, ignore=filter_toolchain_files)
-
-
-def create_asset(target_dir, msvs_version, chrome_path=None):
- """Create the asset."""
- if not os.path.isdir(target_dir):
- os.makedirs(target_dir)
- with utils.tmp_dir() as tmp_dir:
- if not chrome_path:
- print ('Syncing Chrome from scratch. If you already have a checkout, '
- 'specify --chrome_path to save time.')
- chrome_path = os.path.join(tmp_dir.name, 'src')
- if not os.path.isdir(chrome_path):
- subprocess.check_call([utils.GCLIENT, 'config', REPO_CHROME, '--managed'])
- subprocess.check_call([utils.GCLIENT, 'sync'])
-
- gen_toolchain(chrome_path, msvs_version, target_dir)
-
def main():
if sys.platform != 'win32':
print >> sys.stderr, 'This script only runs on Windows.'
sys.exit(1)
parser = argparse.ArgumentParser()
- parser.add_argument('--msvs_version', required=True)
- parser.add_argument('--chrome_path')
+ parser.add_argument('--src_dir', '-s', required=True)
parser.add_argument('--target_dir', '-t', required=True)
args = parser.parse_args()
+ src_dir = os.path.abspath(args.src_dir)
target_dir = os.path.abspath(args.target_dir)
- create_asset(target_dir, args.msvs_version, args.chrome_path)
-
+ shutil.copytree(src_dir, target_dir, ignore=filter_toolchain_files)
if __name__ == '__main__':
main()
diff --git a/infra/bots/assets/win_toolchain/create_and_upload.py b/infra/bots/assets/win_toolchain/create_and_upload.py
index d428ca4..c914d41 100755
--- a/infra/bots/assets/win_toolchain/create_and_upload.py
+++ b/infra/bots/assets/win_toolchain/create_and_upload.py
@@ -20,23 +20,21 @@
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--gsutil')
- parser.add_argument('--chrome_path')
- parser.add_argument('--msvs_version', required=True)
+ parser.add_argument('--src_dir', '-s', required=True)
args = parser.parse_args()
with utils.tmp_dir():
cwd = os.getcwd()
create_script = os.path.join(common.FILE_DIR, 'create.py')
upload_script = os.path.join(common.FILE_DIR, 'upload.py')
+ target_dir = os.path.join(cwd, 'win_toolchain')
try:
cmd = ['python', create_script,
- '-t', cwd,
- '--msvs_version', args.msvs_version]
- if args.chrome_path:
- cmd.extend(['--chrome_path', args.chrome_path])
+ '-t', target_dir,
+ '-s', args.src_dir]
subprocess.check_call(cmd)
- cmd = ['python', upload_script, '-t', cwd]
+ cmd = ['python', upload_script, '-t', target_dir]
if args.gsutil:
cmd.extend(['--gsutil', args.gsutil])
subprocess.check_call(cmd)
diff --git a/infra/bots/bootstrap_win_toolchain_json.py b/infra/bots/bootstrap_win_toolchain_json.py
deleted file mode 100644
index 199abe0..0000000
--- a/infra/bots/bootstrap_win_toolchain_json.py
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-"""Resolve the path placeholders in the win_toolchain.json file."""
-
-
-import argparse
-import win_toolchain_utils
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--win_toolchain_json', required=True)
- parser.add_argument('--depot_tools_parent_dir', required=True)
- args = parser.parse_args()
- win_toolchain_utils.resolve(args.win_toolchain_json,
- args.depot_tools_parent_dir)
-
-
-if __name__ == '__main__':
- main()
diff --git a/infra/bots/recipe_modules/build/default.py b/infra/bots/recipe_modules/build/default.py
index 74acc32..c4e26db 100644
--- a/infra/bots/recipe_modules/build/default.py
+++ b/infra/bots/recipe_modules/build/default.py
@@ -56,13 +56,11 @@
os = api.vars.builder_cfg.get('os', '')
target_arch = api.vars.builder_cfg.get('target_arch', '')
- clang_linux = str(api.vars.slave_dir.join('clang_linux'))
- linux_vulkan_sdk = str(api.vars.slave_dir.join('linux_vulkan_sdk'))
- win_toolchain = str(api.vars.slave_dir.join(
- 't', 'depot_tools', 'win_toolchain', 'vs_files',
- '5454e45bf3764c03d3fc1024b3bf5bc41e3ab62c'))
- win_vulkan_sdk = str(api.vars.slave_dir.join('win_vulkan_sdk'))
- moltenvk = str(api.vars.slave_dir.join('moltenvk'))
+ clang_linux = str(api.vars.slave_dir.join('clang_linux'))
+ linux_vulkan_sdk = str(api.vars.slave_dir.join('linux_vulkan_sdk'))
+ win_toolchain = str(api.vars.slave_dir.join('t'))
+ win_vulkan_sdk = str(api.vars.slave_dir.join('win_vulkan_sdk'))
+ moltenvk = str(api.vars.slave_dir.join('moltenvk'))
cc, cxx = None, None
extra_cflags = []
diff --git a/infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86-Debug-Exceptions.json b/infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86-Debug-Exceptions.json
index b8fff7e..578acae 100644
--- a/infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86-Debug-Exceptions.json
+++ b/infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86-Debug-Exceptions.json
@@ -32,7 +32,7 @@
"[START_DIR]/cache/work/skia/bin/gn",
"gen",
"[START_DIR]/cache/work/skia/out/Build-Win-Clang-x86-Debug-Exceptions/Debug",
- "--args=cc=\"clang\" clang_win=\"[START_DIR]/clang_win\" cxx=\"clang++\" extra_cflags=[\"-O1\", \"/EHsc\", \"-DDUMMY_clang_win_version=42\"] target_cpu=\"x86\" win_sdk=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/5454e45bf3764c03d3fc1024b3bf5bc41e3ab62c/win_sdk\" win_vc=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/5454e45bf3764c03d3fc1024b3bf5bc41e3ab62c/VC\""
+ "--args=cc=\"clang\" clang_win=\"[START_DIR]/clang_win\" cxx=\"clang++\" extra_cflags=[\"-O1\", \"/EHsc\", \"-DDUMMY_clang_win_version=42\"] target_cpu=\"x86\" win_sdk=\"[START_DIR]/t/win_sdk\" win_vc=\"[START_DIR]/t/VC\""
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
diff --git a/infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86_64-Debug-OpenCL.json b/infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86_64-Debug-OpenCL.json
index 058f010..e840c02 100644
--- a/infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86_64-Debug-OpenCL.json
+++ b/infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86_64-Debug-OpenCL.json
@@ -32,7 +32,7 @@
"[START_DIR]/cache/work/skia/bin/gn",
"gen",
"[START_DIR]/cache/work/skia/out/Build-Win-Clang-x86_64-Debug-OpenCL/Debug_x64",
- "--args=cc=\"clang\" clang_win=\"[START_DIR]/clang_win\" cxx=\"clang++\" extra_cflags=[\"-O1\", \"-imsvc[START_DIR]/opencl_headers\", \"-DDUMMY_clang_win_version=42\"] extra_ldflags=[\"/LIBPATH:[START_DIR]/cache/work/skia/third_party/externals/opencl-lib/3-0/lib/x86_64\"] skia_use_opencl=true target_cpu=\"x86_64\" win_sdk=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/5454e45bf3764c03d3fc1024b3bf5bc41e3ab62c/win_sdk\" win_vc=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/5454e45bf3764c03d3fc1024b3bf5bc41e3ab62c/VC\""
+ "--args=cc=\"clang\" clang_win=\"[START_DIR]/clang_win\" cxx=\"clang++\" extra_cflags=[\"-O1\", \"-imsvc[START_DIR]/opencl_headers\", \"-DDUMMY_clang_win_version=42\"] extra_ldflags=[\"/LIBPATH:[START_DIR]/cache/work/skia/third_party/externals/opencl-lib/3-0/lib/x86_64\"] skia_use_opencl=true target_cpu=\"x86_64\" win_sdk=\"[START_DIR]/t/win_sdk\" win_vc=\"[START_DIR]/t/VC\""
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
diff --git a/infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86_64-Release-Vulkan.json b/infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86_64-Release-Vulkan.json
index 04a86b9..3d40fe6 100644
--- a/infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86_64-Release-Vulkan.json
+++ b/infra/bots/recipe_modules/build/examples/full.expected/Build-Win-Clang-x86_64-Release-Vulkan.json
@@ -32,7 +32,7 @@
"[START_DIR]/cache/work/skia/bin/gn",
"gen",
"[START_DIR]/cache/work/skia/out/Build-Win-Clang-x86_64-Release-Vulkan/Release_x64",
- "--args=cc=\"clang\" clang_win=\"[START_DIR]/clang_win\" cxx=\"clang++\" extra_cflags=[\"-DDUMMY_clang_win_version=42\"] is_debug=false skia_enable_vulkan_debug_layers=false skia_vulkan_sdk=\"[START_DIR]/win_vulkan_sdk\" target_cpu=\"x86_64\" win_sdk=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/5454e45bf3764c03d3fc1024b3bf5bc41e3ab62c/win_sdk\" win_vc=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/5454e45bf3764c03d3fc1024b3bf5bc41e3ab62c/VC\""
+ "--args=cc=\"clang\" clang_win=\"[START_DIR]/clang_win\" cxx=\"clang++\" extra_cflags=[\"-DDUMMY_clang_win_version=42\"] is_debug=false skia_enable_vulkan_debug_layers=false skia_vulkan_sdk=\"[START_DIR]/win_vulkan_sdk\" target_cpu=\"x86_64\" win_sdk=\"[START_DIR]/t/win_sdk\" win_vc=\"[START_DIR]/t/VC\""
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
diff --git a/infra/bots/recipes/compile.expected/Build-Win-Clang-x86-Debug.json b/infra/bots/recipes/compile.expected/Build-Win-Clang-x86-Debug.json
index 5695cb0..b7873bb 100644
--- a/infra/bots/recipes/compile.expected/Build-Win-Clang-x86-Debug.json
+++ b/infra/bots/recipes/compile.expected/Build-Win-Clang-x86-Debug.json
@@ -141,7 +141,7 @@
"[START_DIR]/cache/work/skia/bin/gn",
"gen",
"[START_DIR]/cache/work/skia/out/Build-Win-Clang-x86-Debug/Debug",
- "--args=cc=\"clang\" clang_win=\"[START_DIR]/clang_win\" cxx=\"clang++\" extra_cflags=[\"-O1\", \"-DDUMMY_clang_win_version=42\"] target_cpu=\"x86\" win_sdk=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/5454e45bf3764c03d3fc1024b3bf5bc41e3ab62c/win_sdk\" win_vc=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/5454e45bf3764c03d3fc1024b3bf5bc41e3ab62c/VC\""
+ "--args=cc=\"clang\" clang_win=\"[START_DIR]/clang_win\" cxx=\"clang++\" extra_cflags=[\"-O1\", \"-DDUMMY_clang_win_version=42\"] target_cpu=\"x86\" win_sdk=\"[START_DIR]/t/win_sdk\" win_vc=\"[START_DIR]/t/VC\""
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json
index b25e77b..6c5ad5c 100755
--- a/infra/bots/tasks.json
+++ b/infra/bots/tasks.json
@@ -19642,7 +19642,7 @@
{
"name": "skia/bots/win_toolchain",
"path": "t",
- "version": "version:8"
+ "version": "version:9"
}
],
"command": [
diff --git a/infra/bots/win_toolchain_utils.py b/infra/bots/win_toolchain_utils.py
deleted file mode 100644
index f8033e9..0000000
--- a/infra/bots/win_toolchain_utils.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-"""Utilities for manipulating the win_toolchain.json file."""
-
-
-import json
-import os
-import stat
-
-
-PLACEHOLDER = '<(TOOLCHAIN_BASE_DIR)'
-
-
-def _replace_prefix(val, before, after):
- """Replace the given prefix with the given string."""
- if val.startswith(before):
- return val.replace(before, after, 1)
- return val
-
-
-def _replace(val, before, after):
- """Replace occurrences of one string with another within the data."""
- if isinstance(val, basestring):
- return _replace_prefix(val, before, after)
- elif isinstance(val, (list, tuple)):
- return [_replace(elem, before, after) for elem in val]
- elif isinstance(val, dict):
- return {_replace(k, before, after):
- _replace(v, before, after) for k, v in val.iteritems()}
- raise Exception('Cannot replace variable: %s' % val)
-
-
-def _replace_in_file(filename, before, after):
- """Replace occurrences of one string with another within the file."""
- # Make the file writeable, or the below won't work.
- os.chmod(filename, stat.S_IWRITE)
-
- with open(filename) as f:
- contents = json.load(f)
- new_contents = _replace(contents, before, after)
- with open(filename, 'w') as f:
- json.dump(new_contents, f)
-
-
-def abstract(win_toolchain_json, old_path):
- """Replace absolute paths in win_toolchain.json with placeholders."""
- _replace_in_file(win_toolchain_json, old_path, PLACEHOLDER)
-
-
-def resolve(win_toolchain_json, new_path):
- """Replace placeholders in win_toolchain.json with absolute paths."""
- _replace_in_file(win_toolchain_json, PLACEHOLDER, new_path)
diff --git a/site/user/build.md b/site/user/build.md
index 0d3f74a..c44519c 100644
--- a/site/user/build.md
+++ b/site/user/build.md
@@ -257,8 +257,8 @@
You can then pass the VC and SDK paths to GN by setting your GN args:
- win_vc = "C:\toolchain\depot_tools\win_toolchain\vs_files\5454e45bf3764c03d3fc1024b3bf5bc41e3ab62c\VC"
- win_sdk = "C:\toolchain\depot_tools\win_toolchain\vs_files\5454e45bf3764c03d3fc1024b3bf5bc41e3ab62c\win_sdk"
+ win_vc = "C:\toolchain\VC"
+ win_sdk = "C:\toolchain\win_sdk"
This toolchain is the only way we support 32-bit builds, by also setting `target_cpu="x86"`.
There is also a corresponding 2015 toolchain, downloaded via `infra/bots/assets/win_toolchain_2015`.
@@ -302,7 +302,8 @@
There is early, experimental support for [Windows 10 on ARM](https://docs.microsoft.com/en-us/windows/arm/).
This currently requires (a recent version of) MSVC, and the `Visual C++ compilers and libraries for ARM64`
-individual component in the Visual Studio Installer.
+individual component in the Visual Studio Installer. For Googlers, the win_toolchain asset includes the
+ARM64 compiler.
To use that toolchain, set the `target_cpu` GN argument to `"arm64"`. Note that OpenGL is not supported
by Windows 10 on ARM, so Skia's GL backends are stubbed out, and will not work. ANGLE is supported: