| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 1 | // Copyright (C) 2021 The Android Open Source Project | 
|  | 2 | // | 
|  | 3 | // Licensed under the Apache License, Version 2.0 (the "License"); | 
|  | 4 | // you may not use this file except in compliance with the License. | 
|  | 5 | // You may obtain a copy of the License at | 
|  | 6 | // | 
|  | 7 | //      http://www.apache.org/licenses/LICENSE-2.0 | 
|  | 8 | // | 
|  | 9 | // Unless required by applicable law or agreed to in writing, software | 
|  | 10 | // distributed under the License is distributed on an "AS IS" BASIS, | 
|  | 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 
|  | 12 | // See the License for the specific language governing permissions and | 
|  | 13 | // limitations under the License. | 
|  | 14 |  | 
|  | 15 | 'use strict'; | 
|  | 16 |  | 
|  | 17 | // This script takes care of: | 
|  | 18 | // - The build process for the whole UI and the chrome extension. | 
|  | 19 | // - The HTTP dev-server with live-reload capabilities. | 
|  | 20 | // The reason why this is a hand-rolled script rather than a conventional build | 
|  | 21 | // system is keeping incremental build fast and maintaining the set of | 
|  | 22 | // dependencies contained. | 
|  | 23 | // The only way to keep incremental build fast (i.e. O(seconds) for the | 
|  | 24 | // edit-one-line -> reload html cycles) is to run both the TypeScript compiler | 
|  | 25 | // and the rollup bundler in --watch mode. Any other attempt, leads to O(10s) | 
|  | 26 | // incremental-build times. | 
|  | 27 | // This script allows mixing build tools that support --watch mode (tsc and | 
| Primiano Tucci | 9b56703 | 2021-02-12 14:18:12 +0100 | [diff] [blame] | 28 | // rollup) and auto-triggering-on-file-change rules via node-watch. | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 29 | // When invoked without any argument (e.g., for production builds), this script | 
|  | 30 | // just runs all the build tasks serially. It doesn't to do any mtime-based | 
|  | 31 | // check, it always re-runs all the tasks. | 
| Primiano Tucci | 9b56703 | 2021-02-12 14:18:12 +0100 | [diff] [blame] | 32 | // When invoked with --watch, it mounts a pipeline of tasks based on node-watch | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 33 | // and runs them together with tsc --watch and rollup --watch. | 
|  | 34 | // The output directory structure is carefully crafted so that any change to UI | 
|  | 35 | // sources causes cascading triggers of the next steps. | 
|  | 36 | // The overall build graph looks as follows: | 
|  | 37 | // +----------------+      +-----------------------------+ | 
|  | 38 | // | protos/*.proto |----->| pbjs out/tsc/gen/protos.js  |--+ | 
|  | 39 | // +----------------+      +-----------------------------+  | | 
|  | 40 | //                         +-----------------------------+  | | 
|  | 41 | //                         | pbts out/tsc/gen/protos.d.ts|<-+ | 
|  | 42 | //                         +-----------------------------+ | 
|  | 43 | //                             | | 
|  | 44 | //                             V      +-------------------------+ | 
|  | 45 | // +---------+              +-----+   |  out/tsc/frontend/*.js  | | 
|  | 46 | // | ui/*.ts |------------->| tsc |-> +-------------------------+   +--------+ | 
|  | 47 | // +---------+              +-----+   | out/tsc/controller/*.js |-->| rollup | | 
|  | 48 | //                            ^       +-------------------------+   +--------+ | 
|  | 49 | //                +------------+      |   out/tsc/engine/*.js   |       | | 
|  | 50 | // +-----------+  |*.wasm.js   |      +-------------------------+       | | 
|  | 51 | // |ninja *.cc |->|*.wasm.d.ts |                                        | | 
|  | 52 | // +-----------+  |*.wasm      |-----------------+                      | | 
|  | 53 | //                +------------+                 |                      | | 
|  | 54 | //                                               V                      V | 
|  | 55 | // +-----------+  +------+    +------------------------------------------------+ | 
|  | 56 | // | ui/*.scss |->| scss |--->|              Final out/dist/ dir               | | 
|  | 57 | // +-----------+  +------+    +------------------------------------------------+ | 
|  | 58 | // +----------------------+   | +----------+ +---------+ +--------------------+| | 
|  | 59 | // | src/assets/*.png     |   | | assets/  | |*.wasm.js| | frontend_bundle.js || | 
|  | 60 | // +----------------------+   | |  *.css   | |*.wasm   | +--------------------+| | 
|  | 61 | // | buildtools/typefaces |-->| |  *.png   | +---------+ |controller_bundle.js|| | 
|  | 62 | // +----------------------+   | |  *.woff2 |             +--------------------+| | 
|  | 63 | // | buildtools/legacy_tv |   | |  tv.html |             |  engine_bundle.js  || | 
|  | 64 | // +----------------------+   | +----------+             +--------------------+| | 
|  | 65 | //                            +------------------------------------------------+ | 
|  | 66 |  | 
|  | 67 | const argparse = require('argparse'); | 
|  | 68 | const child_process = require('child_process'); | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 69 | var crypto = require('crypto'); | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 70 | const fs = require('fs'); | 
|  | 71 | const http = require('http'); | 
|  | 72 | const path = require('path'); | 
| Primiano Tucci | 9b56703 | 2021-02-12 14:18:12 +0100 | [diff] [blame] | 73 | const fswatch = require('node-watch');  // Like fs.watch(), but works on Linux. | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 74 | const pjoin = path.join; | 
|  | 75 |  | 
|  | 76 | const ROOT_DIR = path.dirname(__dirname);  // The repo root. | 
|  | 77 | const VERSION_SCRIPT = pjoin(ROOT_DIR, 'tools/write_version_header.py'); | 
|  | 78 |  | 
|  | 79 | const cfg = { | 
|  | 80 | watch: false, | 
|  | 81 | verbose: false, | 
|  | 82 | debug: false, | 
|  | 83 | startHttpServer: false, | 
|  | 84 | wasmModules: ['trace_processor', 'trace_to_text'], | 
|  | 85 | testConfigs: ['jest.unit.config.js'], | 
|  | 86 |  | 
|  | 87 | // The fields below will be changed by main() after cmdline parsing. | 
|  | 88 | // Directory structure: | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 89 | // out/xxx/    -> outDir         : Root build dir, for both ninja/wasm and UI. | 
|  | 90 | //   ui/       -> outUiDir       : UI dir. All outputs from this script. | 
|  | 91 | //    tsc/     -> outTscDir      : Transpiled .ts -> .js. | 
|  | 92 | //      gen/   -> outGenDir      : Auto-generated .ts/.js (e.g. protos). | 
|  | 93 | //    dist/    -> outDistRootDir : Only index.html and service_worker.js | 
|  | 94 | //      v1.2/  -> outDistDir     : JS bundles and assets | 
|  | 95 | //    chrome_extension/          : Chrome extension. | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 96 | outDir: pjoin(ROOT_DIR, 'out/ui'), | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 97 | version: '',  // v1.2.3, derived from the CHANGELOG + git. | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 98 | outUiDir: '', | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 99 | outDistRootDir: '', | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 100 | outTscDir: '', | 
|  | 101 | outGenDir: '', | 
|  | 102 | outDistRootDir: '', | 
|  | 103 | outDistDir: '', | 
|  | 104 | outExtDir: '', | 
|  | 105 | }; | 
|  | 106 |  | 
|  | 107 | const RULES = [ | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 108 | {r: /ui\/src\/assets\/index.html/, f: copyIndexHtml}, | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 109 | {r: /ui\/src\/assets\/((.*)[.]png)/, f: copyAssets}, | 
|  | 110 | {r: /buildtools\/typefaces\/(.+[.]woff2)/, f: copyAssets}, | 
|  | 111 | {r: /buildtools\/catapult_trace_viewer\/(.+(js|html))/, f: copyAssets}, | 
|  | 112 | {r: /ui\/src\/assets\/.+[.]scss/, f: compileScss}, | 
|  | 113 | {r: /ui\/src\/assets\/.+[.]scss/, f: compileScss}, | 
|  | 114 | {r: /ui\/src\/chrome_extension\/.*/, f: copyExtensionAssets}, | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 115 | {r: /.*\/dist\/.+\/(?!manifest\.json).*/, f: genServiceWorkerManifestJson}, | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 116 | {r: /.*\/dist\/.*/, f: notifyLiveServer}, | 
|  | 117 | ]; | 
|  | 118 |  | 
|  | 119 | let tasks = []; | 
|  | 120 | let tasksTot = 0, tasksRan = 0; | 
|  | 121 | let serverStarted = false; | 
|  | 122 | let httpWatches = []; | 
|  | 123 | let tStart = Date.now(); | 
|  | 124 | let subprocesses = []; | 
|  | 125 |  | 
|  | 126 | function main() { | 
|  | 127 | const parser = new argparse.ArgumentParser(); | 
|  | 128 | parser.addArgument('--out', {help: 'Output directory'}); | 
|  | 129 | parser.addArgument(['--watch', '-w'], {action: 'storeTrue'}); | 
|  | 130 | parser.addArgument(['--serve', '-s'], {action: 'storeTrue'}); | 
|  | 131 | parser.addArgument(['--verbose', '-v'], {action: 'storeTrue'}); | 
|  | 132 | parser.addArgument(['--no-build', '-n'], {action: 'storeTrue'}); | 
|  | 133 | parser.addArgument(['--no-wasm', '-W'], {action: 'storeTrue'}); | 
|  | 134 | parser.addArgument(['--run-tests', '-t'], {action: 'storeTrue'}); | 
|  | 135 | parser.addArgument(['--debug', '-d'], {action: 'storeTrue'}); | 
|  | 136 |  | 
|  | 137 | const args = parser.parseArgs(); | 
|  | 138 | const clean = !args.no_build; | 
|  | 139 | cfg.outDir = path.resolve(ensureDir(args.out || cfg.outDir)); | 
|  | 140 | cfg.outUiDir = ensureDir(pjoin(cfg.outDir, 'ui'), clean); | 
|  | 141 | cfg.outExtDir = ensureDir(pjoin(cfg.outUiDir, 'chrome_extension')); | 
|  | 142 | cfg.outDistRootDir = ensureDir(pjoin(cfg.outUiDir, 'dist')); | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 143 | const proc = exec('python3', [VERSION_SCRIPT, '--stdout'], {stdout: 'pipe'}); | 
|  | 144 | cfg.version = proc.stdout.toString().trim(); | 
|  | 145 | cfg.outDistDir = ensureDir(pjoin(cfg.outDistRootDir, cfg.version)); | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 146 | cfg.outTscDir = ensureDir(pjoin(cfg.outUiDir, 'tsc')); | 
|  | 147 | cfg.outGenDir = ensureDir(pjoin(cfg.outUiDir, 'tsc/gen')); | 
|  | 148 | cfg.watch = !!args.watch; | 
|  | 149 | cfg.verbose = !!args.verbose; | 
|  | 150 | cfg.debug = !!args.debug; | 
|  | 151 | cfg.startHttpServer = args.serve; | 
|  | 152 |  | 
|  | 153 | process.on('SIGINT', () => { | 
|  | 154 | console.log('\nSIGINT received. Killing all child processes and exiting'); | 
|  | 155 | for (const proc of subprocesses) { | 
|  | 156 | if (proc) proc.kill('SIGINT'); | 
|  | 157 | } | 
|  | 158 | process.exit(130);  // 130 -> Same behavior of bash when killed by SIGINT. | 
|  | 159 | }); | 
|  | 160 |  | 
|  | 161 | // Check that deps are current before starting. | 
|  | 162 | const installBuildDeps = pjoin(ROOT_DIR, 'tools/install-build-deps'); | 
|  | 163 | const depsArgs = ['--check-only', pjoin(cfg.outDir, '.check_deps'), '--ui']; | 
|  | 164 | exec(installBuildDeps, depsArgs); | 
|  | 165 |  | 
|  | 166 | console.log('Entering', cfg.outDir); | 
|  | 167 | process.chdir(cfg.outDir); | 
|  | 168 |  | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 169 | updateSymlinks();  // Links //ui/out -> //out/xxx/ui/ | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 170 |  | 
|  | 171 | // Enqueue empty task. This is needed only for --no-build --serve. The HTTP | 
|  | 172 | // server is started when the task queue reaches quiescence, but it takes at | 
|  | 173 | // least one task for that. | 
|  | 174 | addTask(() => {}); | 
|  | 175 |  | 
|  | 176 | if (!args.no_build) { | 
|  | 177 | buildWasm(args.no_wasm); | 
|  | 178 | scanDir('ui/src/assets'); | 
|  | 179 | scanDir('ui/src/chrome_extension'); | 
|  | 180 | scanDir('buildtools/typefaces'); | 
|  | 181 | scanDir('buildtools/catapult_trace_viewer'); | 
|  | 182 | compileProtos(); | 
|  | 183 | genVersion(); | 
|  | 184 | transpileTsProject('ui'); | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 185 | transpileTsProject('ui/src/service_worker'); | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 186 | bundleJs('rollup.config.js'); | 
|  | 187 | genServiceWorkerManifestJson(); | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 188 |  | 
|  | 189 | // Watches the /dist. When changed: | 
|  | 190 | // - Notifies the HTTP live reload clients. | 
|  | 191 | // - Regenerates the ServiceWorker file map. | 
|  | 192 | scanDir(cfg.outDistRootDir); | 
|  | 193 | } | 
|  | 194 |  | 
|  | 195 | if (args.run_tests) { | 
|  | 196 | runTests(); | 
|  | 197 | } | 
|  | 198 | } | 
|  | 199 |  | 
|  | 200 | // ----------- | 
|  | 201 | // Build rules | 
|  | 202 | // ----------- | 
|  | 203 |  | 
|  | 204 | function runTests() { | 
|  | 205 | const args = | 
|  | 206 | ['--rootDir', cfg.outTscDir, '--verbose', '--runInBand', '--forceExit']; | 
|  | 207 | for (const cfgFile of cfg.testConfigs) { | 
|  | 208 | args.push('--projects', pjoin(ROOT_DIR, 'ui/config', cfgFile)); | 
|  | 209 | } | 
|  | 210 | if (cfg.watch) { | 
|  | 211 | args.push('--watchAll'); | 
|  | 212 | addTask(execNode, ['jest', args, {async: true}]); | 
|  | 213 | } else { | 
|  | 214 | addTask(execNode, ['jest', args]); | 
|  | 215 | } | 
|  | 216 | } | 
|  | 217 |  | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 218 | function copyIndexHtml(src) { | 
|  | 219 | const index_html = () => { | 
|  | 220 | let html = fs.readFileSync(src).toString(); | 
|  | 221 | // First copy the index.html as-is into the dist/v1.2.3/ directory. This is | 
|  | 222 | // only used for archival purporses, so one can open | 
|  | 223 | // ui.perfetto.dev/v1.2.3/ to skip the auto-update and channel logic. | 
|  | 224 | fs.writeFileSync(pjoin(cfg.outDistDir, 'index.html'), html); | 
|  | 225 |  | 
|  | 226 | // Then copy it into the dist/ root by patching the version code. | 
|  | 227 | // TODO(primiano): in next CLs, this script should take a | 
|  | 228 | // --release_map=xxx.json argument, to populate this with multiple channels. | 
|  | 229 | const versionMap = JSON.stringify({'stable': cfg.version}); | 
|  | 230 | const bodyRegex = /data-perfetto_version='[^']*'/; | 
|  | 231 | html = html.replace(bodyRegex, `data-perfetto_version='${versionMap}'`); | 
|  | 232 | fs.writeFileSync(pjoin(cfg.outDistRootDir, 'index.html'), html); | 
|  | 233 | }; | 
|  | 234 | addTask(index_html); | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 235 | } | 
|  | 236 |  | 
|  | 237 | function copyAssets(src, dst) { | 
|  | 238 | addTask(cp, [src, pjoin(cfg.outDistDir, 'assets', dst)]); | 
|  | 239 | } | 
|  | 240 |  | 
|  | 241 | function compileScss() { | 
|  | 242 | const src = pjoin(ROOT_DIR, 'ui/src/assets/perfetto.scss'); | 
|  | 243 | const dst = pjoin(cfg.outDistDir, 'perfetto.css'); | 
|  | 244 | // In watch mode, don't exit(1) if scss fails. It can easily happen by | 
|  | 245 | // having a typo in the css. It will still print an errror. | 
|  | 246 | const noErrCheck = !!cfg.watch; | 
|  | 247 | addTask(execNode, ['node-sass', ['--quiet', src, dst], {noErrCheck}]); | 
|  | 248 | } | 
|  | 249 |  | 
|  | 250 | function compileProtos() { | 
|  | 251 | const dstJs = pjoin(cfg.outGenDir, 'protos.js'); | 
|  | 252 | const dstTs = pjoin(cfg.outGenDir, 'protos.d.ts'); | 
|  | 253 | const inputs = [ | 
|  | 254 | 'protos/perfetto/trace_processor/trace_processor.proto', | 
|  | 255 | 'protos/perfetto/common/trace_stats.proto', | 
|  | 256 | 'protos/perfetto/common/tracing_service_capabilities.proto', | 
|  | 257 | 'protos/perfetto/config/perfetto_config.proto', | 
|  | 258 | 'protos/perfetto/ipc/consumer_port.proto', | 
|  | 259 | 'protos/perfetto/ipc/wire_protocol.proto', | 
|  | 260 | 'protos/perfetto/metrics/metrics.proto', | 
|  | 261 | ]; | 
|  | 262 | const pbjsArgs = [ | 
|  | 263 | '--force-number', | 
|  | 264 | '-t', | 
|  | 265 | 'static-module', | 
|  | 266 | '-w', | 
|  | 267 | 'commonjs', | 
|  | 268 | '-p', | 
|  | 269 | ROOT_DIR, | 
|  | 270 | '-o', | 
|  | 271 | dstJs | 
|  | 272 | ].concat(inputs); | 
|  | 273 | addTask(execNode, ['pbjs', pbjsArgs]); | 
|  | 274 | const pbtsArgs = ['-p', ROOT_DIR, '-o', dstTs, dstJs]; | 
|  | 275 | addTask(execNode, ['pbts', pbtsArgs]); | 
|  | 276 | } | 
|  | 277 |  | 
|  | 278 | // Generates a .ts source that defines the VERSION and SCM_REVISION constants. | 
|  | 279 | function genVersion() { | 
|  | 280 | const cmd = 'python3'; | 
|  | 281 | const args = | 
|  | 282 | [VERSION_SCRIPT, '--ts_out', pjoin(cfg.outGenDir, 'perfetto_version.ts')] | 
|  | 283 | addTask(exec, [cmd, args]); | 
|  | 284 | } | 
|  | 285 |  | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 286 | function updateSymlinks() { | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 287 | mklink(cfg.outUiDir, pjoin(ROOT_DIR, 'ui/out')); | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 288 |  | 
|  | 289 | // Creates a out/dist_version -> out/dist/v1.2.3 symlink, so rollup config | 
|  | 290 | // can point to that without having to know the current version number. | 
|  | 291 | mklink( | 
|  | 292 | path.relative(cfg.outUiDir, cfg.outDistDir), | 
|  | 293 | pjoin(cfg.outUiDir, 'dist_version')); | 
|  | 294 |  | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 295 | mklink( | 
|  | 296 | pjoin(ROOT_DIR, 'ui/node_modules'), pjoin(cfg.outTscDir, 'node_modules')) | 
|  | 297 | } | 
|  | 298 |  | 
|  | 299 | // Invokes ninja for building the {trace_processor, trace_to_text} Wasm modules. | 
|  | 300 | // It copies the .wasm directly into the out/dist/ dir, and the .js/.ts into | 
|  | 301 | // out/tsc/, so the typescript compiler and the bundler can pick them up. | 
|  | 302 | function buildWasm(skipWasmBuild) { | 
|  | 303 | if (!skipWasmBuild) { | 
|  | 304 | const gnArgs = ['gen', `--args=is_debug=${cfg.debug}`, cfg.outDir]; | 
|  | 305 | addTask(exec, [pjoin(ROOT_DIR, 'tools/gn'), gnArgs]); | 
|  | 306 |  | 
|  | 307 | const ninjaArgs = ['-C', cfg.outDir] | 
|  | 308 | ninjaArgs.push(...cfg.wasmModules.map(x => `${x}_wasm`)); | 
|  | 309 | addTask(exec, [pjoin(ROOT_DIR, 'tools/ninja'), ninjaArgs]); | 
|  | 310 | } | 
|  | 311 |  | 
|  | 312 | const wasmOutDir = pjoin(cfg.outDir, 'wasm'); | 
|  | 313 | for (const wasmMod of cfg.wasmModules) { | 
|  | 314 | // The .wasm file goes directly into the dist dir (also .map in debug) | 
|  | 315 | for (const ext of ['.wasm'].concat(cfg.debug ? ['.wasm.map'] : [])) { | 
|  | 316 | const src = `${wasmOutDir}/${wasmMod}${ext}`; | 
|  | 317 | addTask(cp, [src, pjoin(cfg.outDistDir, wasmMod + ext)]); | 
|  | 318 | } | 
|  | 319 | // The .js / .ts go into intermediates, they will be bundled by rollup. | 
|  | 320 | for (const ext of ['.js', '.d.ts']) { | 
|  | 321 | const fname = `${wasmMod}${ext}`; | 
|  | 322 | addTask(cp, [pjoin(wasmOutDir, fname), pjoin(cfg.outGenDir, fname)]); | 
|  | 323 | } | 
|  | 324 | } | 
|  | 325 | } | 
|  | 326 |  | 
|  | 327 | // This transpiles all the sources (frontend, controller, engine, extension) in | 
|  | 328 | // one go. The only project that has a dedicated invocation is service_worker. | 
|  | 329 | function transpileTsProject(project) { | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 330 | const args = ['--project', pjoin(ROOT_DIR, project)]; | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 331 | if (cfg.watch) { | 
|  | 332 | args.push('--watch', '--preserveWatchOutput'); | 
|  | 333 | addTask(execNode, ['tsc', args, {async: true}]); | 
|  | 334 | } else { | 
|  | 335 | addTask(execNode, ['tsc', args]); | 
|  | 336 | } | 
|  | 337 | } | 
|  | 338 |  | 
|  | 339 | // Creates the three {frontend, controller, engine}_bundle.js in one invocation. | 
|  | 340 | function bundleJs(cfgName) { | 
|  | 341 | const rcfg = pjoin(ROOT_DIR, 'ui/config', cfgName) | 
|  | 342 | const args = ['-c', rcfg, '--no-indent']; | 
|  | 343 | args.push(...(cfg.verbose ? [] : ['--silent'])); | 
|  | 344 | if (cfg.watch) { | 
|  | 345 | // --waitForBundleInput is so that we can run tsc --watch and rollup --watch | 
|  | 346 | // together, without having to wait that tsc completes the first build. | 
|  | 347 | args.push('--watch', '--waitForBundleInput', '--no-watch.clearScreen'); | 
|  | 348 | addTask(execNode, ['rollup', args, {async: true}]); | 
|  | 349 | } else { | 
|  | 350 | addTask(execNode, ['rollup', args]); | 
|  | 351 | } | 
|  | 352 | } | 
|  | 353 |  | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 354 | function genServiceWorkerManifestJson() { | 
|  | 355 | function make_manifest() { | 
|  | 356 | const manifest = {resources: {}}; | 
|  | 357 | // When building the subresource manifest skip source maps, the manifest | 
|  | 358 | // itself and the copy of the index.html which is copied under /v1.2.3/. | 
|  | 359 | // The root /index.html will be fetched by service_worker.js separately. | 
|  | 360 | const skipRegex = /(\.map|manifest\.json|index.html)$/; | 
|  | 361 | walk(cfg.outDistDir, absPath => { | 
|  | 362 | const contents = fs.readFileSync(absPath); | 
|  | 363 | const relPath = path.relative(cfg.outDistDir, absPath); | 
|  | 364 | const b64 = crypto.createHash('sha256').update(contents).digest('base64'); | 
|  | 365 | manifest.resources[relPath] = 'sha256-' + b64; | 
|  | 366 | }, skipRegex); | 
|  | 367 | const manifestJson = JSON.stringify(manifest, null, 2); | 
|  | 368 | fs.writeFileSync(pjoin(cfg.outDistDir, 'manifest.json'), manifestJson); | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 369 | } | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 370 | addTask(make_manifest, []); | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 371 | } | 
|  | 372 |  | 
|  | 373 | function startServer() { | 
|  | 374 | const port = 10000; | 
|  | 375 | console.log(`Starting HTTP server on http://localhost:${port}`) | 
|  | 376 | http.createServer(function(req, res) { | 
|  | 377 | console.debug(req.method, req.url); | 
|  | 378 | let uri = req.url.split('?', 1)[0]; | 
|  | 379 | if (uri.endsWith('/')) { | 
|  | 380 | uri += 'index.html'; | 
|  | 381 | } | 
|  | 382 |  | 
|  | 383 | if (uri === '/live_reload') { | 
|  | 384 | // Implements the Server-Side-Events protocol. | 
|  | 385 | const head = { | 
|  | 386 | 'Content-Type': 'text/event-stream', | 
|  | 387 | 'Connection': 'keep-alive', | 
|  | 388 | 'Cache-Control': 'no-cache' | 
|  | 389 | }; | 
|  | 390 | res.writeHead(200, head); | 
|  | 391 | const arrayIdx = httpWatches.length; | 
|  | 392 | // We never remove from the array, the delete leaves an undefined item | 
|  | 393 | // around. It makes keeping track of the index easier at the cost of a | 
|  | 394 | // small leak. | 
|  | 395 | httpWatches.push(res); | 
|  | 396 | req.on('close', () => delete httpWatches[arrayIdx]); | 
|  | 397 | return; | 
|  | 398 | } | 
|  | 399 |  | 
|  | 400 | const absPath = path.normalize(path.join(cfg.outDistRootDir, uri)); | 
|  | 401 | fs.readFile(absPath, function(err, data) { | 
|  | 402 | if (err) { | 
|  | 403 | res.writeHead(404); | 
|  | 404 | res.end(JSON.stringify(err)); | 
|  | 405 | return; | 
|  | 406 | } | 
|  | 407 |  | 
|  | 408 | const mimeMap = { | 
|  | 409 | 'html': 'text/html', | 
|  | 410 | 'css': 'text/css', | 
|  | 411 | 'js': 'application/javascript', | 
|  | 412 | 'wasm': 'application/wasm', | 
|  | 413 | }; | 
|  | 414 | const ext = uri.split('.').pop(); | 
|  | 415 | const cType = mimeMap[ext] || 'octect/stream'; | 
|  | 416 | const head = { | 
|  | 417 | 'Content-Type': cType, | 
|  | 418 | 'Content-Length': data.length, | 
|  | 419 | 'Last-Modified': fs.statSync(absPath).mtime.toUTCString(), | 
|  | 420 | 'Cache-Control': 'no-cache', | 
|  | 421 | }; | 
|  | 422 | res.writeHead(200, head); | 
| Primiano Tucci | 7eead2e | 2021-02-16 11:46:04 +0100 | [diff] [blame] | 423 | res.write(data); | 
|  | 424 | res.end(); | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 425 | }); | 
|  | 426 | }) | 
| Primiano Tucci | 19068d1 | 2021-02-23 20:29:56 +0100 | [diff] [blame^] | 427 | .listen(port, '127.0.0.1'); | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 428 | } | 
|  | 429 |  | 
|  | 430 | // Called whenever a change in the out/dist directory is detected. It sends a | 
|  | 431 | // Server-Side-Event to the live_reload.ts script. | 
|  | 432 | function notifyLiveServer(changedFile) { | 
|  | 433 | for (const cli of httpWatches) { | 
|  | 434 | if (cli === undefined) continue; | 
|  | 435 | cli.write( | 
|  | 436 | 'data: ' + path.relative(cfg.outDistRootDir, changedFile) + '\n\n'); | 
|  | 437 | } | 
|  | 438 | } | 
|  | 439 |  | 
|  | 440 | function copyExtensionAssets() { | 
|  | 441 | addTask(cp, [ | 
|  | 442 | pjoin(ROOT_DIR, 'ui/src/assets/logo-128.png'), | 
|  | 443 | pjoin(cfg.outExtDir, 'logo-128.png') | 
|  | 444 | ]); | 
|  | 445 | addTask(cp, [ | 
|  | 446 | pjoin(ROOT_DIR, 'ui/src/chrome_extension/manifest.json'), | 
|  | 447 | pjoin(cfg.outExtDir, 'manifest.json') | 
|  | 448 | ]); | 
|  | 449 | } | 
|  | 450 |  | 
|  | 451 | // ----------------------- | 
|  | 452 | // Task chaining functions | 
|  | 453 | // ----------------------- | 
|  | 454 |  | 
|  | 455 | function addTask(func, args) { | 
|  | 456 | const task = new Task(func, args); | 
|  | 457 | for (const t of tasks) { | 
|  | 458 | if (t.identity === task.identity) { | 
|  | 459 | return; | 
|  | 460 | } | 
|  | 461 | } | 
|  | 462 | tasks.push(task); | 
|  | 463 | setTimeout(runTasks, 0); | 
|  | 464 | } | 
|  | 465 |  | 
|  | 466 | function runTasks() { | 
|  | 467 | const snapTasks = tasks.splice(0);  // snap = std::move(tasks). | 
|  | 468 | tasksTot += snapTasks.length; | 
|  | 469 | for (const task of snapTasks) { | 
|  | 470 | const DIM = '\u001b[2m'; | 
|  | 471 | const BRT = '\u001b[37m'; | 
|  | 472 | const RST = '\u001b[0m'; | 
|  | 473 | const ms = (new Date(Date.now() - tStart)).toISOString().slice(17, -1); | 
|  | 474 | const ts = `[${DIM}${ms}${RST}]`; | 
|  | 475 | const descr = task.description.substr(0, 80); | 
|  | 476 | console.log(`${ts} ${BRT}${++tasksRan}/${tasksTot}${RST}\t${descr}`); | 
|  | 477 | task.func.apply(/*this=*/ undefined, task.args); | 
|  | 478 | } | 
|  | 479 | // Start the web server once reaching quiescence. | 
|  | 480 | if (tasks.length === 0 && !serverStarted && cfg.startHttpServer) { | 
|  | 481 | serverStarted = true; | 
|  | 482 | startServer(); | 
|  | 483 | } | 
|  | 484 | } | 
|  | 485 |  | 
|  | 486 | // Executes all the RULES that match the given |absPath|. | 
|  | 487 | function scanFile(absPath) { | 
|  | 488 | console.assert(fs.existsSync(absPath)); | 
|  | 489 | console.assert(path.isAbsolute(absPath)); | 
|  | 490 | const normPath = path.relative(ROOT_DIR, absPath); | 
|  | 491 | for (const rule of RULES) { | 
|  | 492 | const match = rule.r.exec(normPath); | 
|  | 493 | if (!match || match[0] !== normPath) continue; | 
|  | 494 | const captureGroup = match.length > 1 ? match[1] : undefined; | 
|  | 495 | rule.f(absPath, captureGroup); | 
|  | 496 | } | 
|  | 497 | } | 
|  | 498 |  | 
|  | 499 | // Walks the passed |dir| recursively and, for each file, invokes the matching | 
| Primiano Tucci | 9b56703 | 2021-02-12 14:18:12 +0100 | [diff] [blame] | 500 | // RULES. If --watch is used, it also installs a fswatch() and re-triggers the | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 501 | // matching RULES on each file change. | 
|  | 502 | function scanDir(dir, regex) { | 
|  | 503 | const filterFn = regex ? absPath => regex.test(absPath) : () => true; | 
|  | 504 | const absDir = path.isAbsolute(dir) ? dir : pjoin(ROOT_DIR, dir); | 
|  | 505 | // Add a fs watch if in watch mode. | 
|  | 506 | if (cfg.watch) { | 
| Primiano Tucci | 9a92cf7 | 2021-02-13 17:42:34 +0100 | [diff] [blame] | 507 | fswatch(absDir, {recursive: true}, (_eventType, filePath) => { | 
| Primiano Tucci | c8be681 | 2021-02-09 18:08:49 +0100 | [diff] [blame] | 508 | if (!filterFn(filePath)) return; | 
|  | 509 | if (cfg.verbose) { | 
|  | 510 | console.log('File change detected', _eventType, filePath); | 
|  | 511 | } | 
|  | 512 | if (fs.existsSync(filePath)) { | 
|  | 513 | scanFile(filePath, filterFn); | 
|  | 514 | } | 
|  | 515 | }); | 
|  | 516 | } | 
|  | 517 | walk(absDir, f => { | 
|  | 518 | if (filterFn(f)) scanFile(f); | 
|  | 519 | }); | 
|  | 520 | } | 
|  | 521 |  | 
|  | 522 | function exec(cmd, args, opts) { | 
|  | 523 | opts = opts || {}; | 
|  | 524 | opts.stdout = opts.stdout || 'inherit'; | 
|  | 525 | if (cfg.verbose) console.log(`${cmd} ${args.join(' ')}\n`); | 
|  | 526 | const spwOpts = {cwd: cfg.outDir, stdio: ['ignore', opts.stdout, 'inherit']}; | 
|  | 527 | const checkExitCode = (code, signal) => { | 
|  | 528 | if (signal === 'SIGINT' || signal === 'SIGTERM') return; | 
|  | 529 | if (code !== 0 && !opts.noErrCheck) { | 
|  | 530 | console.error(`${cmd} ${args.join(' ')} failed with code ${code}`); | 
|  | 531 | process.exit(1); | 
|  | 532 | } | 
|  | 533 | }; | 
|  | 534 | if (opts.async) { | 
|  | 535 | const proc = child_process.spawn(cmd, args, spwOpts); | 
|  | 536 | const procIndex = subprocesses.length; | 
|  | 537 | subprocesses.push(proc); | 
|  | 538 | return new Promise((resolve, _reject) => { | 
|  | 539 | proc.on('exit', (code, signal) => { | 
|  | 540 | delete subprocesses[procIndex]; | 
|  | 541 | checkExitCode(code, signal); | 
|  | 542 | resolve(); | 
|  | 543 | }); | 
|  | 544 | }); | 
|  | 545 | } else { | 
|  | 546 | const spawnRes = child_process.spawnSync(cmd, args, spwOpts); | 
|  | 547 | checkExitCode(spawnRes.status, spawnRes.signal); | 
|  | 548 | return spawnRes; | 
|  | 549 | } | 
|  | 550 | } | 
|  | 551 |  | 
|  | 552 | function execNode(module, args, opts) { | 
|  | 553 | const modPath = pjoin(ROOT_DIR, 'ui/node_modules/.bin', module); | 
|  | 554 | const nodeBin = pjoin(ROOT_DIR, 'tools/node'); | 
|  | 555 | args = [modPath].concat(args || []); | 
|  | 556 | const argsJson = JSON.stringify(args); | 
|  | 557 | return exec(nodeBin, args, opts); | 
|  | 558 | } | 
|  | 559 |  | 
|  | 560 | // ------------------------------------------ | 
|  | 561 | // File system & subprocess utility functions | 
|  | 562 | // ------------------------------------------ | 
|  | 563 |  | 
|  | 564 | class Task { | 
|  | 565 | constructor(func, args) { | 
|  | 566 | this.func = func; | 
|  | 567 | this.args = args || []; | 
|  | 568 | // |identity| is used to dedupe identical tasks in the queue. | 
|  | 569 | this.identity = JSON.stringify([this.func.name, this.args]); | 
|  | 570 | } | 
|  | 571 |  | 
|  | 572 | get description() { | 
|  | 573 | const ret = this.func.name.startsWith('exec') ? [] : [this.func.name]; | 
|  | 574 | const flattenedArgs = [].concat.apply([], this.args); | 
|  | 575 | for (const arg of flattenedArgs) { | 
|  | 576 | const argStr = `${arg}`; | 
|  | 577 | if (argStr.startsWith('/')) { | 
|  | 578 | ret.push(path.relative(cfg.outDir, arg)); | 
|  | 579 | } else { | 
|  | 580 | ret.push(argStr); | 
|  | 581 | } | 
|  | 582 | } | 
|  | 583 | return ret.join(' '); | 
|  | 584 | } | 
|  | 585 | } | 
|  | 586 |  | 
|  | 587 | function walk(dir, callback, skipRegex) { | 
|  | 588 | for (const child of fs.readdirSync(dir)) { | 
|  | 589 | const childPath = pjoin(dir, child); | 
|  | 590 | const stat = fs.lstatSync(childPath); | 
|  | 591 | if (skipRegex !== undefined && skipRegex.test(child)) continue; | 
|  | 592 | if (stat.isDirectory()) { | 
|  | 593 | walk(childPath, callback, skipRegex); | 
|  | 594 | } else if (!stat.isSymbolicLink()) { | 
|  | 595 | callback(childPath); | 
|  | 596 | } | 
|  | 597 | } | 
|  | 598 | } | 
|  | 599 |  | 
|  | 600 | function ensureDir(dirPath, clean) { | 
|  | 601 | const exists = fs.existsSync(dirPath); | 
|  | 602 | if (exists && clean) { | 
|  | 603 | console.log('rm', dirPath); | 
|  | 604 | fs.rmSync(dirPath, {recursive: true}); | 
|  | 605 | } | 
|  | 606 | if (!exists || clean) fs.mkdirSync(dirPath, {recursive: true}); | 
|  | 607 | return dirPath; | 
|  | 608 | } | 
|  | 609 |  | 
|  | 610 | function cp(src, dst) { | 
|  | 611 | ensureDir(path.dirname(dst)); | 
|  | 612 | if (cfg.verbose) { | 
|  | 613 | console.log( | 
|  | 614 | 'cp', path.relative(ROOT_DIR, src), '->', path.relative(ROOT_DIR, dst)); | 
|  | 615 | } | 
|  | 616 | fs.copyFileSync(src, dst); | 
|  | 617 | } | 
|  | 618 |  | 
|  | 619 | function mklink(src, dst) { | 
|  | 620 | // If the symlink already points to the right place don't touch it. This is | 
|  | 621 | // to avoid changing the mtime of the ui/ dir when unnecessary. | 
|  | 622 | if (fs.existsSync(dst)) { | 
|  | 623 | if (fs.lstatSync(dst).isSymbolicLink() && fs.readlinkSync(dst) === src) { | 
|  | 624 | return; | 
|  | 625 | } else { | 
|  | 626 | fs.unlinkSync(dst); | 
|  | 627 | } | 
|  | 628 | } | 
|  | 629 | fs.symlinkSync(src, dst); | 
|  | 630 | } | 
|  | 631 |  | 
|  | 632 | main(); |