Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 1 | #!/usr/bin/env ruby |
Shinichiro Hamaji | 913eb74 | 2016-04-28 17:31:24 +0900 | [diff] [blame] | 2 | # coding: binary |
Shinichiro Hamaji | b69bf8a | 2015-06-10 14:52:06 +0900 | [diff] [blame] | 3 | # |
| 4 | # Copyright 2015 Google Inc. All rights reserved |
| 5 | # |
| 6 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 7 | # you may not use this file except in compliance with the License. |
| 8 | # You may obtain a copy of the License at |
| 9 | # |
| 10 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | # |
| 12 | # Unless required by applicable law or agreed to in writing, software |
| 13 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | # See the License for the specific language governing permissions and |
| 16 | # limitations under the License. |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 17 | |
| 18 | require 'fileutils' |
| 19 | |
Stefan Becker | e4e56f3 | 2016-04-27 11:16:39 +0300 | [diff] [blame] | 20 | # suppress GNU make jobserver magic when calling "make" |
| 21 | ENV.delete('MAKEFLAGS') |
| 22 | ENV.delete('MAKELEVEL') |
| 23 | |
Shinichiro Hamaji | 8ac8171 | 2015-07-16 09:15:54 +0900 | [diff] [blame] | 24 | while true |
| 25 | if ARGV[0] == '-s' |
| 26 | test_serialization = true |
| 27 | ARGV.shift |
| 28 | elsif ARGV[0] == '-c' |
| 29 | ckati = true |
| 30 | ARGV.shift |
| 31 | ENV['KATI_VARIANT'] = 'c' |
| 32 | elsif ARGV[0] == '-n' |
| 33 | via_ninja = true |
| 34 | ARGV.shift |
Colin Cross | df11044 | 2015-10-02 11:06:49 -0700 | [diff] [blame] | 35 | ENV['NINJA_STATUS'] = 'NINJACMD: ' |
Shinichiro Hamaji | 8562057 | 2015-10-03 11:25:15 +0900 | [diff] [blame] | 36 | elsif ARGV[0] == '-a' |
| 37 | gen_all_targets = true |
| 38 | ARGV.shift |
Shinichiro Hamaji | 221a16b | 2015-09-10 12:40:14 +0900 | [diff] [blame] | 39 | elsif ARGV[0] == '-v' |
| 40 | show_failing = true |
| 41 | ARGV.shift |
Sasha Smundak | b4482cb | 2018-08-23 11:21:28 -0700 | [diff] [blame] | 42 | elsif ARGV[0] == "-q" |
| 43 | hide_passing = true |
| 44 | ARGV.shift |
Shinichiro Hamaji | 8ac8171 | 2015-07-16 09:15:54 +0900 | [diff] [blame] | 45 | else |
| 46 | break |
| 47 | end |
Shinichiro Hamaji | 9f4f5d3 | 2015-04-28 03:53:49 +0900 | [diff] [blame] | 48 | end |
| 49 | |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 50 | def get_output_filenames |
| 51 | files = Dir.glob('*') |
| 52 | files.delete('Makefile') |
Shinichiro Hamaji | 8ac8171 | 2015-07-16 09:15:54 +0900 | [diff] [blame] | 53 | files.delete('build.ninja') |
Colin Cross | 27df531 | 2015-11-09 13:39:14 -0800 | [diff] [blame] | 54 | files.delete('env.sh') |
Shinichiro Hamaji | 8ac8171 | 2015-07-16 09:15:54 +0900 | [diff] [blame] | 55 | files.delete('ninja.sh') |
Shinichiro Hamaji | 65489db | 2015-06-29 20:44:26 +0900 | [diff] [blame] | 56 | files.delete('gmon.out') |
Shinichiro Hamaji | 91571ac | 2015-09-07 18:24:00 +0900 | [diff] [blame] | 57 | files.delete('submake') |
Shinichiro Hamaji | b674d58 | 2015-04-28 03:40:57 +0900 | [diff] [blame] | 58 | files.reject!{|f|f =~ /\.json$/} |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 59 | files.reject!{|f|f =~ /^kati\.*/} |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 60 | files |
| 61 | end |
| 62 | |
| 63 | def cleanup |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 64 | (get_output_filenames + Dir.glob('.*')).each do |fname| |
| 65 | next if fname == '.' || fname == '..' |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 66 | FileUtils.rm_rf fname |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 67 | end |
| 68 | end |
| 69 | |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 70 | def move_circular_dep(l) |
| 71 | # We don't care when circular dependency detection happens. |
| 72 | circ = '' |
| 73 | while l.sub!(/Circular .* dropped\.\n/, '') do |
| 74 | circ += $& |
| 75 | end |
| 76 | circ + l |
| 77 | end |
| 78 | |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 79 | expected_failures = [] |
| 80 | unexpected_passes = [] |
| 81 | failures = [] |
| 82 | passes = [] |
| 83 | |
Shinichiro Hamaji | ec42b4e | 2015-05-14 16:20:03 +0900 | [diff] [blame] | 84 | if !ARGV.empty? |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 85 | test_files = ARGV.map do |test| |
| 86 | "testcase/#{File.basename(test)}" |
Shinichiro Hamaji | ec42b4e | 2015-05-14 16:20:03 +0900 | [diff] [blame] | 87 | end |
| 88 | else |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 89 | test_files = Dir.glob('testcase/*.mk').sort |
| 90 | test_files += Dir.glob('testcase/*.sh').sort |
Shinichiro Hamaji | ec42b4e | 2015-05-14 16:20:03 +0900 | [diff] [blame] | 91 | end |
| 92 | |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 93 | def run_in_testdir(test_filename) |
| 94 | c = File.read(test_filename) |
| 95 | name = File.basename(test_filename) |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 96 | dir = "out/#{name}" |
Shinichiro Hamaji | 52e4203 | 2015-03-31 23:20:13 +0900 | [diff] [blame] | 97 | |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 98 | FileUtils.mkdir_p(dir) |
Shinichiro Hamaji | 52e4203 | 2015-03-31 23:20:13 +0900 | [diff] [blame] | 99 | Dir.glob("#{dir}/*").each do |fname| |
| 100 | FileUtils.rm_rf(fname) |
| 101 | end |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 102 | |
| 103 | Dir.chdir(dir) do |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 104 | yield name |
| 105 | end |
| 106 | end |
| 107 | |
Shinichiro Hamaji | 86e1133 | 2015-07-28 15:24:17 +0900 | [diff] [blame] | 108 | def normalize_ninja_log(log, mk) |
Colin Cross | df11044 | 2015-10-02 11:06:49 -0700 | [diff] [blame] | 109 | log.gsub!(/^NINJACMD: .*\n/, '') |
Shinichiro Hamaji | cecc358 | 2015-07-28 15:56:59 +0900 | [diff] [blame] | 110 | log.gsub!(/^ninja: no work to do\.\n/, '') |
Fumitoshi Ukai | 786846d | 2015-07-28 18:08:47 +0900 | [diff] [blame] | 111 | log.gsub!(/^ninja: error: (.*, needed by .*),.*/, |
| 112 | '*** No rule to make target \\1.') |
Dan Willemsen | f3a4ced | 2015-08-25 14:22:35 -0700 | [diff] [blame] | 113 | log.gsub!(/^ninja: warning: multiple rules generate (.*)\. builds involving this target will not be correct.*$/, |
Shinichiro Hamaji | 3c60c12 | 2015-10-05 16:26:38 +0900 | [diff] [blame] | 114 | 'ninja: warning: multiple rules generate \\1.') |
Shinichiro Hamaji | 38892d8 | 2016-05-09 18:51:35 +0900 | [diff] [blame] | 115 | |
Shinichiro Hamaji | 86e1133 | 2015-07-28 15:24:17 +0900 | [diff] [blame] | 116 | if mk =~ /err_error_in_recipe.mk/ |
| 117 | # This test expects ninja fails. Strip ninja specific error logs. |
Shinichiro Hamaji | 38892d8 | 2016-05-09 18:51:35 +0900 | [diff] [blame] | 118 | ninja_failed_subst = '' |
Shinichiro Hamaji | c3a4834 | 2016-03-15 08:27:43 +0900 | [diff] [blame] | 119 | elsif mk =~ /\/fail_/ |
| 120 | # Recipes in these tests fail. |
Shinichiro Hamaji | 38892d8 | 2016-05-09 18:51:35 +0900 | [diff] [blame] | 121 | ninja_failed_subst = "*** [test] Error 1\n" |
| 122 | end |
| 123 | if ninja_failed_subst |
| 124 | log.gsub!(/^FAILED: (.*\n\/bin\/bash)?.*\n/, ninja_failed_subst) |
Shinichiro Hamaji | c3a4834 | 2016-03-15 08:27:43 +0900 | [diff] [blame] | 125 | log.gsub!(/^ninja: .*\n/, '') |
Shinichiro Hamaji | 86e1133 | 2015-07-28 15:24:17 +0900 | [diff] [blame] | 126 | end |
Shinichiro Hamaji | 8ac8171 | 2015-07-16 09:15:54 +0900 | [diff] [blame] | 127 | log |
| 128 | end |
| 129 | |
Shinichiro Hamaji | 913eb74 | 2016-04-28 17:31:24 +0900 | [diff] [blame] | 130 | def normalize_quotes(log) |
| 131 | log.gsub!(/[`'"]/, '"') |
| 132 | # For recent GNU find, which uses Unicode characters. |
| 133 | log.gsub!(/(\xe2\x80\x98|\xe2\x80\x99)/, '"') |
| 134 | log |
| 135 | end |
| 136 | |
Dan Willemsen | f3a4ced | 2015-08-25 14:22:35 -0700 | [diff] [blame] | 137 | def normalize_make_log(expected, mk, via_ninja) |
Shinichiro Hamaji | 913eb74 | 2016-04-28 17:31:24 +0900 | [diff] [blame] | 138 | expected = normalize_quotes(expected) |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 139 | expected.gsub!(/^make(?:\[\d+\])?: (Entering|Leaving) directory.*\n/, '') |
| 140 | expected.gsub!(/^make(?:\[\d+\])?: /, '') |
| 141 | expected = move_circular_dep(expected) |
| 142 | |
| 143 | # Normalizations for old/new GNU make. |
Shinichiro Hamaji | dc258cb | 2016-04-27 18:35:57 +0900 | [diff] [blame] | 144 | expected.gsub!(' recipe for target ', ' commands for target ') |
| 145 | expected.gsub!(' recipe commences ', ' commands commence ') |
| 146 | expected.gsub!('missing rule before recipe.', 'missing rule before commands.') |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 147 | expected.gsub!(' (did you mean TAB instead of 8 spaces?)', '') |
| 148 | expected.gsub!('Extraneous text after', 'extraneous text after') |
| 149 | # Not sure if this is useful. |
| 150 | expected.gsub!(/\s+Stop\.$/, '') |
| 151 | # GNU make 4.0 has this output. |
| 152 | expected.gsub!(/Makefile:\d+: commands for target ".*?" failed\n/, '') |
| 153 | # We treat some warnings as errors. |
Shinichiro Hamaji | 2d2ed95 | 2016-04-28 16:50:46 +0900 | [diff] [blame] | 154 | expected.gsub!(/^\/bin\/(ba)?sh: line 0: /, '') |
Dan Willemsen | f3a4ced | 2015-08-25 14:22:35 -0700 | [diff] [blame] | 155 | # We print out some ninja warnings in some tests to match what we expect |
| 156 | # ninja to produce. Remove them if we're not testing ninja. |
| 157 | if !via_ninja |
| 158 | expected.gsub!(/^ninja: warning: .*\n/, '') |
| 159 | end |
Shinichiro Hamaji | b7be8f1 | 2015-10-05 16:29:58 +0900 | [diff] [blame] | 160 | # Normalization for "include foo" with C++ kati. |
| 161 | expected.gsub!(/(: )(\S+): (No such file or directory)\n\*\*\* No rule to make target "\2"./, '\1\2: \3') |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 162 | |
| 163 | expected |
| 164 | end |
| 165 | |
| 166 | def normalize_kati_log(output) |
Shinichiro Hamaji | 913eb74 | 2016-04-28 17:31:24 +0900 | [diff] [blame] | 167 | output = normalize_quotes(output) |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 168 | output = move_circular_dep(output) |
Shinichiro Hamaji | 913eb74 | 2016-04-28 17:31:24 +0900 | [diff] [blame] | 169 | |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 170 | # kati specific log messages. |
| 171 | output.gsub!(/^\*kati\*.*\n/, '') |
Fumitoshi Ukai | a43b96f | 2015-07-17 19:54:21 +0900 | [diff] [blame] | 172 | output.gsub!(/^c?kati: /, '') |
Fumitoshi Ukai | 08c1e94 | 2015-07-15 16:38:07 +0900 | [diff] [blame] | 173 | output.gsub!(/\/bin\/sh: ([^:]*): command not found/, |
| 174 | "\\1: Command not found") |
Shinichiro Hamaji | a06bee9 | 2015-08-05 17:13:05 +0900 | [diff] [blame] | 175 | output.gsub!(/.*: warning for parse error in an unevaluated line: .*\n/, '') |
Dan Willemsen | e41c755 | 2017-02-22 14:31:16 -0800 | [diff] [blame] | 176 | output.gsub!(/^([^ ]+: )?FindEmulator: /, '') |
Fumitoshi Ukai | 2c6eff6 | 2015-08-11 17:50:56 +0900 | [diff] [blame] | 177 | output.gsub!(/^\/bin\/sh: line 0: /, '') |
Fumitoshi Ukai | b0ede7d | 2015-07-29 10:19:05 +0900 | [diff] [blame] | 178 | output.gsub!(/ (\.\/+)+kati\.\S+/, '') # kati log files in find_command.mk |
| 179 | output.gsub!(/ (\.\/+)+test\S+.json/, '') # json files in find_command.mk |
Shinichiro Hamaji | b7be8f1 | 2015-10-05 16:29:58 +0900 | [diff] [blame] | 180 | # Normalization for "include foo" with Go kati. |
| 181 | output.gsub!(/(: )open (\S+): n(o such file or directory)\nNOTE:.*/, |
| 182 | "\\1\\2: N\\3") |
Dan Willemsen | e6ede25 | 2017-09-29 16:49:02 -0700 | [diff] [blame] | 183 | # Bionic libc has different error messages than glibc |
| 184 | output.gsub!(/Too many symbolic links encountered/, 'Too many levels of symbolic links') |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 185 | output |
| 186 | end |
| 187 | |
Shinichiro Hamaji | 2d2ed95 | 2016-04-28 16:50:46 +0900 | [diff] [blame] | 188 | bash_var = ' SHELL=/bin/bash' |
| 189 | |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 190 | run_make_test = proc do |mk| |
| 191 | c = File.read(mk) |
Shinichiro Hamaji | 6a49292 | 2015-07-27 19:37:20 +0900 | [diff] [blame] | 192 | expected_failure = false |
Fumitoshi Ukai | bfabe63 | 2015-07-28 15:26:56 +0900 | [diff] [blame] | 193 | if c =~ /\A# TODO(?:\(([-a-z|]+)\))?/ |
Shinichiro Hamaji | 6a49292 | 2015-07-27 19:37:20 +0900 | [diff] [blame] | 194 | if $1 |
| 195 | todos = $1.split('|') |
| 196 | if todos.include?('go') && !ckati |
| 197 | expected_failure = true |
| 198 | end |
| 199 | if todos.include?('c') && ckati |
| 200 | expected_failure = true |
| 201 | end |
Fumitoshi Ukai | bfabe63 | 2015-07-28 15:26:56 +0900 | [diff] [blame] | 202 | if todos.include?('go-ninja') && !ckati && via_ninja |
| 203 | expected_failure = true |
| 204 | end |
| 205 | if todos.include?('c-ninja') && ckati && via_ninja |
| 206 | expected_failure = true |
| 207 | end |
Shinichiro Hamaji | 03fa345 | 2016-04-27 17:08:32 +0900 | [diff] [blame] | 208 | if todos.include?('c-exec') && ckati && !via_ninja |
| 209 | expected_failure = true |
| 210 | end |
Shinichiro Hamaji | 6a49292 | 2015-07-27 19:37:20 +0900 | [diff] [blame] | 211 | if todos.include?('ninja') && via_ninja |
| 212 | expected_failure = true |
| 213 | end |
| 214 | else |
| 215 | expected_failure = true |
Shinichiro Hamaji | e919d00 | 2015-06-26 01:15:55 +0900 | [diff] [blame] | 216 | end |
| 217 | end |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 218 | |
| 219 | run_in_testdir(mk) do |name| |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 220 | File.open("Makefile", 'w') do |ofile| |
| 221 | ofile.print(c) |
| 222 | end |
Shinichiro Hamaji | 91571ac | 2015-09-07 18:24:00 +0900 | [diff] [blame] | 223 | File.symlink('../../testcase/submake', 'submake') |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 224 | |
| 225 | expected = '' |
| 226 | output = '' |
| 227 | |
Shinichiro Hamaji | 491e73f | 2015-04-07 12:41:59 +0900 | [diff] [blame] | 228 | testcases = c.scan(/^test\d*/).sort.uniq |
Shinichiro Hamaji | 497754d | 2015-03-31 02:02:11 +0900 | [diff] [blame] | 229 | if testcases.empty? |
| 230 | testcases = [''] |
| 231 | end |
Shinichiro Hamaji | e5be014 | 2015-03-30 19:26:01 +0900 | [diff] [blame] | 232 | |
Shinichiro Hamaji | b6a0454 | 2015-09-09 17:37:41 +0900 | [diff] [blame] | 233 | is_silent_test = mk =~ /\/submake_/ |
| 234 | |
Shinichiro Hamaji | e5be014 | 2015-03-30 19:26:01 +0900 | [diff] [blame] | 235 | cleanup |
| 236 | testcases.each do |tc| |
Shinichiro Hamaji | 8ac8171 | 2015-07-16 09:15:54 +0900 | [diff] [blame] | 237 | cmd = 'make' |
Shinichiro Hamaji | b6a0454 | 2015-09-09 17:37:41 +0900 | [diff] [blame] | 238 | if via_ninja || is_silent_test |
Shinichiro Hamaji | 8ac8171 | 2015-07-16 09:15:54 +0900 | [diff] [blame] | 239 | cmd += ' -s' |
| 240 | end |
Shinichiro Hamaji | 2d2ed95 | 2016-04-28 16:50:46 +0900 | [diff] [blame] | 241 | cmd += bash_var |
Shinichiro Hamaji | 8ac8171 | 2015-07-16 09:15:54 +0900 | [diff] [blame] | 242 | cmd += " #{tc} 2>&1" |
Shinichiro Hamaji | 913eb74 | 2016-04-28 17:31:24 +0900 | [diff] [blame] | 243 | res = IO.popen(cmd, 'r:binary', &:read) |
Dan Willemsen | f3a4ced | 2015-08-25 14:22:35 -0700 | [diff] [blame] | 244 | res = normalize_make_log(res, mk, via_ninja) |
Shinichiro Hamaji | 74a6600 | 2015-04-27 16:42:30 +0900 | [diff] [blame] | 245 | expected += "=== #{tc} ===\n" + res |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 246 | expected_files = get_output_filenames |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 247 | expected += "\n=== FILES ===\n#{expected_files * "\n"}\n" |
Shinichiro Hamaji | e5be014 | 2015-03-30 19:26:01 +0900 | [diff] [blame] | 248 | end |
| 249 | |
| 250 | cleanup |
| 251 | testcases.each do |tc| |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 252 | json = "#{tc.empty? ? 'test' : tc}" |
Fumitoshi Ukai | 0547db6 | 2015-07-29 16:20:59 +0900 | [diff] [blame] | 253 | cmd = "../../kati -save_json=#{json}.json -log_dir=. --use_find_emulator" |
Shinichiro Hamaji | 776ca30 | 2015-06-06 03:52:48 +0900 | [diff] [blame] | 254 | if ckati |
Shinichiro Hamaji | 8ac8171 | 2015-07-16 09:15:54 +0900 | [diff] [blame] | 255 | cmd = "../../ckati --use_find_emulator" |
Shinichiro Hamaji | 776ca30 | 2015-06-06 03:52:48 +0900 | [diff] [blame] | 256 | end |
Shinichiro Hamaji | 8ac8171 | 2015-07-16 09:15:54 +0900 | [diff] [blame] | 257 | if via_ninja |
| 258 | cmd += ' --ninja' |
| 259 | end |
Shinichiro Hamaji | 8562057 | 2015-10-03 11:25:15 +0900 | [diff] [blame] | 260 | if gen_all_targets |
| 261 | if !ckati || !via_ninja |
| 262 | raise "-a should be used with -c -n" |
| 263 | end |
| 264 | cmd += ' --gen_all_targets' |
| 265 | end |
Shinichiro Hamaji | b6a0454 | 2015-09-09 17:37:41 +0900 | [diff] [blame] | 266 | if is_silent_test |
| 267 | cmd += ' -s' |
| 268 | end |
Shinichiro Hamaji | 2d2ed95 | 2016-04-28 16:50:46 +0900 | [diff] [blame] | 269 | cmd += bash_var |
Shinichiro Hamaji | 6ff74ce | 2015-10-03 11:58:35 +0900 | [diff] [blame] | 270 | if !gen_all_targets || mk =~ /makecmdgoals/ |
| 271 | cmd += " #{tc}" |
| 272 | end |
| 273 | cmd += " 2>&1" |
Shinichiro Hamaji | b674d58 | 2015-04-28 03:40:57 +0900 | [diff] [blame] | 274 | res = IO.popen(cmd, 'r:binary', &:read) |
Fumitoshi Ukai | 39837dd | 2015-07-28 12:25:56 +0900 | [diff] [blame] | 275 | if via_ninja && File.exist?('build.ninja') && File.exists?('ninja.sh') |
Shinichiro Hamaji | 8562057 | 2015-10-03 11:25:15 +0900 | [diff] [blame] | 276 | cmd = './ninja.sh -j1 -v' |
| 277 | if gen_all_targets |
| 278 | cmd += " #{tc}" |
| 279 | end |
| 280 | cmd += ' 2>&1' |
| 281 | log = IO.popen(cmd, 'r:binary', &:read) |
Shinichiro Hamaji | 86e1133 | 2015-07-28 15:24:17 +0900 | [diff] [blame] | 282 | res += normalize_ninja_log(log, mk) |
Shinichiro Hamaji | 8ac8171 | 2015-07-16 09:15:54 +0900 | [diff] [blame] | 283 | end |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 284 | res = normalize_kati_log(res) |
Shinichiro Hamaji | 42f42e1 | 2015-04-13 14:26:45 +0900 | [diff] [blame] | 285 | output += "=== #{tc} ===\n" + res |
Shinichiro Hamaji | e5be014 | 2015-03-30 19:26:01 +0900 | [diff] [blame] | 286 | output_files = get_output_filenames |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 287 | output += "\n=== FILES ===\n#{output_files * "\n"}\n" |
| 288 | end |
| 289 | |
| 290 | File.open('out.make', 'w'){|ofile|ofile.print(expected)} |
| 291 | File.open('out.kati', 'w'){|ofile|ofile.print(output)} |
| 292 | |
Shinichiro Hamaji | 74c832f | 2015-04-09 14:54:39 +0900 | [diff] [blame] | 293 | if expected =~ /FAIL/ |
| 294 | puts %Q(#{name} has a string "FAIL" in its expectation) |
| 295 | exit 1 |
| 296 | end |
| 297 | |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 298 | if expected != output |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 299 | if expected_failure |
Sasha Smundak | b4482cb | 2018-08-23 11:21:28 -0700 | [diff] [blame] | 300 | if !hide_passing |
| 301 | puts "#{name}: FAIL (expected)" |
| 302 | end |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 303 | expected_failures << name |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 304 | else |
| 305 | puts "#{name}: FAIL" |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 306 | failures << name |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 307 | end |
Shinichiro Hamaji | 7f3a870 | 2015-09-11 15:37:02 +0900 | [diff] [blame] | 308 | if !expected_failure || show_failing |
Shinichiro Hamaji | 221a16b | 2015-09-10 12:40:14 +0900 | [diff] [blame] | 309 | puts `diff -u out.make out.kati` |
| 310 | end |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 311 | else |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 312 | if expected_failure |
| 313 | puts "#{name}: PASS (unexpected)" |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 314 | unexpected_passes << name |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 315 | else |
Sasha Smundak | b4482cb | 2018-08-23 11:21:28 -0700 | [diff] [blame] | 316 | if !hide_passing |
| 317 | puts "#{name}: PASS" |
| 318 | end |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 319 | passes << name |
Shinichiro Hamaji | 91b105c | 2015-03-30 18:39:05 +0900 | [diff] [blame] | 320 | end |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 321 | end |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 322 | |
Shinichiro Hamaji | e666ac8 | 2015-04-28 04:18:13 +0900 | [diff] [blame] | 323 | if name !~ /^err_/ && test_serialization && !expected_failure |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 324 | testcases.each do |tc| |
| 325 | json = "#{tc.empty? ? 'test' : tc}" |
Fumitoshi Ukai | 6450d0f | 2015-07-10 16:34:06 +0900 | [diff] [blame] | 326 | cmd = "../../kati -save_json=#{json}_2.json -load_json=#{json}.json -n -log_dir=. #{tc} 2>&1" |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 327 | res = IO.popen(cmd, 'r:binary', &:read) |
| 328 | if !File.exist?("#{json}.json") || !File.exist?("#{json}_2.json") |
| 329 | puts "#{name}##{json}: Serialize failure (not exist)" |
Shinichiro Hamaji | 9f4f5d3 | 2015-04-28 03:53:49 +0900 | [diff] [blame] | 330 | puts res |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 331 | else |
| 332 | json1 = File.read("#{json}.json") |
| 333 | json2 = File.read("#{json}_2.json") |
| 334 | if json1 != json2 |
| 335 | puts "#{name}##{json}: Serialize failure" |
Shinichiro Hamaji | 9f4f5d3 | 2015-04-28 03:53:49 +0900 | [diff] [blame] | 336 | puts res |
Shinichiro Hamaji | c48d9ce | 2015-04-28 03:47:39 +0900 | [diff] [blame] | 337 | end |
| 338 | end |
| 339 | end |
| 340 | end |
Fumitoshi Ukai | 119dc91 | 2015-03-30 16:52:41 +0900 | [diff] [blame] | 341 | end |
| 342 | end |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 343 | |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 344 | run_shell_test = proc do |sh| |
Shinichiro Hamaji | 09796ef | 2015-07-30 17:08:29 +0900 | [diff] [blame] | 345 | is_ninja_test = sh =~ /\/ninja_/ |
| 346 | if is_ninja_test && (!ckati || !via_ninja) |
| 347 | next |
| 348 | end |
| 349 | |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 350 | run_in_testdir(sh) do |name| |
| 351 | cleanup |
Sasha Smundak | e8f4d6d | 2018-07-27 10:34:04 -0700 | [diff] [blame] | 352 | cmd = "bash ../../#{sh} make" |
Shinichiro Hamaji | 09796ef | 2015-07-30 17:08:29 +0900 | [diff] [blame] | 353 | if is_ninja_test |
| 354 | cmd += ' -s' |
| 355 | end |
Shinichiro Hamaji | 2d2ed95 | 2016-04-28 16:50:46 +0900 | [diff] [blame] | 356 | cmd += bash_var |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 357 | expected = IO.popen(cmd, 'r:binary', &:read) |
| 358 | cleanup |
Shinichiro Hamaji | 09796ef | 2015-07-30 17:08:29 +0900 | [diff] [blame] | 359 | |
| 360 | if is_ninja_test |
| 361 | if ckati |
Sasha Smundak | e8f4d6d | 2018-07-27 10:34:04 -0700 | [diff] [blame] | 362 | cmd = "bash ../../#{sh} ../../ckati --ninja --regen" |
Shinichiro Hamaji | 09796ef | 2015-07-30 17:08:29 +0900 | [diff] [blame] | 363 | else |
| 364 | next |
| 365 | end |
| 366 | else |
| 367 | if ckati |
Sasha Smundak | e8f4d6d | 2018-07-27 10:34:04 -0700 | [diff] [blame] | 368 | cmd = "bash ../../#{sh} ../../ckati" |
Shinichiro Hamaji | 09796ef | 2015-07-30 17:08:29 +0900 | [diff] [blame] | 369 | else |
Sasha Smundak | e8f4d6d | 2018-07-27 10:34:04 -0700 | [diff] [blame] | 370 | cmd = "bash ../../#{sh} ../../kati --use_cache -log_dir=." |
Shinichiro Hamaji | 09796ef | 2015-07-30 17:08:29 +0900 | [diff] [blame] | 371 | end |
| 372 | end |
Shinichiro Hamaji | 2d2ed95 | 2016-04-28 16:50:46 +0900 | [diff] [blame] | 373 | cmd += bash_var |
Shinichiro Hamaji | 09796ef | 2015-07-30 17:08:29 +0900 | [diff] [blame] | 374 | |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 375 | output = IO.popen(cmd, 'r:binary', &:read) |
| 376 | |
Dan Willemsen | f3a4ced | 2015-08-25 14:22:35 -0700 | [diff] [blame] | 377 | expected = normalize_make_log(expected, sh, is_ninja_test) |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 378 | output = normalize_kati_log(output) |
Shinichiro Hamaji | 09796ef | 2015-07-30 17:08:29 +0900 | [diff] [blame] | 379 | if is_ninja_test |
| 380 | output = normalize_ninja_log(output, sh) |
| 381 | end |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 382 | File.open('out.make', 'w'){|ofile|ofile.print(expected)} |
| 383 | File.open('out.kati', 'w'){|ofile|ofile.print(output)} |
| 384 | |
| 385 | if expected != output |
| 386 | puts "#{name}: FAIL" |
Shinichiro Hamaji | 39e6240 | 2015-05-20 19:34:58 +0900 | [diff] [blame] | 387 | puts `diff -u out.make out.kati` |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 388 | failures << name |
| 389 | else |
Sasha Smundak | b4482cb | 2018-08-23 11:21:28 -0700 | [diff] [blame] | 390 | if !hide_passing |
| 391 | puts "#{name}: PASS" |
| 392 | end |
Shinichiro Hamaji | a9bda7a | 2015-05-20 17:50:42 +0900 | [diff] [blame] | 393 | passes << name |
| 394 | end |
| 395 | end |
| 396 | end |
| 397 | |
| 398 | test_files.each do |test| |
| 399 | if /\.mk$/ =~ test |
| 400 | run_make_test.call(test) |
| 401 | elsif /\.sh$/ =~ test |
| 402 | run_shell_test.call(test) |
| 403 | else |
| 404 | raise "Unknown test type: #{test}" |
| 405 | end |
| 406 | end |
| 407 | |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 408 | puts |
| 409 | |
Sasha Smundak | b4482cb | 2018-08-23 11:21:28 -0700 | [diff] [blame] | 410 | if !expected_failures.empty? && !hide_passing |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 411 | puts "=== Expected failures ===" |
| 412 | expected_failures.each do |n| |
| 413 | puts n |
| 414 | end |
| 415 | end |
| 416 | |
| 417 | if !unexpected_passes.empty? |
| 418 | puts "=== Unexpected passes ===" |
| 419 | unexpected_passes.each do |n| |
| 420 | puts n |
| 421 | end |
| 422 | end |
| 423 | |
| 424 | if !failures.empty? |
| 425 | puts "=== Failures ===" |
| 426 | failures.each do |n| |
| 427 | puts n |
| 428 | end |
| 429 | end |
| 430 | |
| 431 | puts |
| 432 | |
| 433 | if !unexpected_passes.empty? || !failures.empty? |
Shinichiro Hamaji | 776ca30 | 2015-06-06 03:52:48 +0900 | [diff] [blame] | 434 | puts "FAIL! (#{failures.size + unexpected_passes.size} fails #{passes.size} passes)" |
Shinichiro Hamaji | 680e98b | 2015-10-05 17:30:43 +0900 | [diff] [blame] | 435 | exit 1 |
Shinichiro Hamaji | c671360 | 2015-04-01 01:03:34 +0900 | [diff] [blame] | 436 | else |
| 437 | puts 'PASS!' |
| 438 | end |