runtest.rb - platform/build/kati - Gitiles

 #!/usr/bin/env ruby
 #
 # Copyright 2015 Google Inc. All rights reserved
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

 require 'fileutils'

 while true
   if ARGV[0] == '-s'
     test_serialization = true
     ARGV.shift
   elsif ARGV[0] == '-c'
     ckati = true
     ARGV.shift
     ENV['KATI_VARIANT'] = 'c'
   elsif ARGV[0] == '-n'
     via_ninja = true
     ARGV.shift
   else
     break
   end
 end

 def get_output_filenames
   files = Dir.glob('*')
   files.delete('Makefile')
   files.delete('build.ninja')
   files.delete('ninja.sh')
   files.delete('gmon.out')
   files.reject!{|f|f =~ /\.json$/}
   files.reject!{|f|f =~ /^kati\.*/}
   files
 end

 def cleanup
   (get_output_filenames + Dir.glob('.*')).each do |fname|
     next if fname == '.' || fname == '..'
     FileUtils.rm_rf fname
   end
 end

 def move_circular_dep(l)
   # We don't care when circular dependency detection happens.
   circ = ''
   while l.sub!(/Circular .* dropped\.\n/, '') do
     circ += $&
   end
   circ + l
 end

 expected_failures = []
 unexpected_passes = []
 failures = []
 passes = []

 if !ARGV.empty?
   test_files = ARGV.map do |test|
     "testcase/#{File.basename(test)}"
   end
 else
   test_files = Dir.glob('testcase/*.mk').sort
   test_files += Dir.glob('testcase/*.sh').sort
 end

 def run_in_testdir(test_filename)
   c = File.read(test_filename)
   name = File.basename(test_filename)
   dir = "out/#{name}"

   FileUtils.mkdir_p(dir)
   Dir.glob("#{dir}/*").each do |fname|
     FileUtils.rm_rf(fname)
   end

   Dir.chdir(dir) do
     yield name
   end
 end

 def normalize_ninja_log(log, mk)
   log.gsub!(/^\[\d+\/\d+\] .*\n/, '')
   log.gsub!(/^ninja: no work to do\.\n/, '')
   log.gsub!(/^ninja: error: (.*, needed by .*),.*/,
             '*** No rule to make target \\1.')
   if mk =~ /err_error_in_recipe.mk/
     # This test expects ninja fails. Strip ninja specific error logs.
     log.gsub!(/^FAILED: .*\n/, '')
     log.gsub!(/^ninja: .*\n/, '')
   end
   log
 end

 def normalize_make_log(expected)
   expected.gsub!(/^make(?:\[\d+\])?: (Entering|Leaving) directory.*\n/, '')
   expected.gsub!(/^make(?:\[\d+\])?: /, '')
   expected = move_circular_dep(expected)

   # Normalizations for old/new GNU make.
   expected.gsub!(/[`'"]/, '"')
   expected.gsub!(/ (?:commands|recipe) for target /,
                  ' commands for target ')
   expected.gsub!(/ (?:commands|recipe) commences /,
                  ' commands commence ')
   expected.gsub!(' (did you mean TAB instead of 8 spaces?)', '')
   expected.gsub!('Extraneous text after', 'extraneous text after')
   # Not sure if this is useful.
   expected.gsub!(/\s+Stop\.$/, '')
   # GNU make 4.0 has this output.
   expected.gsub!(/Makefile:\d+: commands for target ".*?" failed\n/, '')
   # We treat some warnings as errors.
   expected.gsub!(/Nothing to be done for "test"\.\n/, '')

   expected
 end

 def normalize_kati_log(output)
   output = move_circular_dep(output)
   # kati specific log messages.
   output.gsub!(/^\*kati\*.*\n/, '')
   output.gsub!(/^c?kati: /, '')
   output.gsub!(/[`'"]/, '"')
   output.gsub!(/(: )(?:open )?(\S+): [Nn](o such file or directory)\nNOTE:.*/,
                "\\1\\2: N\\3\n*** No rule to make target \"\\2\".")
   output.gsub!(/\/bin\/sh: ([^:]*): command not found/,
                "\\1: Command not found")
   output.gsub!(/.*: warning for parse error in an unevaluated line: .*\n/, '')
   output.gsub!(/ (\.\/+)+kati\.\S+/, '') # kati log files in find_command.mk
   output.gsub!(/ (\.\/+)+test\S+.json/, '') # json files in find_command.mk
   output
 end

 run_make_test = proc do |mk|
   c = File.read(mk)
   expected_failure = false
   if c =~ /\A# TODO(?:\(([-a-z|]+)\))?/
     if $1
       todos = $1.split('|')
       if todos.include?('go') && !ckati
         expected_failure = true
       end
       if todos.include?('c') && ckati
         expected_failure = true
       end
       if todos.include?('go-ninja') && !ckati && via_ninja
         expected_failure = true
       end
       if todos.include?('c-ninja') && ckati && via_ninja
         expected_failure = true
       end
       if todos.include?('ninja') && via_ninja
         expected_failure = true
       end
     else
       expected_failure = true
     end
   end

   run_in_testdir(mk) do |name|
     # TODO: Fix
     if name =~ /eval_assign/ && ckati
       next
     end

     File.open("Makefile", 'w') do |ofile|
       ofile.print(c)
     end

     expected = ''
     output = ''

     testcases = c.scan(/^test\d*/).sort.uniq
     if testcases.empty?
       testcases = ['']
     end

     cleanup
     testcases.each do |tc|
       cmd = 'make'
       if via_ninja
         cmd += ' -s'
       end
       cmd += " #{tc} 2>&1"
       res = `#{cmd}`
       res = normalize_make_log(res)
       expected += "=== #{tc} ===\n" + res
       expected_files = get_output_filenames
       expected += "\n=== FILES ===\n#{expected_files * "\n"}\n"
     end

     cleanup
     testcases.each do |tc|
       json = "#{tc.empty? ? 'test' : tc}"
       cmd = "../../kati -save_json=#{json}.json -log_dir=. --use_find_emulator"
       if ckati
         cmd = "../../ckati --use_find_emulator"
       end
       if via_ninja
         cmd += ' --ninja'
       end
       cmd += " #{tc} 2>&1"
       res = IO.popen(cmd, 'r:binary', &:read)
       if via_ninja && File.exist?('build.ninja') && File.exists?('ninja.sh')
         log = IO.popen('./ninja.sh -j1 -v 2>&1', 'r:binary', &:read)
         res += normalize_ninja_log(log, mk)
       end
       res = normalize_kati_log(res)
       output += "=== #{tc} ===\n" + res
       output_files = get_output_filenames
       output += "\n=== FILES ===\n#{output_files * "\n"}\n"
     end

     File.open('out.make', 'w'){|ofile|ofile.print(expected)}
     File.open('out.kati', 'w'){|ofile|ofile.print(output)}

     if expected =~ /FAIL/
       puts %Q(#{name} has a string "FAIL" in its expectation)
       exit 1
     end

     if expected != output
       if expected_failure
         puts "#{name}: FAIL (expected)"
         expected_failures << name
       else
         puts "#{name}: FAIL"
         puts `diff -u out.make out.kati`
         failures << name
       end
     else
       if expected_failure
         puts "#{name}: PASS (unexpected)"
         unexpected_passes << name
       else
         puts "#{name}: PASS"
         passes << name
       end
     end

     if name !~ /^err_/ && test_serialization && !expected_failure
       testcases.each do |tc|
         json = "#{tc.empty? ? 'test' : tc}"
         cmd = "../../kati -save_json=#{json}_2.json -load_json=#{json}.json -n -log_dir=. #{tc} 2>&1"
         res = IO.popen(cmd, 'r:binary', &:read)
         if !File.exist?("#{json}.json") || !File.exist?("#{json}_2.json")
           puts "#{name}##{json}: Serialize failure (not exist)"
           puts res
         else
           json1 = File.read("#{json}.json")
           json2 = File.read("#{json}_2.json")
           if json1 != json2
             puts "#{name}##{json}: Serialize failure"
             puts res
           end
         end
       end
     end
   end
 end

 run_shell_test = proc do |sh|
   is_ninja_test = sh =~ /\/ninja_/
   if is_ninja_test && (!ckati || !via_ninja)
     next
   end

   run_in_testdir(sh) do |name|
     cleanup
     cmd = "sh ../../#{sh} make"
     if is_ninja_test
       cmd += ' -s'
     end
     expected = IO.popen(cmd, 'r:binary', &:read)
     cleanup

     if is_ninja_test
       if ckati
         cmd = "sh ../../#{sh} ../../ckati --ninja --regen"
       else
         next
       end
     else
       if ckati
         cmd = "sh ../../#{sh} ../../ckati"
       else
         cmd = "sh ../../#{sh} ../../kati --use_cache -log_dir=."
       end
     end

     output = IO.popen(cmd, 'r:binary', &:read)

     expected = normalize_make_log(expected)
     output = normalize_kati_log(output)
     if is_ninja_test
       output = normalize_ninja_log(output, sh)
     end
     File.open('out.make', 'w'){|ofile|ofile.print(expected)}
     File.open('out.kati', 'w'){|ofile|ofile.print(output)}

     if expected != output
       puts "#{name}: FAIL"
       puts `diff -u out.make out.kati`
       failures << name
     else
       puts "#{name}: PASS"
       passes << name
     end
   end
 end

 test_files.each do |test|
   if /\.mk$/ =~ test
     run_make_test.call(test)
   elsif /\.sh$/ =~ test
     run_shell_test.call(test)
   else
     raise "Unknown test type: #{test}"
   end
 end

 puts

 if !expected_failures.empty?
   puts "=== Expected failures ==="
   expected_failures.each do |n|
     puts n
   end
 end

 if !unexpected_passes.empty?
   puts "=== Unexpected passes ==="
   unexpected_passes.each do |n|
     puts n
   end
 end

 if !failures.empty?
   puts "=== Failures ==="
   failures.each do |n|
     puts n
   end
 end

 puts

 if !unexpected_passes.empty? || !failures.empty?
   puts "FAIL! (#{failures.size + unexpected_passes.size} fails #{passes.size} passes)"
 else
   puts 'PASS!'
 end
	#!/usr/bin/env ruby
	#
	# Copyright 2015 Google Inc. All rights reserved
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	require 'fileutils'

	while true
	if ARGV[0] == '-s'
	test_serialization = true
	ARGV.shift
	elsif ARGV[0] == '-c'
	ckati = true
	ARGV.shift
	ENV['KATI_VARIANT'] = 'c'
	elsif ARGV[0] == '-n'
	via_ninja = true
	ARGV.shift
	else
	break
	end
	end

	def get_output_filenames
	files = Dir.glob('*')
	files.delete('Makefile')
	files.delete('build.ninja')
	files.delete('ninja.sh')
	files.delete('gmon.out')
	files.reject!{\|f\|f =~ /\.json$/}
	files.reject!{\|f\|f =~ /^kati\.*/}
	files
	end

	def cleanup
	(get_output_filenames + Dir.glob('.*')).each do \|fname\|
	next if fname == '.' \|\| fname == '..'
	FileUtils.rm_rf fname
	end
	end

	def move_circular_dep(l)
	# We don't care when circular dependency detection happens.
	circ = ''
	while l.sub!(/Circular .* dropped\.\n/, '') do
	circ += $&
	end
	circ + l
	end

	expected_failures = []
	unexpected_passes = []
	failures = []
	passes = []

	if !ARGV.empty?
	test_files = ARGV.map do \|test\|
	"testcase/#{File.basename(test)}"
	end
	else
	test_files = Dir.glob('testcase/*.mk').sort
	test_files += Dir.glob('testcase/*.sh').sort
	end

	def run_in_testdir(test_filename)
	c = File.read(test_filename)
	name = File.basename(test_filename)
	dir = "out/#{name}"

	FileUtils.mkdir_p(dir)
	Dir.glob("#{dir}/*").each do \|fname\|
	FileUtils.rm_rf(fname)
	end

	Dir.chdir(dir) do
	yield name
	end
	end

	def normalize_ninja_log(log, mk)
	log.gsub!(/^\[\d+\/\d+\] .*\n/, '')
	log.gsub!(/^ninja: no work to do\.\n/, '')
	log.gsub!(/^ninja: error: (., needed by .),.*/,
	'*** No rule to make target \\1.')
	if mk =~ /err_error_in_recipe.mk/
	# This test expects ninja fails. Strip ninja specific error logs.
	log.gsub!(/^FAILED: .*\n/, '')
	log.gsub!(/^ninja: .*\n/, '')
	end
	log
	end

	def normalize_make_log(expected)
	expected.gsub!(/^make(?:\[\d+\])?: (Entering\|Leaving) directory.*\n/, '')
	expected.gsub!(/^make(?:\[\d+\])?: /, '')
	expected = move_circular_dep(expected)

	# Normalizations for old/new GNU make.
	expected.gsub!(/[`'"]/, '"')
	expected.gsub!(/ (?:commands\|recipe) for target /,
	' commands for target ')
	expected.gsub!(/ (?:commands\|recipe) commences /,
	' commands commence ')
	expected.gsub!(' (did you mean TAB instead of 8 spaces?)', '')
	expected.gsub!('Extraneous text after', 'extraneous text after')
	# Not sure if this is useful.
	expected.gsub!(/\s+Stop\.$/, '')
	# GNU make 4.0 has this output.
	expected.gsub!(/Makefile:\d+: commands for target ".*?" failed\n/, '')
	# We treat some warnings as errors.
	expected.gsub!(/Nothing to be done for "test"\.\n/, '')

	expected
	end

	def normalize_kati_log(output)
	output = move_circular_dep(output)
	# kati specific log messages.
	output.gsub!(/^\kati\.*\n/, '')
	output.gsub!(/^c?kati: /, '')
	output.gsub!(/[`'"]/, '"')
	output.gsub!(/(: )(?:open )?(\S+): [Nn](o such file or directory)\nNOTE:.*/,
	"\\1\\2: N\\3\n*** No rule to make target \"\\2\".")
	output.gsub!(/\/bin\/sh: ([^:]*): command not found/,
	"\\1: Command not found")
	output.gsub!(/.: warning for parse error in an unevaluated line: .\n/, '')
	output.gsub!(/ (\.\/+)+kati\.\S+/, '') # kati log files in find_command.mk
	output.gsub!(/ (\.\/+)+test\S+.json/, '') # json files in find_command.mk
	output
	end

	run_make_test = proc do \|mk\|
	c = File.read(mk)
	expected_failure = false
	if c =~ /\A# TODO(?:\(([-a-z\|]+)\))?/
	if $1
	todos = $1.split('\|')
	if todos.include?('go') && !ckati
	expected_failure = true
	end
	if todos.include?('c') && ckati
	expected_failure = true
	end
	if todos.include?('go-ninja') && !ckati && via_ninja
	expected_failure = true
	end
	if todos.include?('c-ninja') && ckati && via_ninja
	expected_failure = true
	end
	if todos.include?('ninja') && via_ninja
	expected_failure = true
	end
	else
	expected_failure = true
	end
	end

	run_in_testdir(mk) do \|name\|
	# TODO: Fix
	if name =~ /eval_assign/ && ckati
	next
	end

	File.open("Makefile", 'w') do \|ofile\|
	ofile.print(c)
	end

	expected = ''
	output = ''

	testcases = c.scan(/^test\d*/).sort.uniq
	if testcases.empty?
	testcases = ['']
	end

	cleanup
	testcases.each do \|tc\|
	cmd = 'make'
	if via_ninja
	cmd += ' -s'
	end
	cmd += " #{tc} 2>&1"
	res = `#{cmd}`
	res = normalize_make_log(res)
	expected += "=== #{tc} ===\n" + res
	expected_files = get_output_filenames
	expected += "\n=== FILES ===\n#{expected_files * "\n"}\n"
	end

	cleanup
	testcases.each do \|tc\|
	json = "#{tc.empty? ? 'test' : tc}"
	cmd = "../../kati -save_json=#{json}.json -log_dir=. --use_find_emulator"
	if ckati
	cmd = "../../ckati --use_find_emulator"
	end
	if via_ninja
	cmd += ' --ninja'
	end
	cmd += " #{tc} 2>&1"
	res = IO.popen(cmd, 'r:binary', &:read)
	if via_ninja && File.exist?('build.ninja') && File.exists?('ninja.sh')
	log = IO.popen('./ninja.sh -j1 -v 2>&1', 'r:binary', &:read)
	res += normalize_ninja_log(log, mk)
	end
	res = normalize_kati_log(res)
	output += "=== #{tc} ===\n" + res
	output_files = get_output_filenames
	output += "\n=== FILES ===\n#{output_files * "\n"}\n"
	end

	File.open('out.make', 'w'){\|ofile\|ofile.print(expected)}
	File.open('out.kati', 'w'){\|ofile\|ofile.print(output)}

	if expected =~ /FAIL/
	puts %Q(#{name} has a string "FAIL" in its expectation)
	exit 1
	end

	if expected != output
	if expected_failure
	puts "#{name}: FAIL (expected)"
	expected_failures << name
	else
	puts "#{name}: FAIL"
	puts `diff -u out.make out.kati`
	failures << name
	end
	else
	if expected_failure
	puts "#{name}: PASS (unexpected)"
	unexpected_passes << name
	else
	puts "#{name}: PASS"
	passes << name
	end
	end

	if name !~ /^err_/ && test_serialization && !expected_failure
	testcases.each do \|tc\|
	json = "#{tc.empty? ? 'test' : tc}"
	cmd = "../../kati -save_json=#{json}_2.json -load_json=#{json}.json -n -log_dir=. #{tc} 2>&1"
	res = IO.popen(cmd, 'r:binary', &:read)
	if !File.exist?("#{json}.json") \|\| !File.exist?("#{json}_2.json")
	puts "#{name}##{json}: Serialize failure (not exist)"
	puts res
	else
	json1 = File.read("#{json}.json")
	json2 = File.read("#{json}_2.json")
	if json1 != json2
	puts "#{name}##{json}: Serialize failure"
	puts res
	end
	end
	end
	end
	end
	end

	run_shell_test = proc do \|sh\|
	is_ninja_test = sh =~ /\/ninja_/
	if is_ninja_test && (!ckati \|\| !via_ninja)
	next
	end

	run_in_testdir(sh) do \|name\|
	cleanup
	cmd = "sh ../../#{sh} make"
	if is_ninja_test
	cmd += ' -s'
	end
	expected = IO.popen(cmd, 'r:binary', &:read)
	cleanup

	if is_ninja_test
	if ckati
	cmd = "sh ../../#{sh} ../../ckati --ninja --regen"
	else
	next
	end
	else
	if ckati
	cmd = "sh ../../#{sh} ../../ckati"
	else
	cmd = "sh ../../#{sh} ../../kati --use_cache -log_dir=."
	end
	end

	output = IO.popen(cmd, 'r:binary', &:read)

	expected = normalize_make_log(expected)
	output = normalize_kati_log(output)
	if is_ninja_test
	output = normalize_ninja_log(output, sh)
	end
	File.open('out.make', 'w'){\|ofile\|ofile.print(expected)}
	File.open('out.kati', 'w'){\|ofile\|ofile.print(output)}

	if expected != output
	puts "#{name}: FAIL"
	puts `diff -u out.make out.kati`
	failures << name
	else
	puts "#{name}: PASS"
	passes << name
	end
	end
	end

	test_files.each do \|test\|
	if /\.mk$/ =~ test
	run_make_test.call(test)
	elsif /\.sh$/ =~ test
	run_shell_test.call(test)
	else
	raise "Unknown test type: #{test}"
	end
	end

	puts

	if !expected_failures.empty?
	puts "=== Expected failures ==="
	expected_failures.each do \|n\|
	puts n
	end
	end

	if !unexpected_passes.empty?
	puts "=== Unexpected passes ==="
	unexpected_passes.each do \|n\|
	puts n
	end
	end

	if !failures.empty?
	puts "=== Failures ==="
	failures.each do \|n\|
	puts n
	end
	end

	puts

	if !unexpected_passes.empty? \|\| !failures.empty?
	puts "FAIL! (#{failures.size + unexpected_passes.size} fails #{passes.size} passes)"
	else
	puts 'PASS!'
	end