Implemented first version of java/dex fuzz testing script.
Test: run_dex_fuzz_test.py
Change-Id: I94bd6c39d8219bcf3ba0150f5537a9690f2820b5
diff --git a/tools/bisection_search/bisection_search.py b/tools/bisection_search/bisection_search.py
index b7f1907..c5971e6 100755
--- a/tools/bisection_search/bisection_search.py
+++ b/tools/bisection_search/bisection_search.py
@@ -24,18 +24,23 @@
import abc
import argparse
+import os
import re
import shlex
-from subprocess import call
import sys
+
+from subprocess import call
from tempfile import NamedTemporaryFile
-from common import DeviceTestEnv
-from common import FatalError
-from common import GetEnvVariableOrError
-from common import HostTestEnv
-from common import LogSeverity
-from common import RetCode
+sys.path.append(os.path.dirname(os.path.dirname(
+ os.path.realpath(__file__))))
+
+from common.common import DeviceTestEnv
+from common.common import FatalError
+from common.common import GetEnvVariableOrError
+from common.common import HostTestEnv
+from common.common import LogSeverity
+from common.common import RetCode
# Passes that are never disabled during search process because disabling them
@@ -157,8 +162,7 @@
'Not recognized output format.')
return [p for p in match_passes if p not in NON_PASSES]
- def _PrepareCmd(self, compiled_methods=None, passes_to_run=None,
- verbose_compiler=False):
+ def _PrepareCmd(self, compiled_methods=None, passes_to_run=None):
"""Prepare command to run."""
cmd = self._base_cmd[0:self._arguments_position]
# insert additional arguments before the first argument
diff --git a/tools/common/__init__.py b/tools/common/__init__.py
new file mode 100644
index 0000000..3955c71
--- /dev/null
+++ b/tools/common/__init__.py
@@ -0,0 +1,17 @@
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This file is intentionally left empty. It indicates that the directory is a Python package.
\ No newline at end of file
diff --git a/tools/bisection_search/common.py b/tools/common/common.py
similarity index 93%
rename from tools/bisection_search/common.py
rename to tools/common/common.py
index 3d92ee5..b822dca 100755
--- a/tools/bisection_search/common.py
+++ b/tools/common/common.py
@@ -23,6 +23,10 @@
import shutil
import time
+from enum import Enum
+from enum import unique
+
+from subprocess import DEVNULL
from subprocess import check_call
from subprocess import PIPE
from subprocess import Popen
@@ -32,9 +36,6 @@
from tempfile import mkdtemp
from tempfile import NamedTemporaryFile
-from enum import Enum
-from enum import unique
-
# Temporary directory path on device.
DEVICE_TMP_PATH = '/data/local/tmp'
@@ -91,7 +92,7 @@
def __lt__(self, other):
if self.__class__ is other.__class__:
return self.value < other.value
- return NotImplemented
+ return NotImplemented
def GetEnvVariableOrError(variable_name):
@@ -115,6 +116,14 @@
return top
+def GetJackClassPath():
+ """Returns Jack's classpath."""
+ top = GetEnvVariableOrError('ANDROID_BUILD_TOP')
+ libdir = top + '/out/host/common/obj/JAVA_LIBRARIES'
+ return libdir + '/core-libart-hostdex_intermediates/classes.jack:' \
+ + libdir + '/core-oj-hostdex_intermediates/classes.jack'
+
+
def _DexArchCachePaths(android_data_path):
"""Returns paths to architecture specific caches.
@@ -172,6 +181,33 @@
CommandListToCommandString(cmd), output, retcode))
+def RunCommand(cmd, out, err, timeout=5):
+ """Executes a command, and returns its return code.
+
+ Args:
+ cmd: list of strings, a command to execute
+ out: string, file name to open for stdout (or None)
+ err: string, file name to open for stderr (or None)
+ timeout: int, time out in seconds
+ Returns:
+ RetCode, return code of running command (forced RetCode.TIMEOUT
+ on timeout)
+ """
+ devnull = DEVNULL
+ outf = devnull
+ if out is not None:
+ outf = open(out, mode='w')
+ errf = devnull
+ if err is not None:
+ errf = open(err, mode='w')
+ (_, _, retcode) = RunCommandForOutput(cmd, None, outf, errf, timeout)
+ if outf != devnull:
+ outf.close()
+ if errf != devnull:
+ errf.close()
+ return retcode
+
+
def CommandListToCommandString(cmd):
"""Converts shell command represented as list of strings to a single string.
diff --git a/tools/javafuzz/README.md b/tools/javafuzz/README.md
index b08075a..a70e4c1 100644
--- a/tools/javafuzz/README.md
+++ b/tools/javafuzz/README.md
@@ -8,7 +8,7 @@
or using various target architectures. Any difference between the outputs
(**divergence**) may indicate a bug in one of the execution modes.
-JavaFuzz can be combined with dexfuzz to get multi-layered fuzz testing.
+JavaFuzz can be combined with DexFuzz to get multi-layered fuzz testing.
How to run JavaFuzz
===================
@@ -36,11 +36,11 @@
jack -cp ${JACK_CLASSPATH} --output-dex . Test.java
art -classpath classes.dex Test
-How to start the JavaFuzz tests
-===============================
+How to start JavaFuzz testing
+=============================
run_java_fuzz_test.py
- [--num_tests=#TESTS]
+ [--num_tests=NUM_TESTS]
[--device=DEVICE]
[--mode1=MODE] [--mode2=MODE]
@@ -56,6 +56,20 @@
tint = Art interpreter on target
topt = Art optimizing on target
+How to start Java/DexFuzz testing (multi-layered)
+=================================================
+
+ run_dex_fuzz_test.py
+ [--num_tests=NUM_TESTS]
+ [--num_inputs=NUM_INPUTS]
+ [--device=DEVICE]
+
+where
+
+ --num_tests : number of tests to run (10000 by default)
+ --num_inputs: number of JavaFuzz programs to generate
+ --device : target device serial number (passed to adb -s)
+
Background
==========
diff --git a/tools/javafuzz/run_dex_fuzz_test.py b/tools/javafuzz/run_dex_fuzz_test.py
new file mode 100755
index 0000000..ff87aa4
--- /dev/null
+++ b/tools/javafuzz/run_dex_fuzz_test.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python3.4
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import shutil
+import sys
+
+from subprocess import check_call
+from tempfile import mkdtemp
+
+sys.path.append(os.path.dirname(os.path.dirname(
+ os.path.realpath(__file__))))
+
+from common.common import FatalError
+from common.common import GetJackClassPath
+from common.common import RetCode
+from common.common import RunCommand
+
+
+#
+# Tester class.
+#
+
+
+class DexFuzzTester(object):
+ """Tester that feeds JavaFuzz programs into DexFuzz testing."""
+
+ def __init__(self, num_tests, num_inputs, device):
+ """Constructor for the tester.
+
+ Args:
+ num_tests: int, number of tests to run
+ num_inputs: int, number of JavaFuzz programs to generate
+ device: string, target device serial number (or None)
+ """
+ self._num_tests = num_tests
+ self._num_inputs = num_inputs
+ self._device = device
+ self._save_dir = None
+ self._results_dir = None
+ self._dexfuzz_dir = None
+ self._inputs_dir = None
+
+ def __enter__(self):
+ """On entry, enters new temp directory after saving current directory.
+
+ Raises:
+ FatalError: error when temp directory cannot be constructed
+ """
+ self._save_dir = os.getcwd()
+ self._results_dir = mkdtemp(dir='/tmp/')
+ self._dexfuzz_dir = mkdtemp(dir=self._results_dir)
+ self._inputs_dir = mkdtemp(dir=self._dexfuzz_dir)
+ if self._results_dir is None or self._dexfuzz_dir is None or \
+ self._inputs_dir is None:
+ raise FatalError('Cannot obtain temp directory')
+ os.chdir(self._dexfuzz_dir)
+ return self
+
+ def __exit__(self, etype, evalue, etraceback):
+ """On exit, re-enters previously saved current directory and cleans up."""
+ os.chdir(self._save_dir)
+ # TODO: detect divergences or shutil.rmtree(self._results_dir)
+
+ def Run(self):
+ """Feeds JavaFuzz programs into DexFuzz testing."""
+ print()
+ print('**\n**** JavaFuzz Testing\n**')
+ print()
+ print('#Tests :', self._num_tests)
+ print('Device :', self._device)
+ print('Directory :', self._results_dir)
+ print()
+ self.GenerateJavaFuzzPrograms()
+ self.RunDexFuzz()
+
+
+ def GenerateJavaFuzzPrograms(self):
+ """Generates JavaFuzzPrograms.
+
+ Raises:
+ FatalError: error when generation fails
+ """
+ os.chdir(self._inputs_dir)
+ for i in range(1, self._num_inputs + 1):
+ jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.', 'Test.java']
+ if RunCommand(['javafuzz'], out='Test.java', err=None) != RetCode.SUCCESS:
+ raise FatalError('Unexpected error while running JavaFuzz')
+ if RunCommand(['jack'] + jack_args, out=None, err='jackerr.txt',
+ timeout=30) != RetCode.SUCCESS:
+ raise FatalError('Unexpected error while running Jack')
+ shutil.move('Test.java', '../Test' + str(i) + '.java')
+ shutil.move('classes.dex', 'classes' + str(i) + '.dex')
+ os.unlink('jackerr.txt')
+
+ def RunDexFuzz(self):
+ """Starts the DexFuzz testing."""
+ os.chdir(self._dexfuzz_dir)
+ os.environ['ANDROID_DATA'] = self._dexfuzz_dir
+ dexfuzz_args = ['--inputs=' + self._inputs_dir, '--execute',
+ '--execute-class=Test', '--repeat=' + str(self._num_tests),
+ '--dump-output', '--interpreter', '--optimizing']
+ if self._device is not None:
+ dexfuzz_args += ['--device=' + self._device, '--allarm']
+ else:
+ dexfuzz_args += ['--host'] # Assume host otherwise.
+ check_call(['dexfuzz'] + dexfuzz_args)
+ # TODO: summarize findings.
+
+
+def main():
+ # Handle arguments.
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--num_tests', default=10000,
+ type=int, help='number of tests to run')
+ parser.add_argument('--num_inputs', default=50,
+ type=int, help='number of JavaFuzz program to generate')
+ parser.add_argument('--device', help='target device serial number')
+ args = parser.parse_args()
+ # Run the DexFuzz tester.
+ with DexFuzzTester(args.num_tests, args.num_inputs, args.device) as fuzzer:
+ fuzzer.Run()
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/javafuzz/run_java_fuzz_test.py b/tools/javafuzz/run_java_fuzz_test.py
index 51d00be..6cf3e85 100755
--- a/tools/javafuzz/run_java_fuzz_test.py
+++ b/tools/javafuzz/run_java_fuzz_test.py
@@ -17,69 +17,28 @@
import abc
import argparse
import filecmp
-
-from glob import glob
-
import os
import shlex
import shutil
-import subprocess
import sys
+from glob import glob
from tempfile import mkdtemp
sys.path.append(os.path.dirname(os.path.dirname(
os.path.realpath(__file__))))
-from bisection_search.common import RetCode
-from bisection_search.common import CommandListToCommandString
-from bisection_search.common import FatalError
-from bisection_search.common import GetEnvVariableOrError
-from bisection_search.common import RunCommandForOutput
-from bisection_search.common import DeviceTestEnv
+from common.common import RetCode
+from common.common import CommandListToCommandString
+from common.common import FatalError
+from common.common import GetJackClassPath
+from common.common import GetEnvVariableOrError
+from common.common import RunCommand
+from common.common import DeviceTestEnv
# Return codes supported by bisection bug search.
BISECTABLE_RET_CODES = (RetCode.SUCCESS, RetCode.ERROR, RetCode.TIMEOUT)
-#
-# Utility methods.
-#
-
-
-def RunCommand(cmd, out, err, timeout=5):
- """Executes a command, and returns its return code.
-
- Args:
- cmd: list of strings, a command to execute
- out: string, file name to open for stdout (or None)
- err: string, file name to open for stderr (or None)
- timeout: int, time out in seconds
- Returns:
- RetCode, return code of running command (forced RetCode.TIMEOUT
- on timeout)
- """
- devnull = subprocess.DEVNULL
- outf = devnull
- if out is not None:
- outf = open(out, mode='w')
- errf = devnull
- if err is not None:
- errf = open(err, mode='w')
- (_, _, retcode) = RunCommandForOutput(cmd, None, outf, errf, timeout)
- if outf != devnull:
- outf.close()
- if errf != devnull:
- errf.close()
- return retcode
-
-
-def GetJackClassPath():
- """Returns Jack's classpath."""
- top = GetEnvVariableOrError('ANDROID_BUILD_TOP')
- libdir = top + '/out/host/common/obj/JAVA_LIBRARIES'
- return libdir + '/core-libart-hostdex_intermediates/classes.jack:' \
- + libdir + '/core-oj-hostdex_intermediates/classes.jack'
-
def GetExecutionModeRunner(device, mode):
"""Returns a runner for the given execution mode.
@@ -104,6 +63,7 @@
return TestRunnerArtOptOnTarget(device)
raise FatalError('Unknown execution mode')
+
#
# Execution mode classes.
#
@@ -335,7 +295,7 @@
#
-# Tester classes.
+# Tester class.
#
@@ -356,7 +316,8 @@
self._runner1 = GetExecutionModeRunner(device, mode1)
self._runner2 = GetExecutionModeRunner(device, mode2)
self._save_dir = None
- self._tmp_dir = None
+ self._results_dir = None
+ self._javafuzz_dir = None
# Statistics.
self._test = 0
self._num_success = 0
@@ -373,16 +334,16 @@
"""
self._save_dir = os.getcwd()
self._results_dir = mkdtemp(dir='/tmp/')
- self._tmp_dir = mkdtemp(dir=self._results_dir)
- if self._tmp_dir is None or self._results_dir is None:
+ self._javafuzz_dir = mkdtemp(dir=self._results_dir)
+ if self._results_dir is None or self._javafuzz_dir is None:
raise FatalError('Cannot obtain temp directory')
- os.chdir(self._tmp_dir)
+ os.chdir(self._javafuzz_dir)
return self
def __exit__(self, etype, evalue, etraceback):
"""On exit, re-enters previously saved current directory and cleans up."""
os.chdir(self._save_dir)
- shutil.rmtree(self._tmp_dir)
+ shutil.rmtree(self._javafuzz_dir)
if self._num_divergences == 0:
shutil.rmtree(self._results_dir)
@@ -408,12 +369,13 @@
def ShowStats(self):
"""Shows current statistics (on same line) while tester is running."""
- print('\rTests:', self._test, \
- 'Success:', self._num_success, \
- 'Not-compiled:', self._num_not_compiled, \
- 'Not-run:', self._num_not_run, \
- 'Timed-out:', self._num_timed_out, \
- 'Divergences:', self._num_divergences, end='')
+ print('\rTests:', self._test,
+ 'Success:', self._num_success,
+ 'Not-compiled:', self._num_not_compiled,
+ 'Not-run:', self._num_not_run,
+ 'Timed-out:', self._num_timed_out,
+ 'Divergences:', self._num_divergences,
+ end='')
sys.stdout.flush()
def RunJavaFuzzTest(self):
@@ -515,12 +477,12 @@
def CleanupTest(self):
"""Cleans up after a single test run."""
- for file_name in os.listdir(self._tmp_dir):
- file_path = os.path.join(self._tmp_dir, file_name)
- if os.path.isfile(file_path):
- os.unlink(file_path)
- elif os.path.isdir(file_path):
- shutil.rmtree(file_path)
+ for file_name in os.listdir(self._javafuzz_dir):
+ file_path = os.path.join(self._javafuzz_dir, file_name)
+ if os.path.isfile(file_path):
+ os.unlink(file_path)
+ elif os.path.isdir(file_path):
+ shutil.rmtree(file_path)
def main():