Merge "Make VTS run on Windows."
diff --git a/Android.mk b/Android.mk
index 9ffc61d..02b62d0 100644
--- a/Android.mk
+++ b/Android.mk
@@ -51,12 +51,24 @@
@rm -f $@.list
# $(hide) unzip $@ -d $(VTS_TESTCASES_OUT)
#
+ @echo "build vts python package for boot HAL"
+ $(hide) find hardware/interfaces/boot/1.0/vts/functional -name '*.py' -or -name '*.config' -or -name '*.push' | sort > $@.list
+ $(hide) $(SOONG_ZIP) -d -o $@ -C hardware/interfaces/boot/1.0/vts/functional -l $@.list
+ @rm -f $@.list
+ $(hide) unzip $@ -d $(VTS_TESTCASES_OUT)
+ #
@echo "build vts python package for NFC HAL"
$(hide) find hardware/interfaces/nfc/1.0/vts/functional -name '*.py' -or -name '*.config' -or -name '*.push' | sort > $@.list
$(hide) $(SOONG_ZIP) -d -o $@ -C hardware/interfaces/nfc/1.0/vts/functional -l $@.list
@rm -f $@.list
$(hide) unzip $@ -d $(VTS_TESTCASES_OUT)
#
+ @echo "build vts python package for power HAL"
+ $(hide) find hardware/interfaces/power/1.0/vts/functional -name '*.py' -or -name '*.config' -or -name '*.push' | sort > $@.list
+ $(hide) $(SOONG_ZIP) -d -o $@ -C hardware/interfaces/power/1.0/vts/functional -l $@.list
+ @rm -f $@.list
+ $(hide) unzip $@ -d $(VTS_TESTCASES_OUT)
+ #
@echo "build vts python package for thermal HAL"
$(hide) find hardware/interfaces/thermal/1.0/vts/functional -name '*.py' -or -name '*.config' -or -name '*.push' | sort > $@.list
$(hide) $(SOONG_ZIP) -d -o $@ -C hardware/interfaces/thermal/1.0/vts/functional -l $@.list
diff --git a/README.md b/README.md
index 7b88886..7d0bccf 100644
--- a/README.md
+++ b/README.md
@@ -8,9 +8,16 @@
VTS has mainly two types of test suites:
- * automated (e.g., fuzzing) and
- * structural (e.g., gTest) testing.
+ * automated (e.g., record-and-replay and fuzzing) and
+ * structural (e.g., gTest and host-driven python) testing.
While the VTS framework and test cases are designed for userdebug or eng build
devices, some test cases can be run on user build devices using its app namely,
-VTS Agent App.
\ No newline at end of file
+VTS Agent App.
+
+__Table of Contents__
+
+1. [VTS User Manual](doc/user_manual.md)
+2. [Where to find VTS test cases](testcases/README.md)
+3. [VTS Test Developer Manual](doc/test_developer_manual.md)
+4. [VTS Framework Developer Manual](doc/framework_developer_manual.md)
diff --git a/doc/framework_developer_manual.md b/doc/framework_developer_manual.md
new file mode 100644
index 0000000..84f2bed
--- /dev/null
+++ b/doc/framework_developer_manual.md
@@ -0,0 +1,8 @@
+# VTS Framework Developer Manual
+
+VTS infrastructure and service design docs will be available publically.
+Until then, please contact Android Partner Engineering.
+
+## 1. Web Dashboard and Notification
+
+[Overview](web/vts_dashboard.md)
\ No newline at end of file
diff --git a/doc/test_developer_manual.md b/doc/test_developer_manual.md
new file mode 100644
index 0000000..7e4d657
--- /dev/null
+++ b/doc/test_developer_manual.md
@@ -0,0 +1,3 @@
+# VTS Test Developer Manual
+
+[Manual](testcase_develop_manual/index.md)
diff --git a/doc/testcase_develop_manual/index.md b/doc/testcase_develop_manual/index.md
index 5b6cf60..c8ad3de 100644
--- a/doc/testcase_develop_manual/index.md
+++ b/doc/testcase_develop_manual/index.md
@@ -11,7 +11,7 @@
As the Android system is being used widely (i.e., requiring frequent security
patching and fast feature launch velocity), we think that it is a right cultural shift for this time.
-Currently, all VTS tests are designed for userdebug build devices only.
+Currently, all VTS tests are designed for userdebug or eng build devices only.
__Table of Contents__
diff --git a/doc/user_manual.md b/doc/user_manual.md
new file mode 100644
index 0000000..70aca3a
--- /dev/null
+++ b/doc/user_manual.md
@@ -0,0 +1,128 @@
+# VTS User Manual
+
+## 1. Setup
+
+### 1.1. Host setup
+
+* [Setup Manual](setup/index.md)
+
+### 1.2. Checkout master git repository
+
+[Download Android Source Code](https://source.android.com/source/downloading.html)
+
+`$ export branch=master`
+
+`$ mkdir ${branch}`
+
+`$ cd ${branch}`
+
+`$ repo init -b ${branch} -u persistent https://android.googlesource.com/platform/manifest`
+
+`$ repo sync -j 8`
+
+### 1.3. Build an Android image
+
+`$ cd ${branch}`
+
+`$ . build/make/envsetup.sh`
+
+`$ lunch aosp_arm64-userdebug # or <your device>-userdebug`
+
+The below is an optional step:
+
+`$ make -j 8`
+
+If this fails, please do:
+
+`$ repo sync -j 8`
+
+`$ make -j 8`
+
+Such can happen because tip of tree (ToT) may not always be buildable.
+
+### 1.4. Build a VTS package
+
+`$ cd ${branch}`
+
+`$ make vts ENABLE_TREBLE=true -j8`
+
+Or use the exact command:
+
+`$ make -j8 vts showcommands dist TARGET_PRODUCT=aosp_arm64 WITH_DEXPREOPT=false TARGET_BUILD_VARIANT=userdebug ENABLE_TREBLE=true`
+
+### 1.5. Connect to an Android device
+
+Let's connect an Android device and a host computer using a USB cable.
+
+* On an Android device, Setting -> About Phone -> Click repeatedly 'Build number' until developer mode is enabled.
+* On an Android device, Setting -> Developer options -> Turn on 'USB debugging'
+* On a host, run `adb devices` from a command line shell.
+* On a Android device, confirm that the host is trusted.
+* On a host, type `adb shell` and if that works, we're ready.
+
+## 2. Run VTS Tests
+
+### 2.1. Run a VTS test plan
+
+`$ vts-tradefed`
+
+`> run vts`
+
+```
+…
+…
+08-16 09:36:03 I/ResultReporter: Saved logs for device_logcat in .../out/host/linux-x86/vts/android-vts/logs/2016.08.16_09.17.13/device_logcat_7912321856562095748.zip
+08-16 09:36:03 I/ResultReporter: Saved logs for host_log in .../out/host/linux-x86/vts/android-vts/logs/2016.08.16_09.17.13/host_log_2775945280523850018.zip
+08-16 09:36:04 I/ResultReporter: Invocation finished in 18m 50s. PASSED: 18, FAILED: 0, NOT EXECUTED: 2, MODULES: 8 of 10
+08-16 09:36:04 I/ResultReporter: Test Result: .../out/host/linux-x86/vts/android-vts/results/2016.08.16_09.17.13/test_result.xml
+08-16 09:36:04 I/ResultReporter: Full Result: .../out/host/linux-x86/vts/android-vts/results/2016.08.16_09.17.13.zip
+```
+
+### 2.2. Test report for APFE (Android Partner Front-End)
+
+The uploadable report xml file can be found at
+
+`out/host/linux-x86/vts/android-vts/results/`
+
+After Android O release, you will be able to upload that xml file to [AFPE](https://partner.android.com)
+and obtain a certificate.
+
+### 2.3. Check the test logs
+
+`$ vi out/host/linux-x86/vts/android-vts/logs/`
+
+Then select a directory which captures the time stamp of your test run (e.g., 2016.08.16_09.17.13).
+
+Then select `host_log_<timestamp>.zip` and host_log.txt in that zip file for host log.
+
+Then select `device_logcat_<timestamp>.zip` and device_logcat.txt in that zip file for device log.
+
+## 3. Run Options for Advanced Users
+
+### 3.1. List of VTS Plans
+
+* __vts__: All default VTS tests (To be finalized before Android O release).
+* __vts-camera-its__: For camera ITS (Image Test Suite) tests ported to VTS.
+* __vts-codelab__: For VTS codelab.
+* __vts-coverage__: For VTS code coverage tests.
+* __vts-fuzz__: For VTS Fuzz tests.
+* __vts-gce__: For VTS tests which can be run on Google Compute Engine (GCE).
+* __vts-hal-hidl-profiling__: For VTS HIDL HAL profiling tasks.
+* __vts-hal-hidl__: For VTS HIDL HAL tests.
+* __vts-host__: For VTS host-driven tests.
+* __vts-kernel__: For VTS kernel tests.
+* __vts-library__: For VTS native library tests.
+* __vts-presubmit__: For VTS pre-submit tests.
+* __vts-security__: For VTS security tests.
+* __vts-system__: For VTS system tests.
+
+### 3.2. Run kernel test cases
+
+LTP (Linux Test Project) is part of vts-kernel.
+[This doc](developer_testing/kernel/ltp.md) shows how to run each LTP test case.
+
+## 4. Debugging
+
+### 4.1. Run VTS tests directly for debugging
+
+[Run directly from command line](testcase_develop_manual/run_vts_directly.md)
diff --git a/sysfuzzer/common/include/fuzz_tester/FuzzerBase.h b/sysfuzzer/common/include/fuzz_tester/FuzzerBase.h
index 9acb566..cb5a776 100644
--- a/sysfuzzer/common/include/fuzz_tester/FuzzerBase.h
+++ b/sysfuzzer/common/include/fuzz_tester/FuzzerBase.h
@@ -64,6 +64,12 @@
return false;
};
+ virtual bool VerifyResults(
+ const vts::FunctionSpecificationMessage& /*expected_result_msg*/,
+ const vts::FunctionSpecificationMessage& /*actual_result_msg*/) {
+ return false;
+ };
+
virtual bool GetAttribute(vts::FunctionSpecificationMessage* /*func_msg*/,
void** /*result*/) {
return false;
diff --git a/sysfuzzer/common/include/replayer/VtsHidlHalReplayer.h b/sysfuzzer/common/include/replayer/VtsHidlHalReplayer.h
index 059fc20..fdafcf7 100644
--- a/sysfuzzer/common/include/replayer/VtsHidlHalReplayer.h
+++ b/sysfuzzer/common/include/replayer/VtsHidlHalReplayer.h
@@ -38,10 +38,11 @@
const char* component_name,
ComponentSpecificationMessage* message);
- // Parses the trace file and stores the parsed sequence of API calls in
- // func_msgs.
+ // Parses the trace file, stores the parsed sequence of API calls in
+ // func_msgs and the corresponding return results in result_msgs.
bool ParseTrace(const char* trace_file,
- vector<FunctionSpecificationMessage>* func_msgs);
+ vector<FunctionSpecificationMessage>* func_msgs,
+ vector<FunctionSpecificationMessage>* result_msgs);
// Replays the API call sequence parsed from the trace file.
bool ReplayTrace(const char* spec_lib_file_path, const char* trace_file,
diff --git a/sysfuzzer/common/replayer/VtsHidlHalReplayer.cpp b/sysfuzzer/common/replayer/VtsHidlHalReplayer.cpp
index 006ca02..584632a 100644
--- a/sysfuzzer/common/replayer/VtsHidlHalReplayer.cpp
+++ b/sysfuzzer/common/replayer/VtsHidlHalReplayer.cpp
@@ -72,8 +72,9 @@
return true;
}
-bool VtsHidlHalReplayer::ParseTrace(
- const char* trace_file, vector<FunctionSpecificationMessage>* func_msgs) {
+bool VtsHidlHalReplayer::ParseTrace(const char* trace_file,
+ vector<FunctionSpecificationMessage>* func_msgs,
+ vector<FunctionSpecificationMessage>* result_msgs) {
std::ifstream in(trace_file, std::ios::in);
bool new_record = true;
std::string record_str;
@@ -89,17 +90,18 @@
unique_ptr<VtsProfilingRecord> record(new VtsProfilingRecord());
if (!google::protobuf::TextFormat::MergeFromString(record_str,
record.get())) {
- cerr << __func__
- << ": Can't parse a given function message: " << record_str
- << endl;
+ cerr << __func__ << ": Can't parse a given function message: "
+ << record_str << endl;
return false;
}
- // Only return entry record.
- // TODO(zhuoyao): Return exit record as well for result verification.
- if (record->event() == InstrumentationEventType::SERVER_API_ENTRY ||
- record->event() == InstrumentationEventType::CLIENT_API_ENTRY ||
- record->event() == InstrumentationEventType::PASSTHROUGH_ENTRY) {
+ if (record->event() == InstrumentationEventType::SERVER_API_ENTRY
+ || record->event() == InstrumentationEventType::CLIENT_API_ENTRY
+ || record->event() == InstrumentationEventType::SYNC_CALLBACK_ENTRY
+ || record->event() == InstrumentationEventType::ASYNC_CALLBACK_ENTRY
+ || record->event() == InstrumentationEventType::PASSTHROUGH_ENTRY) {
func_msgs->push_back(record->func_msg());
+ } else {
+ result_msgs->push_back(record->func_msg());
}
new_record = true;
record_str.clear();
@@ -149,18 +151,30 @@
// Parse the trace file to get the sequence of function calls.
vector<FunctionSpecificationMessage> func_msgs;
- if (!ParseTrace(trace_file, &func_msgs)) {
+ vector<FunctionSpecificationMessage> result_msgs;
+ if (!ParseTrace(trace_file, &func_msgs, &result_msgs)) {
cerr << __func__ << ": couldn't parse trace file: " << trace_file
<< endl;
return false;
}
- // Replay each function call from the trace.
- for (vts::FunctionSpecificationMessage func_msg : func_msgs) {
+ // Replay each function call from the trace and verify the results.
+ for (size_t i = 0; i < func_msgs.size(); i++) {
+ vts::FunctionSpecificationMessage func_msg = func_msgs[i];
+ vts::FunctionSpecificationMessage expected_result_msg = result_msgs[i];
cout << __func__ << ": replay function: " << func_msg.DebugString();
- void* result;
- fuzzer->Fuzz(&func_msg, &result, callback_socket_name_);
+ vts::FunctionSpecificationMessage result_msg;
+ if (!fuzzer->CallFunction(func_msg, callback_socket_name_, &result_msg)) {
+ cerr << __func__ << ": replay function fail." << endl;
+ return false;
+ }
+ if (!fuzzer->VerifyResults(expected_result_msg, result_msg)) {
+ // Verification is not strict, i.e. if fail, output error message and
+ // continue the process.
+ cerr << __func__ << ": verification fail.\nexpected_result: "
+ << expected_result_msg.DebugString() << "\nactual_result: "
+ << result_msg.DebugString() << endl;
+ }
}
- // TODO(zhuoyao): verify return results.
return true;
}
diff --git a/web/dashboard/appengine/servlet/src/main/java/com/android/vts/servlet/VtsPerformanceJobServlet.java b/web/dashboard/appengine/servlet/src/main/java/com/android/vts/servlet/VtsPerformanceJobServlet.java
index f86bfcd..df21b7e 100644
--- a/web/dashboard/appengine/servlet/src/main/java/com/android/vts/servlet/VtsPerformanceJobServlet.java
+++ b/web/dashboard/appengine/servlet/src/main/java/com/android/vts/servlet/VtsPerformanceJobServlet.java
@@ -58,10 +58,11 @@
private static final long MILLI_TO_MICRO = 1000; // conversion factor from milli to micro units
private static final String MEAN = "Mean";
- private static final String MEAN_DELTA = "ΔMean (%)";
+ private static final String MIN = "Min";
+ private static final String MIN_DELTA = "ΔMin (%)";
private static final String STD = "Std";
- private static final String STD_DELTA = "ΔStd (%)";
private static final String SUBJECT_PREFIX = "Daily Performance Digest: ";
+ private static final String LAST_WEEK = "Last Week";
private static final String LABEL_STYLE = "font-family: arial";
private static final String TABLE_STYLE = "border-collapse: collapse; border: 1px solid black; font-size: 12px; font-family: arial;";
private static final String SECTION_LABEL_STYLE = "border: 1px solid black; border-bottom: none; background-color: lightgray;";
@@ -126,7 +127,6 @@
for (ProfilingReportMessage profilingReportMessage :
testReportMessage.getProfilingList()) {
switch(profilingReportMessage.getRegressionMode()) {
- case UNKNOWN_REGRESSION_MODE:
case VTS_REGRESSION_MODE_DISABLED:
continue;
default:
@@ -190,7 +190,7 @@
double delta = test - baseline;
if (baseline != 0) {
double pctChange = delta / baseline;
- alpha = pctChange;
+ alpha = pctChange * 2;
pctChangeString = round(pctChange * 100, N_DIGITS) + " %";
} else if (delta != 0){
// If the percent change is undefined, the cell will be solid red or white
@@ -214,20 +214,18 @@
*/
public static String getPerformanceComparisonHTML(StatSummary baseline, StatSummary test) {
if (test == null || baseline == null) {
- return "<td></td><td></td><td></td><td></td>";
+ return "<td></td><td></td><td></td>";
}
String row = "";
- row += "<td style='" + INNER_CELL_STYLE + "'>" + round(baseline.getMean(), N_DIGITS);
- row += "</td><td style='" + INNER_CELL_STYLE + "'>";
- row += round(baseline.getStd(), N_DIGITS) + "</td>";
// Intensity of red color is a function of the relative (percent) change
// in the new value compared to the previous day's. Intensity is a linear function
// of percentage change, reaching a ceiling at 100% change (e.g. a doubling).
- row += getPercentChangeHTML(baseline.getMean(), test.getMean(),
- INNER_CELL_STYLE, test.getRegressionMode());
- row += getPercentChangeHTML(baseline.getStd(), test.getStd(),
- OUTER_CELL_STYLE,
- VtsProfilingRegressionMode.VTS_REGRESSION_MODE_INCREASING);
+ row += getPercentChangeHTML(baseline.getMin(), test.getMin(),
+ INNER_CELL_STYLE, test.getRegressionMode());
+ row += "<td style='" + INNER_CELL_STYLE + "'>" + round(baseline.getMin(), N_DIGITS);
+ row += "</td><td style='" + INNER_CELL_STYLE + "'>" + round(baseline.getMean(), N_DIGITS);
+ row += "</td><td style='" + OUTER_CELL_STYLE + "'>";
+ row += round(baseline.getStd(), N_DIGITS) + "</td>";
return row;
}
@@ -277,7 +275,7 @@
String content = labels.get(i);
tableHTML += "<th style='" + SECTION_LABEL_STYLE + "' ";
if (i == 0) tableHTML += "colspan='1'";
- else if (i == 1) tableHTML += "colspan='2'";
+ else if (i == 1) tableHTML += "colspan='3'";
else tableHTML += "colspan='4'";
tableHTML += ">" + content + "</th>";
}
@@ -286,17 +284,17 @@
// Format column labels
tableHTML += "<tr>";
for (int i = 0; i < labels.size(); i++) {
+ if (i > 1) {
+ tableHTML += "<th style='" + COL_LABEL_STYLE + "'>" + MIN_DELTA + "</th>";
+ }
if (i == 0) {
tableHTML += "<th style='" + COL_LABEL_STYLE + "'>";
tableHTML += summary.xLabel.toStringUtf8() + "</th>";
} else if (i > 0) {
+ tableHTML += "<th style='" + COL_LABEL_STYLE + "'>" + MIN + "</th>";
tableHTML += "<th style='" + COL_LABEL_STYLE + "'>" + MEAN + "</th>";
tableHTML += "<th style='" + COL_LABEL_STYLE + "'>" + STD + "</th>";
}
- if (i > 1) {
- tableHTML += "<th style='" + COL_LABEL_STYLE + "'>" + MEAN_DELTA + "</th>";
- tableHTML += "<th style='" + COL_LABEL_STYLE + "'>" + STD_DELTA + "</th>";
- }
}
tableHTML += "</tr>";
@@ -305,6 +303,8 @@
ByteString label = stats.getLabel();
tableHTML += "<tr><td style='" + HEADER_COL_STYLE +"'>" + label.toStringUtf8();
tableHTML += "</td><td style='" + INNER_CELL_STYLE + "'>";
+ tableHTML += round(stats.getMin(), N_DIGITS) + "</td>";
+ tableHTML += "<td style='" + INNER_CELL_STYLE + "'>";
tableHTML += round(stats.getMean(), N_DIGITS) + "</td>";
tableHTML += "<td style='" + OUTER_CELL_STYLE + "'>";
tableHTML += round(stats.getStd(), N_DIGITS) + "</td>";
@@ -355,6 +355,12 @@
TimeInterval yesterday = new TimeInterval(oneDayAgo - ONE_DAY/MILLI_TO_MICRO, oneDayAgo, dateStringYesterday);
timeIntervals.add(yesterday);
+ // Add last week as a baseline time interval for analysis
+ long oneWeek = 7 * ONE_DAY/MILLI_TO_MICRO;
+ long oneWeekAgo = now - oneWeek;
+ TimeInterval lastWeek = new TimeInterval(oneWeekAgo - oneWeek, oneWeekAgo, LAST_WEEK);
+ timeIntervals.add(lastWeek);
+
for (String tableName : allTables) {
String body = getPeformanceSummary(tableName, timeIntervals);
if (body == null || body.equals("")) continue;
diff --git a/web/dashboard/appengine/servlet/src/main/java/com/android/vts/util/StatSummary.java b/web/dashboard/appengine/servlet/src/main/java/com/android/vts/util/StatSummary.java
index d283cda..1dbb225 100644
--- a/web/dashboard/appengine/servlet/src/main/java/com/android/vts/util/StatSummary.java
+++ b/web/dashboard/appengine/servlet/src/main/java/com/android/vts/util/StatSummary.java
@@ -24,6 +24,8 @@
*/
public class StatSummary {
private ByteString label;
+ private double min;
+ private double max;
private double mean;
private double var;
private int n;
@@ -40,6 +42,8 @@
*/
public StatSummary(ByteString label, VtsProfilingRegressionMode mode) {
this.label = label;
+ this.min = Double.MAX_VALUE;
+ this.max = Double.MIN_VALUE;
this.mean = 0;
this.var = 0;
this.n = 0;
@@ -55,6 +59,24 @@
double oldMean = mean;
mean = oldMean + (value - oldMean) / n;
var = var + (value - mean) * (value - oldMean);
+ if (value < min) min = value;
+ if (value > max) max = value;
+ }
+
+ /**
+ * Gets the calculated min of the stream.
+ * @return The min.
+ */
+ public double getMin() {
+ return min;
+ }
+
+ /**
+ * Gets the calculated max of the stream.
+ * @return The max.
+ */
+ public double getMax() {
+ return max;
}
/**