Create an initial harness script for benchmarking NullAway overhead (#105)
Fixes #87
We'll probably want to iterate on this more, but it's already useful for measuring NullAway overhead and profiling.
diff --git a/compile-bench/build.gradle b/compile-bench/build.gradle
new file mode 100644
index 0000000..db934b0
--- /dev/null
+++ b/compile-bench/build.gradle
@@ -0,0 +1,50 @@
+/*
+ * Copyright (C) 2017. Uber Technologies
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+plugins {
+ id "net.ltgt.errorprone" version "0.0.13"
+ id "java"
+ id "application"
+}
+
+sourceCompatibility = "1.8"
+targetCompatibility = "1.8"
+
+mainClassName = "com.uber.nullaway.benchmark.NullAwayBenchmarkHarness"
+
+configurations.maybeCreate("epJavac")
+
+dependencies {
+ compile "com.google.errorprone:error_prone_core:2.1.3"
+ compile project(":nullaway")
+
+ errorprone "com.google.errorprone:error_prone_core:2.1.3"
+
+ epJavac "com.google.errorprone:error_prone_core:2.1.3"
+
+}
+
+applicationDefaultJvmArgs = ["-Xbootclasspath/p:${configurations.epJavac.asPath}"]
+
+run {
+ if (project.hasProperty("appArgs")) {
+ args(appArgs.split(' '))
+ }
+}
+compileJava {
+ options.compilerArgs += ["-Xlint:unchecked", "-Xlint:rawtypes", "-Werror"]
+}
+
+
diff --git a/compile-bench/src/main/java/com/uber/nullaway/benchmark/NullAwayBenchmarkHarness.java b/compile-bench/src/main/java/com/uber/nullaway/benchmark/NullAwayBenchmarkHarness.java
new file mode 100644
index 0000000..2d2e5a9
--- /dev/null
+++ b/compile-bench/src/main/java/com/uber/nullaway/benchmark/NullAwayBenchmarkHarness.java
@@ -0,0 +1,71 @@
+package com.uber.nullaway.benchmark;
+
+import com.google.errorprone.ErrorProneCompiler;
+import com.uber.nullaway.NullAway;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/** harness for benchmarking NullAway for some javac task */
+public class NullAwayBenchmarkHarness {
+
+ /**
+ * NOTE: requires Error Prone javac in the bootclasspath via -Xbootclasspath/p: JVM arg
+ *
+ * @param args First argument should be the annotated packages for NullAway. Remaining arguments
+ * are passed directly to javac.
+ */
+ public static void main(String[] args) {
+ String nullawayJar = getJarFileForClass(NullAway.class).getFile();
+ String annotPackages = args[0];
+ String[] javacArgs = Arrays.copyOfRange(args, 1, args.length);
+ // run NullAway first
+ List<String> nullawayArgs =
+ Arrays.asList(
+ "-Xmaxwarns",
+ "1",
+ "-XepDisableAllChecks",
+ "-Xep:NullAway:WARN",
+ "-XepOpt:NullAway:AnnotatedPackages=" + annotPackages,
+ "-processorpath",
+ nullawayJar);
+ List<String> fixedArgs = new ArrayList<>();
+ fixedArgs.addAll(nullawayArgs);
+ fixedArgs.addAll(Arrays.asList(javacArgs));
+ System.out.println("With NullAway");
+ runCompile(fixedArgs, 7, 10);
+ // run without NullAway
+ fixedArgs = new ArrayList<>();
+ fixedArgs.add("-XepDisableAllChecks");
+ fixedArgs.addAll(Arrays.asList(javacArgs));
+ System.out.println("No NullAway");
+ runCompile(fixedArgs, 7, 10);
+ }
+
+ private static void runCompile(List<String> fixedArgs, int warmupRuns, int realRuns) {
+ String[] finalArgs = fixedArgs.toArray(new String[fixedArgs.size()]);
+ for (int i = 0; i < warmupRuns; i++) {
+ System.out.println("Warmup Run " + (i + 1));
+ long startTime = System.nanoTime();
+ ErrorProneCompiler.compile(finalArgs);
+ long endTime = System.nanoTime();
+ System.out.println("Running time " + (((double) endTime - startTime) / 1000000000.0));
+ }
+ long totalRunningTime = 0;
+ for (int i = 0; i < realRuns; i++) {
+ System.out.println("Real Run " + (i + 1));
+ long startTime = System.nanoTime();
+ ErrorProneCompiler.compile(finalArgs);
+ long endTime = System.nanoTime();
+ long runTime = endTime - startTime;
+ System.out.println("Running time " + (((double) runTime) / 1000000000.0));
+ totalRunningTime += runTime;
+ }
+ System.out.println("Average running time " + ((double) totalRunningTime) / realRuns);
+ }
+
+ private static URL getJarFileForClass(Class<?> klass) {
+ return klass.getProtectionDomain().getCodeSource().getLocation();
+ }
+}
diff --git a/nullaway/build.gradle b/nullaway/build.gradle
index 6480c00..590735f 100644
--- a/nullaway/build.gradle
+++ b/nullaway/build.gradle
@@ -70,7 +70,7 @@
}
compileJava {
- options.compilerArgs += ["-Xlint:unchecked", "-Werror"]
+ options.compilerArgs += ["-Xlint:unchecked", "-Xlint:rawtypes", "-Werror"]
}
test {
diff --git a/settings.gradle b/settings.gradle
index 336028d..5480232 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -2,3 +2,4 @@
include ':sample-library-model'
include ':sample'
include ':sample-app'
+include ':compile-bench'
\ No newline at end of file