caliper/src/main/java/com/google/caliper/Runner.java - platform/external/caliper - Gitiles

 /*
  * Copyright (C) 2009 Google Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 package com.google.caliper;

 import com.google.caliper.UserException.DisplayUsageException;
 import com.google.caliper.UserException.ExceptionFromUserCodeException;
 import com.google.caliper.util.InterleavedReader;
 import com.google.common.base.Joiner;
 import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ObjectArrays;
 import com.google.common.io.Closeables;
 import com.google.gson.JsonObject;

 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileFilter;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.PrintStream;
 import java.net.HttpURLConnection;
 import java.net.InetSocketAddress;
 import java.net.Proxy;
 import java.net.URL;
 import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.TimeZone;
 import java.util.regex.Pattern;

 /**
  * Creates, executes and reports benchmark runs.
  */
 public final class Runner {

   private static final FileFilter UPLOAD_FILE_FILTER = new FileFilter() {
     @Override public boolean accept(File file) {
       return file.getName().endsWith(".xml") || file.getName().endsWith(".json");
     }
   };

   private static final String FILE_NAME_DATE_FORMAT = "yyyy-MM-dd'T'HH-mm-ssZ";

   private static final Splitter ARGUMENT_SPLITTER
       = Splitter.on(Pattern.compile("\\s+")).omitEmptyStrings();

   /** Command line arguments to the process */
   private Arguments arguments;
   private ScenarioSelection scenarioSelection;

   private String createFileName(Result result) {
     String timestamp = createTimestamp();
     return String.format("%s.%s.json", result.getRun().getBenchmarkName(), timestamp);
   }

   private String createTimestamp() {
     SimpleDateFormat dateFormat = new SimpleDateFormat(FILE_NAME_DATE_FORMAT, Locale.US);
     dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
     dateFormat.setLenient(true);
     return dateFormat.format(new Date());
   }

   public void run(String... args) {
     this.arguments = Arguments.parse(args);
     File resultsUploadFile = arguments.getUploadResultsFile();
     if (resultsUploadFile != null) {
       uploadResultsFileOrDir(resultsUploadFile);
       return;
     }
     this.scenarioSelection = new ScenarioSelection(arguments);
     if (arguments.getDebug()) {
       debug();
       return;
     }
     Result result = runOutOfProcess();
     new ConsoleReport(result.getRun(), arguments).displayResults();
     boolean saveResultsLocally = arguments.getSaveResultsFile() != null;
     try {
       postResults(result);
     } catch (Exception e) {
       System.out.println();
       System.out.println(e);
       saveResultsLocally = true;
     }

     if (saveResultsLocally) {
       saveResults(result);
     }
   }

   void uploadResultsFileOrDir(File resultsFileOrDir) {
     try {
       if (resultsFileOrDir.isDirectory()) {
         for (File resultsFile : resultsFileOrDir.listFiles(UPLOAD_FILE_FILTER)) {
           uploadResults(resultsFile);
         }
       } else {
         uploadResults(resultsFileOrDir);
       }
     } catch (Exception e) {
       throw new RuntimeException("uploading XML file failed", e);
     }
   }

   private void uploadResults(File resultsUploadFile) throws IOException {
     System.out.println();
     System.out.println("Uploading " + resultsUploadFile.getCanonicalPath());
     InputStream inputStream = new FileInputStream(resultsUploadFile);
     try {
       Result result = new ResultsReader().getResult(inputStream);
       postResults(result);
     } finally {
       inputStream.close();
     }
   }

   private void saveResults(Result result) {
     File resultsFile = arguments.getSaveResultsFile();
     File destinationFile;
     if (resultsFile == null) {
       File dir = new File("./caliper-results");
       dir.mkdirs();
       destinationFile = new File(dir, createFileName(result));
     } else if (resultsFile.exists() && resultsFile.isDirectory()) {
       destinationFile = new File(resultsFile, createFileName(result));
     } else {
       // assume this is a file
       File parent = resultsFile.getParentFile();
       if (parent != null) {
         parent.mkdirs();
       }
       destinationFile = resultsFile;
     }

     PrintStream filePrintStream;
     try {
       filePrintStream = new PrintStream(new FileOutputStream(destinationFile));
     } catch (FileNotFoundException e) {
       throw new RuntimeException("can't open " + destinationFile, e);
     }
     String resultJson = Json.getGsonInstance().toJson(result);
     try {
       System.out.println();
       System.out.println("Writing results to " + destinationFile.getCanonicalPath());
       filePrintStream.print(resultJson);
     } catch (Exception e) {
       System.out.println(e);
       System.out.println("Failed to write results to file, writing to standard out instead:");
       System.out.println(resultJson);
       System.out.flush();
     } finally {
       filePrintStream.close();
     }
   }

   private void postResults(Result result) {
     CaliperRc caliperrc = CaliperRc.INSTANCE;
     String postUrl = caliperrc.getPostUrl();
     String apiKey = caliperrc.getApiKey();
     if (postUrl == null || apiKey == null) {
       // TODO: probably nicer to show a message if only one is null
       return;
     }

     try {
       URL url = new URL(postUrl + apiKey + "/" + result.getRun().getBenchmarkName());
       HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(getProxy());
       urlConnection.setDoOutput(true);
       String resultJson = Json.getGsonInstance().toJson(result);
       urlConnection.getOutputStream().write(resultJson.getBytes());
       if (urlConnection.getResponseCode() == 200) {
         System.out.println("");
         System.out.println("View current and previous benchmark results online:");
         BufferedReader in = new BufferedReader(
             new InputStreamReader(urlConnection.getInputStream()));
         System.out.println("  " + in.readLine());
         in.close();
         return;
       }

       System.out.println("Posting to " + postUrl + " failed: "
           + urlConnection.getResponseMessage());
       BufferedReader reader = new BufferedReader(
           new InputStreamReader(urlConnection.getInputStream()));
       String line;
       while ((line = reader.readLine()) != null) {
         System.out.println(line);
       }
       reader.close();
     } catch (IOException e) {
       throw new RuntimeException("Posting to " + postUrl + " failed.", e);
     }
   }

   private Proxy getProxy() {
     String proxyAddress = CaliperRc.INSTANCE.getProxy();
     if (proxyAddress == null) {
       return Proxy.NO_PROXY;
     }

     String[] proxyHostAndPort = proxyAddress.trim().split(":");
     return new Proxy(Proxy.Type.HTTP, new InetSocketAddress(
         proxyHostAndPort[0], Integer.parseInt(proxyHostAndPort[1])));
   }

   private ScenarioResult runScenario(Scenario scenario) {
     MeasurementResult timeMeasurementResult = measure(scenario, MeasurementType.TIME);
     MeasurementSet allocationMeasurements = null;
     String allocationEventLog = null;
     MeasurementSet memoryMeasurements = null;
     String memoryEventLog = null;
     if (arguments.getMeasureMemory()) {
       MeasurementResult allocationsMeasurementResult =
           measure(scenario, MeasurementType.INSTANCE);
       allocationMeasurements = allocationsMeasurementResult.getMeasurements();
       allocationEventLog = allocationsMeasurementResult.getEventLog();
       MeasurementResult memoryMeasurementResult =
           measure(scenario, MeasurementType.MEMORY);
       memoryMeasurements = memoryMeasurementResult.getMeasurements();
       memoryEventLog = memoryMeasurementResult.getEventLog();
     }

     return new ScenarioResult(timeMeasurementResult.getMeasurements(),
         timeMeasurementResult.getEventLog(),
         allocationMeasurements, allocationEventLog,
         memoryMeasurements, memoryEventLog);
   }

   private static class MeasurementResult {
     private final MeasurementSet measurements;
     private final String eventLog;

     MeasurementResult(MeasurementSet measurements, String eventLog) {
       this.measurements = measurements;
       this.eventLog = eventLog;
     }

     public MeasurementSet getMeasurements() {
       return measurements;
     }

     public String getEventLog() {
       return eventLog;
     }
   }

   private MeasurementResult measure(Scenario scenario, MeasurementType type) {
     Vm vm = new VmFactory().createVm(scenario);
     // this must be done before starting the forked process on certain VMs
     ProcessBuilder processBuilder = createCommand(scenario, vm, type)
         .redirectErrorStream(true);
     Process timeProcess;
     try {
       timeProcess = processBuilder.start();
     } catch (IOException e) {
       throw new RuntimeException("failed to start subprocess", e);
     }

     MeasurementSet measurementSet = null;
     StringBuilder eventLog = new StringBuilder();
     InterleavedReader reader = null;
     try {
       reader = new InterleavedReader(arguments.getMarker(),
           new InputStreamReader(timeProcess.getInputStream()));
       Object o;
       while ((o = reader.read()) != null) {
         if (o instanceof String) {
           eventLog.append(o);
         } else if (measurementSet == null) {
           JsonObject jsonObject = (JsonObject) o;
           measurementSet = Json.measurementSetFromJson(jsonObject);
         } else {
           throw new RuntimeException("Unexpected value: " + o);
         }
       }
     } catch (IOException e) {
       throw new RuntimeException(e);
     } finally {
       Closeables.closeQuietly(reader);
       timeProcess.destroy();
     }

     if (measurementSet == null) {
       String message = "Failed to execute " + Joiner.on(" ").join(processBuilder.command());
       System.err.println("  " + message);
       System.err.println(eventLog.toString());
       throw new ConfigurationException(message);
     }

     return new MeasurementResult(measurementSet, eventLog.toString());
   }

   private ProcessBuilder createCommand(Scenario scenario, Vm vm, MeasurementType type) {
     File workingDirectory = new File(System.getProperty("user.dir"));

     String classPath = System.getProperty("java.class.path");
     if (classPath == null || classPath.length() == 0) {
       throw new IllegalStateException("java.class.path is undefined in " + System.getProperties());
     }

     ImmutableList.Builder<String> vmArgs = ImmutableList.builder();
     vmArgs.addAll(ARGUMENT_SPLITTER.split(scenario.getVariables().get(Scenario.VM_KEY)));
     if (type == MeasurementType.INSTANCE || type == MeasurementType.MEMORY) {
       String allocationJarFile = System.getenv("ALLOCATION_JAR");
       vmArgs.add("-javaagent:" + allocationJarFile);
     }
     vmArgs.addAll(vm.getVmSpecificOptions(type, arguments));

     Map<String, String> vmParameters = scenario.getVariables(
         scenarioSelection.getVmParameterNames());
     for (String vmParameter : vmParameters.values()) {
       vmArgs.addAll(ARGUMENT_SPLITTER.split(vmParameter));
     }

     ImmutableList.Builder<String> caliperArgs = ImmutableList.builder();
     caliperArgs.add("--warmupMillis").add(Long.toString(arguments.getWarmupMillis()));
     caliperArgs.add("--runMillis").add(Long.toString(arguments.getRunMillis()));
     caliperArgs.add("--measurementType").add(type.toString());
     caliperArgs.add("--marker").add(arguments.getMarker());

     Map<String,String> userParameters = scenario.getVariables(
         scenarioSelection.getUserParameterNames());
     for (Entry<String, String> entry : userParameters.entrySet()) {
       caliperArgs.add("-D" + entry.getKey() + "=" + entry.getValue());
     }
     caliperArgs.add(arguments.getSuiteClassName());

     return vm.newProcessBuilder(workingDirectory, classPath,
         vmArgs.build(), InProcessRunner.class.getName(), caliperArgs.build());
   }

   private void debug() {
     try {
       int debugReps = arguments.getDebugReps();
       InProcessRunner runner = new InProcessRunner();
       DebugMeasurer measurer = new DebugMeasurer(debugReps);
       for (Scenario scenario : scenarioSelection.select()) {
         System.out.println("running " + debugReps + " debug reps of " + scenario);
         runner.run(scenarioSelection, scenario, measurer);
       }
     } catch (Exception e) {
       throw new ExceptionFromUserCodeException(e);
     }
   }

   private Result runOutOfProcess() {
     Date executedDate = new Date();
     ImmutableMap.Builder<Scenario, ScenarioResult> resultsBuilder = ImmutableMap.builder();

     try {
       List<Scenario> scenarios = scenarioSelection.select();

       int i = 0;
       for (Scenario scenario : scenarios) {
         beforeMeasurement(i++, scenarios.size(), scenario);
         ScenarioResult scenarioResult = runScenario(scenario);
         afterMeasurement(arguments.getMeasureMemory(), scenarioResult);
         resultsBuilder.put(scenario, scenarioResult);
       }
       System.out.println();

       Environment environment = new EnvironmentGetter().getEnvironmentSnapshot();
       return new Result(
           new Run(resultsBuilder.build(), arguments.getSuiteClassName(), executedDate),
           environment);
     } catch (Exception e) {
       throw new ExceptionFromUserCodeException(e);
     }
   }

   private void beforeMeasurement(int index, int total, Scenario scenario) {
     double percentDone = (double) index / total;
     System.out.printf("%2.0f%% %s", percentDone * 100, scenario);
   }

   private void afterMeasurement(boolean memoryMeasured, ScenarioResult scenarioResult) {
     String memoryMeasurements = "";
     if (memoryMeasured) {
       MeasurementSet instanceMeasurementSet =
           scenarioResult.getMeasurementSet(MeasurementType.INSTANCE);
       String instanceUnit =
         ConsoleReport.UNIT_ORDERING.min(instanceMeasurementSet.getUnitNames().entrySet()).getKey();
       MeasurementSet memoryMeasurementSet = scenarioResult.getMeasurementSet(MeasurementType.MEMORY);
       String memoryUnit =
         ConsoleReport.UNIT_ORDERING.min(memoryMeasurementSet.getUnitNames().entrySet()).getKey();
       memoryMeasurements = String.format(", allocated %s%s for a total of %s%s",
           Math.round(instanceMeasurementSet.medianUnits()), instanceUnit,
           Math.round(memoryMeasurementSet.medianUnits()), memoryUnit);
     }

     MeasurementSet timeMeasurementSet = scenarioResult.getMeasurementSet(MeasurementType.TIME);
     String unit =
         ConsoleReport.UNIT_ORDERING.min(timeMeasurementSet.getUnitNames().entrySet()).getKey();
     System.out.printf(" %.2f %s; \u03C3=%.2f %s @ %d trials%s%n", timeMeasurementSet.medianUnits(),
         unit, timeMeasurementSet.standardDeviationUnits(), unit,
         timeMeasurementSet.getMeasurements().size(), memoryMeasurements);
   }

   public static void main(String[] args) {
     try {
       new Runner().run(args);
       System.exit(0); // user code may have leave non-daemon threads behind!
     } catch (DisplayUsageException e) {
       e.display();
       System.exit(0);
     } catch (UserException e) {
       e.display();
       System.exit(1);
     }
   }

   @SuppressWarnings("unchecked") // temporary fakery
   public static void main(Class<? extends Benchmark> suite, String[] args) {
     main(ObjectArrays.concat(args, suite.getName()));
   }
 }
	/*
	* Copyright (C) 2009 Google Inc.
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	package com.google.caliper;

	import com.google.caliper.UserException.DisplayUsageException;
	import com.google.caliper.UserException.ExceptionFromUserCodeException;
	import com.google.caliper.util.InterleavedReader;
	import com.google.common.base.Joiner;
	import com.google.common.base.Splitter;
	import com.google.common.collect.ImmutableList;
	import com.google.common.collect.ImmutableMap;
	import com.google.common.collect.ObjectArrays;
	import com.google.common.io.Closeables;
	import com.google.gson.JsonObject;

	import java.io.BufferedReader;
	import java.io.File;
	import java.io.FileFilter;
	import java.io.FileInputStream;
	import java.io.FileNotFoundException;
	import java.io.FileOutputStream;
	import java.io.IOException;
	import java.io.InputStream;
	import java.io.InputStreamReader;
	import java.io.PrintStream;
	import java.net.HttpURLConnection;
	import java.net.InetSocketAddress;
	import java.net.Proxy;
	import java.net.URL;
	import java.text.SimpleDateFormat;
	import java.util.Date;
	import java.util.List;
	import java.util.Locale;
	import java.util.Map;
	import java.util.Map.Entry;
	import java.util.TimeZone;
	import java.util.regex.Pattern;

	/**
	* Creates, executes and reports benchmark runs.
	*/
	public final class Runner {

	private static final FileFilter UPLOAD_FILE_FILTER = new FileFilter() {
	@Override public boolean accept(File file) {
	return file.getName().endsWith(".xml") \|\| file.getName().endsWith(".json");
	}
	};

	private static final String FILE_NAME_DATE_FORMAT = "yyyy-MM-dd'T'HH-mm-ssZ";

	private static final Splitter ARGUMENT_SPLITTER
	= Splitter.on(Pattern.compile("\\s+")).omitEmptyStrings();

	/** Command line arguments to the process */
	private Arguments arguments;
	private ScenarioSelection scenarioSelection;

	private String createFileName(Result result) {
	String timestamp = createTimestamp();
	return String.format("%s.%s.json", result.getRun().getBenchmarkName(), timestamp);
	}

	private String createTimestamp() {
	SimpleDateFormat dateFormat = new SimpleDateFormat(FILE_NAME_DATE_FORMAT, Locale.US);
	dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
	dateFormat.setLenient(true);
	return dateFormat.format(new Date());
	}

	public void run(String... args) {
	this.arguments = Arguments.parse(args);
	File resultsUploadFile = arguments.getUploadResultsFile();
	if (resultsUploadFile != null) {
	uploadResultsFileOrDir(resultsUploadFile);
	return;
	}
	this.scenarioSelection = new ScenarioSelection(arguments);
	if (arguments.getDebug()) {
	debug();
	return;
	}
	Result result = runOutOfProcess();
	new ConsoleReport(result.getRun(), arguments).displayResults();
	boolean saveResultsLocally = arguments.getSaveResultsFile() != null;
	try {
	postResults(result);
	} catch (Exception e) {
	System.out.println();
	System.out.println(e);
	saveResultsLocally = true;
	}

	if (saveResultsLocally) {
	saveResults(result);
	}
	}

	void uploadResultsFileOrDir(File resultsFileOrDir) {
	try {
	if (resultsFileOrDir.isDirectory()) {
	for (File resultsFile : resultsFileOrDir.listFiles(UPLOAD_FILE_FILTER)) {
	uploadResults(resultsFile);
	}
	} else {
	uploadResults(resultsFileOrDir);
	}
	} catch (Exception e) {
	throw new RuntimeException("uploading XML file failed", e);
	}
	}

	private void uploadResults(File resultsUploadFile) throws IOException {
	System.out.println();
	System.out.println("Uploading " + resultsUploadFile.getCanonicalPath());
	InputStream inputStream = new FileInputStream(resultsUploadFile);
	try {
	Result result = new ResultsReader().getResult(inputStream);
	postResults(result);
	} finally {
	inputStream.close();
	}
	}

	private void saveResults(Result result) {
	File resultsFile = arguments.getSaveResultsFile();
	File destinationFile;
	if (resultsFile == null) {
	File dir = new File("./caliper-results");
	dir.mkdirs();
	destinationFile = new File(dir, createFileName(result));
	} else if (resultsFile.exists() && resultsFile.isDirectory()) {
	destinationFile = new File(resultsFile, createFileName(result));
	} else {
	// assume this is a file
	File parent = resultsFile.getParentFile();
	if (parent != null) {
	parent.mkdirs();
	}
	destinationFile = resultsFile;
	}

	PrintStream filePrintStream;
	try {
	filePrintStream = new PrintStream(new FileOutputStream(destinationFile));
	} catch (FileNotFoundException e) {
	throw new RuntimeException("can't open " + destinationFile, e);
	}
	String resultJson = Json.getGsonInstance().toJson(result);
	try {
	System.out.println();
	System.out.println("Writing results to " + destinationFile.getCanonicalPath());
	filePrintStream.print(resultJson);
	} catch (Exception e) {
	System.out.println(e);
	System.out.println("Failed to write results to file, writing to standard out instead:");
	System.out.println(resultJson);
	System.out.flush();
	} finally {
	filePrintStream.close();
	}
	}

	private void postResults(Result result) {
	CaliperRc caliperrc = CaliperRc.INSTANCE;
	String postUrl = caliperrc.getPostUrl();
	String apiKey = caliperrc.getApiKey();
	if (postUrl == null \|\| apiKey == null) {
	// TODO: probably nicer to show a message if only one is null
	return;
	}

	try {
	URL url = new URL(postUrl + apiKey + "/" + result.getRun().getBenchmarkName());
	HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(getProxy());
	urlConnection.setDoOutput(true);
	String resultJson = Json.getGsonInstance().toJson(result);
	urlConnection.getOutputStream().write(resultJson.getBytes());
	if (urlConnection.getResponseCode() == 200) {
	System.out.println("");
	System.out.println("View current and previous benchmark results online:");
	BufferedReader in = new BufferedReader(
	new InputStreamReader(urlConnection.getInputStream()));
	System.out.println(" " + in.readLine());
	in.close();
	return;
	}

	System.out.println("Posting to " + postUrl + " failed: "
	+ urlConnection.getResponseMessage());
	BufferedReader reader = new BufferedReader(
	new InputStreamReader(urlConnection.getInputStream()));
	String line;
	while ((line = reader.readLine()) != null) {
	System.out.println(line);
	}
	reader.close();
	} catch (IOException e) {
	throw new RuntimeException("Posting to " + postUrl + " failed.", e);
	}
	}

	private Proxy getProxy() {
	String proxyAddress = CaliperRc.INSTANCE.getProxy();
	if (proxyAddress == null) {
	return Proxy.NO_PROXY;
	}

	String[] proxyHostAndPort = proxyAddress.trim().split(":");
	return new Proxy(Proxy.Type.HTTP, new InetSocketAddress(
	proxyHostAndPort[0], Integer.parseInt(proxyHostAndPort[1])));
	}

	private ScenarioResult runScenario(Scenario scenario) {
	MeasurementResult timeMeasurementResult = measure(scenario, MeasurementType.TIME);
	MeasurementSet allocationMeasurements = null;
	String allocationEventLog = null;
	MeasurementSet memoryMeasurements = null;
	String memoryEventLog = null;
	if (arguments.getMeasureMemory()) {
	MeasurementResult allocationsMeasurementResult =
	measure(scenario, MeasurementType.INSTANCE);
	allocationMeasurements = allocationsMeasurementResult.getMeasurements();
	allocationEventLog = allocationsMeasurementResult.getEventLog();
	MeasurementResult memoryMeasurementResult =
	measure(scenario, MeasurementType.MEMORY);
	memoryMeasurements = memoryMeasurementResult.getMeasurements();
	memoryEventLog = memoryMeasurementResult.getEventLog();
	}

	return new ScenarioResult(timeMeasurementResult.getMeasurements(),
	timeMeasurementResult.getEventLog(),
	allocationMeasurements, allocationEventLog,
	memoryMeasurements, memoryEventLog);
	}

	private static class MeasurementResult {
	private final MeasurementSet measurements;
	private final String eventLog;

	MeasurementResult(MeasurementSet measurements, String eventLog) {
	this.measurements = measurements;
	this.eventLog = eventLog;
	}

	public MeasurementSet getMeasurements() {
	return measurements;
	}

	public String getEventLog() {
	return eventLog;
	}
	}

	private MeasurementResult measure(Scenario scenario, MeasurementType type) {
	Vm vm = new VmFactory().createVm(scenario);
	// this must be done before starting the forked process on certain VMs
	ProcessBuilder processBuilder = createCommand(scenario, vm, type)
	.redirectErrorStream(true);
	Process timeProcess;
	try {
	timeProcess = processBuilder.start();
	} catch (IOException e) {
	throw new RuntimeException("failed to start subprocess", e);
	}

	MeasurementSet measurementSet = null;
	StringBuilder eventLog = new StringBuilder();
	InterleavedReader reader = null;
	try {
	reader = new InterleavedReader(arguments.getMarker(),
	new InputStreamReader(timeProcess.getInputStream()));
	Object o;
	while ((o = reader.read()) != null) {
	if (o instanceof String) {
	eventLog.append(o);
	} else if (measurementSet == null) {
	JsonObject jsonObject = (JsonObject) o;
	measurementSet = Json.measurementSetFromJson(jsonObject);
	} else {
	throw new RuntimeException("Unexpected value: " + o);
	}
	}
	} catch (IOException e) {
	throw new RuntimeException(e);
	} finally {
	Closeables.closeQuietly(reader);
	timeProcess.destroy();
	}

	if (measurementSet == null) {
	String message = "Failed to execute " + Joiner.on(" ").join(processBuilder.command());
	System.err.println(" " + message);
	System.err.println(eventLog.toString());
	throw new ConfigurationException(message);
	}

	return new MeasurementResult(measurementSet, eventLog.toString());
	}

	private ProcessBuilder createCommand(Scenario scenario, Vm vm, MeasurementType type) {
	File workingDirectory = new File(System.getProperty("user.dir"));

	String classPath = System.getProperty("java.class.path");
	if (classPath == null \|\| classPath.length() == 0) {
	throw new IllegalStateException("java.class.path is undefined in " + System.getProperties());
	}

	ImmutableList.Builder<String> vmArgs = ImmutableList.builder();
	vmArgs.addAll(ARGUMENT_SPLITTER.split(scenario.getVariables().get(Scenario.VM_KEY)));
	if (type == MeasurementType.INSTANCE \|\| type == MeasurementType.MEMORY) {
	String allocationJarFile = System.getenv("ALLOCATION_JAR");
	vmArgs.add("-javaagent:" + allocationJarFile);
	}
	vmArgs.addAll(vm.getVmSpecificOptions(type, arguments));

	Map<String, String> vmParameters = scenario.getVariables(
	scenarioSelection.getVmParameterNames());
	for (String vmParameter : vmParameters.values()) {
	vmArgs.addAll(ARGUMENT_SPLITTER.split(vmParameter));
	}

	ImmutableList.Builder<String> caliperArgs = ImmutableList.builder();
	caliperArgs.add("--warmupMillis").add(Long.toString(arguments.getWarmupMillis()));
	caliperArgs.add("--runMillis").add(Long.toString(arguments.getRunMillis()));
	caliperArgs.add("--measurementType").add(type.toString());
	caliperArgs.add("--marker").add(arguments.getMarker());

	Map<String,String> userParameters = scenario.getVariables(
	scenarioSelection.getUserParameterNames());
	for (Entry<String, String> entry : userParameters.entrySet()) {
	caliperArgs.add("-D" + entry.getKey() + "=" + entry.getValue());
	}
	caliperArgs.add(arguments.getSuiteClassName());

	return vm.newProcessBuilder(workingDirectory, classPath,
	vmArgs.build(), InProcessRunner.class.getName(), caliperArgs.build());
	}

	private void debug() {
	try {
	int debugReps = arguments.getDebugReps();
	InProcessRunner runner = new InProcessRunner();
	DebugMeasurer measurer = new DebugMeasurer(debugReps);
	for (Scenario scenario : scenarioSelection.select()) {
	System.out.println("running " + debugReps + " debug reps of " + scenario);
	runner.run(scenarioSelection, scenario, measurer);
	}
	} catch (Exception e) {
	throw new ExceptionFromUserCodeException(e);
	}
	}

	private Result runOutOfProcess() {
	Date executedDate = new Date();
	ImmutableMap.Builder<Scenario, ScenarioResult> resultsBuilder = ImmutableMap.builder();

	try {
	List<Scenario> scenarios = scenarioSelection.select();

	int i = 0;
	for (Scenario scenario : scenarios) {
	beforeMeasurement(i++, scenarios.size(), scenario);
	ScenarioResult scenarioResult = runScenario(scenario);
	afterMeasurement(arguments.getMeasureMemory(), scenarioResult);
	resultsBuilder.put(scenario, scenarioResult);
	}
	System.out.println();

	Environment environment = new EnvironmentGetter().getEnvironmentSnapshot();
	return new Result(
	new Run(resultsBuilder.build(), arguments.getSuiteClassName(), executedDate),
	environment);
	} catch (Exception e) {
	throw new ExceptionFromUserCodeException(e);
	}
	}

	private void beforeMeasurement(int index, int total, Scenario scenario) {
	double percentDone = (double) index / total;
	System.out.printf("%2.0f%% %s", percentDone * 100, scenario);
	}

	private void afterMeasurement(boolean memoryMeasured, ScenarioResult scenarioResult) {
	String memoryMeasurements = "";
	if (memoryMeasured) {
	MeasurementSet instanceMeasurementSet =
	scenarioResult.getMeasurementSet(MeasurementType.INSTANCE);
	String instanceUnit =
	ConsoleReport.UNIT_ORDERING.min(instanceMeasurementSet.getUnitNames().entrySet()).getKey();
	MeasurementSet memoryMeasurementSet = scenarioResult.getMeasurementSet(MeasurementType.MEMORY);
	String memoryUnit =
	ConsoleReport.UNIT_ORDERING.min(memoryMeasurementSet.getUnitNames().entrySet()).getKey();
	memoryMeasurements = String.format(", allocated %s%s for a total of %s%s",
	Math.round(instanceMeasurementSet.medianUnits()), instanceUnit,
	Math.round(memoryMeasurementSet.medianUnits()), memoryUnit);
	}

	MeasurementSet timeMeasurementSet = scenarioResult.getMeasurementSet(MeasurementType.TIME);
	String unit =
	ConsoleReport.UNIT_ORDERING.min(timeMeasurementSet.getUnitNames().entrySet()).getKey();
	System.out.printf(" %.2f %s; \u03C3=%.2f %s @ %d trials%s%n", timeMeasurementSet.medianUnits(),
	unit, timeMeasurementSet.standardDeviationUnits(), unit,
	timeMeasurementSet.getMeasurements().size(), memoryMeasurements);
	}

	public static void main(String[] args) {
	try {
	new Runner().run(args);
	System.exit(0); // user code may have leave non-daemon threads behind!
	} catch (DisplayUsageException e) {
	e.display();
	System.exit(0);
	} catch (UserException e) {
	e.display();
	System.exit(1);
	}
	}

	@SuppressWarnings("unchecked") // temporary fakery
	public static void main(Class<? extends Benchmark> suite, String[] args) {
	main(ObjectArrays.concat(args, suite.getName()));
	}
	}