Store raw json with run results when running safetynet_job.py.
Storing the machine-readable data is a good idea in general, in this
case allowing us to create graphs with the historical runs.
The json now also contains:
- a version number in case there are future changes in the format
- the date and time when the run started
- which profiler was used
- which commits were being compared
- whether a comparison was run or not, that is, if the commit hashes
for before and after differ
Change-Id: I2913beaef30b90669ac4ffe60a656d4b69030588
Reviewed-on: https://pdfium-review.googlesource.com/13370
Commit-Queue: Henrique Nakashima <hnakashima@chromium.org>
Reviewed-by: Nicolás Peña <npm@chromium.org>
diff --git a/testing/tools/safetynet_compare.py b/testing/tools/safetynet_compare.py
index d583312..ce0505f 100755
--- a/testing/tools/safetynet_compare.py
+++ b/testing/tools/safetynet_compare.py
@@ -101,6 +101,7 @@
conclusions = self._DrawConclusions(before, after)
conclusions_dict = conclusions.GetOutputDict()
+ conclusions_dict.setdefault('metadata', {})['profiler'] = self.args.profiler
self._PrintConclusions(conclusions_dict)
diff --git a/testing/tools/safetynet_conclusions.py b/testing/tools/safetynet_conclusions.py
index 3d87587..fdbc10d 100644
--- a/testing/tools/safetynet_conclusions.py
+++ b/testing/tools/safetynet_conclusions.py
@@ -124,6 +124,7 @@
Returns:
A serializable dict with the format illustrated below:
{
+ "version": 1,
"params": {
"threshold": 0.02
},
@@ -176,6 +177,7 @@
}
"""
output_dict = {}
+ output_dict['version'] = 1
output_dict['params'] = {'threshold': self.threshold_significant}
output_dict['summary'] = self.summary.GetOutputDict()
output_dict['comparison_by_case'] = {
diff --git a/testing/tools/safetynet_job.py b/testing/tools/safetynet_job.py
index f3ea9c5..7416533 100755
--- a/testing/tools/safetynet_job.py
+++ b/testing/tools/safetynet_job.py
@@ -14,10 +14,8 @@
import datetime
import json
import os
-import subprocess
import sys
-from common import PrintErr
from common import PrintWithTime
from common import RunCommandPropagateErr
from githelper import GitHelper
@@ -28,14 +26,14 @@
"""Context for a single run, including name and directory paths."""
def __init__(self, args):
- self.run_name = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
+ self.datetime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
self.results_dir = args.results_dir
self.last_revision_covered_file = os.path.join(self.results_dir,
'last_revision_covered')
self.run_output_dir = os.path.join(self.results_dir,
- 'profiles_%s' % self.run_name)
+ 'profiles_%s' % self.datetime)
self.run_output_log_file = os.path.join(self.results_dir,
- '%s.log' % self.run_name)
+ '%s.log' % self.datetime)
class JobRun(object):
@@ -123,13 +121,19 @@
PrintWithTime('Incremental run, current is %s, last is %s'
% (current, last_revision_covered))
+ if not os.path.exists(self.context.run_output_dir):
+ os.makedirs(self.context.run_output_dir)
+
if current == last_revision_covered:
PrintWithTime('No changes seen, finishing job')
+ output_info = {
+ 'metadata': self._BuildRunMetadata(last_revision_covered,
+ current,
+ False)}
+ self._WriteRawJson(output_info)
return 0
# Run compare
- if not os.path.exists(self.context.run_output_dir):
- os.makedirs(self.context.run_output_dir)
cmd = ['testing/tools/safetynet_compare.py',
'--this-repo',
'--machine-readable',
@@ -143,6 +147,13 @@
return 1
output_info = json.loads(json_output)
+
+ run_metadata = self._BuildRunMetadata(last_revision_covered,
+ current,
+ True)
+ output_info.setdefault('metadata', {}).update(run_metadata)
+ self._WriteRawJson(output_info)
+
PrintConclusionsDictHumanReadable(output_info,
colored=(not self.args.output_to_log
and not self.args.no_color),
@@ -165,6 +176,20 @@
return status
+ def _WriteRawJson(self, output_info):
+ json_output_file = os.path.join(self.context.run_output_dir, 'raw.json')
+ with open(json_output_file, 'w') as f:
+ json.dump(output_info, f)
+
+ def _BuildRunMetadata(self, revision_before, revision_after,
+ comparison_performed):
+ return {
+ 'datetime': self.context.datetime,
+ 'revision_before': revision_before,
+ 'revision_after': revision_after,
+ 'comparison_performed': comparison_performed,
+ }
+
def _WriteCheckpoint(self, checkpoint):
if not self.args.no_checkpoint:
with open(self.context.last_revision_covered_file, 'w') as f: