Jan Tattermusch | efd9803 | 2016-04-14 16:29:24 -0700 | [diff] [blame] | 1 | # Copyright 2016, Google Inc. |
| 2 | # All rights reserved. |
| 3 | # |
| 4 | # Redistribution and use in source and binary forms, with or without |
| 5 | # modification, are permitted provided that the following conditions are |
| 6 | # met: |
| 7 | # |
| 8 | # * Redistributions of source code must retain the above copyright |
| 9 | # notice, this list of conditions and the following disclaimer. |
| 10 | # * Redistributions in binary form must reproduce the above |
| 11 | # copyright notice, this list of conditions and the following disclaimer |
| 12 | # in the documentation and/or other materials provided with the |
| 13 | # distribution. |
| 14 | # * Neither the name of Google Inc. nor the names of its |
| 15 | # contributors may be used to endorse or promote products derived from |
| 16 | # this software without specific prior written permission. |
| 17 | # |
| 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 19 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 20 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 21 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 22 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 23 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 24 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 25 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 26 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 27 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 28 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 29 | |
| 30 | # utilities for exporting benchmark results |
| 31 | |
| 32 | import json |
| 33 | import os |
| 34 | import sys |
| 35 | import uuid |
| 36 | |
| 37 | |
| 38 | gcp_utils_dir = os.path.abspath(os.path.join( |
| 39 | os.path.dirname(__file__), '../../gcp/utils')) |
| 40 | sys.path.append(gcp_utils_dir) |
| 41 | import big_query_utils |
| 42 | |
| 43 | |
| 44 | _PROJECT_ID='grpc-testing' |
| 45 | _DATASET_ID='test_dataset' |
| 46 | _RESULTS_TABLE_ID='scenario_results' |
| 47 | |
| 48 | |
| 49 | def upload_scenario_result_to_bigquery(result_file): |
| 50 | bq = big_query_utils.create_big_query() |
| 51 | _create_results_table(bq) |
| 52 | |
| 53 | with open(result_file, 'r') as f: |
| 54 | scenario_result = json.loads(f.read()) |
| 55 | _insert_result(bq, scenario_result) |
| 56 | |
| 57 | |
| 58 | def _insert_result(bq, scenario_result): |
| 59 | _flatten_result_inplace(scenario_result) |
| 60 | |
| 61 | # TODO: handle errors... |
| 62 | row = big_query_utils.make_row(str(uuid.uuid4()), scenario_result) |
| 63 | return big_query_utils.insert_rows(bq, |
| 64 | _PROJECT_ID, |
| 65 | _DATASET_ID, |
| 66 | _RESULTS_TABLE_ID, |
| 67 | [row]) |
| 68 | |
| 69 | |
| 70 | def _create_results_table(bq): |
| 71 | with open(os.path.dirname(__file__) + '/scenario_result_schema.json', 'r') as f: |
| 72 | table_schema = json.loads(f.read()) |
| 73 | desc = 'Results of performance benchmarks.' |
| 74 | return big_query_utils.create_table2(bq, _PROJECT_ID, _DATASET_ID, |
| 75 | _RESULTS_TABLE_ID, table_schema, desc) |
| 76 | |
| 77 | |
| 78 | def _flatten_result_inplace(scenario_result): |
| 79 | """Bigquery is not really great for handling deeply nested data |
| 80 | and repeated fields. To maintain values of some fields while keeping |
| 81 | the schema relatively simple, we artificially leave some of the fields |
| 82 | as JSON strings. |
| 83 | """ |
| 84 | scenario_result['scenario']['clientConfig'] = json.dumps(scenario_result['scenario']['clientConfig']) |
| 85 | scenario_result['scenario']['serverConfig'] = json.dumps(scenario_result['scenario']['serverConfig']) |
| 86 | scenario_result['latencies'] = json.dumps(scenario_result['latencies']) |
| 87 | for stats in scenario_result['clientStats']: |
| 88 | stats['latencies'] = json.dumps(stats['latencies']) |
Jan Tattermusch | 88cc4e2 | 2016-04-14 16:58:50 -0700 | [diff] [blame^] | 89 | scenario_result['serverCores'] = json.dumps(scenario_result['serverCores']) |