David Garcia Quintas | 5468706 | 2017-08-03 19:45:45 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright 2015 gRPC authors. |
| 3 | # |
| 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
| 7 | # |
| 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | # |
| 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
| 15 | |
| 16 | """Detect new flakes introduced in the last 24h hours with respect to the |
| 17 | previous six days""" |
| 18 | |
David Garcia Quintas | 2f39486 | 2017-08-02 21:36:35 -0700 | [diff] [blame] | 19 | from __future__ import absolute_import |
| 20 | from __future__ import division |
| 21 | from __future__ import print_function |
| 22 | |
David Garcia Quintas | f277b17 | 2017-08-09 22:31:21 -0700 | [diff] [blame] | 23 | import datetime |
David Garcia Quintas | 2f39486 | 2017-08-02 21:36:35 -0700 | [diff] [blame] | 24 | import os |
| 25 | import sys |
| 26 | import logging |
| 27 | logging.basicConfig(format='%(asctime)s %(message)s') |
| 28 | |
| 29 | gcp_utils_dir = os.path.abspath( |
| 30 | os.path.join(os.path.dirname(__file__), '../gcp/utils')) |
| 31 | sys.path.append(gcp_utils_dir) |
| 32 | |
| 33 | import big_query_utils |
| 34 | |
David Garcia Quintas | f277b17 | 2017-08-09 22:31:21 -0700 | [diff] [blame] | 35 | def print_table(table): |
David Garcia Quintas | 38a83b3 | 2017-09-08 09:10:29 -0700 | [diff] [blame] | 36 | kokoro_base_url = 'https://kokoro.corp.google.com/job/' |
| 37 | for k, v in table.items(): |
David Garcia Quintas | f277b17 | 2017-08-09 22:31:21 -0700 | [diff] [blame] | 38 | job_name = v[0] |
| 39 | build_id = v[1] |
| 40 | ts = int(float(v[2])) |
| 41 | # TODO(dgq): timezone handling is wrong. We need to determine the timezone |
| 42 | # of the computer running this script. |
| 43 | human_ts = datetime.datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S PDT') |
David Garcia Quintas | 38a83b3 | 2017-09-08 09:10:29 -0700 | [diff] [blame] | 44 | job_path = '{}/{}'.format('/job/'.join(job_name.split('/')), build_id) |
| 45 | full_kokoro_url = kokoro_base_url + job_path |
| 46 | print("Test: {}, Timestamp: {}, url: {}\n".format(k, human_ts, full_kokoro_url)) |
David Garcia Quintas | f277b17 | 2017-08-09 22:31:21 -0700 | [diff] [blame] | 47 | |
David Garcia Quintas | 2f39486 | 2017-08-02 21:36:35 -0700 | [diff] [blame] | 48 | |
David Garcia Quintas | 5468706 | 2017-08-03 19:45:45 -0700 | [diff] [blame] | 49 | def get_flaky_tests(days_lower_bound, days_upper_bound, limit=None): |
David Garcia Quintas | 2f39486 | 2017-08-02 21:36:35 -0700 | [diff] [blame] | 50 | """ period is one of "WEEK", "DAY", etc. |
| 51 | (see https://cloud.google.com/bigquery/docs/reference/standard-sql/functions-and-operators#date_add). """ |
| 52 | |
| 53 | bq = big_query_utils.create_big_query() |
| 54 | query = """ |
| 55 | SELECT |
David Garcia Quintas | f277b17 | 2017-08-09 22:31:21 -0700 | [diff] [blame] | 56 | REGEXP_REPLACE(test_name, r'/\d+', '') AS filtered_test_name, |
| 57 | job_name, |
| 58 | build_id, |
| 59 | timestamp |
| 60 | FROM |
| 61 | [grpc-testing:jenkins_test_results.aggregate_results] |
| 62 | WHERE |
David Garcia Quintas | 4721fc0 | 2017-08-11 14:01:32 -0700 | [diff] [blame] | 63 | timestamp > DATE_ADD(CURRENT_DATE(), {days_lower_bound}, "DAY") |
David Garcia Quintas | 5468706 | 2017-08-03 19:45:45 -0700 | [diff] [blame] | 64 | AND timestamp <= DATE_ADD(CURRENT_DATE(), {days_upper_bound}, "DAY") |
David Garcia Quintas | f277b17 | 2017-08-09 22:31:21 -0700 | [diff] [blame] | 65 | AND NOT REGEXP_MATCH(job_name, '.*portability.*') |
| 66 | AND result != 'PASSED' AND result != 'SKIPPED' |
| 67 | ORDER BY timestamp desc |
David Garcia Quintas | 5468706 | 2017-08-03 19:45:45 -0700 | [diff] [blame] | 68 | """.format(days_lower_bound=days_lower_bound, days_upper_bound=days_upper_bound) |
David Garcia Quintas | 2f39486 | 2017-08-02 21:36:35 -0700 | [diff] [blame] | 69 | if limit: |
| 70 | query += '\n LIMIT {}'.format(limit) |
| 71 | query_job = big_query_utils.sync_query_job(bq, 'grpc-testing', query) |
| 72 | page = bq.jobs().getQueryResults( |
| 73 | pageToken=None, **query_job['jobReference']).execute(num_retries=3) |
David Garcia Quintas | bfd9d80 | 2017-08-16 12:10:48 -0700 | [diff] [blame] | 74 | rows = page.get('rows') |
| 75 | if rows: |
| 76 | return {row['f'][0]['v']: |
| 77 | (row['f'][1]['v'], row['f'][2]['v'], row['f'][3]['v']) |
| 78 | for row in rows} |
| 79 | else: |
| 80 | return {} |
David Garcia Quintas | 2f39486 | 2017-08-02 21:36:35 -0700 | [diff] [blame] | 81 | |
| 82 | |
| 83 | def get_new_flakes(): |
David Garcia Quintas | f277b17 | 2017-08-09 22:31:21 -0700 | [diff] [blame] | 84 | last_week_sans_yesterday = get_flaky_tests(-14, -1) |
David Garcia Quintas | 4721fc0 | 2017-08-11 14:01:32 -0700 | [diff] [blame] | 85 | last_24 = get_flaky_tests(0, +1) |
David Garcia Quintas | 5468706 | 2017-08-03 19:45:45 -0700 | [diff] [blame] | 86 | last_week_sans_yesterday_names = set(last_week_sans_yesterday.keys()) |
David Garcia Quintas | 2f39486 | 2017-08-02 21:36:35 -0700 | [diff] [blame] | 87 | last_24_names = set(last_24.keys()) |
David Garcia Quintas | 5468706 | 2017-08-03 19:45:45 -0700 | [diff] [blame] | 88 | logging.debug('|last_week_sans_yesterday| =', len(last_week_sans_yesterday_names)) |
David Garcia Quintas | 2f39486 | 2017-08-02 21:36:35 -0700 | [diff] [blame] | 89 | logging.debug('|last_24_names| =', len(last_24_names)) |
David Garcia Quintas | 5468706 | 2017-08-03 19:45:45 -0700 | [diff] [blame] | 90 | new_flakes = last_24_names - last_week_sans_yesterday_names |
David Garcia Quintas | 2f39486 | 2017-08-02 21:36:35 -0700 | [diff] [blame] | 91 | logging.debug('|new_flakes| = ', len(new_flakes)) |
| 92 | return {k: last_24[k] for k in new_flakes} |
| 93 | |
| 94 | |
| 95 | def main(): |
David Garcia Quintas | 2f39486 | 2017-08-02 21:36:35 -0700 | [diff] [blame] | 96 | new_flakes = get_new_flakes() |
| 97 | if new_flakes: |
David Garcia Quintas | 5468706 | 2017-08-03 19:45:45 -0700 | [diff] [blame] | 98 | print("Found {} new flakes:".format(len(new_flakes))) |
David Garcia Quintas | f277b17 | 2017-08-09 22:31:21 -0700 | [diff] [blame] | 99 | print_table(new_flakes) |
David Garcia Quintas | bfd9d80 | 2017-08-16 12:10:48 -0700 | [diff] [blame] | 100 | else: |
| 101 | print("No new flakes found!") |
David Garcia Quintas | 2f39486 | 2017-08-02 21:36:35 -0700 | [diff] [blame] | 102 | |
| 103 | |
| 104 | if __name__ == '__main__': |
| 105 | main() |