blob: abf1203b9e3af4ac0b37a7604286b1d5090f4db0 [file] [log] [blame]
beepse0db09d2013-02-24 17:13:21 -08001#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6import logging
7
8import common
9
Prashanth Balasubramanianabd04062014-12-01 15:08:00 -080010import httplib
beeps84e7bb42013-05-31 12:00:06 -070011import httplib2
Fang Deng60184032013-08-16 10:02:08 -070012from autotest_lib.server.cros.dynamic_suite import constants
beeps84e7bb42013-05-31 12:00:06 -070013from chromite.lib import gdata_lib
14
beepse0db09d2013-02-24 17:13:21 -080015try:
beeps84e7bb42013-05-31 12:00:06 -070016 from apiclient.discovery import build as apiclient_build
17 from apiclient import errors as apiclient_errors
18 from oauth2client import file as oauth_client_fileio
beepse0db09d2013-02-24 17:13:21 -080019except ImportError as e:
beeps84e7bb42013-05-31 12:00:06 -070020 apiclient_build = None
Aviv Keshet003ba812013-08-15 10:46:39 -070021 logging.debug("API client for bug filing disabled. %s", e)
beepse0db09d2013-02-24 17:13:21 -080022
23
24class ProjectHostingApiException(Exception):
25 """
26 Raised when an api call fails, since the actual
27 HTTP error can be cryptic.
28 """
29
30
Prashanth Be2dcdf02014-03-02 16:34:59 -080031class BaseIssue(gdata_lib.Issue):
32 """Base issue class with the minimum data to describe a tracker bug.
33 """
34 def __init__(self, t_issue):
35 kwargs = {}
36 kwargs.update((keys, t_issue.get(keys))
37 for keys in gdata_lib.Issue.SlotDefaults.keys())
38 super(BaseIssue, self).__init__(**kwargs)
39
40
41class Issue(BaseIssue):
beeps84e7bb42013-05-31 12:00:06 -070042 """
43 Class representing an Issue and it's related metadata.
44 """
45 def __init__(self, t_issue):
46 """
47 Initialize |self| from tracker issue |t_issue|
48
49 @param t_issue: The base issue we want to use to populate
50 the member variables of this object.
Prashanth Be2dcdf02014-03-02 16:34:59 -080051
52 @raises ProjectHostingApiException: If the tracker issue doesn't
53 contain all expected fields needed to create a complete issue.
beeps84e7bb42013-05-31 12:00:06 -070054 """
Prashanth Be2dcdf02014-03-02 16:34:59 -080055 super(Issue, self).__init__(t_issue)
beeps84e7bb42013-05-31 12:00:06 -070056
Prashanth Be2dcdf02014-03-02 16:34:59 -080057 try:
58 # The value keyed under 'summary' in the tracker issue
59 # is, unfortunately, not the summary but the title. The
60 # actual summary is the update at index 0.
61 self.summary = t_issue.get('updates')[0]
62 self.comments = t_issue.get('updates')[1:]
beeps84e7bb42013-05-31 12:00:06 -070063
Prashanth Be2dcdf02014-03-02 16:34:59 -080064 # open or closed statuses are classified according to labels like
65 # unconfirmed, verified, fixed etc just like through the front end.
66 self.state = t_issue.get(constants.ISSUE_STATE)
67 self.merged_into = None
68 if (t_issue.get(constants.ISSUE_STATUS)
69 == constants.ISSUE_DUPLICATE and
70 constants.ISSUE_MERGEDINTO in t_issue):
71 parent_issue_dict = t_issue.get(constants.ISSUE_MERGEDINTO)
72 self.merged_into = parent_issue_dict.get('issueId')
73 except KeyError as e:
74 raise ProjectHostingApiException('Cannot create a '
75 'complete issue %s, tracker issue: %s' % (e, t_issue))
Fang Deng60184032013-08-16 10:02:08 -070076
beeps84e7bb42013-05-31 12:00:06 -070077
beepse0db09d2013-02-24 17:13:21 -080078class ProjectHostingApiClient():
79 """
80 Client class for interaction with the project hosting api.
81 """
82
Prashanth Be2dcdf02014-03-02 16:34:59 -080083 # Maximum number of results we would like when querying the tracker.
beeps84e7bb42013-05-31 12:00:06 -070084 _max_results_for_issue = 50
Prashanth Be2dcdf02014-03-02 16:34:59 -080085 _start_index = 1
beepse0db09d2013-02-24 17:13:21 -080086
beeps84e7bb42013-05-31 12:00:06 -070087
88 def __init__(self, oauth_credentials, project_name):
89 if apiclient_build is None:
90 raise ProjectHostingApiException('Cannot get apiclient library.')
91
92 storage = oauth_client_fileio.Storage(oauth_credentials)
93 credentials = storage.get()
94 if credentials is None or credentials.invalid:
95 raise ProjectHostingApiException('Invalid credentials for Project '
96 'Hosting api. Cannot file bugs.')
97
98 http = credentials.authorize(httplib2.Http())
99 self._codesite_service = apiclient_build('projecthosting',
100 'v2', http=http)
beepse0db09d2013-02-24 17:13:21 -0800101 self._project_name = project_name
102
103
104 def _execute_request(self, request):
105 """
beeps84e7bb42013-05-31 12:00:06 -0700106 Executes an api request.
beepse0db09d2013-02-24 17:13:21 -0800107
beeps84e7bb42013-05-31 12:00:06 -0700108 @param request: An apiclient.http.HttpRequest object representing the
109 request to be executed.
110 @raises: ProjectHostingApiException if we fail to execute the request.
111 This could happen if we receive an http response that is not a
112 2xx, or if the http object itself encounters an error.
113
114 @return: A deserialized object model of the response body returned for
115 the request.
beepse0db09d2013-02-24 17:13:21 -0800116 """
117 try:
118 return request.execute()
Prashanth Balasubramanianabd04062014-12-01 15:08:00 -0800119 except (apiclient_errors.Error, httplib2.HttpLib2Error,
120 httplib.BadStatusLine) as e:
beeps84e7bb42013-05-31 12:00:06 -0700121 msg = 'Unable to execute your request: %s'
122 raise ProjectHostingApiException(msg % e)
beepse0db09d2013-02-24 17:13:21 -0800123
124
125 def _get_field(self, field):
126 """
127 Gets a field from the project.
128
129 This method directly queries the project hosting API using bugdroids1's,
130 api key.
131
132 @param field: A selector, which corresponds loosely to a field in the
133 new bug description of the crosbug frontend.
beeps84e7bb42013-05-31 12:00:06 -0700134 @raises: ProjectHostingApiException, if the request execution fails.
beepse0db09d2013-02-24 17:13:21 -0800135
136 @return: A json formatted python dict of the specified field's options,
137 or None if we can't find the api library. This dictionary
138 represents the javascript literal used by the front end tracker
139 and can hold multiple filds.
140
141 The returned dictionary follows a template, but it's structure
142 is only loosely defined as it needs to match whatever the front
143 end describes via javascript.
144 For a new issue interface which looks like:
145
146 field 1: text box
147 drop down: predefined value 1 = description
148 predefined value 2 = description
149 field 2: text box
150 similar structure as field 1
151
152 you will get a dictionary like:
153 {
154 'field name 1': {
155 'project realted config': 'config value'
156 'property': [
157 {predefined value for property 1, description},
158 {predefined value for property 2, description}
159 ]
160 },
161
162 'field name 2': {
163 similar structure
164 }
165 ...
166 }
167 """
beeps84e7bb42013-05-31 12:00:06 -0700168 project = self._codesite_service.projects()
beepse0db09d2013-02-24 17:13:21 -0800169 request = project.get(projectId=self._project_name,
170 fields=field)
171 return self._execute_request(request)
172
173
beeps84e7bb42013-05-31 12:00:06 -0700174 def _list_updates(self, issue_id):
175 """
176 Retrieve all updates for a given issue including comments, changes to
177 it's labels, status etc. The first element in the dictionary returned
178 by this method, is by default, the 0th update on the bug; which is the
179 entry that created it. All the text in a given update is keyed as
180 'content', and updates that contain no text, eg: a change to the status
181 of a bug, will contain the emtpy string instead.
182
183 @param issue_id: The id of the issue we want detailed information on.
184 @raises: ProjectHostingApiException, if the request execution fails.
185
186 @return: A json formatted python dict that has an entry for each update
187 performed on this issue.
188 """
189 issue_comments = self._codesite_service.issues().comments()
190 request = issue_comments.list(projectId=self._project_name,
191 issueId=issue_id,
192 maxResults=self._max_results_for_issue)
193 return self._execute_request(request)
194
195
Fang Deng60184032013-08-16 10:02:08 -0700196 def _get_issue(self, issue_id):
197 """
198 Gets an issue given it's id.
199
200 @param issue_id: A string representing the issue id.
201 @raises: ProjectHostingApiException, if failed to get the issue.
202
203 @return: A json formatted python dict that has the issue content.
204 """
205 issues = self._codesite_service.issues()
206 try:
207 request = issues.get(projectId=self._project_name,
208 issueId=issue_id)
209 except TypeError as e:
210 raise ProjectHostingApiException(
211 'Unable to get issue %s from project %s: %s' %
212 (issue_id, self._project_name, str(e)))
213 return self._execute_request(request)
214
215
Prashanth Be2dcdf02014-03-02 16:34:59 -0800216 def set_max_results(self, max_results):
217 """Set the max results to return.
218
219 @param max_results: An integer representing the maximum number of
220 matching results to return per query.
221 """
222 self._max_results_for_issue = max_results
223
224
225 def set_start_index(self, start_index):
226 """Set the start index, for paging.
227
228 @param start_index: The new start index to use.
229 """
230 self._start_index = start_index
231
232
233 def list_issues(self, **kwargs):
beeps84e7bb42013-05-31 12:00:06 -0700234 """
235 List issues containing the search marker. This method will only list
236 the summary, title and id of an issue, though it searches through the
237 comments. Eg: if we're searching for the marker '123', issues that
238 contain a comment of '123' will appear in the output, but the string
239 '123' itself may not, because the output only contains issue summaries.
240
Prashanth Be2dcdf02014-03-02 16:34:59 -0800241 @param kwargs:
242 q: The anchor string used in the search.
243 can: a string representing the search space that is passed to the
244 google api, can be 'all', 'new', 'open', 'owned', 'reported',
245 'starred', or 'to-verify', defaults to 'all'.
246 label: A string representing a single label to match.
beeps84e7bb42013-05-31 12:00:06 -0700247
248 @return: A json formatted python dict of all matching issues.
Prashanth Be2dcdf02014-03-02 16:34:59 -0800249
250 @raises: ProjectHostingApiException, if the request execution fails.
beeps84e7bb42013-05-31 12:00:06 -0700251 """
252 issues = self._codesite_service.issues()
Prashanth Be2dcdf02014-03-02 16:34:59 -0800253
254 # Asking for issues with None or '' labels will restrict the query
255 # to those issues without labels.
256 if not kwargs['label']:
257 del kwargs['label']
258
beeps84e7bb42013-05-31 12:00:06 -0700259 request = issues.list(projectId=self._project_name,
Prashanth Be2dcdf02014-03-02 16:34:59 -0800260 startIndex=self._start_index,
261 maxResults=self._max_results_for_issue,
262 **kwargs)
beeps84e7bb42013-05-31 12:00:06 -0700263 return self._execute_request(request)
264
265
beepse0db09d2013-02-24 17:13:21 -0800266 def _get_property_values(self, prop_dict):
267 """
268 Searches a dictionary as returned by _get_field for property lists,
269 then returns each value in the list. Effectively this gives us
beeps84e7bb42013-05-31 12:00:06 -0700270 all the accepted values for a property. For example, in crosbug,
271 'properties' map to things like Status, Labels, Owner etc, each of these
272 will have a list within the issuesConfig dict.
beepse0db09d2013-02-24 17:13:21 -0800273
274 @param prop_dict: dictionary which contains a list of properties.
beepse0db09d2013-02-24 17:13:21 -0800275 @yield: each value in a property list. This can be a dict or any other
276 type of datastructure, the caller is responsible for handling
277 it correctly.
278 """
279 for name, property in prop_dict.iteritems():
280 if isinstance(property, list):
281 for values in property:
282 yield values
283
284
285 def _get_cros_labels(self, prop_dict):
286 """
287 Helper function to isolate labels from the labels dictionary. This
288 dictionary is of the form:
289 {
290 "label": "Cr-OS-foo",
291 "description": "description"
292 },
293 And maps to the frontend like so:
294 Labels: Cr-???
295 Cr-OS-foo = description
296 where Cr-OS-foo is a conveniently predefined value for Label Cr-OS-???.
297
298 @param prop_dict: a dictionary we expect the Cros label to be in.
299 @return: A lower case product area, eg: video, factory, ui.
300 """
301 label = prop_dict.get('label')
302 if label and 'Cr-OS-' in label:
303 return label.split('Cr-OS-')[1]
304
305
306 def get_areas(self):
307 """
308 Parse issue options and return a list of 'Cr-OS' labels.
309
310 @return: a list of Cr-OS labels from crosbug, eg: ['kernel', 'systems']
311 """
beeps84e7bb42013-05-31 12:00:06 -0700312 if apiclient_build is None:
beepse0db09d2013-02-24 17:13:21 -0800313 logging.error('Missing Api-client import. Cannot get area-labels.')
314 return []
315
316 try:
317 issue_options_dict = self._get_field('issuesConfig')
318 except ProjectHostingApiException as e:
319 logging.error('Unable to determine area labels: %s', str(e))
320 return []
321
322 # Since we can request multiple fields at once we need to
323 # retrieve each one from the field options dictionary, even if we're
324 # really only asking for one field.
325 issue_options = issue_options_dict.get('issuesConfig')
beepsc8a875b2013-03-25 10:20:38 -0700326 if issue_options is None:
327 logging.error('The IssueConfig field does not contain issue '
328 'configuration as a member anymore; The project '
329 'hosting api might have changed.')
330 return []
331
beepse0db09d2013-02-24 17:13:21 -0800332 return filter(None, [self._get_cros_labels(each)
333 for each in self._get_property_values(issue_options)
334 if isinstance(each, dict)])
335
beeps84e7bb42013-05-31 12:00:06 -0700336
337 def create_issue(self, request_body):
338 """
339 Convert the request body into an issue on the frontend tracker.
340
341 @param request_body: A python dictionary with key-value pairs
342 that represent the fields of the issue.
343 eg: {
344 'title': 'bug title',
345 'description': 'bug description',
346 'labels': ['Type-Bug'],
347 'owner': {'name': 'owner@'},
348 'cc': [{'name': 'cc1'}, {'name': 'cc2'}]
349 }
350 Note the title and descriptions fields of a
351 new bug are not optional, all other fields are.
352 @raises: ProjectHostingApiException, if request execution fails.
353
354 @return: The response body, which will contain the metadata of the
355 issue created, or an error response code and information
356 about a failure.
357 """
358 issues = self._codesite_service.issues()
359 request = issues.insert(projectId=self._project_name, sendEmail=True,
360 body=request_body)
361 return self._execute_request(request)
362
363
364 def update_issue(self, issue_id, request_body):
365 """
366 Convert the request body into an update on an issue.
367
368 @param request_body: A python dictionary with key-value pairs
369 that represent the fields of the update.
370 eg:
371 {
372 'content': 'comment to add',
373 'updates':
374 {
375 'labels': ['Type-Bug', 'another label'],
376 'owner': 'owner@',
377 'cc': ['cc1@', cc2@'],
378 }
379 }
380 Note the owner and cc fields need to be email
381 addresses the tracker recognizes.
382 @param issue_id: The id of the issue to update.
383 @raises: ProjectHostingApiException, if request execution fails.
384
385 @return: The response body, which will contain information about the
386 update of said issue, or an error response code and information
387 about a failure.
388 """
389 issues = self._codesite_service.issues()
390 request = issues.comments().insert(projectId=self._project_name,
beepse41cf4c2013-06-20 14:43:50 -0700391 issueId=issue_id, sendEmail=False,
beeps84e7bb42013-05-31 12:00:06 -0700392 body=request_body)
393 return self._execute_request(request)
394
395
396 def _populate_issue_updates(self, t_issue):
397 """
398 Populates a tracker issue with updates.
399
400 Any issue is useless without it's updates, since the updates will
401 contain both the summary and the comments. We need at least one of
402 those to successfully dedupe. The Api doesn't allow us to grab all this
403 information in one shot because viewing the comments on an issue
404 requires more authority than just viewing it's title.
405
406 @param t_issue: The basic tracker issue, to populate with updates.
407 @raises: ProjectHostingApiException, if request execution fails.
408
409 @returns: A tracker issue, with it's updates.
410 """
411 updates = self._list_updates(t_issue['id'])
412 t_issue['updates'] = [update['content'] for update in
413 self._get_property_values(updates)
414 if update.get('content')]
415 return t_issue
416
417
418 def get_tracker_issues_by_text(self, search_text, full_text=True,
Prashanth Be2dcdf02014-03-02 16:34:59 -0800419 include_dupes=False, label=None):
beeps84e7bb42013-05-31 12:00:06 -0700420 """
421 Find all Tracker issues that contain the specified search text.
422
423 @param search_text: Anchor text to use in the search.
424 @param full_text: True if we would like an extensive search through
425 issue comments. If False the search will be restricted
426 to just summaries and titles.
Fang Deng60184032013-08-16 10:02:08 -0700427 @param include_dupes: If True, search over both open issues as well as
428 closed issues whose status is 'Duplicate'. If False,
429 only search over open issues.
Prashanth Be2dcdf02014-03-02 16:34:59 -0800430 @param label: A string representing a single label to match.
431
beeps84e7bb42013-05-31 12:00:06 -0700432 @return: A list of issues that contain the search text, or an empty list
433 when we're either unable to list issues or none match the text.
434 """
435 issue_list = []
436 try:
Fang Deng60184032013-08-16 10:02:08 -0700437 search_space = 'all' if include_dupes else 'open'
Prashanth Be2dcdf02014-03-02 16:34:59 -0800438 feed = self.list_issues(q=search_text, can=search_space,
439 label=label)
beeps84e7bb42013-05-31 12:00:06 -0700440 except ProjectHostingApiException as e:
441 logging.error('Unable to search for issues with marker %s: %s',
442 search_text, e)
443 return issue_list
444
445 for t_issue in self._get_property_values(feed):
Fang Deng60184032013-08-16 10:02:08 -0700446 state = t_issue.get(constants.ISSUE_STATE)
447 status = t_issue.get(constants.ISSUE_STATUS)
448 is_open_or_dup = (state == constants.ISSUE_OPEN or
449 (state == constants.ISSUE_CLOSED
450 and status == constants.ISSUE_DUPLICATE))
beeps84e7bb42013-05-31 12:00:06 -0700451 # All valid issues will have an issue id we can use to retrieve
452 # more information about it. If we encounter a failure mode that
453 # returns a bad Http response code but doesn't throw an exception
454 # we won't find an issue id in the returned json.
Fang Deng60184032013-08-16 10:02:08 -0700455 if t_issue.get('id') and is_open_or_dup:
Prashanth Be2dcdf02014-03-02 16:34:59 -0800456 # TODO(beeps): If this method turns into a performance
457 # bottle neck yield each issue and refactor the reporter.
458 # For now passing all issues allows us to detect when
459 # deduping fails, because multiple issues will match a
460 # given query exactly.
beeps84e7bb42013-05-31 12:00:06 -0700461 try:
Prashanth Be2dcdf02014-03-02 16:34:59 -0800462 if full_text:
463 issue = Issue(self._populate_issue_updates(t_issue))
464 else:
465 issue = BaseIssue(t_issue)
beeps84e7bb42013-05-31 12:00:06 -0700466 except ProjectHostingApiException as e:
467 logging.error('Unable to list the updates of issue %s: %s',
Prashanth Be2dcdf02014-03-02 16:34:59 -0800468 t_issue.get('id'), str(e))
469 else:
470 issue_list.append(issue)
beeps84e7bb42013-05-31 12:00:06 -0700471 return issue_list
Fang Deng60184032013-08-16 10:02:08 -0700472
473
474 def get_tracker_issue_by_id(self, issue_id):
475 """
476 Returns an issue object given the id.
477
478 @param issue_id: A string representing the issue id.
479
480 @return: An Issue object on success or None on failure.
481 """
482 try:
483 t_issue = self._get_issue(issue_id)
484 return Issue(self._populate_issue_updates(t_issue))
485 except ProjectHostingApiException as e:
486 logging.error('Creation of an Issue object for %s fails: %s',
487 issue_id, str(e))
488 return None