(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 1 | # Copyright (C) 2018 The Android Open Source Project |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
| 14 | """Module to update packages from GitHub archive.""" |
| 15 | |
| 16 | |
| 17 | import json |
| 18 | import re |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 19 | import urllib.request |
| 20 | |
| 21 | import archive_utils |
| 22 | import fileutils |
Haibo Huang | 8845e1e | 2018-09-06 17:02:45 -0700 | [diff] [blame] | 23 | import git_utils |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 24 | import metadata_pb2 # pylint: disable=import-error |
| 25 | import updater_utils |
| 26 | |
| 27 | GITHUB_URL_PATTERN = (r'^https:\/\/github.com\/([-\w]+)\/([-\w]+)\/' + |
| 28 | r'(releases\/download\/|archive\/)') |
| 29 | GITHUB_URL_RE = re.compile(GITHUB_URL_PATTERN) |
| 30 | |
| 31 | |
Haibo Huang | 9dcade4 | 2018-08-03 11:52:25 -0700 | [diff] [blame] | 32 | def _edit_distance(str1, str2): |
| 33 | prev = list(range(0, len(str2) + 1)) |
| 34 | for i, chr1 in enumerate(str1): |
| 35 | cur = [i + 1] |
| 36 | for j, chr2 in enumerate(str2): |
| 37 | if chr1 == chr2: |
| 38 | cur.append(prev[j]) |
| 39 | else: |
| 40 | cur.append(min(prev[j + 1], prev[j], cur[j]) + 1) |
| 41 | prev = cur |
| 42 | return prev[len(str2)] |
| 43 | |
| 44 | |
| 45 | def choose_best_url(urls, previous_url): |
| 46 | """Returns the best url to download from a list of candidate urls. |
| 47 | |
| 48 | This function calculates similarity between previous url and each of new |
| 49 | urls. And returns the one best matches previous url. |
| 50 | |
| 51 | Similarity is measured by editing distance. |
| 52 | |
| 53 | Args: |
| 54 | urls: Array of candidate urls. |
| 55 | previous_url: String of the url used previously. |
| 56 | |
| 57 | Returns: |
| 58 | One url from `urls`. |
| 59 | """ |
| 60 | return min(urls, default=None, |
| 61 | key=lambda url: _edit_distance( |
| 62 | url, previous_url)) |
| 63 | |
| 64 | |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 65 | class GithubArchiveUpdater(): |
| 66 | """Updater for archives from GitHub. |
| 67 | |
| 68 | This updater supports release archives in GitHub. Version is determined by |
| 69 | release name in GitHub. |
| 70 | """ |
| 71 | |
| 72 | VERSION_FIELD = 'tag_name' |
| 73 | |
| 74 | def __init__(self, url, proj_path, metadata): |
| 75 | self.proj_path = proj_path |
| 76 | self.metadata = metadata |
| 77 | self.old_url = url |
| 78 | self.owner = None |
| 79 | self.repo = None |
Haibo Huang | 8845e1e | 2018-09-06 17:02:45 -0700 | [diff] [blame] | 80 | self.new_version = None |
| 81 | self.new_url = None |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 82 | self._parse_url(url) |
| 83 | |
| 84 | def _parse_url(self, url): |
| 85 | if url.type != metadata_pb2.URL.ARCHIVE: |
| 86 | raise ValueError('Only archive url from Github is supported.') |
| 87 | match = GITHUB_URL_RE.match(url.value) |
| 88 | if match is None: |
| 89 | raise ValueError('Url format is not supported.') |
| 90 | try: |
| 91 | self.owner, self.repo = match.group(1, 2) |
| 92 | except IndexError: |
| 93 | raise ValueError('Url format is not supported.') |
| 94 | |
Haibo Huang | 39aaab6 | 2019-01-25 12:23:03 -0800 | [diff] [blame] | 95 | def _fetch_latest_version(self): |
Haibo Huang | 8845e1e | 2018-09-06 17:02:45 -0700 | [diff] [blame] | 96 | """Checks upstream and gets the latest release tag.""" |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 97 | |
| 98 | url = 'https://api.github.com/repos/{}/{}/releases/latest'.format( |
| 99 | self.owner, self.repo) |
| 100 | with urllib.request.urlopen(url) as request: |
Haibo Huang | 8845e1e | 2018-09-06 17:02:45 -0700 | [diff] [blame] | 101 | data = json.loads(request.read().decode()) |
| 102 | self.new_version = data[self.VERSION_FIELD] |
| 103 | |
| 104 | supported_assets = [ |
| 105 | a['browser_download_url'] for a in data['assets'] |
| 106 | if archive_utils.is_supported_archive(a['browser_download_url'])] |
| 107 | |
| 108 | # Adds source code urls. |
| 109 | supported_assets.append( |
| 110 | 'https://github.com/{}/{}/archive/{}.tar.gz'.format( |
| 111 | self.owner, self.repo, data.get('tag_name'))) |
| 112 | supported_assets.append( |
| 113 | 'https://github.com/{}/{}/archive/{}.zip'.format( |
| 114 | self.owner, self.repo, data.get('tag_name'))) |
| 115 | |
| 116 | self.new_url = choose_best_url(supported_assets, self.old_url.value) |
| 117 | |
Haibo Huang | 39aaab6 | 2019-01-25 12:23:03 -0800 | [diff] [blame] | 118 | def _fetch_latest_commit(self): |
Haibo Huang | 8845e1e | 2018-09-06 17:02:45 -0700 | [diff] [blame] | 119 | """Checks upstream and gets the latest commit to master.""" |
| 120 | |
| 121 | url = 'https://api.github.com/repos/{}/{}/commits/master'.format( |
| 122 | self.owner, self.repo) |
| 123 | with urllib.request.urlopen(url) as request: |
| 124 | data = json.loads(request.read().decode()) |
| 125 | self.new_version = data['sha'] |
| 126 | self.new_url = 'https://github.com/{}/{}/archive/{}.zip'.format( |
| 127 | self.owner, self.repo, self.new_version) |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 128 | |
Haibo Huang | 39aaab6 | 2019-01-25 12:23:03 -0800 | [diff] [blame] | 129 | def get_current_version(self): |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 130 | """Returns the latest version name recorded in METADATA.""" |
| 131 | return self.metadata.third_party.version |
| 132 | |
Haibo Huang | 39aaab6 | 2019-01-25 12:23:03 -0800 | [diff] [blame] | 133 | def get_latest_version(self): |
| 134 | """Returns the latest version name in upstream.""" |
| 135 | return self.new_version |
| 136 | |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 137 | def _write_metadata(self, url, path): |
| 138 | updated_metadata = metadata_pb2.MetaData() |
| 139 | updated_metadata.CopyFrom(self.metadata) |
Haibo Huang | 8845e1e | 2018-09-06 17:02:45 -0700 | [diff] [blame] | 140 | updated_metadata.third_party.version = self.new_version |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 141 | for metadata_url in updated_metadata.third_party.url: |
| 142 | if metadata_url == self.old_url: |
| 143 | metadata_url.value = url |
| 144 | fileutils.write_metadata(path, updated_metadata) |
| 145 | |
Haibo Huang | 24950e7 | 2018-06-29 14:53:39 -0700 | [diff] [blame] | 146 | def check(self): |
| 147 | """Checks update for package. |
| 148 | |
| 149 | Returns True if a new version is available. |
| 150 | """ |
Haibo Huang | 39aaab6 | 2019-01-25 12:23:03 -0800 | [diff] [blame] | 151 | current = self.get_current_version() |
Haibo Huang | 8845e1e | 2018-09-06 17:02:45 -0700 | [diff] [blame] | 152 | if git_utils.is_commit(current): |
Haibo Huang | 39aaab6 | 2019-01-25 12:23:03 -0800 | [diff] [blame] | 153 | self._fetch_latest_commit() |
Haibo Huang | 8845e1e | 2018-09-06 17:02:45 -0700 | [diff] [blame] | 154 | else: |
Haibo Huang | 39aaab6 | 2019-01-25 12:23:03 -0800 | [diff] [blame] | 155 | self._fetch_latest_version() |
Haibo Huang | 24950e7 | 2018-06-29 14:53:39 -0700 | [diff] [blame] | 156 | print('Current version: {}. Latest version: {}'.format( |
Haibo Huang | 8845e1e | 2018-09-06 17:02:45 -0700 | [diff] [blame] | 157 | current, self.new_version), end='') |
Haibo Huang | 24950e7 | 2018-06-29 14:53:39 -0700 | [diff] [blame] | 158 | |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 159 | def update(self): |
| 160 | """Updates the package. |
| 161 | |
Haibo Huang | 24950e7 | 2018-06-29 14:53:39 -0700 | [diff] [blame] | 162 | Has to call check() before this function. |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 163 | """ |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 164 | temporary_dir = None |
| 165 | try: |
Haibo Huang | 8845e1e | 2018-09-06 17:02:45 -0700 | [diff] [blame] | 166 | temporary_dir = archive_utils.download_and_extract(self.new_url) |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 167 | package_dir = archive_utils.find_archive_root(temporary_dir) |
Haibo Huang | 8845e1e | 2018-09-06 17:02:45 -0700 | [diff] [blame] | 168 | self._write_metadata(self.new_url, package_dir) |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 169 | updater_utils.replace_package(package_dir, self.proj_path) |
| 170 | finally: |
Elliott Hughes | 1c3a4f4 | 2018-08-03 15:35:48 -0700 | [diff] [blame] | 171 | # Don't remove the temporary directory, or it'll be impossible |
| 172 | # to debug the failure... |
| 173 | # shutil.rmtree(temporary_dir, ignore_errors=True) |
(raulenrique) | dfdda47 | 2018-06-04 12:02:29 -0700 | [diff] [blame] | 174 | urllib.request.urlcleanup() |