Jack Neus | c474c9c | 2021-07-26 23:08:54 +0000 | [diff] [blame] | 1 | # Copyright (C) 2021 The Android Open Source Project |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | # you may not use this file except in compliance with the License. |
| 5 | # You may obtain a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. |
| 14 | |
| 15 | """This module contains functions used to fetch files from various sources.""" |
| 16 | |
| 17 | import subprocess |
| 18 | import sys |
| 19 | from urllib.parse import urlparse |
Matt Story | 11b30b9 | 2021-10-26 10:56:13 -0400 | [diff] [blame] | 20 | from urllib.request import urlopen |
| 21 | |
Jack Neus | c474c9c | 2021-07-26 23:08:54 +0000 | [diff] [blame] | 22 | |
Jack Neus | 1988385 | 2021-10-25 22:38:44 +0000 | [diff] [blame] | 23 | def fetch_file(url, verbose=False): |
Jack Neus | c474c9c | 2021-07-26 23:08:54 +0000 | [diff] [blame] | 24 | """Fetch a file from the specified source using the appropriate protocol. |
| 25 | |
| 26 | Returns: |
| 27 | The contents of the file as bytes. |
| 28 | """ |
| 29 | scheme = urlparse(url).scheme |
| 30 | if scheme == 'gs': |
| 31 | cmd = ['gsutil', 'cat', url] |
| 32 | try: |
| 33 | result = subprocess.run( |
Jack Neus | 1988385 | 2021-10-25 22:38:44 +0000 | [diff] [blame] | 34 | cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, |
| 35 | check=True) |
| 36 | if result.stderr and verbose: |
| 37 | print('warning: non-fatal error running "gsutil": %s' % result.stderr, |
| 38 | file=sys.stderr) |
Jack Neus | c474c9c | 2021-07-26 23:08:54 +0000 | [diff] [blame] | 39 | return result.stdout |
| 40 | except subprocess.CalledProcessError as e: |
Jack Neus | 1988385 | 2021-10-25 22:38:44 +0000 | [diff] [blame] | 41 | print('fatal: error running "gsutil": %s' % e.stderr, |
Jack Neus | c474c9c | 2021-07-26 23:08:54 +0000 | [diff] [blame] | 42 | file=sys.stderr) |
| 43 | sys.exit(1) |
Matt Story | 11b30b9 | 2021-10-26 10:56:13 -0400 | [diff] [blame] | 44 | with urlopen(url) as f: |
| 45 | return f.read() |