Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2018 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | package com.android.tradefed.config; |
| 17 | |
| 18 | import com.android.annotations.VisibleForTesting; |
Julien Desprez | 63cc539 | 2019-12-05 18:36:44 +0000 | [diff] [blame] | 19 | import com.android.tradefed.build.BuildRetrievalError; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 20 | import com.android.tradefed.config.OptionSetter.OptionFieldsForName; |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 21 | import com.android.tradefed.config.remote.GcsRemoteFileResolver; |
Julien Desprez | fb7b4f6 | 2019-07-03 14:05:29 -0700 | [diff] [blame] | 22 | import com.android.tradefed.config.remote.HttpRemoteFileResolver; |
| 23 | import com.android.tradefed.config.remote.HttpsRemoteFileResolver; |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 24 | import com.android.tradefed.config.remote.IRemoteFileResolver; |
Julien Desprez | 6f8cb8d | 2019-06-05 10:57:48 -0700 | [diff] [blame] | 25 | import com.android.tradefed.config.remote.LocalFileResolver; |
Julien Desprez | 2d01ae3 | 2020-02-27 09:46:23 -0800 | [diff] [blame] | 26 | import com.android.tradefed.device.ITestDevice; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 27 | import com.android.tradefed.log.LogUtil.CLog; |
| 28 | import com.android.tradefed.util.FileUtil; |
Julien Desprez | e30f0d9 | 2019-01-07 17:32:14 -0800 | [diff] [blame] | 29 | import com.android.tradefed.util.MultiMap; |
Julien Desprez | c18493d | 2019-06-13 09:49:29 -0700 | [diff] [blame] | 30 | import com.android.tradefed.util.ZipUtil; |
| 31 | import com.android.tradefed.util.ZipUtil2; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 32 | |
| 33 | import java.io.File; |
Julien Desprez | c18493d | 2019-06-13 09:49:29 -0700 | [diff] [blame] | 34 | import java.io.IOException; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 35 | import java.lang.reflect.Field; |
Julien Desprez | 13138de | 2019-06-10 16:57:19 -0700 | [diff] [blame] | 36 | import java.net.URI; |
| 37 | import java.net.URISyntaxException; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 38 | import java.util.ArrayList; |
| 39 | import java.util.Collection; |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 40 | import java.util.HashMap; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 41 | import java.util.HashSet; |
Julien Desprez | 8f0432f | 2019-01-04 13:47:54 -0800 | [diff] [blame] | 42 | import java.util.LinkedHashMap; |
Julien Desprez | e30f0d9 | 2019-01-07 17:32:14 -0800 | [diff] [blame] | 43 | import java.util.List; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 44 | import java.util.Map; |
Julien Desprez | 8f0432f | 2019-01-04 13:47:54 -0800 | [diff] [blame] | 45 | import java.util.Map.Entry; |
Julien Desprez | a71befa | 2020-02-10 14:24:17 -0800 | [diff] [blame] | 46 | import java.util.ServiceLoader; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 47 | import java.util.Set; |
Julien Desprez | 15d9c7d | 2019-01-02 11:15:18 -0800 | [diff] [blame] | 48 | import java.util.concurrent.atomic.AtomicBoolean; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 49 | |
| 50 | /** |
| 51 | * Class that helps resolving path to remote files. |
| 52 | * |
| 53 | * <p>For example: gs://bucket/path/file.txt will be resolved by downloading the file from the GCS |
| 54 | * bucket. |
Julien Desprez | a71befa | 2020-02-10 14:24:17 -0800 | [diff] [blame] | 55 | * |
| 56 | * <p>New protocols should be added to META_INF/services. |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 57 | */ |
| 58 | public class DynamicRemoteFileResolver { |
| 59 | |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 60 | private static final Map<String, IRemoteFileResolver> PROTOCOL_SUPPORT = new HashMap<>(); |
| 61 | |
| 62 | static { |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 63 | PROTOCOL_SUPPORT.put(GcsRemoteFileResolver.PROTOCOL, new GcsRemoteFileResolver()); |
Julien Desprez | 6f8cb8d | 2019-06-05 10:57:48 -0700 | [diff] [blame] | 64 | PROTOCOL_SUPPORT.put(LocalFileResolver.PROTOCOL, new LocalFileResolver()); |
Julien Desprez | fb7b4f6 | 2019-07-03 14:05:29 -0700 | [diff] [blame] | 65 | PROTOCOL_SUPPORT.put(HttpRemoteFileResolver.PROTOCOL_HTTP, new HttpRemoteFileResolver()); |
| 66 | PROTOCOL_SUPPORT.put(HttpsRemoteFileResolver.PROTOCOL_HTTPS, new HttpsRemoteFileResolver()); |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 67 | } |
Julien Desprez | 15d9c7d | 2019-01-02 11:15:18 -0800 | [diff] [blame] | 68 | // The configuration map being static, we only need to update it once per TF instance. |
| 69 | private static AtomicBoolean sIsUpdateDone = new AtomicBoolean(false); |
Julien Desprez | c18493d | 2019-06-13 09:49:29 -0700 | [diff] [blame] | 70 | // Query key for requesting to unzip a downloaded file automatically. |
| 71 | public static final String UNZIP_KEY = "unzip"; |
Julien Desprez | a0a4aa7 | 2019-06-18 11:36:22 -0700 | [diff] [blame] | 72 | // Query key for requesting a download to be optional, so if it fails we don't replace it. |
| 73 | public static final String OPTIONAL_KEY = "optional"; |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 74 | |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 75 | private Map<String, OptionFieldsForName> mOptionMap; |
Julien Desprez | 28f4af1 | 2020-02-28 11:30:00 -0800 | [diff] [blame] | 76 | // Populated from {@link ICommandOptions#getDynamicDownloadArgs()} |
| 77 | private Map<String, String> mExtraArgs = new LinkedHashMap<>(); |
Julien Desprez | 2d01ae3 | 2020-02-27 09:46:23 -0800 | [diff] [blame] | 78 | private ITestDevice mDevice; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 79 | |
| 80 | /** Sets the map of options coming from {@link OptionSetter} */ |
| 81 | public void setOptionMap(Map<String, OptionFieldsForName> optionMap) { |
| 82 | mOptionMap = optionMap; |
| 83 | } |
| 84 | |
Julien Desprez | 2d01ae3 | 2020-02-27 09:46:23 -0800 | [diff] [blame] | 85 | /** Sets the device under tests */ |
| 86 | public void setDevice(ITestDevice device) { |
| 87 | mDevice = device; |
| 88 | } |
| 89 | |
Julien Desprez | 28f4af1 | 2020-02-28 11:30:00 -0800 | [diff] [blame] | 90 | /** Add extra args for the query. */ |
| 91 | public void addExtraArgs(Map<String, String> extraArgs) { |
| 92 | mExtraArgs.putAll(extraArgs); |
| 93 | } |
| 94 | |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 95 | /** |
| 96 | * Runs through all the {@link File} option type and check if their path should be resolved. |
| 97 | * |
| 98 | * @return The list of {@link File} that was resolved that way. |
Julien Desprez | 63cc539 | 2019-12-05 18:36:44 +0000 | [diff] [blame] | 99 | * @throws BuildRetrievalError |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 100 | */ |
Julien Desprez | 63cc539 | 2019-12-05 18:36:44 +0000 | [diff] [blame] | 101 | public final Set<File> validateRemoteFilePath() throws BuildRetrievalError { |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 102 | Set<File> downloadedFiles = new HashSet<>(); |
| 103 | try { |
Julien Desprez | cf0f0b2 | 2019-09-18 14:34:28 -0700 | [diff] [blame] | 104 | Map<Field, Object> fieldSeen = new HashMap<>(); |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 105 | for (Map.Entry<String, OptionFieldsForName> optionPair : mOptionMap.entrySet()) { |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 106 | final OptionFieldsForName optionFields = optionPair.getValue(); |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 107 | for (Map.Entry<Object, Field> fieldEntry : optionFields) { |
Julien Desprez | a0a4aa7 | 2019-06-18 11:36:22 -0700 | [diff] [blame] | 108 | |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 109 | final Object obj = fieldEntry.getKey(); |
| 110 | final Field field = fieldEntry.getValue(); |
| 111 | final Option option = field.getAnnotation(Option.class); |
| 112 | if (option == null) { |
| 113 | continue; |
| 114 | } |
| 115 | // At this point, we know this is an option field; make sure it's set |
| 116 | field.setAccessible(true); |
| 117 | final Object value; |
| 118 | try { |
| 119 | value = field.get(obj); |
Julien Desprez | c054188 | 2019-07-10 08:36:02 -0700 | [diff] [blame] | 120 | if (value == null) { |
| 121 | continue; |
| 122 | } |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 123 | } catch (IllegalAccessException e) { |
Julien Desprez | 63cc539 | 2019-12-05 18:36:44 +0000 | [diff] [blame] | 124 | throw new BuildRetrievalError( |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 125 | String.format("internal error: %s", e.getMessage())); |
| 126 | } |
| 127 | |
Julien Desprez | cf0f0b2 | 2019-09-18 14:34:28 -0700 | [diff] [blame] | 128 | if (fieldSeen.get(field) != null && fieldSeen.get(field).equals(obj)) { |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 129 | continue; |
Julien Desprez | c054188 | 2019-07-10 08:36:02 -0700 | [diff] [blame] | 130 | } |
| 131 | // Keep track of the field set on each object |
Julien Desprez | cf0f0b2 | 2019-09-18 14:34:28 -0700 | [diff] [blame] | 132 | fieldSeen.put(field, obj); |
Julien Desprez | c054188 | 2019-07-10 08:36:02 -0700 | [diff] [blame] | 133 | |
| 134 | if (value instanceof File) { |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 135 | File consideredFile = (File) value; |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 136 | File downloadedFile = resolveRemoteFiles(consideredFile, option); |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 137 | if (downloadedFile != null) { |
| 138 | downloadedFiles.add(downloadedFile); |
| 139 | // Replace the field value |
| 140 | try { |
| 141 | field.set(obj, downloadedFile); |
| 142 | } catch (IllegalAccessException e) { |
| 143 | CLog.e(e); |
Julien Desprez | 63cc539 | 2019-12-05 18:36:44 +0000 | [diff] [blame] | 144 | throw new BuildRetrievalError( |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 145 | String.format( |
Julien Desprez | 6524966 | 2018-12-27 16:50:29 -0800 | [diff] [blame] | 146 | "Failed to download %s due to '%s'", |
| 147 | consideredFile.getPath(), e.getMessage()), |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 148 | e); |
| 149 | } |
| 150 | } |
| 151 | } else if (value instanceof Collection) { |
| 152 | Collection<Object> c = (Collection<Object>) value; |
| 153 | Collection<Object> copy = new ArrayList<>(c); |
| 154 | for (Object o : copy) { |
| 155 | if (o instanceof File) { |
| 156 | File consideredFile = (File) o; |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 157 | File downloadedFile = resolveRemoteFiles(consideredFile, option); |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 158 | if (downloadedFile != null) { |
| 159 | downloadedFiles.add(downloadedFile); |
| 160 | // TODO: See if order could be preserved. |
| 161 | c.remove(consideredFile); |
| 162 | c.add(downloadedFile); |
| 163 | } |
| 164 | } |
| 165 | } |
Julien Desprez | 8f0432f | 2019-01-04 13:47:54 -0800 | [diff] [blame] | 166 | } else if (value instanceof Map) { |
| 167 | Map<Object, Object> m = (Map<Object, Object>) value; |
| 168 | Map<Object, Object> copy = new LinkedHashMap<>(m); |
| 169 | for (Entry<Object, Object> entry : copy.entrySet()) { |
| 170 | Object key = entry.getKey(); |
| 171 | Object val = entry.getValue(); |
| 172 | |
| 173 | Object finalKey = key; |
| 174 | Object finalVal = val; |
| 175 | if (key instanceof File) { |
| 176 | key = resolveRemoteFiles((File) key, option); |
| 177 | if (key != null) { |
| 178 | downloadedFiles.add((File) key); |
| 179 | finalKey = key; |
| 180 | } |
| 181 | } |
| 182 | if (val instanceof File) { |
| 183 | val = resolveRemoteFiles((File) val, option); |
| 184 | if (val != null) { |
| 185 | downloadedFiles.add((File) val); |
| 186 | finalVal = val; |
| 187 | } |
| 188 | } |
| 189 | |
| 190 | m.remove(entry.getKey()); |
| 191 | m.put(finalKey, finalVal); |
| 192 | } |
Julien Desprez | e30f0d9 | 2019-01-07 17:32:14 -0800 | [diff] [blame] | 193 | } else if (value instanceof MultiMap) { |
| 194 | MultiMap<Object, Object> m = (MultiMap<Object, Object>) value; |
Julien Desprez | e218c72 | 2020-03-27 13:34:45 -0700 | [diff] [blame] | 195 | synchronized (m) { |
| 196 | MultiMap<Object, Object> copy = new MultiMap<>(m); |
| 197 | for (Object key : copy.keySet()) { |
| 198 | List<Object> mapValues = copy.get(key); |
Julien Desprez | e30f0d9 | 2019-01-07 17:32:14 -0800 | [diff] [blame] | 199 | |
Julien Desprez | e218c72 | 2020-03-27 13:34:45 -0700 | [diff] [blame] | 200 | m.remove(key); |
| 201 | Object finalKey = key; |
| 202 | if (key instanceof File) { |
| 203 | key = resolveRemoteFiles((File) key, option); |
| 204 | if (key != null) { |
| 205 | downloadedFiles.add((File) key); |
| 206 | finalKey = key; |
Julien Desprez | e30f0d9 | 2019-01-07 17:32:14 -0800 | [diff] [blame] | 207 | } |
| 208 | } |
Julien Desprez | e218c72 | 2020-03-27 13:34:45 -0700 | [diff] [blame] | 209 | for (Object mapValue : mapValues) { |
| 210 | if (mapValue instanceof File) { |
| 211 | File f = resolveRemoteFiles((File) mapValue, option); |
| 212 | if (f != null) { |
| 213 | downloadedFiles.add(f); |
| 214 | mapValue = f; |
| 215 | } |
| 216 | } |
| 217 | m.put(finalKey, mapValue); |
| 218 | } |
Julien Desprez | e30f0d9 | 2019-01-07 17:32:14 -0800 | [diff] [blame] | 219 | } |
| 220 | } |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 221 | } |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 222 | } |
| 223 | } |
Julien Desprez | e218c72 | 2020-03-27 13:34:45 -0700 | [diff] [blame] | 224 | } catch (RuntimeException | BuildRetrievalError e) { |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 225 | // Clean up the files before throwing |
| 226 | for (File f : downloadedFiles) { |
| 227 | FileUtil.recursiveDelete(f); |
| 228 | } |
| 229 | throw e; |
| 230 | } |
| 231 | return downloadedFiles; |
| 232 | } |
| 233 | |
Dan Shi | f663f59 | 2019-06-25 16:17:16 -0700 | [diff] [blame] | 234 | /** |
| 235 | * Download the files matching given filters in a remote zip file. |
| 236 | * |
| 237 | * <p>A file inside the remote zip file is only downloaded if its path matches any of the |
| 238 | * include filters but not the exclude filters. |
| 239 | * |
| 240 | * @param destDir the file to place the downloaded contents into. |
| 241 | * @param remoteZipFilePath the remote path to the zip file to download, relative to an |
| 242 | * implementation specific root. |
| 243 | * @param includeFilters a list of regex strings to download matching files. A file's path |
| 244 | * matching any filter will be downloaded. |
| 245 | * @param excludeFilters a list of regex strings to skip downloading matching files. A file's |
| 246 | * path matching any filter will not be downloaded. |
Julien Desprez | 63cc539 | 2019-12-05 18:36:44 +0000 | [diff] [blame] | 247 | * @throws BuildRetrievalError if files could not be downloaded. |
Dan Shi | f663f59 | 2019-06-25 16:17:16 -0700 | [diff] [blame] | 248 | */ |
| 249 | public void resolvePartialDownloadZip( |
| 250 | File destDir, |
| 251 | String remoteZipFilePath, |
| 252 | List<String> includeFilters, |
| 253 | List<String> excludeFilters) |
Julien Desprez | 63cc539 | 2019-12-05 18:36:44 +0000 | [diff] [blame] | 254 | throws BuildRetrievalError { |
Dan Shi | f663f59 | 2019-06-25 16:17:16 -0700 | [diff] [blame] | 255 | Map<String, String> queryArgs; |
| 256 | String protocol; |
| 257 | try { |
| 258 | URI uri = new URI(remoteZipFilePath); |
| 259 | protocol = uri.getScheme(); |
| 260 | queryArgs = parseQuery(uri.getQuery()); |
| 261 | } catch (URISyntaxException e) { |
Julien Desprez | 63cc539 | 2019-12-05 18:36:44 +0000 | [diff] [blame] | 262 | throw new BuildRetrievalError( |
Dan Shi | f663f59 | 2019-06-25 16:17:16 -0700 | [diff] [blame] | 263 | String.format( |
| 264 | "Failed to parse the remote zip file path: %s", remoteZipFilePath), |
| 265 | e); |
| 266 | } |
| 267 | IRemoteFileResolver resolver = getResolver(protocol); |
| 268 | |
| 269 | queryArgs.put("partial_download_dir", destDir.getAbsolutePath()); |
| 270 | if (includeFilters != null) { |
| 271 | queryArgs.put("include_filters", String.join(";", includeFilters)); |
| 272 | } |
| 273 | if (excludeFilters != null) { |
| 274 | queryArgs.put("exclude_filters", String.join(";", excludeFilters)); |
| 275 | } |
| 276 | // Downloaded individual files should be saved to destDir, return value is not needed. |
| 277 | try { |
Julien Desprez | 2d01ae3 | 2020-02-27 09:46:23 -0800 | [diff] [blame] | 278 | resolver.setPrimaryDevice(mDevice); |
Julien Desprez | 3b318ad | 2020-01-13 10:45:12 -0800 | [diff] [blame] | 279 | resolver.resolveRemoteFiles(new File(remoteZipFilePath), queryArgs); |
Julien Desprez | 63cc539 | 2019-12-05 18:36:44 +0000 | [diff] [blame] | 280 | } catch (BuildRetrievalError e) { |
Dan Shi | f663f59 | 2019-06-25 16:17:16 -0700 | [diff] [blame] | 281 | if (isOptional(queryArgs)) { |
| 282 | CLog.d( |
| 283 | "Failed to partially download '%s' but marked optional so skipping: %s", |
| 284 | remoteZipFilePath, e.getMessage()); |
| 285 | } else { |
| 286 | throw e; |
| 287 | } |
| 288 | } |
| 289 | } |
| 290 | |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 291 | @VisibleForTesting |
Julien Desprez | 9b47723 | 2019-03-07 11:08:25 -0800 | [diff] [blame] | 292 | protected IRemoteFileResolver getResolver(String protocol) { |
Julien Desprez | 15d9c7d | 2019-01-02 11:15:18 -0800 | [diff] [blame] | 293 | if (updateProtocols()) { |
Julien Desprez | a71befa | 2020-02-10 14:24:17 -0800 | [diff] [blame] | 294 | // Use the service loader to find all the implementations. |
| 295 | ServiceLoader<IRemoteFileResolver> serviceLoader = |
| 296 | ServiceLoader.load(IRemoteFileResolver.class); |
| 297 | for (IRemoteFileResolver resolver : serviceLoader) { |
| 298 | PROTOCOL_SUPPORT.putIfAbsent(resolver.getSupportedProtocol(), resolver); |
| 299 | } |
Julien Desprez | 15d9c7d | 2019-01-02 11:15:18 -0800 | [diff] [blame] | 300 | } |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 301 | return PROTOCOL_SUPPORT.get(protocol); |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 302 | } |
| 303 | |
Julien Desprez | 15d9c7d | 2019-01-02 11:15:18 -0800 | [diff] [blame] | 304 | @VisibleForTesting |
Julien Desprez | 9b47723 | 2019-03-07 11:08:25 -0800 | [diff] [blame] | 305 | protected boolean updateProtocols() { |
Julien Desprez | 15d9c7d | 2019-01-02 11:15:18 -0800 | [diff] [blame] | 306 | return sIsUpdateDone.compareAndSet(false, true); |
| 307 | } |
| 308 | |
| 309 | @VisibleForTesting |
| 310 | IGlobalConfiguration getGlobalConfig() { |
| 311 | return GlobalConfiguration.getInstance(); |
| 312 | } |
| 313 | |
Julien Desprez | c18493d | 2019-06-13 09:49:29 -0700 | [diff] [blame] | 314 | /** |
| 315 | * Utility that allows to check whether or not a file should be unzip and unzip it if required. |
| 316 | */ |
| 317 | public static final File unzipIfRequired(File downloadedFile, Map<String, String> query) |
| 318 | throws IOException { |
| 319 | String unzipValue = query.get(UNZIP_KEY); |
| 320 | if (unzipValue != null && "true".equals(unzipValue.toLowerCase())) { |
| 321 | // File was requested to be unzipped. |
| 322 | if (ZipUtil.isZipFileValid(downloadedFile, false)) { |
| 323 | File unzipped = |
| 324 | ZipUtil2.extractZipToTemp( |
| 325 | downloadedFile, FileUtil.getBaseName(downloadedFile.getName())); |
| 326 | FileUtil.deleteFile(downloadedFile); |
| 327 | return unzipped; |
| 328 | } else { |
| 329 | CLog.w("%s was requested to be unzipped but is not a valid zip.", downloadedFile); |
| 330 | } |
| 331 | } |
| 332 | // Return the original file untouched |
| 333 | return downloadedFile; |
| 334 | } |
| 335 | |
Julien Desprez | 63cc539 | 2019-12-05 18:36:44 +0000 | [diff] [blame] | 336 | private File resolveRemoteFiles(File consideredFile, Option option) throws BuildRetrievalError { |
Julien Desprez | 13138de | 2019-06-10 16:57:19 -0700 | [diff] [blame] | 337 | File fileToResolve; |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 338 | String path = consideredFile.getPath(); |
Julien Desprez | 13138de | 2019-06-10 16:57:19 -0700 | [diff] [blame] | 339 | String protocol; |
| 340 | Map<String, String> query; |
| 341 | try { |
| 342 | URI uri = new URI(path); |
| 343 | protocol = uri.getScheme(); |
| 344 | query = parseQuery(uri.getQuery()); |
| 345 | fileToResolve = new File(protocol + ":" + uri.getPath()); |
| 346 | } catch (URISyntaxException e) { |
| 347 | CLog.e(e); |
| 348 | return null; |
| 349 | } |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 350 | IRemoteFileResolver resolver = getResolver(protocol); |
| 351 | if (resolver != null) { |
Julien Desprez | a0a4aa7 | 2019-06-18 11:36:22 -0700 | [diff] [blame] | 352 | try { |
Julien Desprez | 9b9baf8 | 2019-12-17 11:43:33 -0800 | [diff] [blame] | 353 | CLog.d( |
| 354 | "Considering option '%s' with path: '%s' for download.", |
| 355 | option.name(), path); |
Julien Desprez | 28f4af1 | 2020-02-28 11:30:00 -0800 | [diff] [blame] | 356 | // Overrides query args |
| 357 | query.putAll(mExtraArgs); |
Hadrien Zalek | 536f846 | 2020-05-02 00:45:47 -0700 | [diff] [blame^] | 358 | resolver.setPrimaryDevice(mDevice); |
Julien Desprez | 3b318ad | 2020-01-13 10:45:12 -0800 | [diff] [blame] | 359 | return resolver.resolveRemoteFiles(fileToResolve, query); |
Julien Desprez | 63cc539 | 2019-12-05 18:36:44 +0000 | [diff] [blame] | 360 | } catch (BuildRetrievalError e) { |
Julien Desprez | a0a4aa7 | 2019-06-18 11:36:22 -0700 | [diff] [blame] | 361 | if (isOptional(query)) { |
| 362 | CLog.d( |
| 363 | "Failed to resolve '%s' but marked optional so skipping: %s", |
| 364 | fileToResolve, e.getMessage()); |
| 365 | } else { |
| 366 | throw e; |
| 367 | } |
| 368 | } |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 369 | } |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 370 | // Not a remote file |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 371 | return null; |
| 372 | } |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 373 | |
| 374 | /** |
Julien Desprez | 13138de | 2019-06-10 16:57:19 -0700 | [diff] [blame] | 375 | * Parse a URL query style. Delimited by &, and map values represented by =. Example: |
| 376 | * ?key=value&key2=value2 |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 377 | */ |
Julien Desprez | 13138de | 2019-06-10 16:57:19 -0700 | [diff] [blame] | 378 | private Map<String, String> parseQuery(String query) { |
| 379 | Map<String, String> values = new HashMap<>(); |
| 380 | if (query == null) { |
| 381 | return values; |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 382 | } |
Julien Desprez | 13138de | 2019-06-10 16:57:19 -0700 | [diff] [blame] | 383 | for (String maps : query.split("&")) { |
| 384 | String[] keyVal = maps.split("="); |
| 385 | values.put(keyVal[0], keyVal[1]); |
| 386 | } |
| 387 | return values; |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 388 | } |
Julien Desprez | a0a4aa7 | 2019-06-18 11:36:22 -0700 | [diff] [blame] | 389 | |
| 390 | /** Whether or not a link was requested as optional. */ |
| 391 | private boolean isOptional(Map<String, String> query) { |
| 392 | String value = query.get(OPTIONAL_KEY); |
| 393 | if (value == null) { |
| 394 | return false; |
| 395 | } |
| 396 | return "true".equals(value.toLowerCase()); |
| 397 | } |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 398 | } |