Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2018 The Android Open Source Project |
| 3 | * |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | */ |
| 16 | package com.android.tradefed.config; |
| 17 | |
| 18 | import com.android.annotations.VisibleForTesting; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 19 | import com.android.tradefed.config.OptionSetter.OptionFieldsForName; |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 20 | import com.android.tradefed.config.remote.GcsRemoteFileResolver; |
| 21 | import com.android.tradefed.config.remote.IRemoteFileResolver; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 22 | import com.android.tradefed.log.LogUtil.CLog; |
| 23 | import com.android.tradefed.util.FileUtil; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 24 | |
| 25 | import java.io.File; |
| 26 | import java.lang.reflect.Field; |
| 27 | import java.util.ArrayList; |
| 28 | import java.util.Collection; |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 29 | import java.util.HashMap; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 30 | import java.util.HashSet; |
Julien Desprez | 8f0432f | 2019-01-04 13:47:54 -0800 | [diff] [blame^] | 31 | import java.util.LinkedHashMap; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 32 | import java.util.Map; |
Julien Desprez | 8f0432f | 2019-01-04 13:47:54 -0800 | [diff] [blame^] | 33 | import java.util.Map.Entry; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 34 | import java.util.Set; |
Julien Desprez | 15d9c7d | 2019-01-02 11:15:18 -0800 | [diff] [blame] | 35 | import java.util.concurrent.atomic.AtomicBoolean; |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 36 | |
| 37 | /** |
| 38 | * Class that helps resolving path to remote files. |
| 39 | * |
| 40 | * <p>For example: gs://bucket/path/file.txt will be resolved by downloading the file from the GCS |
| 41 | * bucket. |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 42 | */ |
| 43 | public class DynamicRemoteFileResolver { |
| 44 | |
Julien Desprez | 15d9c7d | 2019-01-02 11:15:18 -0800 | [diff] [blame] | 45 | public static final String DYNAMIC_RESOLVER = "dynamic-resolver"; |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 46 | private static final Map<String, IRemoteFileResolver> PROTOCOL_SUPPORT = new HashMap<>(); |
| 47 | |
| 48 | static { |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 49 | PROTOCOL_SUPPORT.put(GcsRemoteFileResolver.PROTOCOL, new GcsRemoteFileResolver()); |
| 50 | } |
Julien Desprez | 15d9c7d | 2019-01-02 11:15:18 -0800 | [diff] [blame] | 51 | // The configuration map being static, we only need to update it once per TF instance. |
| 52 | private static AtomicBoolean sIsUpdateDone = new AtomicBoolean(false); |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 53 | |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 54 | private Map<String, OptionFieldsForName> mOptionMap; |
| 55 | |
| 56 | /** Sets the map of options coming from {@link OptionSetter} */ |
| 57 | public void setOptionMap(Map<String, OptionFieldsForName> optionMap) { |
| 58 | mOptionMap = optionMap; |
| 59 | } |
| 60 | |
| 61 | /** |
| 62 | * Runs through all the {@link File} option type and check if their path should be resolved. |
| 63 | * |
| 64 | * @return The list of {@link File} that was resolved that way. |
| 65 | * @throws ConfigurationException |
| 66 | */ |
| 67 | public final Set<File> validateRemoteFilePath() throws ConfigurationException { |
| 68 | Set<File> downloadedFiles = new HashSet<>(); |
| 69 | try { |
| 70 | for (Map.Entry<String, OptionFieldsForName> optionPair : mOptionMap.entrySet()) { |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 71 | final OptionFieldsForName optionFields = optionPair.getValue(); |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 72 | for (Map.Entry<Object, Field> fieldEntry : optionFields) { |
| 73 | final Object obj = fieldEntry.getKey(); |
| 74 | final Field field = fieldEntry.getValue(); |
| 75 | final Option option = field.getAnnotation(Option.class); |
| 76 | if (option == null) { |
| 77 | continue; |
| 78 | } |
| 79 | // At this point, we know this is an option field; make sure it's set |
| 80 | field.setAccessible(true); |
| 81 | final Object value; |
| 82 | try { |
| 83 | value = field.get(obj); |
| 84 | } catch (IllegalAccessException e) { |
| 85 | throw new ConfigurationException( |
| 86 | String.format("internal error: %s", e.getMessage())); |
| 87 | } |
| 88 | |
| 89 | if (value == null) { |
| 90 | continue; |
| 91 | } else if (value instanceof File) { |
| 92 | File consideredFile = (File) value; |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 93 | File downloadedFile = resolveRemoteFiles(consideredFile, option); |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 94 | if (downloadedFile != null) { |
| 95 | downloadedFiles.add(downloadedFile); |
| 96 | // Replace the field value |
| 97 | try { |
| 98 | field.set(obj, downloadedFile); |
| 99 | } catch (IllegalAccessException e) { |
| 100 | CLog.e(e); |
| 101 | throw new ConfigurationException( |
| 102 | String.format( |
Julien Desprez | 6524966 | 2018-12-27 16:50:29 -0800 | [diff] [blame] | 103 | "Failed to download %s due to '%s'", |
| 104 | consideredFile.getPath(), e.getMessage()), |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 105 | e); |
| 106 | } |
| 107 | } |
| 108 | } else if (value instanceof Collection) { |
| 109 | Collection<Object> c = (Collection<Object>) value; |
| 110 | Collection<Object> copy = new ArrayList<>(c); |
| 111 | for (Object o : copy) { |
| 112 | if (o instanceof File) { |
| 113 | File consideredFile = (File) o; |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 114 | File downloadedFile = resolveRemoteFiles(consideredFile, option); |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 115 | if (downloadedFile != null) { |
| 116 | downloadedFiles.add(downloadedFile); |
| 117 | // TODO: See if order could be preserved. |
| 118 | c.remove(consideredFile); |
| 119 | c.add(downloadedFile); |
| 120 | } |
| 121 | } |
| 122 | } |
Julien Desprez | 8f0432f | 2019-01-04 13:47:54 -0800 | [diff] [blame^] | 123 | } else if (value instanceof Map) { |
| 124 | Map<Object, Object> m = (Map<Object, Object>) value; |
| 125 | Map<Object, Object> copy = new LinkedHashMap<>(m); |
| 126 | for (Entry<Object, Object> entry : copy.entrySet()) { |
| 127 | Object key = entry.getKey(); |
| 128 | Object val = entry.getValue(); |
| 129 | |
| 130 | Object finalKey = key; |
| 131 | Object finalVal = val; |
| 132 | if (key instanceof File) { |
| 133 | key = resolveRemoteFiles((File) key, option); |
| 134 | if (key != null) { |
| 135 | downloadedFiles.add((File) key); |
| 136 | finalKey = key; |
| 137 | } |
| 138 | } |
| 139 | if (val instanceof File) { |
| 140 | val = resolveRemoteFiles((File) val, option); |
| 141 | if (val != null) { |
| 142 | downloadedFiles.add((File) val); |
| 143 | finalVal = val; |
| 144 | } |
| 145 | } |
| 146 | |
| 147 | m.remove(entry.getKey()); |
| 148 | m.put(finalKey, finalVal); |
| 149 | } |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 150 | } |
Julien Desprez | 8f0432f | 2019-01-04 13:47:54 -0800 | [diff] [blame^] | 151 | // TODO: add support for multimap |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 152 | } |
| 153 | } |
| 154 | } catch (ConfigurationException e) { |
| 155 | // Clean up the files before throwing |
| 156 | for (File f : downloadedFiles) { |
| 157 | FileUtil.recursiveDelete(f); |
| 158 | } |
| 159 | throw e; |
| 160 | } |
| 161 | return downloadedFiles; |
| 162 | } |
| 163 | |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 164 | @VisibleForTesting |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 165 | IRemoteFileResolver getResolver(String protocol) { |
Julien Desprez | 15d9c7d | 2019-01-02 11:15:18 -0800 | [diff] [blame] | 166 | if (updateProtocols()) { |
| 167 | IGlobalConfiguration globalConfig = getGlobalConfig(); |
| 168 | Object o = globalConfig.getConfigurationObject(DYNAMIC_RESOLVER); |
| 169 | if (o != null) { |
| 170 | if (o instanceof IRemoteFileResolver) { |
| 171 | IRemoteFileResolver resolver = (IRemoteFileResolver) o; |
| 172 | CLog.d("Adding %s to supported remote file resolver", resolver); |
| 173 | PROTOCOL_SUPPORT.put(resolver.getSupportedProtocol(), resolver); |
| 174 | } else { |
| 175 | CLog.e("%s is not of type IRemoteFileResolver", o); |
| 176 | } |
| 177 | } |
| 178 | } |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 179 | return PROTOCOL_SUPPORT.get(protocol); |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 180 | } |
| 181 | |
Julien Desprez | 15d9c7d | 2019-01-02 11:15:18 -0800 | [diff] [blame] | 182 | @VisibleForTesting |
| 183 | boolean updateProtocols() { |
| 184 | return sIsUpdateDone.compareAndSet(false, true); |
| 185 | } |
| 186 | |
| 187 | @VisibleForTesting |
| 188 | IGlobalConfiguration getGlobalConfig() { |
| 189 | return GlobalConfiguration.getInstance(); |
| 190 | } |
| 191 | |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 192 | private File resolveRemoteFiles(File consideredFile, Option option) |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 193 | throws ConfigurationException { |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 194 | String path = consideredFile.getPath(); |
| 195 | String protocol = getProtocol(path); |
| 196 | IRemoteFileResolver resolver = getResolver(protocol); |
| 197 | if (resolver != null) { |
| 198 | return resolver.resolveRemoteFiles(consideredFile, option); |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 199 | } |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 200 | // Not a remote file |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 201 | return null; |
| 202 | } |
Julien Desprez | 27bcd4b | 2018-12-19 12:49:08 -0800 | [diff] [blame] | 203 | |
| 204 | /** |
| 205 | * Java URL doesn't recognize 'gs' as a protocol and throws an exception so we do the protocol |
| 206 | * extraction ourselves. |
| 207 | */ |
| 208 | private String getProtocol(String path) { |
| 209 | int index = path.indexOf(":/"); |
| 210 | if (index == -1) { |
| 211 | return ""; |
| 212 | } |
| 213 | return path.substring(0, index); |
| 214 | } |
Julien Desprez | d65e691 | 2018-12-13 15:20:52 -0800 | [diff] [blame] | 215 | } |