[fix] Change GCSFileDownloader to use gcs library.
Test: Unit test, function test.
Bug: 113347158
Change-Id: I33b2b32dfcc3dc22aa37097a3c1506cd15220cec
diff --git a/.classpath b/.classpath
index 890e9d5..fe8a63d 100644
--- a/.classpath
+++ b/.classpath
@@ -25,7 +25,7 @@
<classpathentry kind="var" path="TRADEFED_ROOT/out/soong/.intermediates/external/perfetto/perfetto_config-full/linux_glibc_common/combined/perfetto_config-full.jar"/>
<classpathentry kind="var" path="TRADEFED_ROOT/out/soong/.intermediates/tools/tradefederation/core/tradefed-grpc-lib-1.0.1/linux_glibc_common/combined/tradefed-grpc-lib-1.0.1.jar"/>
<classpathentry kind="var" path="TRADEFED_ROOT/out/soong/.intermediates/platform_testing/libraries/aoa-helper/aoa-helper/linux_glibc_common/combined/aoa-helper.jar"/>
- <classpathentry kind="var" path="TRADEFED_ROOT/prebuilts/tools/common/google-api-java-client/1.23.0/google-api-java-client-min-repackaged-1.23.0.jar"/>
+ <classpathentry exported="true" kind="var" path="TRADEFED_ROOT/prebuilts/tools/common/google-api-java-client/1.23.0/google-api-java-client-min-repackaged-1.23.0.jar"/>
<classpathentry kind="var" path="TRADEFED_ROOT/out/soong/.intermediates/prebuilts/tools/common/google-api-services-compute/google-api-services-compute/linux_glibc_common/combined/google-api-services-compute.jar"/>
<classpathentry kind="var" path="TRADEFED_ROOT/prebuilts/misc/common/frameworks/platformprotos-prebuilt.jar"/>
<classpathentry kind="output" path="bin"/>
diff --git a/src/com/android/tradefed/util/FileUtil.java b/src/com/android/tradefed/util/FileUtil.java
index 607c4fe..2ba2ba8 100644
--- a/src/com/android/tradefed/util/FileUtil.java
+++ b/src/com/android/tradefed/util/FileUtil.java
@@ -1036,6 +1036,18 @@
}
/**
+ * Helper method to calculate base64 md5 for a file.
+ *
+ * @param file
+ * @return md5 of the file
+ * @throws IOException
+ */
+ public static String calculateBase64Md5(File file) throws IOException {
+ FileInputStream inputSource = new FileInputStream(file);
+ return StreamUtil.calculateBase64Md5(inputSource);
+ }
+
+ /**
* Converts an integer representing unix mode to a set of {@link PosixFilePermission}s
*/
public static Set<PosixFilePermission> unixModeToPosix(int mode) {
diff --git a/src/com/android/tradefed/util/GCSBucketUtil.java b/src/com/android/tradefed/util/GCSBucketUtil.java
index 614bcf2..0854274 100644
--- a/src/com/android/tradefed/util/GCSBucketUtil.java
+++ b/src/com/android/tradefed/util/GCSBucketUtil.java
@@ -32,8 +32,9 @@
/**
* File manager to download and upload files from Google Cloud Storage (GCS).
*
- * This class should NOT be used from the scope of a test (i.e., IRemoteTest).
+ * <p>This class should NOT be used from the scope of a test (i.e., IRemoteTest).
*/
+@Deprecated
public class GCSBucketUtil {
// https://cloud.google.com/storage/docs/gsutil
diff --git a/src/com/android/tradefed/util/GCSFileDownloader.java b/src/com/android/tradefed/util/GCSFileDownloader.java
index 8a54add..5fbb98b 100644
--- a/src/com/android/tradefed/util/GCSFileDownloader.java
+++ b/src/com/android/tradefed/util/GCSFileDownloader.java
@@ -19,16 +19,25 @@
import com.android.tradefed.build.BuildRetrievalError;
import com.android.tradefed.build.IFileDownloader;
import com.android.tradefed.log.LogUtil.CLog;
-import com.android.tradefed.util.GCSBucketUtil.GCSFileMetadata;
+import com.google.auth.Credentials;
+import com.google.auth.oauth2.ServiceAccountCredentials;
+import com.google.auth.oauth2.UserCredentials;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.Bucket;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.Storage.BlobListOption;
+import com.google.cloud.storage.StorageException;
+import com.google.cloud.storage.StorageOptions;
import com.google.common.annotations.VisibleForTesting;
-import java.io.ByteArrayInputStream;
import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.nio.file.Path;
+import java.nio.channels.Channels;
import java.nio.file.Paths;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
@@ -39,12 +48,18 @@
public static final String GCS_PREFIX = "gs://";
public static final String GCS_APPROX_PREFIX = "gs:/";
- private static final long TIMEOUT = 10 * 60 * 1000; // 10minutes
- private static final long RETRY_INTERVAL = 1000; // 1s
- private static final int ATTETMPTS = 3;
- private static final Pattern GCS_PATH_PATTERN = Pattern.compile("gs://([^/]*)(/.*)");
+ private static final Pattern GCS_PATH_PATTERN = Pattern.compile("gs://([^/]*)/(.*)");
private static final String PATH_SEP = "/";
+ private File mJsonKeyFile = null;
+ private Storage mStorage;
+
+ public GCSFileDownloader(File jsonKeyFile) {
+ mJsonKeyFile = jsonKeyFile;
+ }
+
+ public GCSFileDownloader() {}
+
/**
* Download a file from a GCS bucket file.
*
@@ -53,10 +68,46 @@
* @return {@link InputStream} with the file content.
*/
public InputStream downloadFile(String bucketName, String filename) throws IOException {
- GCSBucketUtil bucket = getGCSBucketUtil(bucketName);
- Path path = Paths.get(filename);
- String contents = bucket.pullContents(path);
- return new ByteArrayInputStream(contents.getBytes());
+ try {
+ Blob blob = getBucket(bucketName).get(filename);
+ if (blob == null) {
+ throw new IOException(
+ String.format("gs://%s/%s doesn't exist.", bucketName, filename));
+ }
+ return Channels.newInputStream(blob.reader());
+ } catch (StorageException e) {
+ throw new IOException(e);
+ }
+ }
+
+ Storage getStorage() throws IOException {
+ if (mStorage == null) {
+ Credentials credential = null;
+ if (mJsonKeyFile != null && mJsonKeyFile.exists()) {
+ CLog.d("Using json key file %s.", mJsonKeyFile);
+ credential =
+ ServiceAccountCredentials.fromStream(new FileInputStream(mJsonKeyFile));
+ } else {
+ CLog.d("Using local authentication.");
+ try {
+ credential = UserCredentials.getApplicationDefault();
+ } catch (IOException e) {
+ CLog.e(e.getMessage());
+ CLog.e("Try 'gcloud auth application-default login' to login.");
+ throw e;
+ }
+ }
+ mStorage = StorageOptions.newBuilder().setCredentials(credential).build().getService();
+ }
+ return mStorage;
+ }
+
+ Bucket getBucket(String bucketName) throws IOException, StorageException {
+ Bucket bucket = getStorage().get(bucketName);
+ if (bucket == null) {
+ throw new IOException(String.format("Bucket %s doesn't exist.", bucketName));
+ }
+ return bucket;
}
/**
@@ -86,16 +137,74 @@
downloadFile(pathParts[0], pathParts[1], destFile);
}
+
+ private boolean isFileFresh(File localFile, Blob remoteFile) throws IOException {
+ if (localFile == null && remoteFile == null) {
+ return true;
+ }
+ if (localFile == null || remoteFile == null) {
+ return false;
+ }
+ return remoteFile.getMd5().equals(FileUtil.calculateBase64Md5(localFile));
+ }
+
@Override
public boolean isFresh(File localFile, String remotePath) throws BuildRetrievalError {
String[] pathParts = parseGcsPath(remotePath);
try {
- return recursiveCheckFreshness(localFile, pathParts[0], Paths.get(pathParts[1]));
- } catch (IOException e) {
+ return recursiveCheckFolderFreshness(getBucket(pathParts[0]), pathParts[1], localFile);
+ } catch (IOException | StorageException e) {
throw new BuildRetrievalError(e.getMessage(), e);
}
}
+ /**
+ * For GCS, if it's a file, we use file content's md5 hash to check if the local file is the
+ * same as the remote file. If it's a folder, we will check all the files in the folder are the
+ * same and all the sub-folders also have the same files.
+ *
+ * @param bucket is the gcs bucket.
+ * @param remoteFilename is the relative path to the bucket.
+ * @param localFile is the local file
+ * @return true if local file is the same as remote file, otherwise false.
+ * @throws IOException
+ * @throws StorageException
+ */
+ private boolean recursiveCheckFolderFreshness(
+ Bucket bucket, String remoteFilename, File localFile)
+ throws IOException, StorageException {
+ if (!localFile.exists()) {
+ return false;
+ }
+ if (localFile.isFile()) {
+ return isFileFresh(localFile, bucket.get(remoteFilename));
+ }
+ // localFile is a folder.
+ Set<String> subFilenames = new HashSet<>(Arrays.asList(localFile.list()));
+ remoteFilename = sanitizeDirectoryName(remoteFilename);
+
+ for (Blob subRemoteFile : listRemoteFilesUnderFolder(bucket, remoteFilename)) {
+ if (subRemoteFile.getName().equals(remoteFilename)) {
+ // Skip the current folder.
+ continue;
+ }
+ String subFilename = Paths.get(subRemoteFile.getName()).getFileName().toString();
+ if (!recursiveCheckFolderFreshness(
+ bucket, subRemoteFile.getName(), new File(localFile, subFilename))) {
+ return false;
+ }
+ subFilenames.remove(subFilename);
+ }
+ return subFilenames.isEmpty();
+ }
+
+ Iterable<Blob> listRemoteFilesUnderFolder(Bucket bucket, String folder) {
+ return bucket.list(
+ BlobListOption.prefix(sanitizeDirectoryName(folder)),
+ BlobListOption.currentDirectory())
+ .iterateAll();
+ }
+
String[] parseGcsPath(String remotePath) throws BuildRetrievalError {
if (remotePath.startsWith(GCS_APPROX_PREFIX) && !remotePath.startsWith(GCS_PREFIX)) {
// File object remove double // so we have to rebuild it in some cases
@@ -109,98 +218,66 @@
return new String[] {m.group(1), m.group(2)};
}
- /**
- * For GCS, if it's a file, we use file content's md5 hash to check if the local file is the
- * same as the remote file. If it's a folder, we will check all the files in the folder are the
- * same and all the sub-folders also have the same files.
- *
- * @param localFile is the local file
- * @param bucketName is the remote file's GCS bucket name
- * @param remotePath is the relative path to the bucket.
- * @return true if local file is the same as remote file, otherwise false.
- * @throws IOException
- */
- private boolean recursiveCheckFreshness(File localFile, String bucketName, Path remotePath)
- throws IOException {
- GCSBucketUtil bucketUtil = getGCSBucketUtil(bucketName);
- if (localFile.isFile()) {
- GCSFileMetadata fileInfo = bucketUtil.stat(remotePath);
- boolean isFileFresh = fileInfo.mMd5Hash.equals(bucketUtil.md5Hash(localFile));
- if (!isFileFresh) {
- CLog.d("Local file for %s is not fresh.", remotePath);
- }
- return isFileFresh;
- } else if (localFile.isDirectory()) {
- Set<String> remoteUriSets = new HashSet<String>(bucketUtil.ls(remotePath));
- String remoteUri = sanitizeDirectoryName(bucketUtil.getUriForGcsPath(remotePath));
- // If the folder has files inside it, "ls" will include the folder itself.
- // If the folder only has folders or has nothing inside it, "ls" will not include the
- // folder itself. That said depends on folder's content, "ls" may or may not list the
- // current folder. Since the current folder should always exists (otherwise the "ls"
- // already throws exception), we don't bother to check it is in the "ls" result or not.
- remoteUriSets.remove(remoteUri);
-
- for (File subFile : localFile.listFiles()) {
- Path remoteSubPath = remotePath.resolve(subFile.getName());
- String remoteSubUri = bucketUtil.getUriForGcsPath(remoteSubPath);
- if (subFile.isDirectory()) {
- remoteSubUri = sanitizeDirectoryName(remoteSubUri);
- }
- if (!remoteUriSets.contains(remoteSubUri)) {
- CLog.d("GCS doesn't have %s.", remoteSubUri);
- return false;
- }
- remoteUriSets.remove(remoteSubUri);
- }
- if (!remoteUriSets.isEmpty()) {
- CLog.d("GCS has these files but local doesn't: %s", remoteUriSets);
- return false;
- }
- for (File subFile : localFile.listFiles()) {
- if (!recursiveCheckFreshness(
- subFile, bucketName, remotePath.resolve(subFile.getName()))) {
- return false;
- }
- }
- return true;
- }
- return false;
- }
-
- /** Folder name should end with "/" */
String sanitizeDirectoryName(String name) {
+ /** Folder name should end with "/" */
if (!name.endsWith(PATH_SEP)) {
name += PATH_SEP;
}
return name;
}
+ /** check given filename is a folder or not. */
+ private boolean isFolder(Bucket bucket, String filename) throws StorageException {
+ filename = sanitizeDirectoryName(filename);
+ return bucket.list(BlobListOption.prefix(filename), BlobListOption.currentDirectory())
+ .iterateAll()
+ .iterator()
+ .hasNext();
+ }
+
@VisibleForTesting
void downloadFile(String bucketName, String filename, File localFile)
throws BuildRetrievalError {
- CLog.i("Downloading %s %s to %s", bucketName, filename, localFile.getAbsolutePath());
-
- GCSBucketUtil bucketUtil = getGCSBucketUtil(bucketName);
try {
- if (!bucketUtil.isFile(filename)) {
- filename = sanitizeDirectoryName(filename);
- filename += "*";
- localFile.mkdirs();
- bucketUtil.setRecursive(true);
- }
- bucketUtil.pull(Paths.get(filename), localFile);
- } catch (IOException e) {
- CLog.e("Failed to download %s, clean up.", localFile.getAbsoluteFile());
+ recursiveDownload(getStorage().get(bucketName), filename, localFile);
+ } catch (IOException | StorageException e) {
+ CLog.e("Failed to download gs://%s/%s, clean up.", bucketName, filename);
throw new BuildRetrievalError(e.getMessage(), e);
}
}
- private GCSBucketUtil getGCSBucketUtil(String bucketName) {
- GCSBucketUtil bucketUtil = new GCSBucketUtil(bucketName);
- bucketUtil.setTimeoutMs(TIMEOUT);
- bucketUtil.setRetryInterval(RETRY_INTERVAL);
- bucketUtil.setAttempts(ATTETMPTS);
- return bucketUtil;
+ private void recursiveDownload(Bucket bucket, String filepath, File localFile)
+ throws StorageException, IOException {
+ CLog.d(
+ "Downloading gs://%s/%s to %s",
+ bucket.getName(), filepath, localFile.getAbsolutePath());
+ if (!isFolder(bucket, filepath)) {
+ Blob blob = bucket.get(filepath);
+ if (blob == null) {
+ throw new IOException(
+ String.format("gs://%s/%s doesn't exist.", bucket.getName(), filepath));
+ }
+ blob.downloadTo(localFile.toPath());
+ return;
+ }
+ // Remote file is a folder.
+ filepath = sanitizeDirectoryName(filepath);
+ if (!localFile.exists()) {
+ FileUtil.mkdirsRWX(localFile);
+ }
+ Set<String> subFilenames = new HashSet<>(Arrays.asList(localFile.list()));
+ for (Blob subRemoteFile : listRemoteFilesUnderFolder(bucket, filepath)) {
+ if (subRemoteFile.getName().equals(filepath)) {
+ // Skip the current folder.
+ continue;
+ }
+ String subFilename = Paths.get(subRemoteFile.getName()).getFileName().toString();
+ recursiveDownload(bucket, subRemoteFile.getName(), new File(localFile, subFilename));
+ subFilenames.remove(subFilename);
+ }
+ for (String subFilename : subFilenames) {
+ FileUtil.recursiveDelete(new File(localFile, subFilename));
+ }
}
/**
diff --git a/src/com/android/tradefed/util/StreamUtil.java b/src/com/android/tradefed/util/StreamUtil.java
index 6520aac..a853c15 100644
--- a/src/com/android/tradefed/util/StreamUtil.java
+++ b/src/com/android/tradefed/util/StreamUtil.java
@@ -33,6 +33,7 @@
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
import java.util.Objects;
import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipOutputStream;
@@ -305,6 +306,22 @@
* @throws IOException
*/
public static String calculateMd5(InputStream inputSource) throws IOException {
+ return bytesToHexString(calculateMd5Digest(inputSource));
+ }
+
+ /**
+ * Helper method to calculate base64 md5 for a inputStream. The inputStream will be consumed and
+ * closed.
+ *
+ * @param inputSource used to create inputStream
+ * @return base64 md5 of the stream
+ * @throws IOException
+ */
+ public static String calculateBase64Md5(InputStream inputSource) throws IOException {
+ return Base64.getEncoder().encodeToString(calculateMd5Digest(inputSource));
+ }
+
+ private static byte[] calculateMd5Digest(InputStream inputSource) throws IOException {
MessageDigest md = null;
try {
md = MessageDigest.getInstance("md5");
@@ -318,8 +335,7 @@
// Read through the stream to update digest.
}
input.close();
- String md5 = bytesToHexString(md.digest());
- return md5;
+ return md.digest();
}
private static final char[] HEX_CHARS = {
diff --git a/tests/src/com/android/tradefed/util/FileUtilFuncTest.java b/tests/src/com/android/tradefed/util/FileUtilFuncTest.java
index 9f6aa06..a08d320 100644
--- a/tests/src/com/android/tradefed/util/FileUtilFuncTest.java
+++ b/tests/src/com/android/tradefed/util/FileUtilFuncTest.java
@@ -342,6 +342,24 @@
}
}
+ /**
+ * Verify {@link FileUtil#calculateBase64Md5(File)} works.
+ *
+ * @throws IOException
+ */
+ public void testCalculateBase64Md5() throws IOException {
+ final String source = "testtesttesttesttest";
+ final String base64Md5 = "8xf2gvr+AwnGpCOvC076WQ==";
+ File tmpFile = FileUtil.createTempFile("testCalculateMd5", ".txt");
+ try {
+ FileUtil.writeToFile(source, tmpFile);
+ String actualBase64Md5 = FileUtil.calculateBase64Md5(tmpFile);
+ assertEquals(base64Md5, actualBase64Md5);
+ } finally {
+ FileUtil.deleteFile(tmpFile);
+ }
+ }
+
/** Test that {@link FileUtil#recursiveSymlink(File, File)} properly simlink files. */
public void testRecursiveSymlink() throws IOException {
File dir1 = null;
diff --git a/tests/src/com/android/tradefed/util/GCSFileDownloaderFuncTest.java b/tests/src/com/android/tradefed/util/GCSFileDownloaderFuncTest.java
index a9a925c..327653b 100644
--- a/tests/src/com/android/tradefed/util/GCSFileDownloaderFuncTest.java
+++ b/tests/src/com/android/tradefed/util/GCSFileDownloaderFuncTest.java
@@ -18,6 +18,10 @@
import com.android.tradefed.build.BuildRetrievalError;
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.Bucket;
+import com.google.cloud.storage.Storage.BlobListOption;
+
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -42,22 +46,15 @@
private static final String FOLDER_NAME1 = "folder1";
private static final String FOLDER_NAME2 = "folder2";
private static final String FILE_CONTENT = "Hello World!";
- private static final long TIMEOUT = 30000L;
private GCSFileDownloader mDownloader;
+ private Bucket mBucket;
private String mRemoteRoot;
private File mLocalRoot;
- private static void createFile(String content, String bucketName, String... pathSegs)
- throws IOException {
+ private static void createFile(String content, Bucket bucket, String... pathSegs) {
String path = String.join("/", pathSegs);
- getGCSBucketUtil(bucketName).pushString(content, Paths.get(path));
- }
-
- private static GCSBucketUtil getGCSBucketUtil(String bucketName) {
- GCSBucketUtil bucket = new GCSBucketUtil(bucketName);
- bucket.setTimeoutMs(TIMEOUT);
- return bucket;
+ bucket.create(path, content.getBytes());
}
@Before
@@ -66,12 +63,6 @@
FileUtil.createTempFile(GCSFileDownloaderFuncTest.class.getSimpleName(), "");
mRemoteRoot = tempFile.getName();
FileUtil.deleteFile(tempFile);
- createFile(FILE_CONTENT, BUCKET_NAME, mRemoteRoot, FILE_NAME1);
- createFile(FILE_NAME2, BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FILE_NAME2);
- createFile(FILE_NAME3, BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FILE_NAME3);
- createFile(FILE_NAME4, BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FOLDER_NAME2, FILE_NAME4);
-
- mLocalRoot = FileUtil.createTempDir(GCSFileDownloaderFuncTest.class.getSimpleName());
mDownloader =
new GCSFileDownloader() {
@@ -88,14 +79,21 @@
}
}
};
+ mBucket = mDownloader.getStorage().get(BUCKET_NAME);
+ createFile(FILE_CONTENT, mBucket, mRemoteRoot, FILE_NAME1);
+ createFile(FILE_NAME2, mBucket, mRemoteRoot, FOLDER_NAME1, FILE_NAME2);
+ createFile(FILE_NAME3, mBucket, mRemoteRoot, FOLDER_NAME1, FILE_NAME3);
+ createFile(FILE_NAME4, mBucket, mRemoteRoot, FOLDER_NAME1, FOLDER_NAME2, FILE_NAME4);
+ mLocalRoot = FileUtil.createTempDir(GCSFileDownloaderFuncTest.class.getSimpleName());
+
}
@After
- public void tearDown() throws IOException {
+ public void tearDown() {
FileUtil.recursiveDelete(mLocalRoot);
- GCSBucketUtil bucket = new GCSBucketUtil(BUCKET_NAME);
- bucket.setTimeoutMs(TIMEOUT);
- bucket.remove(mRemoteRoot, true);
+ for (Blob blob : mBucket.list(BlobListOption.prefix(mRemoteRoot)).iterateAll()) {
+ blob.delete();
+ }
}
@Test
@@ -126,10 +124,33 @@
}
@Test
+ public void testDownloadFile_nonExist() throws Exception {
+ try {
+ mDownloader.downloadFile(
+ String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, "non_exist_file"));
+ Assert.fail("Should throw BuildRetrievalError.");
+ } catch (BuildRetrievalError e) {
+ // Expect BuildRetrievalError
+ }
+ }
+
+ @Test
public void testDownloadFile_folder() throws Exception {
File localFile =
mDownloader.downloadFile(
+ String.format("gs://%s/%s/%s/", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1));
+ checkDownloadedFolder(localFile);
+ }
+
+ @Test
+ public void testDownloadFile_folderNotsanitize() throws Exception {
+ File localFile =
+ mDownloader.downloadFile(
String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1));
+ checkDownloadedFolder(localFile);
+ }
+
+ private void checkDownloadedFolder(File localFile) throws Exception {
Assert.assertTrue(localFile.isDirectory());
Assert.assertEquals(3, localFile.list().length);
for (String filename : localFile.list()) {
@@ -158,6 +179,17 @@
}
@Test
+ public void testDownloadFile_folder_nonExist() throws Exception {
+ try {
+ mDownloader.downloadFile(
+ String.format("gs://%s/%s/%s/", BUCKET_NAME, "mRemoteRoot", "nonExistFolder"));
+ Assert.fail("Should throw BuildRetrievalError.");
+ } catch (BuildRetrievalError e) {
+ // Expect BuildRetrievalError
+ }
+ }
+
+ @Test
public void testCheckFreshness() throws Exception {
String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1);
File localFile = mDownloader.downloadFile(remotePath);
@@ -169,7 +201,7 @@
String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FILE_NAME1);
File localFile = mDownloader.downloadFile(remotePath);
// Change the remote file.
- createFile("New content.", BUCKET_NAME, mRemoteRoot, FILE_NAME1);
+ createFile("New content.", mBucket, mRemoteRoot, FILE_NAME1);
Assert.assertFalse(mDownloader.isFresh(localFile, remotePath));
}
@@ -184,8 +216,7 @@
public void testCheckFreshness_folder_addFile() throws Exception {
String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1);
File localFolder = mDownloader.downloadFile(remotePath);
- createFile(
- "A new file", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FOLDER_NAME2, "new_file.txt");
+ createFile("A new file", mBucket, mRemoteRoot, FOLDER_NAME1, FOLDER_NAME2, "new_file.txt");
Assert.assertFalse(mDownloader.isFresh(localFolder, remotePath));
}
@@ -193,8 +224,7 @@
public void testCheckFreshness_folder_removeFile() throws Exception {
String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1);
File localFolder = mDownloader.downloadFile(remotePath);
- getGCSBucketUtil(BUCKET_NAME)
- .remove(Paths.get(mRemoteRoot, FOLDER_NAME1, FILE_NAME3), true);
+ mBucket.get(Paths.get(mRemoteRoot, FOLDER_NAME1, FILE_NAME3).toString()).delete();
Assert.assertFalse(mDownloader.isFresh(localFolder, remotePath));
}
@@ -202,7 +232,7 @@
public void testCheckFreshness_folder_changeFile() throws Exception {
String remotePath = String.format("gs://%s/%s/%s", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1);
File localFolder = mDownloader.downloadFile(remotePath);
- createFile("New content", BUCKET_NAME, mRemoteRoot, FOLDER_NAME1, FILE_NAME3);
+ createFile("New content", mBucket, mRemoteRoot, FOLDER_NAME1, FILE_NAME3);
Assert.assertFalse(mDownloader.isFresh(localFolder, remotePath));
}
}
diff --git a/tests/src/com/android/tradefed/util/GCSFileDownloaderTest.java b/tests/src/com/android/tradefed/util/GCSFileDownloaderTest.java
index 78dd1fb..54b4d8d 100644
--- a/tests/src/com/android/tradefed/util/GCSFileDownloaderTest.java
+++ b/tests/src/com/android/tradefed/util/GCSFileDownloaderTest.java
@@ -61,7 +61,7 @@
try {
localFile = mGCSFileDownloader.downloadFile("gs://bucket/this/is/a/file.txt");
String content = FileUtil.readStringFromFile(localFile);
- Assert.assertEquals("bucket\n/this/is/a/file.txt", content);
+ Assert.assertEquals("bucket\nthis/is/a/file.txt", content);
} finally {
FileUtil.deleteFile(localFile);
}
@@ -91,7 +91,7 @@
public void testParseGcsPath() throws Exception {
String[] parts = mGCSFileDownloader.parseGcsPath("gs://bucketname/path/to/file");
Assert.assertEquals("bucketname", parts[0]);
- Assert.assertEquals("/path/to/file", parts[1]);
+ Assert.assertEquals("path/to/file", parts[1]);
}
@Test
@@ -100,7 +100,7 @@
String gcsPath = "gs:/bucketName/path/to/file";
String[] parts = mGCSFileDownloader.parseGcsPath(gcsPath);
Assert.assertEquals("bucketName", parts[0]);
- Assert.assertEquals("/path/to/file", parts[1]);
+ Assert.assertEquals("path/to/file", parts[1]);
}
@Test
diff --git a/tests/src/com/android/tradefed/util/StreamUtilTest.java b/tests/src/com/android/tradefed/util/StreamUtilTest.java
index 0bc4cce..f1a238c 100644
--- a/tests/src/com/android/tradefed/util/StreamUtilTest.java
+++ b/tests/src/com/android/tradefed/util/StreamUtilTest.java
@@ -17,7 +17,6 @@
import com.android.tradefed.result.ByteArrayInputStreamSource;
import com.android.tradefed.result.InputStreamSource;
-import com.android.tradefed.util.StreamUtil;
import junit.framework.TestCase;
@@ -29,13 +28,12 @@
import java.io.OutputStreamWriter;
import java.io.Writer;
-/**
- * Unit tests for the {@link StreamUtil} utility class
- */
+/** Unit tests for the {@link com.android.tradefed.util.StreamUtil} utility class */
public class StreamUtilTest extends TestCase {
/**
- * Verify that {@link StreamUtil#getByteArrayListFromSource} works as expected.
+ * Verify that {@link com.android.tradefed.util.StreamUtil#getByteArrayListFromSource} works as
+ * expected.
*/
public void testGetByteArrayListFromSource() throws Exception {
final String contents = "this is a string";
@@ -53,7 +51,8 @@
}
/**
- * Verify that {@link StreamUtil#getByteArrayListFromStream} works as expected.
+ * Verify that {@link com.android.tradefed.util.StreamUtil#getByteArrayListFromStream} works as
+ * expected.
*/
public void testGetByteArrayListFromStream() throws Exception {
final String contents = "this is a string";
@@ -69,7 +68,8 @@
}
/**
- * Verify that {@link StreamUtil#getStringFromSource} works as expected.
+ * Verify that {@link com.android.tradefed.util.StreamUtil#getStringFromSource} works as
+ * expected.
*/
public void testGetStringFromSource() throws Exception {
final String contents = "this is a string";
@@ -81,7 +81,8 @@
}
/**
- * Verify that {@link StreamUtil#getBufferedReaderFromStreamSrc} works as expected.
+ * Verify that {@link com.android.tradefed.util.StreamUtil#getBufferedReaderFromStreamSrc} works
+ * as expected.
*/
public void testGetBufferedReaderFromInputStream() throws Exception {
final String contents = "this is a string";
@@ -97,7 +98,8 @@
}
/**
- * Verify that {@link StreamUtil#countLinesFromSource} works as expected.
+ * Verify that {@link com.android.tradefed.util.StreamUtil#countLinesFromSource} works as
+ * expected.
*/
public void testCountLinesFromSource() throws Exception {
final String contents = "foo\nbar\n\foo\n";
@@ -106,7 +108,8 @@
}
/**
- * Verify that {@link StreamUtil#getStringFromStream} works as expected.
+ * Verify that {@link com.android.tradefed.util.StreamUtil#getStringFromStream} works as
+ * expected.
*/
public void testGetStringFromStream() throws Exception {
final String contents = "this is a string";
@@ -116,7 +119,9 @@
}
/**
- * Verify that {@link StreamUtil#calculateMd5(InputStream)} works as expected.
+ * Verify that {@link com.android.tradefed.util.StreamUtil#calculateMd5(InputStream)} works as
+ * expected.
+ *
* @throws IOException
*/
public void testCalculateMd5() throws IOException {
@@ -127,6 +132,20 @@
assertEquals(md5, actualMd5);
}
+ /**
+ * Verify that {@link com.android.tradefed.util.StreamUtil#calculateBase64Md5(InputStream)}
+ * works as expected.
+ *
+ * @throws IOException
+ */
+ public void testCalculateBase64Md5() throws IOException {
+ final String source = "testtesttesttesttest";
+ final String base64Md5 = "8xf2gvr+AwnGpCOvC076WQ==";
+ ByteArrayInputStream inputSource = new ByteArrayInputStream(source.getBytes());
+ String actualBase64Md5 = StreamUtil.calculateBase64Md5(inputSource);
+ assertEquals(base64Md5, actualBase64Md5);
+ }
+
public void testCopyStreams() throws Exception {
String text = getLargeText();
ByteArrayInputStream bais = new ByteArrayInputStream(text.getBytes());