Merge changes I8111e11f,I3407c85f
* changes:
Relax rules to match the target_files zip.
Support local paths in fetch_cvd
diff --git a/host/commands/assemble_cvd/super_image_mixer.cc b/host/commands/assemble_cvd/super_image_mixer.cc
index 1960d41..9544670 100644
--- a/host/commands/assemble_cvd/super_image_mixer.cc
+++ b/host/commands/assemble_cvd/super_image_mixer.cc
@@ -44,8 +44,8 @@
if (file_info.source != source) {
continue;
}
- std::string expected_filename = "target_files-" + file_iter.second.build_id + ".zip";
- if (!android::base::EndsWith(file_path, expected_filename)) {
+ std::string expected_filename = "target_files-" + file_iter.second.build_id;
+ if (file_path.find(expected_filename) != std::string::npos) {
continue;
}
return file_path;;
diff --git a/host/commands/fetcher/build_api.cc b/host/commands/fetcher/build_api.cc
index ff2b65e..fb685c7 100644
--- a/host/commands/fetcher/build_api.cc
+++ b/host/commands/fetcher/build_api.cc
@@ -15,13 +15,19 @@
#include "build_api.h"
+#include <dirent.h>
+#include <unistd.h>
+
#include <chrono>
#include <set>
#include <string>
#include <thread>
+#include <android-base/strings.h>
#include <glog/logging.h>
+#include "common/libs/utils/files.h"
+
namespace {
const std::string BUILD_API =
@@ -56,6 +62,16 @@
return out << "(id=\"" << build.id << "\", target=\"" << build.target << "\")";
}
+std::ostream& operator<<(std::ostream& out, const DirectoryBuild& build) {
+ auto paths = android::base::Join(build.paths, ":");
+ return out << "(paths=\"" << paths << "\", target=\"" << build.target << "\")";
+}
+
+std::ostream& operator<<(std::ostream& out, const Build& build) {
+ std::visit([&out](auto&& arg) { out << arg; }, build);
+ return out;
+}
+
BuildApi::BuildApi(std::unique_ptr<CredentialSource> credential_source)
: credential_source(std::move(credential_source)) {}
@@ -108,6 +124,26 @@
return artifacts;
}
+struct CloseDir {
+ void operator()(DIR* dir) {
+ closedir(dir);
+ }
+};
+
+using UniqueDir = std::unique_ptr<DIR, CloseDir>;
+
+std::vector<Artifact> BuildApi::Artifacts(const DirectoryBuild& build) {
+ std::vector<Artifact> artifacts;
+ for (const auto& path : build.paths) {
+ auto dir = UniqueDir(opendir(path.c_str()));
+ CHECK(dir != nullptr) << "Could not read files from \"" << path << "\"";
+ for (auto entity = readdir(dir.get()); entity != nullptr; entity = readdir(dir.get())) {
+ artifacts.emplace_back(std::string(entity->d_name));
+ }
+ }
+ return artifacts;
+}
+
bool BuildApi::ArtifactToFile(const DeviceBuild& build,
const std::string& artifact,
const std::string& path) {
@@ -116,9 +152,35 @@
return curl.DownloadToFile(url, path, Headers());
}
-DeviceBuild ArgumentToBuild(BuildApi* build_api, const std::string& arg,
- const std::string& default_build_target,
- const std::chrono::seconds& retry_period) {
+bool BuildApi::ArtifactToFile(const DirectoryBuild& build,
+ const std::string& artifact,
+ const std::string& destination) {
+ for (const auto& path : build.paths) {
+ auto source = path + "/" + artifact;
+ if (!cvd::FileExists(source)) {
+ continue;
+ }
+ unlink(destination.c_str());
+ if (symlink(source.c_str(), destination.c_str())) {
+ int error_num = errno;
+ LOG(ERROR) << "Could not create symlink from " << source << " to "
+ << destination << ": " << strerror(error_num);
+ return false;
+ }
+ return true;
+ }
+ return false;
+}
+
+Build ArgumentToBuild(BuildApi* build_api, const std::string& arg,
+ const std::string& default_build_target,
+ const std::chrono::seconds& retry_period) {
+ if (arg.find(":") != std::string::npos) {
+ std::vector<std::string> dirs = android::base::Split(arg, ":");
+ std::string id = dirs.back();
+ dirs.pop_back();
+ return DirectoryBuild(dirs, id);
+ }
size_t slash_pos = arg.find('/');
if (slash_pos != std::string::npos
&& arg.find('/', slash_pos + 1) != std::string::npos) {
diff --git a/host/commands/fetcher/build_api.h b/host/commands/fetcher/build_api.h
index bb89e81..54bbd72 100644
--- a/host/commands/fetcher/build_api.h
+++ b/host/commands/fetcher/build_api.h
@@ -20,6 +20,7 @@
#include <memory>
#include <ostream>
#include <string>
+#include <variant>
#include "credential_source.h"
#include "curl_wrapper.h"
@@ -35,6 +36,7 @@
unsigned int crc32;
public:
Artifact(const Json::Value&);
+ Artifact(const std::string& name) : name(name) {}
const std::string& Name() const { return name; }
size_t Size() const { return size; }
@@ -58,6 +60,23 @@
std::ostream& operator<<(std::ostream&, const DeviceBuild&);
+struct DirectoryBuild {
+ // TODO(schuffelen): Support local builds other than "eng"
+ DirectoryBuild(const std::vector<std::string>& paths,
+ const std::string& target)
+ : paths(paths), target(target), id("eng") {}
+
+ std::vector<std::string> paths;
+ std::string target;
+ std::string id;
+};
+
+std::ostream& operator<<(std::ostream&, const DirectoryBuild&);
+
+using Build = std::variant<DeviceBuild, DirectoryBuild>;
+
+std::ostream& operator<<(std::ostream&, const Build&);
+
class BuildApi {
CurlWrapper curl;
std::unique_ptr<CredentialSource> credential_source;
@@ -76,8 +95,24 @@
bool ArtifactToFile(const DeviceBuild& build, const std::string& artifact,
const std::string& path);
+
+ std::vector<Artifact> Artifacts(const DirectoryBuild&);
+
+ bool ArtifactToFile(const DirectoryBuild& build, const std::string& artifact,
+ const std::string& path);
+
+ std::vector<Artifact> Artifacts(const Build& build) {
+ return std::visit([this](auto&& arg) { return Artifacts(arg); }, build);
+ }
+
+ bool ArtifactToFile(const Build& build, const std::string& artifact,
+ const std::string& path) {
+ return std::visit([this, &artifact, &path](auto&& arg) {
+ return ArtifactToFile(arg, artifact, path);
+ }, build);
+ }
};
-DeviceBuild ArgumentToBuild(BuildApi* api, const std::string& arg,
- const std::string& default_build_target,
- const std::chrono::seconds& retry_period);
+Build ArgumentToBuild(BuildApi* api, const std::string& arg,
+ const std::string& default_build_target,
+ const std::chrono::seconds& retry_period);
diff --git a/host/commands/fetcher/fetch_cvd.cc b/host/commands/fetcher/fetch_cvd.cc
index abaede1..7532810 100644
--- a/host/commands/fetcher/fetch_cvd.cc
+++ b/host/commands/fetcher/fetch_cvd.cc
@@ -69,28 +69,32 @@
* For example, for a target "aosp_cf_x86_phone-userdebug" at a build "5824130",
* the image zip file would be "aosp_cf_x86_phone-img-5824130.zip"
*/
-std::string target_build_zip(const DeviceBuild& build, const std::string& name) {
- std::string target = build.target;
+std::string TargetBuildZipFromArtifacts(
+ const Build& build, const std::string& name,
+ const std::vector<Artifact>& artifacts) {
+ std::string target = std::visit([](auto&& arg) { return arg.target; }, build);
size_t dash_pos = target.find("-");
if (dash_pos != std::string::npos) {
target.replace(dash_pos, target.size() - dash_pos, "");
}
- return target + "-" + name + "-" + build.id + ".zip";
+ auto id = std::visit([](auto&& arg) { return arg.id; }, build);
+ auto match = target + "-" + name + "-" + id;
+ for (const auto& artifact : artifacts) {
+ if (artifact.Name().find(match) != std::string::npos) {
+ return artifact.Name();
+ }
+ }
+ return "";
}
std::vector<std::string> download_images(BuildApi* build_api,
- const DeviceBuild& build,
+ const Build& build,
const std::string& target_directory,
const std::vector<std::string>& images) {
- std::string img_zip_name = target_build_zip(build, "img");
auto artifacts = build_api->Artifacts(build);
- bool has_image_zip = false;
- for (const auto& artifact : artifacts) {
- has_image_zip |= artifact.Name() == img_zip_name;
- }
- if (!has_image_zip) {
- LOG(FATAL) << "Target " << build.target << " at id " << build.id
- << " did not have " << img_zip_name;
+ std::string img_zip_name = TargetBuildZipFromArtifacts(build, "img", artifacts);
+ if (img_zip_name.size() == 0) {
+ LOG(FATAL) << "Target " << build << " did not have an img zip";
return {};
}
std::string local_path = target_directory + "/" + img_zip_name;
@@ -112,23 +116,18 @@
return files;
}
std::vector<std::string> download_images(BuildApi* build_api,
- const DeviceBuild& build,
+ const Build& build,
const std::string& target_directory) {
return download_images(build_api, build, target_directory, {});
}
std::vector<std::string> download_target_files(BuildApi* build_api,
- const DeviceBuild& build,
+ const Build& build,
const std::string& target_directory) {
- std::string target_zip = target_build_zip(build, "target_files");
auto artifacts = build_api->Artifacts(build);
- bool has_target_zip = false;
- for (const auto& artifact : artifacts) {
- has_target_zip |= artifact.Name() == target_zip;
- }
- if (!has_target_zip) {
- LOG(FATAL) << "Target " << build.target << " at id " << build.id
- << " did not have " << target_zip;
+ std::string target_zip = TargetBuildZipFromArtifacts(build, "target_files", artifacts);
+ if (target_zip.size() == 0) {
+ LOG(FATAL) << "Target " << build << " did not have a target files zip";
return {};
}
std::string local_path = target_directory + "/" + target_zip;
@@ -141,7 +140,7 @@
}
std::vector<std::string> download_host_package(BuildApi* build_api,
- const DeviceBuild& build,
+ const Build& build,
const std::string& target_directory) {
auto artifacts = build_api->Artifacts(build);
bool has_host_package = false;
@@ -149,8 +148,7 @@
has_host_package |= artifact.Name() == HOST_TOOLS;
}
if (!has_host_package) {
- LOG(FATAL) << "Target " << build.target << " at id " << build.id
- << " did not have " << HOST_TOOLS;
+ LOG(FATAL) << "Target " << build << " did not have " << HOST_TOOLS;
return {};
}
std::string local_path = target_directory + "/" + HOST_TOOLS;
@@ -193,7 +191,7 @@
}
std::vector<std::string> download_ota_tools(BuildApi* build_api,
- const DeviceBuild& build,
+ const Build& build,
const std::string& target_directory) {
auto artifacts = build_api->Artifacts(build);
bool has_host_package = false;
@@ -201,8 +199,7 @@
has_host_package |= artifact.Name() == OTA_TOOLS;
}
if (!has_host_package) {
- LOG(ERROR) << "Target " << build.target << " at id " << build.id
- << " did not have " << OTA_TOOLS;
+ LOG(ERROR) << "Target " << build << " did not have " << OTA_TOOLS;
return {};
}
std::string local_path = target_directory + "/" + OTA_TOOLS;
@@ -231,11 +228,14 @@
return files;
}
-void AddFilesToConfig(cvd::FileSource purpose, const DeviceBuild& build,
+void AddFilesToConfig(cvd::FileSource purpose, const Build& build,
const std::vector<std::string>& paths, cvd::FetcherConfig* config,
bool override_entry = false) {
for (const std::string& path : paths) {
- cvd::CvdFile file(purpose, build.id, build.target, path);
+ // TODO(schuffelen): Do better for local builds here.
+ auto id = std::visit([](auto&& arg) { return arg.id; }, build);
+ auto target = std::visit([](auto&& arg) { return arg.target; }, build);
+ cvd::CvdFile file(purpose, id, target, path);
bool added = config->add_cvd_file(file, override_entry);
if (!added) {
LOG(ERROR) << "Duplicate file " << file;
@@ -280,9 +280,9 @@
}
BuildApi build_api(std::move(credential_source));
- DeviceBuild default_build = ArgumentToBuild(&build_api, FLAGS_default_build,
- DEFAULT_BUILD_TARGET,
- retry_period);
+ auto default_build = ArgumentToBuild(&build_api, FLAGS_default_build,
+ DEFAULT_BUILD_TARGET,
+ retry_period);
std::vector<std::string> host_package_files =
download_host_package(&build_api, default_build, target_dir);
@@ -292,7 +292,7 @@
AddFilesToConfig(cvd::FileSource::DEFAULT_BUILD, default_build, host_package_files, &config);
if (FLAGS_system_build != "" || FLAGS_kernel_build != "" || FLAGS_otatools_build != "") {
- DeviceBuild ota_build = default_build;
+ auto ota_build = default_build;
if (FLAGS_otatools_build != "") {
ota_build = ArgumentToBuild(&build_api, FLAGS_otatools_build,
DEFAULT_BUILD_TARGET, retry_period);
@@ -314,8 +314,12 @@
AddFilesToConfig(cvd::FileSource::DEFAULT_BUILD, default_build, image_files, &config);
}
if (FLAGS_system_build != "" || FLAGS_download_target_files_zip) {
+ std::string default_target_dir = target_dir + "/default";
+ if (mkdir(default_target_dir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) < 0) {
+ LOG(FATAL) << "Could not create " << default_target_dir;
+ }
std::vector<std::string> target_files =
- download_target_files(&build_api, default_build, target_dir);
+ download_target_files(&build_api, default_build, default_target_dir);
if (target_files.empty()) {
LOG(FATAL) << "Could not download target files for " << default_build;
}
@@ -323,9 +327,9 @@
}
if (FLAGS_system_build != "") {
- DeviceBuild system_build = ArgumentToBuild(&build_api, FLAGS_system_build,
- DEFAULT_BUILD_TARGET,
- retry_period);
+ auto system_build = ArgumentToBuild(&build_api, FLAGS_system_build,
+ DEFAULT_BUILD_TARGET,
+ retry_period);
if (FLAGS_download_img_zip) {
std::vector<std::string> image_files =
download_images(&build_api, system_build, target_dir, {"system.img"});
@@ -338,8 +342,12 @@
&config, true);
}
}
+ std::string system_target_dir = target_dir + "/system";
+ if (mkdir(system_target_dir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) < 0) {
+ LOG(FATAL) << "Could not create " << system_target_dir;
+ }
std::vector<std::string> target_files =
- download_target_files(&build_api, system_build, target_dir);
+ download_target_files(&build_api, system_build, system_target_dir);
if (target_files.empty()) {
LOG(FATAL) << "Could not download target files for " << system_build;
}
@@ -347,8 +355,8 @@
}
if (FLAGS_kernel_build != "") {
- DeviceBuild kernel_build = ArgumentToBuild(&build_api, FLAGS_kernel_build,
- "kernel", retry_period);
+ auto kernel_build = ArgumentToBuild(&build_api, FLAGS_kernel_build,
+ "kernel", retry_period);
std::string local_path = target_dir + "/kernel";
if (build_api.ArtifactToFile(kernel_build, "bzImage", local_path)) {
diff --git a/host/commands/fetcher/install_zip.cc b/host/commands/fetcher/install_zip.cc
index bd6294f..3ffd123 100644
--- a/host/commands/fetcher/install_zip.cc
+++ b/host/commands/fetcher/install_zip.cc
@@ -20,6 +20,7 @@
#include <string>
#include <vector>
+#include <android-base/strings.h>
#include <glog/logging.h>
#include "common/libs/utils/archive.h"
@@ -77,8 +78,14 @@
extraction_success = false;
}
}
- for (auto& file : files) {
- file = target_directory + "/" + file;
+ auto it = files.begin();
+ while (it != files.end()) {
+ if (*it == "" || android::base::EndsWith(*it, "/")) {
+ it = files.erase(it);
+ } else {
+ *it = target_directory + "/" + *it;
+ it++;
+ }
}
return extraction_success ? files : std::vector<std::string>{};
}