update_engine: Refactor payload generation request args in a class.
This patch introduces a new struct PayloadGenerationConfig encapsulating
the payload generation request performed by the user in the command
line.
Since we have several payload generation strategies that are triggered
based on the source image version and if there is an image version at
all, encapsulating the request in a single struct helps splitting the
different strategies in independent classes and pass the payload
generation configuration struct with the request to those classes.
BUG=chromium:430950
TEST=FEATURES=test emerge-link update_engine
Change-Id: Ie960c6510bf02d8e10f5e1adaa847a2048c59598
Reviewed-on: https://chromium-review.googlesource.com/258520
Reviewed-by: Alex Vakulenko <avakulenko@chromium.org>
Tested-by: Alex Deymo <deymo@chromium.org>
Commit-Queue: Alex Deymo <deymo@chromium.org>
diff --git a/delta_performer_unittest.cc b/delta_performer_unittest.cc
index 6aa61a3..4698256 100644
--- a/delta_performer_unittest.cc
+++ b/delta_performer_unittest.cc
@@ -13,8 +13,8 @@
#include <base/files/file_path.h>
#include <base/files/file_util.h>
-#include <base/strings/string_util.h>
#include <base/strings/stringprintf.h>
+#include <base/strings/string_util.h>
#include <google/protobuf/repeated_field.h>
#include <gtest/gtest.h>
@@ -107,6 +107,9 @@
kValidOperationData,
};
+// Chuck size used for full payloads during test.
+size_t kDefaultFullChunkSize = 1024 * 1024;
+
} // namespace
static void CompareFilesByBlock(const string& a_file, const string& b_file) {
@@ -469,21 +472,38 @@
ScopedLoopMounter b_mounter(state->b_img, &b_mnt, MS_RDONLY);
const string private_key =
signature_test == kSignatureGenerator ? kUnittestPrivateKeyPath : "";
+
+ PayloadGenerationConfig payload_config;
+ payload_config.is_delta = !full_rootfs;
+ payload_config.chunk_size = chunk_size;
+ if (!full_rootfs) {
+ payload_config.source.rootfs_part = state->a_img;
+ payload_config.source.rootfs_mountpt = a_mnt;
+ if (!full_kernel)
+ payload_config.source.kernel_part = state->old_kernel;
+ payload_config.source.image_info = old_image_info;
+ EXPECT_TRUE(payload_config.source.LoadImageSize());
+
+ payload_config.minor_version =
+ DeltaPerformer::kSupportedMinorPayloadVersion;
+ } else {
+ payload_config.minor_version = DeltaPerformer::kFullPayloadMinorVersion;
+ if (payload_config.chunk_size == -1)
+ payload_config.chunk_size = kDefaultFullChunkSize;
+ }
+ payload_config.target.rootfs_part = state->b_img;
+ payload_config.target.rootfs_mountpt = b_mnt;
+ payload_config.target.kernel_part = state->new_kernel;
+ payload_config.target.image_info = new_image_info;
+ EXPECT_TRUE(payload_config.target.LoadImageSize());
+
+ EXPECT_TRUE(payload_config.Validate());
EXPECT_TRUE(
DeltaDiffGenerator::GenerateDeltaUpdateFile(
- full_rootfs ? "" : a_mnt,
- full_rootfs ? "" : state->a_img,
- b_mnt,
- state->b_img,
- full_kernel ? "" : state->old_kernel,
- state->new_kernel,
+ payload_config,
state->delta_path,
private_key,
- chunk_size,
kRootFSPartitionSize,
- DeltaPerformer::kSupportedMinorPayloadVersion,
- full_rootfs ? nullptr : &old_image_info,
- &new_image_info,
&state->metadata_size));
}
diff --git a/payload_generator/delta_diff_generator.cc b/payload_generator/delta_diff_generator.cc
index 3683860..ab21282 100644
--- a/payload_generator/delta_diff_generator.cc
+++ b/payload_generator/delta_diff_generator.cc
@@ -53,8 +53,7 @@
namespace {
-const uint64_t kVersionNumber = 1;
-const uint64_t kFullUpdateChunkSize = 1024 * 1024; // bytes
+const uint64_t kMajorVersionNumber = 1;
// The maximum destination size allowed for bsdiff. In general, bsdiff should
// work for arbitrary big files, but the payload generation and payload
@@ -868,30 +867,27 @@
return true;
}
-bool InitializePartitionInfos(const string& old_kernel,
- const string& new_kernel,
- const string& old_rootfs,
- const string& new_rootfs,
+bool InitializePartitionInfos(const PayloadGenerationConfig& config,
DeltaArchiveManifest* manifest) {
- if (!old_kernel.empty()) {
+ if (!config.source.kernel_part.empty()) {
TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo(
true,
- old_kernel,
+ config.source.kernel_part,
manifest->mutable_old_kernel_info()));
}
TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo(
true,
- new_kernel,
+ config.target.kernel_part,
manifest->mutable_new_kernel_info()));
- if (!old_rootfs.empty()) {
+ if (!config.source.rootfs_part.empty()) {
TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo(
false,
- old_rootfs,
+ config.source.rootfs_part,
manifest->mutable_old_rootfs_info()));
}
TEST_AND_RETURN_FALSE(DeltaDiffGenerator::InitializePartitionInfo(
false,
- new_rootfs,
+ config.target.rootfs_part,
manifest->mutable_new_rootfs_info()));
return true;
}
@@ -976,58 +972,35 @@
}
bool DeltaDiffGenerator::GenerateDeltaUpdateFile(
- const string& old_root,
- const string& old_image,
- const string& new_root,
- const string& new_image,
- const string& old_kernel_part,
- const string& new_kernel_part,
+ const PayloadGenerationConfig& config,
const string& output_path,
const string& private_key_path,
- off_t chunk_size,
size_t rootfs_partition_size,
- uint32_t minor_version,
- const ImageInfo* old_image_info,
- const ImageInfo* new_image_info,
uint64_t* metadata_size) {
- TEST_AND_RETURN_FALSE(chunk_size == -1 || chunk_size % kBlockSize == 0);
- int old_image_block_count = 0, old_image_block_size = 0;
- int new_image_block_count = 0, new_image_block_size = 0;
- TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(new_image,
- &new_image_block_count,
- &new_image_block_size));
- if (!old_image.empty()) {
- TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(old_image,
- &old_image_block_count,
- &old_image_block_size));
- TEST_AND_RETURN_FALSE(old_image_block_size == new_image_block_size);
+ int old_image_block_count = config.source.rootfs_size / config.block_size;
+ int new_image_block_count = config.target.rootfs_size / config.block_size;
+
+ if (config.is_delta) {
LOG_IF(WARNING, old_image_block_count != new_image_block_count)
<< "Old and new images have different block counts.";
-
- // If new_image_info is present, old_image_info must be present.
- TEST_AND_RETURN_FALSE(!old_image_info == !new_image_info);
- } else {
- // old_image_info must not be present for a full update.
- TEST_AND_RETURN_FALSE(!old_image_info);
+ // TODO(deymo): Our tools only support growing the filesystem size during
+ // an update. Remove this check when that's fixed. crbug.com/192136
+ LOG_IF(FATAL, old_image_block_count > new_image_block_count)
+ << "Shirking the rootfs size is not supported at the moment.";
}
// Sanity checks for the partition size.
- TEST_AND_RETURN_FALSE(rootfs_partition_size % kBlockSize == 0);
- size_t fs_size = static_cast<size_t>(new_image_block_size) *
- new_image_block_count;
+ TEST_AND_RETURN_FALSE(rootfs_partition_size % config.block_size == 0);
LOG(INFO) << "Rootfs partition size: " << rootfs_partition_size;
- LOG(INFO) << "Actual filesystem size: " << fs_size;
- TEST_AND_RETURN_FALSE(rootfs_partition_size >= fs_size);
-
- // Sanity check kernel partition arg
- TEST_AND_RETURN_FALSE(utils::FileSize(new_kernel_part) >= 0);
+ LOG(INFO) << "Actual filesystem size: " << config.target.rootfs_size;
+ TEST_AND_RETURN_FALSE(rootfs_partition_size >= config.target.rootfs_size);
vector<Block> blocks(max(old_image_block_count, new_image_block_count));
LOG(INFO) << "Invalid block index: " << Vertex::kInvalidIndex;
LOG(INFO) << "Block count: " << blocks.size();
- for (vector<Block>::size_type i = 0; i < blocks.size(); i++) {
- CHECK(blocks[i].reader == Vertex::kInvalidIndex);
- CHECK(blocks[i].writer == Vertex::kInvalidIndex);
+ for (const Block& block : blocks) {
+ CHECK(block.reader == Vertex::kInvalidIndex);
+ CHECK(block.writer == Vertex::kInvalidIndex);
}
Graph graph;
CheckGraph(graph);
@@ -1047,46 +1020,47 @@
vector<Vertex::Index> final_order;
Vertex::Index scratch_vertex = Vertex::kInvalidIndex;
{
- int fd;
+ int data_file_fd;
TEST_AND_RETURN_FALSE(
- utils::MakeTempFile(kTempFileTemplate, &temp_file_path, &fd));
+ utils::MakeTempFile(kTempFileTemplate, &temp_file_path, &data_file_fd));
temp_file_unlinker.reset(new ScopedPathUnlinker(temp_file_path));
- TEST_AND_RETURN_FALSE(fd >= 0);
- ScopedFdCloser fd_closer(&fd);
- if (!old_image.empty()) {
+ TEST_AND_RETURN_FALSE(data_file_fd >= 0);
+ ScopedFdCloser data_file_fd_closer(&data_file_fd);
+ if (config.is_delta) {
// Delta update
// Set the minor version for this payload.
LOG(INFO) << "Adding Delta Minor Version.";
- manifest.set_minor_version(minor_version);
- if (minor_version == kInPlaceMinorPayloadVersion) {
+ manifest.set_minor_version(config.minor_version);
+ if (config.minor_version == kInPlaceMinorPayloadVersion) {
TEST_AND_RETURN_FALSE(DeltaReadFiles(&graph,
&blocks,
- old_root,
- new_root,
- chunk_size,
- fd,
+ config.source.rootfs_mountpt,
+ config.target.rootfs_mountpt,
+ config.chunk_size,
+ data_file_fd,
&data_file_size,
false)); // src_ops_allowed
LOG(INFO) << "done reading normal files";
CheckGraph(graph);
LOG(INFO) << "Starting metadata processing";
- TEST_AND_RETURN_FALSE(Metadata::DeltaReadMetadata(&graph,
- &blocks,
- old_image,
- new_image,
- fd,
- &data_file_size));
+ TEST_AND_RETURN_FALSE(Metadata::DeltaReadMetadata(
+ &graph,
+ &blocks,
+ config.source.rootfs_part,
+ config.target.rootfs_part,
+ data_file_fd,
+ &data_file_size));
LOG(INFO) << "Done metadata processing";
CheckGraph(graph);
graph.resize(graph.size() + 1);
TEST_AND_RETURN_FALSE(ReadUnwrittenBlocks(blocks,
- fd,
+ data_file_fd,
&data_file_size,
- old_image,
- new_image,
+ config.source.rootfs_part,
+ config.target.rootfs_part,
&graph.back()));
if (graph.back().op.data_length() == 0) {
LOG(INFO) << "No unwritten blocks to write, omitting operation";
@@ -1105,10 +1079,10 @@
// Read kernel partition
TEST_AND_RETURN_FALSE(
- DeltaCompressKernelPartition(old_kernel_part,
- new_kernel_part,
+ DeltaCompressKernelPartition(config.source.kernel_part,
+ config.target.kernel_part,
&kernel_ops,
- fd,
+ data_file_fd,
&data_file_size,
false)); // src_ops_allowed
@@ -1122,18 +1096,18 @@
TEST_AND_RETURN_FALSE(InplaceGenerator::ConvertGraphToDag(
&graph,
- new_root,
- fd,
+ config.target.rootfs_mountpt,
+ data_file_fd,
&data_file_size,
&final_order,
scratch_vertex));
- } else if (minor_version == kSourceMinorPayloadVersion) {
+ } else if (config.minor_version == kSourceMinorPayloadVersion) {
TEST_AND_RETURN_FALSE(DeltaReadFiles(&graph,
&blocks,
- old_root,
- new_root,
- chunk_size,
- fd,
+ config.source.rootfs_mountpt,
+ config.target.rootfs_mountpt,
+ config.chunk_size,
+ data_file_fd,
&data_file_size,
true)); // src_ops_allowed
@@ -1142,10 +1116,10 @@
// Read kernel partition
TEST_AND_RETURN_FALSE(
- DeltaCompressKernelPartition(old_kernel_part,
- new_kernel_part,
+ DeltaCompressKernelPartition(config.source.kernel_part,
+ config.target.kernel_part,
&kernel_ops,
- fd,
+ data_file_fd,
&data_file_size,
true)); // src_ops_allowed
LOG(INFO) << "done reading kernel";
@@ -1153,10 +1127,10 @@
graph.resize(graph.size() + 1);
TEST_AND_RETURN_FALSE(ReadUnwrittenBlocks(blocks,
- fd,
+ data_file_fd,
&data_file_size,
- old_image,
- new_image,
+ config.source.rootfs_part,
+ config.target.rootfs_part,
&graph.back()));
if (graph.back().op.data_length() == 0) {
LOG(INFO) << "No unwritten blocks to write, omitting operation";
@@ -1166,21 +1140,14 @@
final_order = OrderIndices(graph);
} else {
LOG(ERROR) << "Unsupported minor version given for delta payload: "
- << minor_version;
+ << config.minor_version;
return false;
}
} else {
- // Full update
- off_t new_image_size =
- static_cast<off_t>(new_image_block_count) * new_image_block_size;
- TEST_AND_RETURN_FALSE(FullUpdateGenerator::Run(&graph,
- new_kernel_part,
- new_image,
- new_image_size,
- fd,
+ TEST_AND_RETURN_FALSE(FullUpdateGenerator::Run(config,
+ data_file_fd,
&data_file_size,
- kFullUpdateChunkSize,
- kBlockSize,
+ &graph,
&kernel_ops,
&final_order));
@@ -1190,11 +1157,11 @@
}
}
- if (old_image_info)
- *(manifest.mutable_old_image_info()) = *old_image_info;
+ if (!config.source.ImageInfoIsEmpty())
+ *(manifest.mutable_old_image_info()) = config.source.image_info;
- if (new_image_info)
- *(manifest.mutable_new_image_info()) = *new_image_info;
+ if (!config.target.ImageInfoIsEmpty())
+ *(manifest.mutable_new_image_info()) = config.target.image_info;
OperationNameMap op_name_map;
CheckGraph(graph);
@@ -1248,11 +1215,7 @@
AddSignatureOp(next_blob_offset, signature_blob_length, &manifest);
}
- TEST_AND_RETURN_FALSE(InitializePartitionInfos(old_kernel_part,
- new_kernel_part,
- old_image,
- new_image,
- &manifest));
+ TEST_AND_RETURN_FALSE(InitializePartitionInfos(config, &manifest));
// Serialize protobuf
string serialized_manifest;
@@ -1271,8 +1234,8 @@
// Write header
TEST_AND_RETURN_FALSE(writer.Write(kDeltaMagic, strlen(kDeltaMagic)));
- // Write version number
- TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer, kVersionNumber));
+ // Write major version number
+ TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer, kMajorVersionNumber));
// Write protobuf length
TEST_AND_RETURN_FALSE(WriteUint64AsBigEndian(&writer,
diff --git a/payload_generator/delta_diff_generator.h b/payload_generator/delta_diff_generator.h
index 2f1b356..5698b74 100644
--- a/payload_generator/delta_diff_generator.h
+++ b/payload_generator/delta_diff_generator.h
@@ -15,6 +15,7 @@
#include "update_engine/payload_constants.h"
#include "update_engine/payload_generator/graph_types.h"
#include "update_engine/payload_generator/graph_utils.h"
+#include "update_engine/payload_generator/payload_generation_config.h"
#include "update_engine/update_metadata.pb.h"
// There is one function in DeltaDiffGenerator of importance to users
@@ -36,22 +37,6 @@
// The minor version used by the A to B delta generator algorithm.
extern const uint32_t kSourceMinorPayloadVersion;
-// This struct stores all relevant info for an edge that is cut between
-// nodes old_src -> old_dst by creating new vertex new_vertex. The new
-// relationship is:
-// old_src -(read before)-> new_vertex <-(write before)- old_dst
-// new_vertex is a MOVE operation that moves some existing blocks into
-// temp space. The temp extents are, by necessity, stored in new_vertex
-// (as dst extents) and old_dst (as src extents), but they are also broken
-// out into tmp_extents, as the nodes themselves may contain many more
-// extents.
-struct CutEdgeVertexes {
- Vertex::Index new_vertex;
- Vertex::Index old_src;
- Vertex::Index old_dst;
- std::vector<Extent> tmp_extents;
-};
-
class DeltaDiffGenerator {
public:
// Represents a disk block on the install partition.
@@ -70,32 +55,21 @@
};
// This is the only function that external users of the class should call.
- // old_image and new_image are paths to two image files. They should be
- // mounted read-only at paths old_root and new_root respectively.
- // {old,new}_kernel_part are paths to the old and new kernel partition
- // images, respectively.
- // private_key_path points to a private key used to sign the update.
+ // The |config| describes the payload generation request, describing both
+ // old and new images for delta payloads and only the new image for full
+ // payloads.
+ // For delta payloads, the images should be already mounted read-only at
+ // the respective rootfs_mountpt.
+ // |private_key_path| points to a private key used to sign the update.
// Pass empty string to not sign the update.
- // output_path is the filename where the delta update should be written.
- // If |chunk_size| is not -1, the delta payload is generated based on
- // |chunk_size| chunks rather than whole files.
+ // |output_path| is the filename where the delta update should be written.
// This method computes scratch space based on |rootfs_partition_size|.
- // |minor_version| indicates the payload minor version for a delta update.
// Returns true on success. Also writes the size of the metadata into
// |metadata_size|.
- static bool GenerateDeltaUpdateFile(const std::string& old_root,
- const std::string& old_image,
- const std::string& new_root,
- const std::string& new_image,
- const std::string& old_kernel_part,
- const std::string& new_kernel_part,
+ static bool GenerateDeltaUpdateFile(const PayloadGenerationConfig& config,
const std::string& output_path,
const std::string& private_key_path,
- off_t chunk_size,
size_t rootfs_partition_size,
- uint32_t minor_version,
- const ImageInfo* old_image_info,
- const ImageInfo* new_image_info,
uint64_t* metadata_size);
// These functions are public so that the unit tests can access them:
diff --git a/payload_generator/full_update_generator.cc b/payload_generator/full_update_generator.cc
index f1d6b23..c634a70 100644
--- a/payload_generator/full_update_generator.cc
+++ b/payload_generator/full_update_generator.cc
@@ -11,8 +11,8 @@
#include <deque>
#include <memory>
-#include <base/strings/string_util.h>
#include <base/strings/stringprintf.h>
+#include <base/strings/string_util.h>
#include "update_engine/bzip.h"
#include "update_engine/utils.h"
@@ -108,31 +108,23 @@
} // namespace
bool FullUpdateGenerator::Run(
- Graph* graph,
- const string& new_kernel_part,
- const string& new_image,
- off_t image_size,
+ const PayloadGenerationConfig& config,
int fd,
off_t* data_file_size,
- off_t chunk_size,
- off_t block_size,
+ Graph* graph,
vector<DeltaArchiveManifest_InstallOperation>* kernel_ops,
vector<Vertex::Index>* final_order) {
- TEST_AND_RETURN_FALSE(chunk_size > 0);
- TEST_AND_RETURN_FALSE((chunk_size % block_size) == 0);
+ TEST_AND_RETURN_FALSE(config.Validate());
+ // FullUpdateGenerator requires a positive chuck_size, otherwise there will
+ // be only one operation with the whole partition which should not be allowed.
+ TEST_AND_RETURN_FALSE(config.chunk_size > 0);
+ const ImageConfig& target = config.target; // Shortcut.
size_t max_threads = std::max(sysconf(_SC_NPROCESSORS_ONLN), 4L);
LOG(INFO) << "Max threads: " << max_threads;
- // Get the sizes early in the function, so we can fail fast if the user
- // passed us bad paths.
- TEST_AND_RETURN_FALSE(image_size >= 0 &&
- image_size <= utils::FileSize(new_image));
- const off_t kernel_size = utils::FileSize(new_kernel_part);
- TEST_AND_RETURN_FALSE(kernel_size >= 0);
-
- off_t part_sizes[] = { image_size, kernel_size };
- string paths[] = { new_image, new_kernel_part };
+ uint64_t part_sizes[] = { target.rootfs_size, target.kernel_size };
+ string paths[] = { target.rootfs_part, target.kernel_part };
for (int partition = 0; partition < 2; ++partition) {
const string& path = paths[partition];
@@ -148,11 +140,11 @@
while (threads.size() < max_threads && bytes_left > 0) {
shared_ptr<ChunkProcessor> processor(
new ChunkProcessor(in_fd, offset,
- std::min(bytes_left, chunk_size)));
+ std::min(bytes_left, config.chunk_size)));
threads.push_back(processor);
TEST_AND_RETURN_FALSE(processor->Start());
- bytes_left -= chunk_size;
- offset += chunk_size;
+ bytes_left -= config.chunk_size;
+ offset += config.chunk_size;
}
// Need to wait for a chunk processor to complete and process its output
@@ -185,8 +177,8 @@
*data_file_size += use_buf.size();
op->set_data_length(use_buf.size());
Extent* dst_extent = op->add_dst_extents();
- dst_extent->set_start_block(processor->offset() / block_size);
- dst_extent->set_num_blocks(chunk_size / block_size);
+ dst_extent->set_start_block(processor->offset() / config.block_size);
+ dst_extent->set_num_blocks(config.chunk_size / config.block_size);
int progress = static_cast<int>(
(processor->offset() + processor->buffer_in().size()) * 100.0 /
diff --git a/payload_generator/full_update_generator.h b/payload_generator/full_update_generator.h
index 888fb6d..b741163 100644
--- a/payload_generator/full_update_generator.h
+++ b/payload_generator/full_update_generator.h
@@ -5,32 +5,26 @@
#ifndef UPDATE_ENGINE_PAYLOAD_GENERATOR_FULL_UPDATE_GENERATOR_H_
#define UPDATE_ENGINE_PAYLOAD_GENERATOR_FULL_UPDATE_GENERATOR_H_
-#include <glib.h>
-
#include <string>
#include <vector>
#include "update_engine/payload_generator/graph_types.h"
+#include "update_engine/payload_generator/payload_generation_config.h"
namespace chromeos_update_engine {
class FullUpdateGenerator {
public:
- // Given a new rootfs and kernel (|new_image|, |new_kernel_part|), reads them
- // sequentially, creating a full update of chunk_size chunks. Populates
- // |graph|, |kernel_ops|, and |final_order|, with data about the update
- // operations, and writes relevant data to |fd|, updating |data_file_size| as
- // it does. Only the first |image_size| bytes are read from |new_image|
- // assuming that this is the actual file system.
+ // Creates a full update for the target image defined in |config|. |config|
+ // must be a valid payload generation configuration for a full payload.
+ // Populates |graph|, |kernel_ops|, and |final_order|, with data about the
+ // update operations, and writes relevant data to |data_file_fd|, updating
+ // |data_file_size| as it does.
static bool Run(
- Graph* graph,
- const std::string& new_kernel_part,
- const std::string& new_image,
- off_t image_size,
- int fd,
+ const PayloadGenerationConfig& config,
+ int data_file_fd,
off_t* data_file_size,
- off_t chunk_size,
- off_t block_size,
+ Graph* graph,
std::vector<DeltaArchiveManifest_InstallOperation>* kernel_ops,
std::vector<Vertex::Index>* final_order);
diff --git a/payload_generator/full_update_generator_unittest.cc b/payload_generator/full_update_generator_unittest.cc
index 9f4aad5..c43ec49 100644
--- a/payload_generator/full_update_generator_unittest.cc
+++ b/payload_generator/full_update_generator_unittest.cc
@@ -9,6 +9,7 @@
#include <gtest/gtest.h>
+#include "update_engine/delta_performer.h"
#include "update_engine/test_utils.h"
using chromeos_update_engine::test_utils::FillWithData;
@@ -17,36 +18,42 @@
namespace chromeos_update_engine {
-namespace {
- const size_t kBlockSize = 4096;
-} // namespace
+class FullUpdateGeneratorTest : public ::testing::Test {
+ protected:
+ void SetUp() override {
+ config_.is_delta = false;
+ config_.minor_version = DeltaPerformer::kFullPayloadMinorVersion;
+ config_.chunk_size = 128 * 1024;
+ config_.block_size = 4096;
+ }
+
+ PayloadGenerationConfig config_;
+};
-class FullUpdateGeneratorTest : public ::testing::Test { };
-
-
-TEST(FullUpdateGeneratorTest, RunTest) {
+TEST_F(FullUpdateGeneratorTest, RunTest) {
chromeos::Blob new_root(20 * 1024 * 1024);
chromeos::Blob new_kern(16 * 1024 * 1024);
- const off_t kChunkSize = 128 * 1024;
FillWithData(&new_root);
FillWithData(&new_kern);
+
// Assume hashes take 2 MiB beyond the rootfs.
- off_t new_rootfs_size = new_root.size() - 2 * 1024 * 1024;
+ config_.target.rootfs_size = new_root.size() - 2 * 1024 * 1024;
+ config_.target.kernel_size = new_kern.size();
- string new_root_path;
EXPECT_TRUE(utils::MakeTempFile("NewFullUpdateTest_R.XXXXXX",
- &new_root_path,
+ &config_.target.rootfs_part,
nullptr));
- ScopedPathUnlinker new_root_path_unlinker(new_root_path);
- EXPECT_TRUE(test_utils::WriteFileVector(new_root_path, new_root));
+ ScopedPathUnlinker rootfs_part_unlinker(config_.target.rootfs_part);
+ EXPECT_TRUE(test_utils::WriteFileVector(config_.target.rootfs_part,
+ new_root));
- string new_kern_path;
EXPECT_TRUE(utils::MakeTempFile("NewFullUpdateTest_K.XXXXXX",
- &new_kern_path,
+ &config_.target.kernel_part,
nullptr));
- ScopedPathUnlinker new_kern_path_unlinker(new_kern_path);
- EXPECT_TRUE(test_utils::WriteFileVector(new_kern_path, new_kern));
+ ScopedPathUnlinker kernel_path_unlinker(config_.target.kernel_part);
+ EXPECT_TRUE(test_utils::WriteFileVector(config_.target.kernel_part,
+ new_kern));
string out_blobs_path;
int out_blobs_fd;
@@ -57,30 +64,27 @@
ScopedFdCloser out_blobs_fd_closer(&out_blobs_fd);
off_t out_blobs_length = 0;
-
Graph graph;
vector<DeltaArchiveManifest_InstallOperation> kernel_ops;
vector<Vertex::Index> final_order;
- EXPECT_TRUE(FullUpdateGenerator::Run(&graph,
- new_kern_path,
- new_root_path,
- new_rootfs_size,
+ EXPECT_TRUE(FullUpdateGenerator::Run(config_,
out_blobs_fd,
&out_blobs_length,
- kChunkSize,
- kBlockSize,
+ &graph,
&kernel_ops,
&final_order));
- EXPECT_EQ(new_rootfs_size / kChunkSize, graph.size());
- EXPECT_EQ(new_rootfs_size / kChunkSize, final_order.size());
- EXPECT_EQ(new_kern.size() / kChunkSize, kernel_ops.size());
- for (off_t i = 0; i < (new_rootfs_size / kChunkSize); ++i) {
+ int64_t target_rootfs_chucks =
+ config_.target.rootfs_size / config_.chunk_size;
+ EXPECT_EQ(target_rootfs_chucks, graph.size());
+ EXPECT_EQ(target_rootfs_chucks, final_order.size());
+ EXPECT_EQ(new_kern.size() / config_.chunk_size, kernel_ops.size());
+ for (off_t i = 0; i < target_rootfs_chucks; ++i) {
EXPECT_EQ(i, final_order[i]);
EXPECT_EQ(1, graph[i].op.dst_extents_size());
- EXPECT_EQ(i * kChunkSize / kBlockSize,
+ EXPECT_EQ(i * config_.chunk_size / config_.block_size,
graph[i].op.dst_extents(0).start_block()) << "i = " << i;
- EXPECT_EQ(kChunkSize / kBlockSize,
+ EXPECT_EQ(config_.chunk_size / config_.block_size,
graph[i].op.dst_extents(0).num_blocks());
if (graph[i].op.type() !=
DeltaArchiveManifest_InstallOperation_Type_REPLACE) {
diff --git a/payload_generator/generate_delta_main.cc b/payload_generator/generate_delta_main.cc
index 9f7c6ae..0403d73 100644
--- a/payload_generator/generate_delta_main.cc
+++ b/payload_generator/generate_delta_main.cc
@@ -20,6 +20,7 @@
#include "update_engine/delta_performer.h"
#include "update_engine/payload_generator/delta_diff_generator.h"
+#include "update_engine/payload_generator/payload_generation_config.h"
#include "update_engine/payload_generator/payload_signer.h"
#include "update_engine/payload_verifier.h"
#include "update_engine/prefs.h"
@@ -253,7 +254,7 @@
DEFINE_int64(rootfs_partition_size,
chromeos_update_engine::kRootFSPartitionSize,
"RootFS partition size for the image once installed");
- DEFINE_int32(minor_version, 0,
+ DEFINE_int32(minor_version, DeltaPerformer::kFullPayloadMinorVersion,
"The minor version of the payload being generated");
DEFINE_string(old_channel, "",
@@ -337,14 +338,33 @@
FLAGS_prefs_dir);
return 0;
}
- CHECK(!FLAGS_new_image.empty());
+
+ // A payload generation was requested. Convert the flags to a
+ // PayloadGenerationConfig.
+ PayloadGenerationConfig payload_config;
+ payload_config.source.rootfs_part = FLAGS_old_image;
+ payload_config.source.rootfs_mountpt = FLAGS_old_dir;
+ payload_config.source.kernel_part = FLAGS_old_kernel;
+
+ payload_config.target.rootfs_part = FLAGS_new_image;
+ payload_config.target.rootfs_mountpt = FLAGS_new_dir;
+ payload_config.target.kernel_part = FLAGS_new_kernel;
+
+ payload_config.chunk_size = FLAGS_chunk_size;
+ payload_config.block_size = kBlockSize;
+
+ // The kernel and rootfs size is never passed to the delta_generator, so we
+ // need to detect those from the provided files.
+ if (!FLAGS_old_image.empty()) {
+ CHECK(payload_config.source.LoadImageSize());
+ }
+ if (!FLAGS_new_image.empty()) {
+ CHECK(payload_config.target.LoadImageSize());
+ }
+
+ payload_config.is_delta = !FLAGS_old_image.empty();
+
CHECK(!FLAGS_out_file.empty());
- CHECK(!FLAGS_new_kernel.empty());
-
- bool is_delta = !FLAGS_old_image.empty();
-
- ImageInfo old_image_info;
- ImageInfo new_image_info;
// Ignore failures. These are optional arguments.
ParseImageInfo(FLAGS_new_channel,
@@ -353,7 +373,7 @@
FLAGS_new_key,
FLAGS_new_build_channel,
FLAGS_new_build_version,
- &new_image_info);
+ &payload_config.target.image_info);
// Ignore failures. These are optional arguments.
ParseImageInfo(FLAGS_old_channel,
@@ -362,64 +382,48 @@
FLAGS_old_key,
FLAGS_old_build_channel,
FLAGS_old_build_version,
- &old_image_info);
+ &payload_config.source.image_info);
- if (is_delta) {
- LOG(INFO) << "Generating delta update";
- CHECK(!FLAGS_old_dir.empty());
- CHECK(!FLAGS_new_dir.empty());
- if (!utils::IsDir(FLAGS_old_dir.c_str()) ||
- !utils::IsDir(FLAGS_new_dir.c_str())) {
- LOG(FATAL) << "old_dir or new_dir not directory";
- }
- } else {
- LOG(INFO) << "Generating full update";
- }
-
+ payload_config.minor_version = FLAGS_minor_version;
// Look for the minor version in the old image if it was not given as an
// argument.
- if (!CommandLine::ForCurrentProcess()->HasSwitch("minor_version")) {
+ if (payload_config.is_delta &&
+ !CommandLine::ForCurrentProcess()->HasSwitch("minor_version")) {
uint32_t minor_version;
base::FilePath image_path(FLAGS_old_dir);
base::FilePath conf_loc("etc/update_engine.conf");
base::FilePath conf_path = image_path.Append(conf_loc);
if (utils::GetMinorVersion(conf_path, &minor_version)) {
- FLAGS_minor_version = minor_version;
+ payload_config.minor_version = minor_version;
} else {
- if (is_delta) {
- FLAGS_minor_version = kInPlaceMinorPayloadVersion;
- } else {
- FLAGS_minor_version = DeltaPerformer::kFullPayloadMinorVersion;
- }
+ payload_config.minor_version = kInPlaceMinorPayloadVersion;
}
}
- if (static_cast<uint32_t>(FLAGS_minor_version) ==
- kSourceMinorPayloadVersion ||
- static_cast<uint32_t>(FLAGS_minor_version) ==
- kInPlaceMinorPayloadVersion ||
- static_cast<uint32_t>(FLAGS_minor_version) ==
- DeltaPerformer::kFullPayloadMinorVersion) {
- uint64_t metadata_size;
- if (!DeltaDiffGenerator::GenerateDeltaUpdateFile(
- FLAGS_old_dir,
- FLAGS_old_image,
- FLAGS_new_dir,
- FLAGS_new_image,
- FLAGS_old_kernel,
- FLAGS_new_kernel,
- FLAGS_out_file,
- FLAGS_private_key,
- FLAGS_chunk_size,
- FLAGS_rootfs_partition_size,
- FLAGS_minor_version,
- is_delta ? &old_image_info : nullptr,
- &new_image_info,
- &metadata_size)) {
- return 1;
- }
+ // Full payloads use a hard-coded chunk_size of 1 MiB.
+ if (!payload_config.is_delta) {
+ payload_config.chunk_size = 1024 * 1024;
+ }
+
+ if (payload_config.is_delta) {
+ LOG(INFO) << "Generating delta update";
} else {
- LOG(FATAL) << "Unsupported minor payload version: " << FLAGS_minor_version;
+ LOG(INFO) << "Generating full update";
+ }
+
+ // From this point, all the options have been parsed.
+ if (!payload_config.Validate()) {
+ LOG(FATAL) << "Invalid options passed. See errors above.";
+ }
+
+ uint64_t metadata_size;
+ if (!DeltaDiffGenerator::GenerateDeltaUpdateFile(
+ payload_config,
+ FLAGS_out_file,
+ FLAGS_private_key,
+ FLAGS_rootfs_partition_size,
+ &metadata_size)) {
+ return 1;
}
return 0;
diff --git a/payload_generator/inplace_generator.h b/payload_generator/inplace_generator.h
index 73a791b..a120196 100644
--- a/payload_generator/inplace_generator.h
+++ b/payload_generator/inplace_generator.h
@@ -18,6 +18,22 @@
namespace chromeos_update_engine {
+// This struct stores all relevant info for an edge that is cut between
+// nodes old_src -> old_dst by creating new vertex new_vertex. The new
+// relationship is:
+// old_src -(read before)-> new_vertex <-(write before)- old_dst
+// new_vertex is a MOVE operation that moves some existing blocks into
+// temp space. The temp extents are, by necessity, stored in new_vertex
+// (as dst extents) and old_dst (as src extents), but they are also broken
+// out into tmp_extents, as the nodes themselves may contain many more
+// extents.
+struct CutEdgeVertexes {
+ Vertex::Index new_vertex;
+ Vertex::Index old_src;
+ Vertex::Index old_dst;
+ std::vector<Extent> tmp_extents;
+};
+
class InplaceGenerator {
public:
// Modifies blocks read by 'op' so that any blocks referred to by
diff --git a/payload_generator/payload_generation_config.cc b/payload_generator/payload_generation_config.cc
new file mode 100644
index 0000000..8194970
--- /dev/null
+++ b/payload_generator/payload_generation_config.cc
@@ -0,0 +1,115 @@
+// Copyright 2015 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "update_engine/payload_generator/payload_generation_config.h"
+
+#include "update_engine/delta_performer.h"
+#include "update_engine/payload_generator/delta_diff_generator.h"
+#include "update_engine/utils.h"
+
+namespace chromeos_update_engine {
+
+bool ImageConfig::ValidateRootfsExists() const {
+ TEST_AND_RETURN_FALSE(!rootfs_part.empty());
+ TEST_AND_RETURN_FALSE(utils::FileExists(rootfs_part.c_str()));
+ TEST_AND_RETURN_FALSE(rootfs_size > 0);
+ // The requested size is within the limits of the file.
+ TEST_AND_RETURN_FALSE(static_cast<off_t>(rootfs_size) <=
+ utils::FileSize(rootfs_part.c_str()));
+ return true;
+}
+
+bool ImageConfig::ValidateKernelExists() const {
+ TEST_AND_RETURN_FALSE(!kernel_part.empty());
+ TEST_AND_RETURN_FALSE(utils::FileExists(kernel_part.c_str()));
+ TEST_AND_RETURN_FALSE(kernel_size > 0);
+ TEST_AND_RETURN_FALSE(static_cast<off_t>(kernel_size) <=
+ utils::FileSize(kernel_part.c_str()));
+ return true;
+}
+
+bool ImageConfig::ValidateIsEmpty() const {
+ TEST_AND_RETURN_FALSE(ImageInfoIsEmpty());
+
+ TEST_AND_RETURN_FALSE(rootfs_part.empty());
+ TEST_AND_RETURN_FALSE(rootfs_size == 0);
+ TEST_AND_RETURN_FALSE(rootfs_mountpt.empty());
+ TEST_AND_RETURN_FALSE(kernel_part.empty());
+ TEST_AND_RETURN_FALSE(kernel_size == 0);
+ return true;
+}
+
+bool ImageConfig::LoadImageSize() {
+ TEST_AND_RETURN_FALSE(!rootfs_part.empty());
+ int rootfs_block_count, rootfs_block_size;
+ TEST_AND_RETURN_FALSE(utils::GetFilesystemSize(rootfs_part,
+ &rootfs_block_count,
+ &rootfs_block_size));
+ rootfs_size = static_cast<size_t>(rootfs_block_count) * rootfs_block_size;
+ if (!kernel_part.empty())
+ kernel_size = utils::FileSize(kernel_part);
+
+ // TODO(deymo): The delta generator algorithm doesn't support a block size
+ // different than 4 KiB. Remove this check once that's fixed. crbug.com/455045
+ if (rootfs_block_size != 4096) {
+ LOG(ERROR) << "The filesystem provided in " << rootfs_part
+ << " has a block size of " << rootfs_block_size
+ << " but delta_generator only supports 4096.";
+ return false;
+ }
+ return true;
+}
+
+bool ImageConfig::ImageInfoIsEmpty() const {
+ return image_info.board().empty()
+ && image_info.key().empty()
+ && image_info.channel().empty()
+ && image_info.version().empty()
+ && image_info.build_channel().empty()
+ && image_info.build_version().empty();
+}
+
+bool PayloadGenerationConfig::Validate() const {
+ if (is_delta) {
+ TEST_AND_RETURN_FALSE(source.ValidateRootfsExists());
+ TEST_AND_RETURN_FALSE(source.rootfs_size % block_size == 0);
+
+ if (!source.kernel_part.empty()) {
+ TEST_AND_RETURN_FALSE(source.ValidateKernelExists());
+ TEST_AND_RETURN_FALSE(source.kernel_size % block_size == 0);
+ }
+
+ // For deltas, we also need to check that the image is mounted in order to
+ // inspect the contents.
+ TEST_AND_RETURN_FALSE(!source.rootfs_mountpt.empty());
+ TEST_AND_RETURN_FALSE(utils::IsDir(source.rootfs_mountpt.c_str()));
+
+ TEST_AND_RETURN_FALSE(!target.rootfs_mountpt.empty());
+ TEST_AND_RETURN_FALSE(utils::IsDir(target.rootfs_mountpt.c_str()));
+
+ // Check for the supported minor_version values.
+ TEST_AND_RETURN_FALSE(minor_version == kInPlaceMinorPayloadVersion ||
+ minor_version == kSourceMinorPayloadVersion);
+
+ // If new_image_info is present, old_image_info must be present.
+ TEST_AND_RETURN_FALSE(source.ImageInfoIsEmpty() ==
+ target.ImageInfoIsEmpty());
+ } else {
+ // All the "source" image fields must be empty for full payloads.
+ TEST_AND_RETURN_FALSE(source.ValidateIsEmpty());
+ TEST_AND_RETURN_FALSE(minor_version ==
+ DeltaPerformer::kFullPayloadMinorVersion);
+ }
+
+ // In all cases, the target image must exists.
+ TEST_AND_RETURN_FALSE(target.ValidateRootfsExists());
+ TEST_AND_RETURN_FALSE(target.ValidateKernelExists());
+ TEST_AND_RETURN_FALSE(target.rootfs_size % block_size == 0);
+ TEST_AND_RETURN_FALSE(target.kernel_size % block_size == 0);
+
+ TEST_AND_RETURN_FALSE(chunk_size == -1 || chunk_size % block_size == 0);
+ return true;
+}
+
+} // namespace chromeos_update_engine
diff --git a/payload_generator/payload_generation_config.h b/payload_generator/payload_generation_config.h
new file mode 100644
index 0000000..1fd8aae
--- /dev/null
+++ b/payload_generator/payload_generation_config.h
@@ -0,0 +1,103 @@
+// Copyright 2015 The Chromium OS Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef UPDATE_ENGINE_PAYLOAD_GENERATOR_PAYLOAD_GENERATION_CONFIG_H_
+#define UPDATE_ENGINE_PAYLOAD_GENERATOR_PAYLOAD_GENERATION_CONFIG_H_
+
+#include <cstddef>
+
+#include <string>
+#include <vector>
+
+#include "update_engine/update_metadata.pb.h"
+
+namespace chromeos_update_engine {
+
+// The ImageConfig struct describes a pair of binaries kernel and rootfs and the
+// metadata associated with the image they are part of, like build number, size,
+// etc.
+struct ImageConfig {
+ // Returns whether the ImageConfig is an empty image.
+ bool ValidateIsEmpty() const;
+
+ // Returns whether the ImageConfig is not an empty image and all the rootfs
+ // or kernel fields are set correctly. Source images are allowed to have an
+ // empty kernel_part meaning that the delta update ships a full kernel.
+ bool ValidateRootfsExists() const;
+ bool ValidateKernelExists() const;
+
+ // Load |rootfs_size| and |kernel_size| from the respective image files. For
+ // the kernel, the whole |kernel_part| file is assumed. For the rootfs, the
+ // size is detected from the filesystem.
+ // Returns whether the image size was properly detected.
+ bool LoadImageSize();
+
+ // Returns whether the |image_info| field is empty.
+ bool ImageInfoIsEmpty() const;
+
+ // The ImageInfo message defined in the update_metadata.proto file describes
+ // the metadata of the image.
+ ImageInfo image_info;
+
+ // The path to the rootfs partition. This can be a regular file or a block
+ // device such as a loop device.
+ std::string rootfs_part;
+
+ // The size of the filesystem in rootfs_part. If rootfs verification is used
+ // (verity) this value should match the size of the verity device. This can be
+ // smaller than the partition and is the size of the data update_engine
+ // assumes verified for the source image, and the size of that data it
+ // should generate for the target image.
+ uint64_t rootfs_size = 0;
+
+ // The mount point where the rootfs_part is mounted. This is required for
+ // delta payloads that iterate the filesystem using the kernel API.
+ std::string rootfs_mountpt;
+
+ // The path to the kernel partition. This can be a regular file or a block
+ // device such as a loop device.
+ std::string kernel_part;
+
+ // The size of the verified part of the kernel partition. This is normally the
+ // whole partition.
+ uint64_t kernel_size = 0;
+};
+
+// The PayloadGenerationConfig struct encapsulates all the configuration to
+// build the requested payload. This includes information about the old and new
+// image as well as the restrictions applied to the payload (like minor-version
+// and full/delta payload).
+struct PayloadGenerationConfig {
+ // Returns whether the PayloadGenerationConfig is valid.
+ bool Validate() const;
+
+ // Image information about the new image that's the target of this payload.
+ ImageConfig target;
+
+ // Image information pertaining the old image, if any. This is only valid
+ // if is_full is false, so we are requested a delta payload.
+ ImageConfig source;
+
+ // Wheter the requested payload is a delta payload.
+ bool is_delta = false;
+
+ // The minor_version of the requested payload.
+ uint32_t minor_version;
+
+ // The chunk size is the maximum size that a single operation should write in
+ // the destination. Operations bigger than chunk_size should be split. A value
+ // of -1 means no chunk_size limit.
+ off_t chunk_size = -1;
+
+ // TODO(deymo): Remove the block_size member and maybe replace it with a
+ // minimum alignment size for blocks (if needed). Algorithms shold be able to
+ // pick the block_size they want, but for now only 4 KiB is supported.
+
+ // The block size used for all the operations in the manifest.
+ size_t block_size = 4096;
+};
+
+} // namespace chromeos_update_engine
+
+#endif // UPDATE_ENGINE_PAYLOAD_GENERATOR_PAYLOAD_GENERATION_CONFIG_H_
diff --git a/update_engine.gyp b/update_engine.gyp
index b0f5110..e6add56 100644
--- a/update_engine.gyp
+++ b/update_engine.gyp
@@ -283,6 +283,7 @@
'payload_generator/graph_utils.cc',
'payload_generator/inplace_generator.cc',
'payload_generator/metadata.cc',
+ 'payload_generator/payload_generation_config.cc',
'payload_generator/payload_signer.cc',
'payload_generator/tarjan.cc',
'payload_generator/topological_sort.cc',