Alex Deymo | aea4c1c | 2015-08-19 20:24:43 -0700 | [diff] [blame] | 1 | // |
| 2 | // Copyright (C) 2012 The Android Open Source Project |
| 3 | // |
| 4 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | // you may not use this file except in compliance with the License. |
| 6 | // You may obtain a copy of the License at |
| 7 | // |
| 8 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | // |
| 10 | // Unless required by applicable law or agreed to in writing, software |
| 11 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | // See the License for the specific language governing permissions and |
| 14 | // limitations under the License. |
| 15 | // |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 16 | |
Alex Deymo | 161c4a1 | 2014-05-16 15:56:21 -0700 | [diff] [blame] | 17 | #include "update_engine/payload_generator/full_update_generator.h" |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 18 | |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 19 | #include <fcntl.h> |
Alex Deymo | 161c4a1 | 2014-05-16 15:56:21 -0700 | [diff] [blame] | 20 | #include <inttypes.h> |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 21 | |
Alex Vakulenko | 072359c | 2014-07-18 11:41:07 -0700 | [diff] [blame] | 22 | #include <algorithm> |
| 23 | #include <deque> |
Alex Deymo | bc91a27 | 2014-05-20 16:45:33 -0700 | [diff] [blame] | 24 | #include <memory> |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 25 | |
Alex Deymo | de942f3 | 2015-03-13 11:57:15 -0700 | [diff] [blame] | 26 | #include <base/format_macros.h> |
Alex Deymo | f1cbe17 | 2015-03-05 15:58:37 -0800 | [diff] [blame] | 27 | #include <base/strings/string_util.h> |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 28 | #include <base/strings/stringprintf.h> |
| 29 | #include <base/synchronization/lock.h> |
| 30 | #include <base/threading/simple_thread.h> |
Alex Vakulenko | 3f39d5c | 2015-10-13 09:27:13 -0700 | [diff] [blame] | 31 | #include <brillo/secure_blob.h> |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 32 | |
Alex Deymo | 39910dc | 2015-11-09 17:04:30 -0800 | [diff] [blame] | 33 | #include "update_engine/common/utils.h" |
Alex Deymo | 0bc2611 | 2015-10-19 20:54:57 -0700 | [diff] [blame] | 34 | #include "update_engine/payload_generator/bzip.h" |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 35 | |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 36 | using std::vector; |
| 37 | |
| 38 | namespace chromeos_update_engine { |
| 39 | |
| 40 | namespace { |
| 41 | |
Alex Deymo | de942f3 | 2015-03-13 11:57:15 -0700 | [diff] [blame] | 42 | const size_t kDefaultFullChunkSize = 1024 * 1024; // 1 MiB |
| 43 | |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 44 | // This class encapsulates a full update chunk processing thread work. The |
| 45 | // processor reads a chunk of data from the input file descriptor and compresses |
| 46 | // it. The processor will destroy itself when the work is done. |
| 47 | class ChunkProcessor : public base::DelegateSimpleThread::Delegate { |
| 48 | public: |
| 49 | // Read a chunk of |size| bytes from |fd| starting at offset |offset|. |
| 50 | ChunkProcessor(int fd, off_t offset, size_t size, |
| 51 | BlobFileWriter* blob_file, AnnotatedOperation* aop) |
| 52 | : fd_(fd), |
| 53 | offset_(offset), |
| 54 | size_(size), |
| 55 | blob_file_(blob_file), |
| 56 | aop_(aop) {} |
| 57 | // We use a default move constructor since all the data members are POD types. |
| 58 | ChunkProcessor(ChunkProcessor&&) = default; |
| 59 | ~ChunkProcessor() override = default; |
| 60 | |
| 61 | // Overrides DelegateSimpleThread::Delegate. |
| 62 | // Run() handles the read from |fd| in a thread-safe way, and stores the |
| 63 | // new operation to generate the region starting at |offset| of size |size| |
| 64 | // in the output operation |aop|. The associated blob data is stored in |
| 65 | // |blob_fd| and |blob_file_size| is updated. |
| 66 | void Run() override; |
| 67 | |
| 68 | private: |
| 69 | bool ProcessChunk(); |
| 70 | |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 71 | // Work parameters. |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 72 | int fd_; |
| 73 | off_t offset_; |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 74 | size_t size_; |
| 75 | BlobFileWriter* blob_file_; |
| 76 | AnnotatedOperation* aop_; |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 77 | |
| 78 | DISALLOW_COPY_AND_ASSIGN(ChunkProcessor); |
| 79 | }; |
| 80 | |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 81 | void ChunkProcessor::Run() { |
| 82 | if (!ProcessChunk()) { |
| 83 | LOG(ERROR) << "Error processing region at " << offset_ << " of size " |
| 84 | << size_; |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 85 | } |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 86 | } |
| 87 | |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 88 | bool ChunkProcessor::ProcessChunk() { |
Alex Vakulenko | 3f39d5c | 2015-10-13 09:27:13 -0700 | [diff] [blame] | 89 | brillo::Blob buffer_in_(size_); |
| 90 | brillo::Blob op_blob; |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 91 | ssize_t bytes_read = -1; |
| 92 | TEST_AND_RETURN_FALSE(utils::PReadAll(fd_, |
| 93 | buffer_in_.data(), |
| 94 | buffer_in_.size(), |
| 95 | offset_, |
| 96 | &bytes_read)); |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 97 | TEST_AND_RETURN_FALSE(bytes_read == static_cast<ssize_t>(size_)); |
| 98 | TEST_AND_RETURN_FALSE(BzipCompress(buffer_in_, &op_blob)); |
| 99 | |
Alex Deymo | a12ee11 | 2015-08-12 22:19:32 -0700 | [diff] [blame] | 100 | InstallOperation_Type op_type = InstallOperation::REPLACE_BZ; |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 101 | |
| 102 | if (op_blob.size() >= buffer_in_.size()) { |
| 103 | // A REPLACE is cheaper than a REPLACE_BZ in this case. |
| 104 | op_blob = std::move(buffer_in_); |
Alex Deymo | a12ee11 | 2015-08-12 22:19:32 -0700 | [diff] [blame] | 105 | op_type = InstallOperation::REPLACE; |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 106 | } |
| 107 | |
Sen Jiang | 5456c19 | 2015-08-19 15:01:16 -0700 | [diff] [blame] | 108 | TEST_AND_RETURN_FALSE(aop_->SetOperationBlob(&op_blob, blob_file_)); |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 109 | aop_->op.set_type(op_type); |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 110 | return true; |
| 111 | } |
| 112 | |
| 113 | } // namespace |
| 114 | |
Alex Deymo | 477aec2 | 2015-03-24 23:40:48 -0700 | [diff] [blame] | 115 | bool FullUpdateGenerator::GenerateOperations( |
Alex Deymo | f1cbe17 | 2015-03-05 15:58:37 -0800 | [diff] [blame] | 116 | const PayloadGenerationConfig& config, |
Sen Jiang | ebdf17d | 2015-08-19 11:53:27 -0700 | [diff] [blame] | 117 | const PartitionConfig& old_part, |
| 118 | const PartitionConfig& new_part, |
Sen Jiang | 8cc502d | 2015-08-10 10:04:54 -0700 | [diff] [blame] | 119 | BlobFileWriter* blob_file, |
Sen Jiang | ebdf17d | 2015-08-19 11:53:27 -0700 | [diff] [blame] | 120 | vector<AnnotatedOperation>* aops) { |
| 121 | TEST_AND_RETURN_FALSE(new_part.ValidateExists()); |
Alex Deymo | de942f3 | 2015-03-13 11:57:15 -0700 | [diff] [blame] | 122 | |
Alex Deymo | 477aec2 | 2015-03-24 23:40:48 -0700 | [diff] [blame] | 123 | // FullUpdateGenerator requires a positive chunk_size, otherwise there will |
Alex Deymo | f1cbe17 | 2015-03-05 15:58:37 -0800 | [diff] [blame] | 124 | // be only one operation with the whole partition which should not be allowed. |
Alex Deymo | 2d3b2d6 | 2015-07-17 17:34:36 -0700 | [diff] [blame] | 125 | // For performance reasons, we force a small default hard limit of 1 MiB. This |
| 126 | // limit can be changed in the config, and we will use the smaller of the two |
| 127 | // soft/hard limits. |
| 128 | size_t full_chunk_size; |
| 129 | if (config.hard_chunk_size >= 0) { |
| 130 | full_chunk_size = std::min(static_cast<size_t>(config.hard_chunk_size), |
| 131 | config.soft_chunk_size); |
Alex Deymo | de942f3 | 2015-03-13 11:57:15 -0700 | [diff] [blame] | 132 | } else { |
Alex Deymo | 2d3b2d6 | 2015-07-17 17:34:36 -0700 | [diff] [blame] | 133 | full_chunk_size = std::min(kDefaultFullChunkSize, config.soft_chunk_size); |
Alex Deymo | de942f3 | 2015-03-13 11:57:15 -0700 | [diff] [blame] | 134 | LOG(INFO) << "No chunk_size provided, using the default chunk_size for the " |
| 135 | << "full operations: " << full_chunk_size << " bytes."; |
| 136 | } |
| 137 | TEST_AND_RETURN_FALSE(full_chunk_size > 0); |
| 138 | TEST_AND_RETURN_FALSE(full_chunk_size % config.block_size == 0); |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 139 | |
Sen Jiang | ebdf17d | 2015-08-19 11:53:27 -0700 | [diff] [blame] | 140 | size_t chunk_blocks = full_chunk_size / config.block_size; |
Alex Deymo | f329b93 | 2014-10-30 01:37:48 -0700 | [diff] [blame] | 141 | size_t max_threads = std::max(sysconf(_SC_NPROCESSORS_ONLN), 4L); |
Sen Jiang | 625406c | 2015-09-16 16:35:23 -0700 | [diff] [blame] | 142 | LOG(INFO) << "Compressing partition " << new_part.name |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 143 | << " from " << new_part.path << " splitting in chunks of " |
Sen Jiang | ebdf17d | 2015-08-19 11:53:27 -0700 | [diff] [blame] | 144 | << chunk_blocks << " blocks (" << config.block_size |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 145 | << " bytes each) using " << max_threads << " threads"; |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 146 | |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 147 | int in_fd = open(new_part.path.c_str(), O_RDONLY, 0); |
| 148 | TEST_AND_RETURN_FALSE(in_fd >= 0); |
| 149 | ScopedFdCloser in_fd_closer(&in_fd); |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 150 | |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 151 | // We potentially have all the ChunkProcessors in memory but only |
| 152 | // |max_threads| will actually hold a block in memory while we process. |
Sen Jiang | ebdf17d | 2015-08-19 11:53:27 -0700 | [diff] [blame] | 153 | size_t partition_blocks = new_part.size / config.block_size; |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 154 | size_t num_chunks = (partition_blocks + chunk_blocks - 1) / chunk_blocks; |
| 155 | aops->resize(num_chunks); |
| 156 | vector<ChunkProcessor> chunk_processors; |
| 157 | chunk_processors.reserve(num_chunks); |
Sen Jiang | 8cc502d | 2015-08-10 10:04:54 -0700 | [diff] [blame] | 158 | blob_file->SetTotalBlobs(num_chunks); |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 159 | |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 160 | for (size_t i = 0; i < num_chunks; ++i) { |
| 161 | size_t start_block = i * chunk_blocks; |
| 162 | // The last chunk could be smaller. |
| 163 | size_t num_blocks = std::min(chunk_blocks, |
| 164 | partition_blocks - i * chunk_blocks); |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 165 | |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 166 | // Preset all the static information about the operations. The |
| 167 | // ChunkProcessor will set the rest. |
| 168 | AnnotatedOperation* aop = aops->data() + i; |
| 169 | aop->name = base::StringPrintf("<%s-operation-%" PRIuS ">", |
Sen Jiang | 625406c | 2015-09-16 16:35:23 -0700 | [diff] [blame] | 170 | new_part.name.c_str(), i); |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 171 | Extent* dst_extent = aop->op.add_dst_extents(); |
| 172 | dst_extent->set_start_block(start_block); |
| 173 | dst_extent->set_num_blocks(num_blocks); |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 174 | |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 175 | chunk_processors.emplace_back( |
| 176 | in_fd, |
Sen Jiang | ebdf17d | 2015-08-19 11:53:27 -0700 | [diff] [blame] | 177 | static_cast<off_t>(start_block) * config.block_size, |
| 178 | num_blocks * config.block_size, |
Sen Jiang | 8cc502d | 2015-08-10 10:04:54 -0700 | [diff] [blame] | 179 | blob_file, |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 180 | aop); |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 181 | } |
| 182 | |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 183 | // Thread pool used for worker threads. |
| 184 | base::DelegateSimpleThreadPool thread_pool("full-update-generator", |
| 185 | max_threads); |
| 186 | thread_pool.Start(); |
| 187 | for (ChunkProcessor& processor : chunk_processors) |
| 188 | thread_pool.AddWork(&processor); |
| 189 | thread_pool.JoinAll(); |
Sen Jiang | af1575f | 2015-11-09 13:34:16 -0800 | [diff] [blame] | 190 | |
| 191 | // All the work done, disable logging. |
| 192 | blob_file->SetTotalBlobs(0); |
| 193 | |
Alex Deymo | 7fad7b7 | 2015-07-21 22:22:47 -0700 | [diff] [blame] | 194 | // All the operations must have a type set at this point. Otherwise, a |
| 195 | // ChunkProcessor failed to complete. |
| 196 | for (const AnnotatedOperation& aop : *aops) { |
| 197 | if (!aop.op.has_type()) |
| 198 | return false; |
| 199 | } |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 200 | return true; |
| 201 | } |
| 202 | |
| 203 | } // namespace chromeos_update_engine |